Merge branch 'dev'

dev_scripts/cflow.py

@@ -36,7 +36,7 @@ class Result(Enum):
 def edit_pyc_lines(pyc: PYCFile, src_lines: list[str]):
     if pyc.version == (3, 10):
         pyc.replace_duplicated_returns10(src_lines)
-    elif pyc.version == (3, 12):
+    elif pyc.version >= (3, 12):
         pyc.replace_duplicated_returns12(src_lines)
     seen_lines = set()
     # multiple instructions can start the same lno, but the segmentation model will only assign the lno to the first one

dev_scripts/dataset_generation/bytecode2csv.py

@@ -115,7 +115,7 @@ def bytecode2csv(py_path: pathlib.Path, pyc_path: pathlib.Path) -> tuple[list, l
     pyc = PYCFile(str(pyc_path.resolve()))
     if pyc.version == (3, 10):
         pyc.replace_duplicated_returns10(py_path.read_text().split("\n"))
-    elif pyc.version == (3, 12):
+    elif pyc.version >= (3, 12):
         pyc.replace_duplicated_returns12(py_path.read_text().split("\n"))
     global_masker = create_global_masker(pyc)
 

dev_scripts/statement/tokenizer/special_tokens_map.json

@@ -1085,6 +1085,13 @@
       "normalized": false,
       "single_word": false
     },
+    {
+      "content": "FORMAT_WITH_SPEC",
+      "lstrip": false,
+      "rstrip": true,
+      "normalized": false,
+      "single_word": false
+    },
     {
       "content": "GEN_START",
       "lstrip": false,

pylingual/control_flow_reconstruction/cfg.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import itertools
 from typing import TYPE_CHECKING
 from pathlib import Path
 
@@ -75,6 +76,55 @@ class CFG(DiGraph_CFT):
         self._create_dominator_tree()
         return nx.dfs_postorder_nodes(self, source=self.start, sort_neighbors=lambda nodes: sorted(nodes, key=lambda x: x.offset, reverse=True))
 
+    def is_loop_header(self, node):
+        # Check all predecessors
+        for predecessor in self.predecessors(node):
+            # A back edge exists if the predecessor is dominated by this node
+            if self.dominates(node, predecessor):
+                return True
+        return False
+
+    def dfs_labeled_edges_no_loop(self, source=None, depth_limit=None, *, sort_neighbors=None):
+        if source is None:
+            # edges for all components
+            nodes = self
+        else:
+            # edges for components with source
+            nodes = [source]
+        if depth_limit is None:
+            depth_limit = len(self)
+
+        get_children = self.neighbors if sort_neighbors is None else lambda n: iter(sort_neighbors(self.neighbors(n)))
+
+        visited = set()
+        for start in nodes:
+            if start in visited:
+                continue
+            yield start, start, "forward"
+            visited.add(start)
+            stack = [(start, get_children(start))]
+            depth_now = 1
+            while stack:
+                parent, children = stack[-1]
+                for child in children:
+                    if child in visited or self.is_loop_header(child) or not all(p in visited for p in self.predecessors(child)):
+                        yield parent, child, "nontree"
+                    else:
+                        yield parent, child, "forward"
+                        visited.add(child)
+                        if depth_now < depth_limit:
+                            stack.append((child, iter(get_children(child))))
+                            depth_now += 1
+                            break
+                        else:
+                            yield parent, child, "reverse-depth_limit"
+                else:
+                    stack.pop()
+                    depth_now -= 1
+                    if stack:
+                        yield stack[-1][0], parent, "reverse"
+            yield start, start, "reverse"
+
     def apply_graphs(self):
         graphs = self.iteration_graphs.pop()
         if self.iteration_graphs:
@@ -130,3 +180,16 @@ class CFG(DiGraph_CFT):
             dot.write(out, prog=["neato", "-n"], format=CFG.graph_format)
         else:
             self.iteration_graphs[-1].append(dot.to_string())
+
+    def cdg(self) -> CFG:
+        pdt = nx.create_empty_copy(self)
+        pdt.add_edges_from((B, A) for A, B in nx.immediate_dominators(self.reverse(), self.end).items())
+        pdt.remove_edge(self.end, self.end)
+        pdr = nx.transitive_closure_dag(pdt)
+        postdominates = lambda A, B: pdr.has_edge(A, B) or A == B
+        control_dependent = lambda A, B: 0 < sum(postdominates(A, succ) for succ in self.successors(B)) < self.out_degree(B)
+        cdg = nx.create_empty_copy(self)
+        cdg.add_edges_from((B, A, {"kind": EdgeKind.Fall}) for A, B in itertools.product(self.nodes, self.nodes) if A != B and control_dependent(A, B))
+        cdg.remove_node(self.end)
+        cdg.add_edges_from(((self.start, n) for n in cdg.nodes if cdg.in_degree(n) == 0 and n != self.start), kind=EdgeKind.Fall)
+        return cdg

pylingual/control_flow_reconstruction/structure.py

@@ -1,5 +1,3 @@
-import pdb
-
 from pylingual.editable_bytecode import EditableBytecode
 from pylingual.editable_bytecode.control_flow_graph import bytecode_to_control_flow_graph
 
@@ -27,13 +25,8 @@ def structure_control_flow(cfg: nx.DiGraph, bytecode: EditableBytecode) -> Contr
     cfg = CFG.from_graph(cfg, bytecode)
     runs = get_template_runs(bytecode.version[:2])
 
-    try:
-        while len(cfg) > 1:
-            if not iteration(cfg, runs):
-                return MetaTemplate("\x1b[31mirreducible cflow\x1b[0m", bytecode.codeobj)
-    except Exception:
-        if hasattr(pdb, "xpm"):
-            pdb.xpm()  # type: ignore
-        raise
+    while len(cfg) > 1:
+        if not iteration(cfg, runs):
+            return MetaTemplate("\x1b[31mirreducible cflow\x1b[0m", bytecode.codeobj)
 
     return next(iter(cfg.nodes))

pylingual/control_flow_reconstruction/templates/Block.py

@@ -4,8 +4,10 @@ from typing import TYPE_CHECKING, override
 from itertools import chain
 from pylingual.editable_bytecode import Inst
 
+import networkx as nx
+
 from ..cft import ControlFlowTemplate, EdgeKind, SourceContext, SourceLine, register_template, EdgeCategory, out_edge_dict, MetaTemplate, indent_str
-from ..utils import E, N, T, defer_source_to, remove_nodes
+from ..utils import E, N, T, defer_source_to, remove_nodes, versions_from, without_instructions, has_no_lines, exact_instructions, make_try_match
 
 if TYPE_CHECKING:
     from pylingual.control_flow_reconstruction.cfg import CFG
@@ -37,6 +39,71 @@ class EndTemplate(ControlFlowTemplate):
     to_indented_source = defer_source_to("body")
 
 
+@register_template(3, 0)
+class RemoveUnreachable(ControlFlowTemplate):
+    @override
+    @classmethod
+    def try_match(cls, cfg, node) -> ControlFlowTemplate | None:
+        if node is not cfg.start:
+            return None
+
+        valid = list(nx.dfs_preorder_nodes(cfg, source=cfg.start))
+        invalid = [n for n in cfg.nodes if n not in valid]
+        if invalid:
+            cfg.remove_nodes_from(invalid)
+            return node
+
+
+@register_template(0, 0, (3, 13))
+class JumpTemplate(ControlFlowTemplate):
+    template = T(
+        body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")),
+        jump=N("tail", "block?")
+        .with_in_deg(1)
+        .with_cond(
+            exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"),
+            exact_instructions("POP_JUMP_IF_TRUE"),
+            exact_instructions("JUMP_FORWARD"),
+            exact_instructions("JUMP_BACKWARD"),
+            exact_instructions("POP_JUMP_IF_NOT_NONE"),
+            exact_instructions("POP_JUMP_IF_NONE"),
+            exact_instructions("POP_JUMP_IF_FALSE"),
+        ),
+        block=N.tail(),
+        tail=N.tail(),
+    )
+
+    try_match = make_try_match(
+        {
+            EdgeKind.Fall: "tail",
+            EdgeKind.TrueJump: "block",
+        },
+        "body",
+        "jump",
+    )
+
+    to_indented_source = defer_source_to("body")
+
+
+@register_template(0, 0, *versions_from(3, 11))
+class NopTemplate(ControlFlowTemplate):
+    template = T(
+        body=~N("nop", None).with_cond(without_instructions("CLEANUP_THROW")),
+        nop=N("tail", None).with_in_deg(1).with_cond(exact_instructions("NOP")).with_cond(has_no_lines),
+        tail=N.tail(),
+    )
+
+    try_match = make_try_match(
+        {
+            EdgeKind.Fall: "tail",
+        },
+        "body",
+        "nop",
+    )
+
+    to_indented_source = defer_source_to("body")
+
+
 @register_template(0, 20)
 @register_template(2, 20)
 class BlockTemplate(ControlFlowTemplate):

pylingual/control_flow_reconstruction/templates/CDG.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, override
+
+import networkx as nx
+
+from ..cft import ControlFlowTemplate, register_template
+
+if TYPE_CHECKING:
+    from pylingual.control_flow_reconstruction.cfg import CFG
+
+
+# it's better than nothing
+@register_template(101, 0)
+class CDG(ControlFlowTemplate):
+    def __init__(self, cfg: CFG):
+        self.cdg = cfg.cdg()
+        self.start = cfg.start
+        self.blame = cfg.start.blame
+        self.header_lines = self.line("# irreducible cflow, using cdg fallback", meta=True)
+
+    @override
+    @classmethod
+    def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None:
+        cdg = CDG(cfg)
+
+        if cfg.visualize == cfg._visualize:
+            cfg.remove_edges_from(list(cfg.edges))
+            cfg.add_edges_from(cdg.cdg.edges(data=True))
+            cfg.remove_node(cfg.end)
+            cfg.layout_nodes()
+            cfg.visualize()
+
+        cfg.clear()
+        cfg.add_node(cdg)
+        return cdg
+
+    @override
+    def get_instructions(self):
+        return []
+
+    @override
+    def to_indented_source(self, source):
+        cdg = self.cdg
+        for p, n in nx.dfs_edges(cdg, self.start):
+            cdg.nodes[n]["indent"] = cdg.nodes[p].get("indent", -1) + 1
+        cdg.remove_node(self.start)
+        src = []
+        for n in sorted(cdg.nodes, key=lambda x: x.offset):
+            src.extend(source[n, cdg.nodes[n].get("indent", 0)])
+        return src

pylingual/control_flow_reconstruction/templates/Conditional.py

@@ -1,13 +1,36 @@
 from ..cft import ControlFlowTemplate, EdgeKind, register_template
 from ..utils import T, N, defer_source_to, run_is, has_no_lines, with_instructions, has_instval, starting_instructions, to_indented_source, make_try_match, without_top_level_instructions
+from .Loop import BreakTemplate
 
 
 @register_template(1, 40)
 class IfElse(ControlFlowTemplate):
     template = T(
         if_header=~N("if_body", "else_body").with_cond(without_top_level_instructions("WITH_EXCEPT_START", "CHECK_EXC_MATCH", "FOR_ITER")),
+        if_body=N.tail().with_in_deg(1).of_type(BreakTemplate) | ~N("tail.").with_in_deg(1),
+        else_body=N.tail().with_in_deg(1).of_type(BreakTemplate) | ~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1),
+        tail=N.tail(),
+    )
+
+    try_match = make_try_match({EdgeKind.Fall: "tail"}, "if_header", "if_body", "else_body")
+
+    @to_indented_source
+    def to_indented_source():
+        """
+        {if_header}
+            {if_body}
+        {else_body?else:}
+            {else_body}
+        """
+
+
+@register_template(1, 39, (3, 12), (3, 13))
+class IfElseLoop(ControlFlowTemplate):
+    template = T(
+        if_header=~N("else_body", "if_body").with_cond(without_top_level_instructions("WITH_EXCEPT_START", "CHECK_EXC_MATCH", "FOR_ITER")),
         if_body=~N("tail.").with_in_deg(1),
-        else_body=~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1),
+        else_body=~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1).with_cond(has_no_lines),
+        for_iter=N.tail().with_cond(with_instructions("FOR_ITER")),
         tail=N.tail(),
     )
 

pylingual/control_flow_reconstruction/templates/Exception.py

@@ -13,6 +13,8 @@ from ..utils import (
     with_instructions,
     without_instructions,
     ending_instructions,
+    has_no_lines,
+    has_some_lines,
     exact_instructions,
     no_back_edges,
     without_top_level_instructions,
@@ -39,8 +41,8 @@ class Except3_11(ControlFlowTemplate):
             return x
 
 
-@register_template(0, 0, *versions_from(3, 11))
-class Try3_11(ControlFlowTemplate):
+@register_template(0, 0, *versions_from(3, 12))
+class Try3_12(ControlFlowTemplate):
     template = T(
         try_header=N("try_body"),
         try_body=N("tail.", None, "except_body"),
@@ -71,8 +73,8 @@ class Try3_11(ControlFlowTemplate):
         """
 
 
-@register_template(0, 0, *versions_from(3, 11))
-class TryElse3_11(ControlFlowTemplate):
+@register_template(0, 0, *versions_from(3, 12))
+class TryElse3_12(ControlFlowTemplate):
     template = T(
         try_header=N("try_body"),
         try_body=N("try_else.", None, "except_body"),
@@ -107,6 +109,76 @@ class TryElse3_11(ControlFlowTemplate):
         """
 
 
+@register_template(0, 0, (3, 11))
+class Try3_11(ControlFlowTemplate):
+    template = T(
+        try_header=N("try_body"),
+        try_body=N("try_else.", None, "except_body"),
+        except_body=N("tail.", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11),
+        try_else=~N("tail.").with_in_deg(1).with_cond(has_no_lines),
+        reraise=reraise,
+        tail=N.tail(),
+    )
+
+    try_match = revert_on_fail(
+        make_try_match(
+            {
+                EdgeKind.Fall: "tail",
+            },
+            "try_header",
+            "try_else",
+            "try_body",
+            "except_body",
+            "reraise",
+        )
+    )
+
+    @to_indented_source
+    def to_indented_source():
+        """
+        {try_header}
+        try:
+            {try_body}
+        {except_body}
+        """
+
+
+@register_template(0, 0, (3, 11))
+class TryElse3_11(ControlFlowTemplate):
+    template = T(
+        try_header=N("try_body"),
+        try_body=N("try_else.", None, "except_body"),
+        except_body=N("tail.", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11),
+        try_else=~N("tail.").with_in_deg(1).with_cond(has_some_lines),
+        reraise=reraise,
+        tail=N.tail(),
+    )
+
+    try_match = revert_on_fail(
+        make_try_match(
+            {
+                EdgeKind.Fall: "tail",
+            },
+            "try_header",
+            "try_body",
+            "except_body",
+            "try_else",
+            "reraise",
+        )
+    )
+
+    @to_indented_source
+    def to_indented_source():
+        """
+        {try_header}
+        try:
+            {try_body}
+        {except_body}
+        else:
+            {try_else}
+        """
+
+
 class BareExcept3_11(Except3_11):
     template = T(
         except_body=N("except_footer.", None, "reraise").with_cond(without_top_level_instructions("RERAISE")),
@@ -185,7 +257,7 @@ class ExceptExc3_11(Except3_11):
         except_header=N("except_body", "no_match", "reraise").with_cond(ending_instructions("CHECK_EXC_MATCH", "POP_JUMP_FORWARD_IF_FALSE"), ending_instructions("CHECK_EXC_MATCH", "POP_JUMP_IF_FALSE")),
         except_body=N("except_footer.", None, "reraise").of_subtemplate(ExcBody3_11).with_in_deg(1),
         no_match=N("tail?", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11),
-        except_footer=~N("tail.").with_in_deg(1).with_cond(starting_instructions("POP_EXCEPT")),
+        except_footer=~N("tail.").with_in_deg(1).with_cond(starting_instructions("SWAP", "POP_EXCEPT"), starting_instructions("POP_EXCEPT")),
         reraise=reraise,
         tail=N.tail(),
     )
@@ -225,7 +297,7 @@ class TryFinally3_11(ControlFlowTemplate):
         tail=N.tail(),
     )
     template2 = T(
-        try_except=N("finally_body", None, "fail_body").of_type(Try3_11, TryElse3_11),
+        try_except=N("finally_body", None, "fail_body").of_type(Try3_11, TryElse3_11, Try3_12, TryElse3_12),
         finally_body=~N("tail.").with_in_deg(1).with_cond(no_back_edges),
         fail_body=N(E.exc("reraise")).with_cond(ending_instructions("POP_TOP", "RERAISE")),
         reraise=reraise,
@@ -280,7 +352,7 @@ class TryFinally3_11(ControlFlowTemplate):
     def to_indented_source(self, source: SourceContext) -> list[SourceLine]:
         header = source[self.try_header]
         body = source[self.try_body, 1]
-        if isinstance(self.try_header, (Try3_11, TryElse3_11)) and self.members["try_body"] is None:
+        if isinstance(self.try_header, (Try3_11, TryElse3_11, Try3_12, TryElse3_12)) and self.members["try_body"] is None:
             s = header
         else:
             s = chain(header, self.line("try:"), body)
@@ -310,20 +382,6 @@ class Except3_9(ControlFlowTemplate):
             return node
 
 
-class Except3_9(ControlFlowTemplate):
-    @classmethod
-    @override
-    def try_match(cls, cfg, node) -> ControlFlowTemplate | None:
-        if [x.opname for x in node.get_instructions()] == ["RERAISE"]:
-            return node
-        if x := ExceptExc3_9.try_match(cfg, node):
-            return x
-        if x := BareExcept3_9.try_match(cfg, node):
-            return x
-        if isinstance(node, Except3_9):
-            return node
-
-
 @register_template(0, 0, (3, 9), (3, 10))
 class Try3_9(ControlFlowTemplate):
     template = T(

pylingual/control_flow_reconstruction/templates/Generator.py

@@ -1,5 +1,5 @@
 from ..cft import ControlFlowTemplate, EdgeKind, MetaTemplate, register_template
-from ..utils import E, T, N, defer_source_to, exact_instructions, no_back_edges, to_indented_source, make_try_match
+from ..utils import E, T, N, defer_source_to, ending_instructions, exact_instructions, no_back_edges, to_indented_source, make_try_match
 
 
 @register_template(0, 0)
@@ -38,7 +38,7 @@ class AwaitWith3_12(ControlFlowTemplate):
 @register_template(0, 0)
 class Generator3_12(ControlFlowTemplate):
     template = T(
-        entry=N("body").with_cond(exact_instructions("RETURN_GENERATOR", "POP_TOP")),
+        entry=N("body").with_cond(ending_instructions("RETURN_GENERATOR", "POP_TOP")),
         body=N(E.exc("gen_cleanup"), E.meta("end?")),
         gen_cleanup=N(E.meta("end")).with_cond(exact_instructions("CALL_INTRINSIC_1", "RERAISE")),
         end=N().of_type(MetaTemplate),

pylingual/control_flow_reconstruction/templates/Loop.py

@@ -1,13 +1,22 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
 from ..cft import ControlFlowTemplate, EdgeKind, register_template
 from ..utils import (
     T,
     N,
+    with_instructions,
+    exact_instructions,
+    has_no_lines,
+    condense_mapping,
     defer_source_to,
     starting_instructions,
     to_indented_source,
     make_try_match,
 )
 
+if TYPE_CHECKING:
+    from pylingual.control_flow_reconstruction.cfg import CFG
+
 
 @register_template(0, 1)
 class ForLoop(ControlFlowTemplate):
@@ -41,6 +50,24 @@ class SelfLoop(ControlFlowTemplate):
         """
 
 
+@register_template(0, 2)
+class TrueSelfLoop(ControlFlowTemplate):
+    template = T(loop_body=~N("tail.", "loop_body"), tail=N.tail())
+
+    try_match = make_try_match(
+        {
+            EdgeKind.Fall: "tail",
+        },
+        "loop_body",
+    )
+
+    @to_indented_source
+    def to_indented_source():
+        """
+        {loop_body}
+        """
+
+
 @register_template(0, 3)
 class InlinedComprehensionTemplate(ControlFlowTemplate):
     template = T(
@@ -58,3 +85,96 @@ class InlinedComprehensionTemplate(ControlFlowTemplate):
     )
 
     to_indented_source = defer_source_to("comp")
+
+
+class BreakTemplate(ControlFlowTemplate):
+    @classmethod
+    def try_match(cls, cfg, node):
+        if isinstance(node, BreakTemplate) or has_no_lines(cfg, node) or with_instructions("RAISE_VARARGS")(cfg, node):
+            return None
+        return condense_mapping(cls, cfg, {"child": node}, "child")
+
+    def to_indented_source(self, source):
+        return self.child.to_indented_source(source) + self.line("break")
+
+
+class ContinueTemplate(ControlFlowTemplate):
+    @classmethod
+    def try_match(cls, cfg, node):
+        if isinstance(node, ContinueTemplate) or has_no_lines(cfg, node):
+            return None
+        instruction = node.get_instructions()[-1].opname
+        if instruction in {"JUMP_ABSOLUTE", "JUMP_BACKWARD", "CONTINUE_LOOP"} and (node.get_instructions()[-1].starts_line is not None or node.get_instructions()[-2].starts_line is not None):
+            return condense_mapping(cls, cfg, {"child": node}, "child")
+        return None
+
+    def to_indented_source(self, source):
+        return self.child.to_indented_source(source) + self.line("continue")
+
+
+@register_template(0, 0)
+class FixLoop(ControlFlowTemplate):
+    @classmethod
+    def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None:
+        # check that its a loop that we need to fix
+        # find the end of the loop
+        # find all nodes that belong to the loop
+        # find nodes in loop that go to end
+        # replace those edges with meta edges to the end
+        # find nodes in loop that go to header
+        # replace all but last of those edges with meta edge to end
+
+        # a node is a loop header if there are back-edges to it
+        # a latching node is a node with a back-edge to the loop header
+        # a back-edge is an edge from any node that is dominated by this node
+        back_edges = []
+        for predecessor in cfg.predecessors(node):
+            # A back edge exists if the predecessor is reachable from the node (node dominates predecessor)
+            if cfg.dominates(node, predecessor):
+                back_edges.append(predecessor)
+
+        if not back_edges:
+            return None
+
+        # Get all nodes encompassed by the loop excluding source node and initial false jump
+        loopnode = None
+        for succ in cfg.successors(node):
+            if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.Fall:
+                loopnode = succ
+                break
+
+        dfs_edges = cfg.dfs_labeled_edges_no_loop(source=loopnode)
+        encompassed_nodes = [v for u, v, d in dfs_edges if d == "forward"]
+
+        edges_to_remove = []
+
+        # Find the candidate end that break connects to
+        candidate_end = None
+        for succ in cfg.successors(node):
+            if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.FalseJump and cfg.out_degree(succ) <= 1:
+                candidate_end = succ
+
+                # Candidate end is a buffer node
+                if cfg.in_degree(candidate_end) == 1 and all(x.opname in {"POP_TOP", "POP_BLOCK", "END_FOR", "RETURN_CONST", "LOAD_CONST", "RETURN_VALUE", "JUMP_BACKWARD"} for x in candidate_end.get_instructions()):
+                    for ss in cfg.successors(candidate_end):
+                        if cfg.get_edge_data(candidate_end, ss).get("kind") != EdgeKind.Exception:
+                            candidate_end = ss
+                            break
+
+        if encompassed_nodes is not None:
+            for succ in encompassed_nodes:
+                if cfg.get_edge_data(succ, candidate_end) != None:
+                    edges_to_remove.append((succ, candidate_end))
+
+        for pred, succ in edges_to_remove:
+            break_node = BreakTemplate.try_match(cfg, pred)
+            if break_node is not None:
+                cfg.remove_edge(break_node, succ)
+
+        for candidate in back_edges:
+            cont_node = ContinueTemplate.try_match(cfg, candidate)
+            if cont_node is not None and cfg.in_degree(node) > 2:
+                cfg.remove_edge(cont_node, node)
+
+        cfg.iterate()
+        return

pylingual/control_flow_reconstruction/utils.py

@@ -134,6 +134,10 @@ def has_no_lines(cfg: CFG, node: ControlFlowTemplate | None) -> bool:
     return node is None or all(i.starts_line is None for i in node.get_instructions())
 
 
+def has_some_lines(cfg: CFG, node: ControlFlowTemplate | None) -> bool:
+    return node is None or any(i.starts_line is not None for i in node.get_instructions())
+
+
 def run_is(n: int):
     def check_run(cfg: CFG, node: ControlFlowTemplate | None) -> bool:
         return cfg.run == n

pylingual/main.py

@@ -130,11 +130,7 @@ def main(files: list[str], out_dir: Path | None, config_file: Path | None, versi
                 pyc = result.original_pyc
                 print_result(f"Equivalence Results for {pyc.pyc_path.name if pyc.pyc_path else repr(pyc)}", result.equivalence_results)
             except Exception:
-                import pdb
-
                 live.stop()
-                if hasattr(pdb, "xpm"):
-                    pdb.xpm()  # type: ignore
                 logger.exception(f"Failed to decompile {pyc_path}")
             console.rule()
 

pylingual/masking/global_masker.py

@@ -238,7 +238,7 @@ class Masker:
         if inst.is_jump_target:
             view = f"{inst.offset} {view}"
             # create list of offsets greater than or less than inst, for all jump origins to this inst
-            jumps_greater_or_less = [inst.offset < inst.offset for inst in inst.jumped_to_from_insts]
+            jumps_greater_or_less = [i.offset < inst.offset for i in inst.jumped_to_from_insts]
             if any(jumps_greater_or_less):  # 1 in list
                 view = f"^~> {view}"
             if not all(jumps_greater_or_less):  # 0 in list

test/Conditional.py

@@ -302,3 +302,24 @@ def f_nofallthru_if_pass():
     if a > b:
         pass
     print("end")
+
+
+def g1_ifElseLoop():
+    for a in range(3):
+        if a > b:
+            print(1)
+
+
+def g2_ifElseLoop():
+    for a in range(3):
+        if a > b:
+            print(1)
+        print(2)
+
+
+def g3_ifElseLoop():
+    for a in range(3):
+        if a > b:
+            print(1)
+        else:
+            print(2)

test/Exception.py

@@ -1,4 +1,11 @@
-def a_TryExcept():
+def a0_bare_try_except():
+    try:
+        print(1)
+    except:
+        print(2)
+
+
+def a1_bare_try_except_fallthrough():
     try:
         print(1)
     except:
@@ -6,7 +13,8 @@ def a_TryExcept():
     print(3)
 
 
-def b_TryExceptBareNested():
+# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match
+def b0_nested_try_except():
     try:
         print(1)
     except:
@@ -17,7 +25,7 @@ def b_TryExceptBareNested():
             print(4)
 
 
-def b1_TryExceptBareNestedFallthrough():
+def b1_nested_try_except_fallthrough():
     try:
         print(1)
     except:
@@ -29,7 +37,8 @@ def b1_TryExceptBareNestedFallthrough():
         print(5)
 
 
-def b2_TryExceptBareNestedEarlyFallthrough():
+# 3.13 Duplicate blocks
+def b2_nested_try_except_early_fallthrough():
     try:
         print(1)
     except:
@@ -41,7 +50,7 @@ def b2_TryExceptBareNestedEarlyFallthrough():
     print(5)
 
 
-def b3_TryExceptBareNestedDoubleFallthrough():
+def b3_nested_try_except_double_fallthrough():
     try:
         print(1)
     except:
@@ -54,7 +63,8 @@ def b3_TryExceptBareNestedDoubleFallthrough():
     print(6)
 
 
-def c_TryExceptBareMultiNested():
+# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match
+def c0_multi_except_nested():
     try:
         print(1)
     except a:
@@ -71,7 +81,8 @@ def c_TryExceptBareMultiNested():
             print(7)
 
 
-def c1_TryExceptBareMultiNestedFallthrough():
+# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match
+def c1_multi_except_nested_fallthrough():
     try:
         print(1)
     except a:
@@ -89,7 +100,7 @@ def c1_TryExceptBareMultiNestedFallthrough():
             print(8)
 
 
-def c2_TryExceptBareMultiNestedFallthrough2():
+def c2_multi_except_nested_fallthrough2():
     try:
         print(1)
     except a:
@@ -108,7 +119,8 @@ def c2_TryExceptBareMultiNestedFallthrough2():
         print(9)
 
 
-def c3_TryExceptBareMultiNestedEarlyFallthrough():
+# 3.13 Duplicate blocks
+def c3_multi_except_nested_early_fallthrough():
     try:
         print(1)
     except a:
@@ -126,7 +138,7 @@ def c3_TryExceptBareMultiNestedEarlyFallthrough():
     print(8)
 
 
-def c4_TryExceptBareMultiNestedAllFallthrough():
+def c4_multi_except_nested_all_fallthrough():
     try:
         print(1)
     except a:
@@ -146,7 +158,9 @@ def c4_TryExceptBareMultiNestedAllFallthrough():
     print(10)
 
 
-def d_TryExceptBareNestedNamed():
+# 3.10/3.11/3.12/3.13 Duplicate blocks causing templates to not match
+# Discussed in issue 41
+def d0_named_except_nested():
     try:
         print(1)
     except A as a:
@@ -157,7 +171,7 @@ def d_TryExceptBareNestedNamed():
             print(4)
 
 
-def d1_TryExceptBareNestedNamedFallthrough():
+def d1_named_except_nested_fallthrough():
     try:
         print(1)
     except A as a:
@@ -169,7 +183,8 @@ def d1_TryExceptBareNestedNamedFallthrough():
         print(5)
 
 
-def d2_TryExceptBareNestedNamedEarlyFallthrough():
+# 3.13 Duplicate blocks
+def d2_named_except_nested_early_fallthrough():
     try:
         print(1)
     except A as a:
@@ -181,7 +196,7 @@ def d2_TryExceptBareNestedNamedEarlyFallthrough():
     print(5)
 
 
-def d3_TryExceptBareNestedNamedDoubleFallthrough():
+def d3_named_except_nested_double_fallthrough():
     try:
         print(1)
     except A as a:
@@ -194,7 +209,7 @@ def d3_TryExceptBareNestedNamedDoubleFallthrough():
     print(6)
 
 
-def e_TryExceptElseBare():
+def e0_try_except_else():
     try:
         print(1)
     except:
@@ -204,7 +219,7 @@ def e_TryExceptElseBare():
     print(4)
 
 
-def f_TryExceptElseFinallyBare():
+def f0_try_except_else_finally():
     try:
         print(1)
     except:
@@ -216,7 +231,7 @@ def f_TryExceptElseFinallyBare():
     print(5)
 
 
-def g_TryExceptElseMulti():
+def g0_multi_except_with_else():
     try:
         print(1)
     except a:
@@ -228,7 +243,7 @@ def g_TryExceptElseMulti():
     print(5)
 
 
-def h_TryExceptElseMultiFallback():
+def h0_multi_except_fallback_with_else():
     try:
         print(1)
     except a:
@@ -242,7 +257,7 @@ def h_TryExceptElseMultiFallback():
     print(6)
 
 
-def i_TryExceptElseMultiNamedAndUnnamed():
+def i0_mixed_named_unnamed_except_with_else():
     try:
         print(1)
     except A as a:
@@ -256,7 +271,7 @@ def i_TryExceptElseMultiNamedAndUnnamed():
     print(6)
 
 
-def j_TryExceptElseNamed():
+def j0_named_except_with_else():
     try:
         print(1)
     except A as a:
@@ -268,7 +283,7 @@ def j_TryExceptElseNamed():
     print(5)
 
 
-def k_TryExceptFinallyBare():
+def k0_try_except_finally():
     try:
         print(1)
     except:
@@ -278,7 +293,7 @@ def k_TryExceptFinallyBare():
     print(4)
 
 
-def l_TryExceptFinallyBareSpecific():
+def l0_specific_except_finally():
     try:
         print(1)
     except a:
@@ -288,7 +303,7 @@ def l_TryExceptFinallyBareSpecific():
     print(4)
 
 
-def m_TryExceptMulti():
+def m0_multi_except():
     try:
         print(1)
     except a:
@@ -300,7 +315,7 @@ def m_TryExceptMulti():
     print(5)
 
 
-def n_TryExceptMultiFallback():
+def n0_multi_except_with_fallback():
     try:
         print(1)
     except a:
@@ -312,7 +327,7 @@ def n_TryExceptMultiFallback():
     print(5)
 
 
-def o_TryExceptMultiFallbackFinally():
+def o0_multi_except_fallback_finally():
     try:
         print(1)
     except a:
@@ -324,7 +339,7 @@ def o_TryExceptMultiFallbackFinally():
     print(5)
 
 
-def p_TryExceptMultiNamed():
+def p0_multi_named_except():
     try:
         print(1)
     except A as a:
@@ -336,7 +351,7 @@ def p_TryExceptMultiNamed():
     print(5)
 
 
-def q_TryExceptMultiNamedAndUnnamed():
+def q0_mixed_named_unnamed_except():
     try:
         print(1)
     except A as a:
@@ -348,7 +363,7 @@ def q_TryExceptMultiNamedAndUnnamed():
     print(5)
 
 
-def r_TryExceptMultiNamedAndUnnamedFinally():
+def r0_mixed_named_unnamed_except_finally():
     try:
         print(1)
     except A as a:
@@ -362,7 +377,7 @@ def r_TryExceptMultiNamedAndUnnamedFinally():
     print(6)
 
 
-def s_TryExceptMultiNamedFallback():
+def s0_named_except_fallback():
     try:
         print(1)
     except A as a:
@@ -372,7 +387,7 @@ def s_TryExceptMultiNamedFallback():
     print(4)
 
 
-def t_TryExceptMultiNamedFallbackFinally():
+def t0_named_except_fallback_finally():
     try:
         print(1)
     except A as a:
@@ -384,7 +399,7 @@ def t_TryExceptMultiNamedFallbackFinally():
     print(5)
 
 
-def u_TryExceptMultiNamedFinally():
+def u0_multi_named_except_finally():
     try:
         print(1)
     except A as a:
@@ -396,7 +411,7 @@ def u_TryExceptMultiNamedFinally():
     print(5)
 
 
-def v_TryExceptMultiFinally():
+def v0_multi_except_finally():
     try:
         print(1)
     except a:
@@ -408,7 +423,7 @@ def v_TryExceptMultiFinally():
     print(5)
 
 
-def w_TryExceptRaise():
+def w0_try_except_raise():
     try:
         print(1)
     except:
@@ -416,7 +431,7 @@ def w_TryExceptRaise():
         raise Exc
 
 
-def x_TryExceptRaiseMulti():
+def x0_multi_except_raise():
     try:
         print(1)
     except a:
@@ -427,7 +442,7 @@ def x_TryExceptRaiseMulti():
         raise Exc
 
 
-def y_TryExceptRaiseNamed():
+def y0_named_except_raise():
     try:
         print(1)
     except A as a:
@@ -435,7 +450,8 @@ def y_TryExceptRaiseNamed():
         raise Exc
 
 
-def z_TryExceptReturn():
+# 3.11 Try return getting left outside of TryExcept
+def z0_try_except_return():
     try:
         print(1)
         return 2
@@ -443,7 +459,8 @@ def z_TryExceptReturn():
         print(2)
 
 
-def z1_TryExceptReturn():
+# 3.11 Try return getting left outside of TryExcept
+def z1_try_except_return_both():
     try:
         print(1)
         return 2
@@ -452,7 +469,8 @@ def z1_TryExceptReturn():
         return 3
 
 
-def aa_TryExceptReturnMulti():
+# 3.11 Try return getting left outside of TryExcept
+def aa0_multi_except_return():
     try:
         print(1)
         return 2
@@ -462,7 +480,10 @@ def aa_TryExceptReturnMulti():
         print(3)
 
 
-def aa1_TryExceptReturnMulti():
+# 3.6/3.7/3.8 ExceptExc abandons tail node.
+# Could be fixed (?) but breaks other test cases
+# 3.11 Try return getting left outside of TryExcept
+def aa1_multi_except_return_both():
     try:
         print(1)
         return 2
@@ -473,7 +494,8 @@ def aa1_TryExceptReturnMulti():
         print(3)
 
 
-def ab_TryExceptReturnNamed():
+# 3.11 Try return getting left outside of TryExcept
+def ab0_named_except_raise_return():
     try:
         print(1)
         return 2
@@ -485,14 +507,33 @@ def ab_TryExceptReturnNamed():
         raise Exc
 
 
-def TryEmptryFinally():
+# 3.8 Double natural edge graph error (?)
+# 3.11 Try return getting left outside of TryExcept
+def ab1_named_except_return():
+    try:
+        print(1)
+        return 2
+    except A as a:
+        print(2)
+        return 3
+
+
+def ab2_named_except_return():
+    try:
+        return a
+    except Exception as a:
+        return b
+
+
+# 3.11/3.12/3.13 No template match
+def ac0_empty_try_finally():
     try:
         pass
     finally:
         print(1)
 
 
-def TryMultiple():
+def ad0_multiple_try_blocks():
     try:
         print(1)
     except:
@@ -504,7 +545,7 @@ def TryMultiple():
         print(4)
 
 
-def TryExceptElseTry():
+def ae0_try_except_else_nested_try():
     try:
         print(1)
     except:
@@ -516,7 +557,9 @@ def TryExceptElseTry():
             print(4)
 
 
-def TryFinallyNestedExcept():
+# 3.9 Duplicate blocks (?)
+# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?)
+def af0_try_finally_nested_except():
     try:
         print(1)
     finally:
@@ -526,35 +569,40 @@ def TryFinallyNestedExcept():
             print(3)
 
 
-def TryExceptTuple():
+def ag0_try_except_tuple():
     try:
         print(1)
     except (A, B):
         print(2)
 
 
-def TryFinallyReturn():
+# 3.9 Difficult template ambiguity between Try/TryFinally
+# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?)
+def ah0_try_finally_return():
     try:
         print(1)
     finally:
         return 2
 
 
-def TryReturnFinally():
+# 3.11/3.12/3.13 No template match (?)
+def ai0_try_return_finally():
     try:
         return 1
     finally:
         print(2)
 
 
-def TryReturnFinallyReturn():
+# 3.9/3.10 Difficult template ambiguity between Try/TryFinally
+# 3.11/3.12/3.13 No template match (?)
+def aj0_try_return_finally_return():
     try:
         return 1
     finally:
         return 2
 
 
-def TryExceptRaise():
+def ak0_try_except_raise_return():
     try:
         print(1)
         return 2
@@ -562,8 +610,8 @@ def TryExceptRaise():
         raise Exception()
 
 
-"""
-def TryExceptReturnFinally():
+# 3.8/3.9/3.10 No template match
+def al0_try_except_return_finally():
     try:
         raise Exception()
     except:
@@ -571,28 +619,19 @@ def TryExceptReturnFinally():
         return 2
     finally:
         print(3)
-"""
 
-"""
-def TryFinallyRaise():
+
+# 3.8/3.9/3.10 No template match
+# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?)
+def am0_try_finally_raise():
     try:
         print(1)
         return 2
     finally:
         raise Exception()
-"""
 
 
-def ab1_TryExceptReturnNamed():
-    try:
-        print(1)
-        return 2
-    except A as a:
-        print(2)
-        return 3
-
-
-def ac_TryFinallyBareFallthrough():
+def an0_try_finally_fallthrough():
     try:
         print(1)
     finally:
@@ -600,7 +639,7 @@ def ac_TryFinallyBareFallthrough():
     print(3)
 
 
-def ad_TryFinallyBare():
+def ao0_try_finally_simple():
     try:
         print(1)
     finally:

test/Loop.py

@@ -1,31 +1,78 @@
-# FOR LOOP TESTS
-
-
-def a_for_over_list():
+def a0_for_over_list():
     for x in [1, 2, 3]:
         print("for over list")
 
 
-def b_for_over_tuples():
+def a1_for_over_list_nofallthru():
+    for x in [1, 2, 3]:
+        print("for over list")
+    print("end")
+
+
+def b0_for_over_tuples():
     for a, b in [(1, 2), (3, 4)]:
         print("tuples")
 
 
-def c_for_else():
+def b1_for_over_tuples_nofallthru():
+    for a, b in [(1, 2), (3, 4)]:
+        print("tuples")
+    print("end")
+
+
+# 3.6/3.7 No else template
+def c0_for_else():
     for i in range(3):
         print("for body")
     else:
         print("for else")
 
 
-def d_for_with_break():
+# 3.6/3.7 No else template
+def c1_for_else_nofallthru():
+    for i in range(3):
+        print("for body")
+    else:
+        print("for else")
+    print("end")
+
+
+# 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end
+# 3.9 Naive break detection, an unexpected buffer block to end
+def d0_for_with_break():
     for x in range(10):
         if x == 5:
             print("breaking")
             break
 
 
-def e_for_with_continue():
+# 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end
+def d1_for_with_break_nofallthru():
+    for x in range(10):
+        if x == 5:
+            print("breaking")
+            break
+    print("end")
+
+
+# Help to implement break
+def d2_for_without_break():
+    for x in range(10):
+        if x == 5:
+            print("not breaking")
+    print("end")
+
+
+# Help to implement break
+def d3_for_return():
+    for x in range(10):
+        if x == 5:
+            print("not breaking")
+            return
+    print("end")
+
+
+def e0_for_with_continue():
     for x in range(5):
         if x % 2 == 0:
             print("continuing")
@@ -33,13 +80,29 @@ def e_for_with_continue():
         print("after continue")
 
 
-def f_nested_for_loops():
+def e1_for_with_continue_nofallthru():
+    for x in range(5):
+        if x % 2 == 0:
+            print("continuing")
+            continue
+        print("after continue")
+    print("end")
+
+
+def f0_nested_for_loops():
     for i in range(2):
         for j in range(3):
             print(f"nested {i},{j}")
 
 
-def g_for_with_try_except():
+def f1_nested_for_loops_nofallthru():
+    for i in range(2):
+        for j in range(3):
+            print(f"nested {i},{j}")
+    print("end")
+
+
+def g0_for_with_try_except():
     for x in range(2):
         try:
             print("try block")
@@ -47,13 +110,29 @@ def g_for_with_try_except():
             print("except block")
 
 
-def h_for_with_with_statement():
+def g1_for_with_try_except_nofallthru():
+    for x in range(2):
+        try:
+            print("try block")
+        except Exception:
+            print("except block")
+    print("end")
+
+
+def h0_for_with_with_statement():
     for _ in range(1):
         with a:
             print("inside with")
 
 
-def i_for_with_function_call_iterable():
+def h1_for_with_with_statement_nofallthru():
+    for _ in range(1):
+        with a:
+            print("inside with")
+    print("end")
+
+
+def i0_for_with_function_call_iterable():
     def get_items():
         return [1, 2, 3]
 
@@ -61,147 +140,8 @@ def i_for_with_function_call_iterable():
         print(f"item: {item}")
 
 
-def j_for_with_empty_body_ellipsis():
-    for _ in range(3):
-        ...
-
-
-def k_while_true_with_break():
-    while True:
-        print("while true")
-        break
-
-
-def l_while_with_else():
-    i = 0
-    while i < 3:
-        print(f"looping {i}")
-        i += 1
-    else:
-        print("while else")
-
-
-def m_while_with_continue():
-    i = 0
-    while i < 5:
-        i += 1
-        if i % 2 == 0:
-            print("continue")
-            continue
-        print("after continue")
-
-
-def n_while_with_break():
-    i = 0
-    while True:
-        print("break in while")
-        break
-
-
-def o_nested_while_loops():
-    i = 0
-    while i < 2:
-        j = 0
-        while j < 2:
-            print(f"nested while {i},{j}")
-            j += 1
-        i += 1
-
-
-def p_while_with_try_except():
-    while True:
-        try:
-            print("try in while")
-        except:
-            print("except in while")
-
-
-def q_while_with_with_statement():
-    while True:
-        with a:
-            print("inside while with")
-
-
-def r_for_inside_while():
-    while True:
-        for x in [1, 2]:
-            print("for in while")
-
-
-def s_while_inside_for():
-    for _ in range(1):
-        while True:
-            print("while in for")
-            break
-
-
-def t_while_with_empty_body_ellipsis():
-    while True:
-        ...
-
-
-def a_nofallthru_for_over_list():
-    for x in [1, 2, 3]:
-        print("for over list")
-    print("end")
-
-
-def b_nofallthru_for_over_tuples():
-    for a, b in [(1, 2), (3, 4)]:
-        print("tuples")
-    print("end")
-
-
-def c_nofallthru_for_else():
-    for i in range(3):
-        print("for body")
-    else:
-        print("for else")
-    print("end")
-
-
-def d_nofallthru_for_with_break():
-    for x in range(10):
-        if x == 5:
-            print("breaking")
-            break
-    print("end")
-
-
-def e_nofallthru_for_with_continue():
-    for x in range(5):
-        if x % 2 == 0:
-            print("continuing")
-            continue
-        print("after continue")
-    print("end")
-
-
-def f_nofallthru_nested_for_loops():
-    for i in range(2):
-        for j in range(3):
-            print(f"nested {i},{j}")
-    print("end")
-
-
-def g_nofallthru_for_with_try_except():
-    for x in range(2):
-        try:
-            print("try block")
-        except Exception:
-            print("except block")
-    print("end")
-
-
-def h_nofallthru_for_with_with_statement():
-    for _ in range(1):
-        with a:
-            print("inside with")
-    print("end")
-
-
-def i_nofallthru_for_with_function_call_iterable():
-    def g_nofallthruet_items():
+def i1_for_with_function_call_iterable_nofallthru():
+    def get_items():
         return [1, 2, 3]
 
     for item in get_items():
@@ -209,20 +149,54 @@ def i_nofallthru_for_with_function_call_iterable():
     print("end")
 
 
-def j_nofallthru_for_with_empty_body_ellipsis():
+def j0_for_with_empty_body_ellipsis():
+    for _ in range(3):
+        ...
+
+
+def j1_for_with_empty_body_ellipsis_nofallthru():
     for _ in range(3):
         ...
     print("end")
 
 
-def k_nofallthru_while_true_with_break():
+# 3.6/3.7 Naive break detection, no back edge
+# 3.9/3.11 No while loop detection, self false_jump edge & naive break detection
+def k0_while_true_with_break():
+    x = 0
     while True:
         print("while true")
-        break
+        x += 1
+        if x >= 1:
+            break
+
+
+# 3.6/3.7 Naive break detection, no back edge
+# 3.9/3.11 No while loop detection, self false_jump edge & naive break detection
+def k1_while_true_with_break_nofallthru():
+    x = 0
+    while True:
+        print("while true")
+        x += 1
+        if x >= 1:
+            break
     print("end")
 
 
-def l_nofallthru_while_with_else():
+# 3.6/3.7 No else template
+# 3.11 No while loop detection, self false_jump edge
+def l0_while_with_else():
+    i = 0
+    while i < 3:
+        print(f"looping {i}")
+        i += 1
+    else:
+        print("while else")
+
+
+# 3.6/3.7 No else template
+# 3.11 No while loop detection, self false_jump edge
+def l1_while_with_else_nofallthru():
     i = 0
     while i < 3:
         print(f"looping {i}")
@@ -232,7 +206,19 @@ def l_nofallthru_while_with_else():
     print("end")
 
 
-def m_nofallthru_while_with_continue():
+# 3.11 No continue
+def m0_while_with_continue():
+    i = 0
+    while i < 5:
+        i += 1
+        if i % 2 == 0:
+            print("continue")
+            continue
+        print("after continue")
+
+
+# 3.11 No continue
+def m1_while_with_continue_nofallthru():
     i = 0
     while i < 5:
         i += 1
@@ -243,7 +229,16 @@ def m_nofallthru_while_with_continue():
     print("end")
 
 
-def n_nofallthru_while_with_break():
+# 3.6/3.7 Naive break detection, no back edge
+def n0_while_with_break():
+    i = 0
+    while True:
+        print("break in while")
+        break
+
+
+# 3.6/3.7 Naive break detection, no back edge
+def n1_while_with_break_nofallthru():
     i = 0
     while True:
         print("break in while")
@@ -251,7 +246,19 @@ def n_nofallthru_while_with_break():
     print("end")
 
 
-def o_nofallthru_nested_while_loops():
+# 3.11 While template broke
+def o0_nested_while_loops():
+    i = 0
+    while i < 2:
+        j = 0
+        while j < 2:
+            print(f"nested while {i},{j}")
+            j += 1
+        i += 1
+
+
+# 3.11 While template broke
+def o1_nested_while_loops_nofallthru():
     i = 0
     while i < 2:
         j = 0
@@ -262,7 +269,19 @@ def o_nofallthru_nested_while_loops():
     print("end")
 
 
-def p_nofallthru_while_with_try_except():
+# 3.6/3.7 While template broke (?)
+# 3.9 Disconnected with MetaTemplate[end] (?)
+def p0_while_with_try_except():
+    while True:
+        try:
+            print("try in while")
+        except:
+            print("except in while")
+
+
+# 3.6/3.7 While template broke (?)
+# 3.9 Disconnected with MetaTemplate[end] (?)
+def p1_while_with_try_except_nofallthru():
     while True:
         try:
             print("try in while")
@@ -271,21 +290,46 @@ def p_nofallthru_while_with_try_except():
     print("end")
 
 
-def q_nofallthru_while_with_with_statement():
+# 3.6/3.7 While template broke (?) abandoning nodes
+def q0_while_with_with_statement():
+    while True:
+        with a:
+            print("inside while with")
+
+
+# 3.6/3.7 While template broke (?) abandoning nodes
+def q1_while_with_with_statement_nofallthru():
     while True:
         with a:
             print("inside while with")
     print("end")
 
 
-def r_nofallthru_for_inside_while():
+# 3.6/3.7 While template broke
+def r0_for_inside_while():
+    while True:
+        for x in [1, 2]:
+            print("for in while")
+
+
+# 3.6/3.7 While template broke
+def r1_for_inside_while_nofallthru():
     while True:
         for x in [1, 2]:
             print("for in while")
     print("end")
 
 
-def s_nofallthru_while_inside_for():
+# 3.6/3.7 While template broke
+def s0_while_inside_for():
+    for _ in range(1):
+        while True:
+            print("while in for")
+            break
+
+
+# 3.6/3.7 While template broke
+def s1_while_inside_for_nofallthru():
     for _ in range(1):
         while True:
             print("while in for")
@@ -293,7 +337,151 @@ def s_nofallthru_while_inside_for():
     print("end")
 
 
-def t_nofallthru_while_with_empty_body_ellipsis():
+# 3.6/3.7 While template broke
+def t0_while_with_empty_body_ellipsis():
+    while True:
+        ...
+
+
+# 3.6/3.7 While template broke
+def t1_while_with_empty_body_ellipsis_nofallthru():
     while True:
         ...
     print("end")
+
+
+def u0_break_in_nested_for():
+    for i in range(3):
+        for j in range(3):
+            if i == 1 and j == 1:
+                print("Breaking inner loop")
+                break
+            print(f"i={i}, j={j}")
+
+
+def u1_break_in_nested_for_nofallthru():
+    for i in range(3):
+        for j in range(3):
+            if i == 1 and j == 1:
+                print("Breaking inner loop")
+                break
+            print(f"i={i}, j={j}")
+    print("end")
+
+
+def v0_continue_in_nested_for():
+    for i in range(3):
+        for j in range(3):
+            if j == 1:
+                continue
+            print(f"Processing i={i}, j={j}")
+
+
+def v1_continue_in_nested_for_nofallthru():
+    for i in range(3):
+        for j in range(3):
+            if j == 1:
+                continue
+            print(f"Processing i={i}, j={j}")
+    print("end")
+
+
+# 3.13 if statement putting code in the else block
+def w0_break_with_else():
+    for i in range(5):
+        if i == 3:
+            print("Breaking before else")
+            break
+    else:
+        print("This won't execute due to break")
+
+
+# 3.13 if statement putting code in the else block
+def w1_break_with_else_nofallthru():
+    for i in range(5):
+        if i == 3:
+            print("Breaking before else")
+            break
+    else:
+        print("This won't execute due to break")
+    print("end")
+
+
+# 3.6/3.7 No continue detection
+def x0_continue_with_else():
+    for i in range(3):
+        if i == 1:
+            continue
+        print(f"Processing {i}")
+    else:
+        print("Else clause still executes after continue")
+
+
+# 3.6/3.7 No continue detection
+def x1_continue_with_else_nofallthru():
+    for i in range(3):
+        if i == 1:
+            continue
+        print(f"Processing {i}")
+    else:
+        print("Else clause still executes after continue")
+    print("end")
+
+
+# 3.9/3.11 Naive break detection, break statement is further up
+def y0_break_in_try_except():
+    for i in range(5):
+        try:
+            if i == 3:
+                break
+            print(f"Value: {i}")
+        except:
+            print("Exception occurred")
+
+
+# 3.9 Naive break detection, break statement is further up
+def y1_break_in_try_except_nofallthru():
+    for i in range(5):
+        try:
+            if i == 3:
+                break
+            print(f"Value: {i}")
+        except:
+            print("Exception occurred")
+    print("end")
+
+
+# 3.9 Naive break detection, break statement is further up
+def y2_return_in_try_except_nofallthru():
+    for i in range(5):
+        try:
+            if i == 3:
+                print(f"Value: {i}")
+            else:
+                break
+        except:
+            print("Exception occurred")
+    print("end")
+
+
+# 3.6/3.9 No continue detection
+def z0_continue_in_try_except():
+    for i in range(5):
+        try:
+            if i == 2:
+                continue
+            print(f"Value: {i}")
+        except:
+            print("Exception occurred")
+
+
+# 3.6/3.9/3.11 No continue detection
+def z1_continue_in_try_except_nofallthru():
+    for i in range(5):
+        try:
+            if i == 2:
+                continue
+            print(f"Value: {i}")
+        except:
+            print("Exception occurred")
+    print("end")

test/With.py

@@ -1,165 +1,81 @@
-def bare_with():
+def a0_bare_with():
     with a:
         print(1)
 
 
-def bare_with_fallthrough():
+def a1_bare_with_fallthrough():
     with a:
         print(1)
     print(2)
 
 
-## Known to fail on 3.10
-def multi_with():
+def b0_multi_with():
     with a, b:
         print(1)
 
 
-def multi_with_fallthrough():
+def b1_multi_with_fallthrough():
     with a, b:
         print(1)
     print(2)
 
 
-def with_as():
+def c0_with_as():
     with a as c:
         print(1)
 
 
-def with_as_fallthrough():
+def c1_with_as_fallthrough():
     with a as c:
         print(1)
     print(2)
 
 
-def multi_with_as():
+def d0_multi_with_as():
     with a, b as c:
         print(1)
 
 
-def multi_with_as_fallthrough():
+def d1_multi_with_as_fallthrough():
     with a, b as c:
         print(1)
     print(2)
 
 
-def with_multi_as():
+def e0_with_multi_as():
     with a as b, c:
         print(1)
 
 
-def with_multi_as_fallthrough():
+def e1_with_multi_as_fallthrough():
     with a as b, c:
         print(1)
     print(2)
 
 
-def multi_with_multi_as():
+def f0_multi_with_multi_as():
     with a as b, c as d:
         print(1)
 
 
-# Known to fail on 3.10
-def multi_with_multi_as_fallthrough():
+def f1_multi_with_multi_as_fallthrough():
     with a as b, c as d:
         print(1)
     print(2)
 
 
-def multi_with_multi_as_alt():
+def g0_multi_with_multi_as_alt():
     with a, b as c, d:
         print(1)
 
 
-# Known to fail on 3.10
-def multi_with_multi_as_fallthrough_alt():
+def g1_multi_with_multi_as_fallthrough_alt():
     with a, b as c, d:
         print(1)
     print(2)
 
 
-async def bare_async_with():
-    async with a:
-        print(1)
-
-
-async def bare_async_with_fallthrough():
-    async with a:
-        print(1)
-    print(2)
-
-
-## Known to fail on 3.10
-async def multi_async_with():
-    async with a, b:
-        print(1)
-
-
-async def multi_async_with_fallthrough():
-    async with a, b:
-        print(1)
-    print(2)
-
-
-async def with_as():
-    async with a as c:
-        print(1)
-
-
-async def with_as_fallthrough():
-    async with a as c:
-        print(1)
-    print(2)
-
-
-async def multi_async_with_as():
-    async with a, b as c:
-        print(1)
-
-
-async def multi_async_with_as_fallthrough():
-    async with a, b as c:
-        print(1)
-    print(2)
-
-
-async def with_multi_as():
-    async with a as b, c:
-        print(1)
-
-
-async def with_multi_as_fallthrough():
-    async with a as b, c:
-        print(1)
-    print(2)
-
-
-async def multi_async_with_multi_as():
-    async with a as b, c as d:
-        print(1)
-
-
-# Known to fail on 3.10
-async def multi_async_with_multi_as_fallthrough():
-    async with a as b, c as d:
-        print(1)
-    print(2)
-
-
-async def multi_async_with_multi_as_alt():
-    async with a, b as c, d:
-        print(1)
-
-
-# Known to fail on 3.10
-async def multi_async_with_multi_as_fallthrough_alt():
-    async with a, b as c, d:
-        print(1)
-    print(2)
-
-
-def try_with_except():
-    # With statement with outer exception handler
+def h0_try_with_except():
     try:
         with a:
             print(1)
@@ -168,15 +84,90 @@ def try_with_except():
     print(3)
 
 
-def with_return():
-    # With statement with return
+def i0_with_return():
     with a:
         return 1
     print(1)
 
 
-def with_raise():
-    # With statement with raise
+def j0_with_raise():
     with a:
         raise Exc
     print(1)
+
+
+async def k0_bare_async_with():
+    async with a:
+        print(1)
+
+
+async def k1_bare_async_with_fallthrough():
+    async with a:
+        print(1)
+    print(2)
+
+
+async def l0_multi_async_with():
+    async with a, b:
+        print(1)
+
+
+async def l1_multi_async_with_fallthrough():
+    async with a, b:
+        print(1)
+    print(2)
+
+
+async def m0_async_with_as():
+    async with a as c:
+        print(1)
+
+
+async def m1_async_with_as_fallthrough():
+    async with a as c:
+        print(1)
+    print(2)
+
+
+async def n0_multi_async_with_as():
+    async with a, b as c:
+        print(1)
+
+
+async def n1_multi_async_with_as_fallthrough():
+    async with a, b as c:
+        print(1)
+    print(2)
+
+
+async def o0_async_with_multi_as():
+    async with a as b, c:
+        print(1)
+
+
+async def o1_async_with_multi_as_fallthrough():
+    async with a as b, c:
+        print(1)
+    print(2)
+
+
+async def p0_multi_async_with_multi_as():
+    async with a as b, c as d:
+        print(1)
+
+
+async def p1_multi_async_with_multi_as_fallthrough():
+    async with a as b, c as d:
+        print(1)
+    print(2)
+
+
+async def q0_multi_async_with_multi_as_alt():
+    async with a, b as c, d:
+        print(1)
+
+
+async def q1_multi_async_with_multi_as_fallthrough_alt():
+    async with a, b as c, d:
+        print(1)
+    print(2)