diff --git a/dev_scripts/cflow.py b/dev_scripts/cflow.py index 8df7538..72e84ad 100644 --- a/dev_scripts/cflow.py +++ b/dev_scripts/cflow.py @@ -36,7 +36,7 @@ class Result(Enum): def edit_pyc_lines(pyc: PYCFile, src_lines: list[str]): if pyc.version == (3, 10): pyc.replace_duplicated_returns10(src_lines) - elif pyc.version == (3, 12): + elif pyc.version >= (3, 12): pyc.replace_duplicated_returns12(src_lines) seen_lines = set() # multiple instructions can start the same lno, but the segmentation model will only assign the lno to the first one diff --git a/dev_scripts/dataset_generation/bytecode2csv.py b/dev_scripts/dataset_generation/bytecode2csv.py index 130751b..1ab0c77 100644 --- a/dev_scripts/dataset_generation/bytecode2csv.py +++ b/dev_scripts/dataset_generation/bytecode2csv.py @@ -115,7 +115,7 @@ def bytecode2csv(py_path: pathlib.Path, pyc_path: pathlib.Path) -> tuple[list, l pyc = PYCFile(str(pyc_path.resolve())) if pyc.version == (3, 10): pyc.replace_duplicated_returns10(py_path.read_text().split("\n")) - elif pyc.version == (3, 12): + elif pyc.version >= (3, 12): pyc.replace_duplicated_returns12(py_path.read_text().split("\n")) global_masker = create_global_masker(pyc) diff --git a/dev_scripts/statement/tokenizer/special_tokens_map.json b/dev_scripts/statement/tokenizer/special_tokens_map.json index 5d7a5d0..47193ed 100644 --- a/dev_scripts/statement/tokenizer/special_tokens_map.json +++ b/dev_scripts/statement/tokenizer/special_tokens_map.json @@ -1085,6 +1085,13 @@ "normalized": false, "single_word": false }, + { + "content": "FORMAT_WITH_SPEC", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, { "content": "GEN_START", "lstrip": false, diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index 1e6c21d..2ae8494 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -1,5 +1,6 @@ from __future__ import annotations +import itertools from typing import TYPE_CHECKING from pathlib import Path @@ -75,6 +76,55 @@ class CFG(DiGraph_CFT): self._create_dominator_tree() return nx.dfs_postorder_nodes(self, source=self.start, sort_neighbors=lambda nodes: sorted(nodes, key=lambda x: x.offset, reverse=True)) + def is_loop_header(self, node): + # Check all predecessors + for predecessor in self.predecessors(node): + # A back edge exists if the predecessor is dominated by this node + if self.dominates(node, predecessor): + return True + return False + + def dfs_labeled_edges_no_loop(self, source=None, depth_limit=None, *, sort_neighbors=None): + if source is None: + # edges for all components + nodes = self + else: + # edges for components with source + nodes = [source] + if depth_limit is None: + depth_limit = len(self) + + get_children = self.neighbors if sort_neighbors is None else lambda n: iter(sort_neighbors(self.neighbors(n))) + + visited = set() + for start in nodes: + if start in visited: + continue + yield start, start, "forward" + visited.add(start) + stack = [(start, get_children(start))] + depth_now = 1 + while stack: + parent, children = stack[-1] + for child in children: + if child in visited or self.is_loop_header(child) or not all(p in visited for p in self.predecessors(child)): + yield parent, child, "nontree" + else: + yield parent, child, "forward" + visited.add(child) + if depth_now < depth_limit: + stack.append((child, iter(get_children(child)))) + depth_now += 1 + break + else: + yield parent, child, "reverse-depth_limit" + else: + stack.pop() + depth_now -= 1 + if stack: + yield stack[-1][0], parent, "reverse" + yield start, start, "reverse" + def apply_graphs(self): graphs = self.iteration_graphs.pop() if self.iteration_graphs: @@ -130,3 +180,16 @@ class CFG(DiGraph_CFT): dot.write(out, prog=["neato", "-n"], format=CFG.graph_format) else: self.iteration_graphs[-1].append(dot.to_string()) + + def cdg(self) -> CFG: + pdt = nx.create_empty_copy(self) + pdt.add_edges_from((B, A) for A, B in nx.immediate_dominators(self.reverse(), self.end).items()) + pdt.remove_edge(self.end, self.end) + pdr = nx.transitive_closure_dag(pdt) + postdominates = lambda A, B: pdr.has_edge(A, B) or A == B + control_dependent = lambda A, B: 0 < sum(postdominates(A, succ) for succ in self.successors(B)) < self.out_degree(B) + cdg = nx.create_empty_copy(self) + cdg.add_edges_from((B, A, {"kind": EdgeKind.Fall}) for A, B in itertools.product(self.nodes, self.nodes) if A != B and control_dependent(A, B)) + cdg.remove_node(self.end) + cdg.add_edges_from(((self.start, n) for n in cdg.nodes if cdg.in_degree(n) == 0 and n != self.start), kind=EdgeKind.Fall) + return cdg diff --git a/pylingual/control_flow_reconstruction/structure.py b/pylingual/control_flow_reconstruction/structure.py index b7d1ea8..98a26eb 100644 --- a/pylingual/control_flow_reconstruction/structure.py +++ b/pylingual/control_flow_reconstruction/structure.py @@ -1,5 +1,3 @@ -import pdb - from pylingual.editable_bytecode import EditableBytecode from pylingual.editable_bytecode.control_flow_graph import bytecode_to_control_flow_graph @@ -27,13 +25,8 @@ def structure_control_flow(cfg: nx.DiGraph, bytecode: EditableBytecode) -> Contr cfg = CFG.from_graph(cfg, bytecode) runs = get_template_runs(bytecode.version[:2]) - try: - while len(cfg) > 1: - if not iteration(cfg, runs): - return MetaTemplate("\x1b[31mirreducible cflow\x1b[0m", bytecode.codeobj) - except Exception: - if hasattr(pdb, "xpm"): - pdb.xpm() # type: ignore - raise + while len(cfg) > 1: + if not iteration(cfg, runs): + return MetaTemplate("\x1b[31mirreducible cflow\x1b[0m", bytecode.codeobj) return next(iter(cfg.nodes)) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 83397a4..1788a43 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -4,8 +4,10 @@ from typing import TYPE_CHECKING, override from itertools import chain from pylingual.editable_bytecode import Inst +import networkx as nx + from ..cft import ControlFlowTemplate, EdgeKind, SourceContext, SourceLine, register_template, EdgeCategory, out_edge_dict, MetaTemplate, indent_str -from ..utils import E, N, T, defer_source_to, remove_nodes +from ..utils import E, N, T, defer_source_to, remove_nodes, versions_from, without_instructions, has_no_lines, exact_instructions, make_try_match if TYPE_CHECKING: from pylingual.control_flow_reconstruction.cfg import CFG @@ -37,6 +39,71 @@ class EndTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("body") +@register_template(3, 0) +class RemoveUnreachable(ControlFlowTemplate): + @override + @classmethod + def try_match(cls, cfg, node) -> ControlFlowTemplate | None: + if node is not cfg.start: + return None + + valid = list(nx.dfs_preorder_nodes(cfg, source=cfg.start)) + invalid = [n for n in cfg.nodes if n not in valid] + if invalid: + cfg.remove_nodes_from(invalid) + return node + + +@register_template(0, 0, (3, 13)) +class JumpTemplate(ControlFlowTemplate): + template = T( + body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), + jump=N("tail", "block?") + .with_in_deg(1) + .with_cond( + exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), + exact_instructions("POP_JUMP_IF_TRUE"), + exact_instructions("JUMP_FORWARD"), + exact_instructions("JUMP_BACKWARD"), + exact_instructions("POP_JUMP_IF_NOT_NONE"), + exact_instructions("POP_JUMP_IF_NONE"), + exact_instructions("POP_JUMP_IF_FALSE"), + ), + block=N.tail(), + tail=N.tail(), + ) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + EdgeKind.TrueJump: "block", + }, + "body", + "jump", + ) + + to_indented_source = defer_source_to("body") + + +@register_template(0, 0, *versions_from(3, 11)) +class NopTemplate(ControlFlowTemplate): + template = T( + body=~N("nop", None).with_cond(without_instructions("CLEANUP_THROW")), + nop=N("tail", None).with_in_deg(1).with_cond(exact_instructions("NOP")).with_cond(has_no_lines), + tail=N.tail(), + ) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + }, + "body", + "nop", + ) + + to_indented_source = defer_source_to("body") + + @register_template(0, 20) @register_template(2, 20) class BlockTemplate(ControlFlowTemplate): diff --git a/pylingual/control_flow_reconstruction/templates/CDG.py b/pylingual/control_flow_reconstruction/templates/CDG.py new file mode 100644 index 0000000..5cd96f9 --- /dev/null +++ b/pylingual/control_flow_reconstruction/templates/CDG.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, override + +import networkx as nx + +from ..cft import ControlFlowTemplate, register_template + +if TYPE_CHECKING: + from pylingual.control_flow_reconstruction.cfg import CFG + + +# it's better than nothing +@register_template(101, 0) +class CDG(ControlFlowTemplate): + def __init__(self, cfg: CFG): + self.cdg = cfg.cdg() + self.start = cfg.start + self.blame = cfg.start.blame + self.header_lines = self.line("# irreducible cflow, using cdg fallback", meta=True) + + @override + @classmethod + def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: + cdg = CDG(cfg) + + if cfg.visualize == cfg._visualize: + cfg.remove_edges_from(list(cfg.edges)) + cfg.add_edges_from(cdg.cdg.edges(data=True)) + cfg.remove_node(cfg.end) + cfg.layout_nodes() + cfg.visualize() + + cfg.clear() + cfg.add_node(cdg) + return cdg + + @override + def get_instructions(self): + return [] + + @override + def to_indented_source(self, source): + cdg = self.cdg + for p, n in nx.dfs_edges(cdg, self.start): + cdg.nodes[n]["indent"] = cdg.nodes[p].get("indent", -1) + 1 + cdg.remove_node(self.start) + src = [] + for n in sorted(cdg.nodes, key=lambda x: x.offset): + src.extend(source[n, cdg.nodes[n].get("indent", 0)]) + return src diff --git a/pylingual/control_flow_reconstruction/templates/Conditional.py b/pylingual/control_flow_reconstruction/templates/Conditional.py index f163981..41989cc 100644 --- a/pylingual/control_flow_reconstruction/templates/Conditional.py +++ b/pylingual/control_flow_reconstruction/templates/Conditional.py @@ -1,13 +1,36 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import T, N, defer_source_to, run_is, has_no_lines, with_instructions, has_instval, starting_instructions, to_indented_source, make_try_match, without_top_level_instructions +from .Loop import BreakTemplate @register_template(1, 40) class IfElse(ControlFlowTemplate): template = T( if_header=~N("if_body", "else_body").with_cond(without_top_level_instructions("WITH_EXCEPT_START", "CHECK_EXC_MATCH", "FOR_ITER")), + if_body=N.tail().with_in_deg(1).of_type(BreakTemplate) | ~N("tail.").with_in_deg(1), + else_body=N.tail().with_in_deg(1).of_type(BreakTemplate) | ~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1), + tail=N.tail(), + ) + + try_match = make_try_match({EdgeKind.Fall: "tail"}, "if_header", "if_body", "else_body") + + @to_indented_source + def to_indented_source(): + """ + {if_header} + {if_body} + {else_body?else:} + {else_body} + """ + + +@register_template(1, 39, (3, 12), (3, 13)) +class IfElseLoop(ControlFlowTemplate): + template = T( + if_header=~N("else_body", "if_body").with_cond(without_top_level_instructions("WITH_EXCEPT_START", "CHECK_EXC_MATCH", "FOR_ITER")), if_body=~N("tail.").with_in_deg(1), - else_body=~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1), + else_body=~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1).with_cond(has_no_lines), + for_iter=N.tail().with_cond(with_instructions("FOR_ITER")), tail=N.tail(), ) diff --git a/pylingual/control_flow_reconstruction/templates/Exception.py b/pylingual/control_flow_reconstruction/templates/Exception.py index 79be386..7aa5619 100644 --- a/pylingual/control_flow_reconstruction/templates/Exception.py +++ b/pylingual/control_flow_reconstruction/templates/Exception.py @@ -13,6 +13,8 @@ from ..utils import ( with_instructions, without_instructions, ending_instructions, + has_no_lines, + has_some_lines, exact_instructions, no_back_edges, without_top_level_instructions, @@ -39,8 +41,8 @@ class Except3_11(ControlFlowTemplate): return x -@register_template(0, 0, *versions_from(3, 11)) -class Try3_11(ControlFlowTemplate): +@register_template(0, 0, *versions_from(3, 12)) +class Try3_12(ControlFlowTemplate): template = T( try_header=N("try_body"), try_body=N("tail.", None, "except_body"), @@ -71,8 +73,8 @@ class Try3_11(ControlFlowTemplate): """ -@register_template(0, 0, *versions_from(3, 11)) -class TryElse3_11(ControlFlowTemplate): +@register_template(0, 0, *versions_from(3, 12)) +class TryElse3_12(ControlFlowTemplate): template = T( try_header=N("try_body"), try_body=N("try_else.", None, "except_body"), @@ -107,6 +109,76 @@ class TryElse3_11(ControlFlowTemplate): """ +@register_template(0, 0, (3, 11)) +class Try3_11(ControlFlowTemplate): + template = T( + try_header=N("try_body"), + try_body=N("try_else.", None, "except_body"), + except_body=N("tail.", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11), + try_else=~N("tail.").with_in_deg(1).with_cond(has_no_lines), + reraise=reraise, + tail=N.tail(), + ) + + try_match = revert_on_fail( + make_try_match( + { + EdgeKind.Fall: "tail", + }, + "try_header", + "try_else", + "try_body", + "except_body", + "reraise", + ) + ) + + @to_indented_source + def to_indented_source(): + """ + {try_header} + try: + {try_body} + {except_body} + """ + + +@register_template(0, 0, (3, 11)) +class TryElse3_11(ControlFlowTemplate): + template = T( + try_header=N("try_body"), + try_body=N("try_else.", None, "except_body"), + except_body=N("tail.", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11), + try_else=~N("tail.").with_in_deg(1).with_cond(has_some_lines), + reraise=reraise, + tail=N.tail(), + ) + + try_match = revert_on_fail( + make_try_match( + { + EdgeKind.Fall: "tail", + }, + "try_header", + "try_body", + "except_body", + "try_else", + "reraise", + ) + ) + + @to_indented_source + def to_indented_source(): + """ + {try_header} + try: + {try_body} + {except_body} + else: + {try_else} + """ + + class BareExcept3_11(Except3_11): template = T( except_body=N("except_footer.", None, "reraise").with_cond(without_top_level_instructions("RERAISE")), @@ -185,7 +257,7 @@ class ExceptExc3_11(Except3_11): except_header=N("except_body", "no_match", "reraise").with_cond(ending_instructions("CHECK_EXC_MATCH", "POP_JUMP_FORWARD_IF_FALSE"), ending_instructions("CHECK_EXC_MATCH", "POP_JUMP_IF_FALSE")), except_body=N("except_footer.", None, "reraise").of_subtemplate(ExcBody3_11).with_in_deg(1), no_match=N("tail?", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11), - except_footer=~N("tail.").with_in_deg(1).with_cond(starting_instructions("POP_EXCEPT")), + except_footer=~N("tail.").with_in_deg(1).with_cond(starting_instructions("SWAP", "POP_EXCEPT"), starting_instructions("POP_EXCEPT")), reraise=reraise, tail=N.tail(), ) @@ -225,7 +297,7 @@ class TryFinally3_11(ControlFlowTemplate): tail=N.tail(), ) template2 = T( - try_except=N("finally_body", None, "fail_body").of_type(Try3_11, TryElse3_11), + try_except=N("finally_body", None, "fail_body").of_type(Try3_11, TryElse3_11, Try3_12, TryElse3_12), finally_body=~N("tail.").with_in_deg(1).with_cond(no_back_edges), fail_body=N(E.exc("reraise")).with_cond(ending_instructions("POP_TOP", "RERAISE")), reraise=reraise, @@ -280,7 +352,7 @@ class TryFinally3_11(ControlFlowTemplate): def to_indented_source(self, source: SourceContext) -> list[SourceLine]: header = source[self.try_header] body = source[self.try_body, 1] - if isinstance(self.try_header, (Try3_11, TryElse3_11)) and self.members["try_body"] is None: + if isinstance(self.try_header, (Try3_11, TryElse3_11, Try3_12, TryElse3_12)) and self.members["try_body"] is None: s = header else: s = chain(header, self.line("try:"), body) @@ -310,20 +382,6 @@ class Except3_9(ControlFlowTemplate): return node -class Except3_9(ControlFlowTemplate): - @classmethod - @override - def try_match(cls, cfg, node) -> ControlFlowTemplate | None: - if [x.opname for x in node.get_instructions()] == ["RERAISE"]: - return node - if x := ExceptExc3_9.try_match(cfg, node): - return x - if x := BareExcept3_9.try_match(cfg, node): - return x - if isinstance(node, Except3_9): - return node - - @register_template(0, 0, (3, 9), (3, 10)) class Try3_9(ControlFlowTemplate): template = T( diff --git a/pylingual/control_flow_reconstruction/templates/Generator.py b/pylingual/control_flow_reconstruction/templates/Generator.py index 87be641..9027460 100644 --- a/pylingual/control_flow_reconstruction/templates/Generator.py +++ b/pylingual/control_flow_reconstruction/templates/Generator.py @@ -1,5 +1,5 @@ from ..cft import ControlFlowTemplate, EdgeKind, MetaTemplate, register_template -from ..utils import E, T, N, defer_source_to, exact_instructions, no_back_edges, to_indented_source, make_try_match +from ..utils import E, T, N, defer_source_to, ending_instructions, exact_instructions, no_back_edges, to_indented_source, make_try_match @register_template(0, 0) @@ -38,7 +38,7 @@ class AwaitWith3_12(ControlFlowTemplate): @register_template(0, 0) class Generator3_12(ControlFlowTemplate): template = T( - entry=N("body").with_cond(exact_instructions("RETURN_GENERATOR", "POP_TOP")), + entry=N("body").with_cond(ending_instructions("RETURN_GENERATOR", "POP_TOP")), body=N(E.exc("gen_cleanup"), E.meta("end?")), gen_cleanup=N(E.meta("end")).with_cond(exact_instructions("CALL_INTRINSIC_1", "RERAISE")), end=N().of_type(MetaTemplate), diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index 3f56a0c..fbdf78c 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -1,13 +1,22 @@ +from __future__ import annotations +from typing import TYPE_CHECKING from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, N, + with_instructions, + exact_instructions, + has_no_lines, + condense_mapping, defer_source_to, starting_instructions, to_indented_source, make_try_match, ) +if TYPE_CHECKING: + from pylingual.control_flow_reconstruction.cfg import CFG + @register_template(0, 1) class ForLoop(ControlFlowTemplate): @@ -41,6 +50,24 @@ class SelfLoop(ControlFlowTemplate): """ +@register_template(0, 2) +class TrueSelfLoop(ControlFlowTemplate): + template = T(loop_body=~N("tail.", "loop_body"), tail=N.tail()) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + }, + "loop_body", + ) + + @to_indented_source + def to_indented_source(): + """ + {loop_body} + """ + + @register_template(0, 3) class InlinedComprehensionTemplate(ControlFlowTemplate): template = T( @@ -58,3 +85,96 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): ) to_indented_source = defer_source_to("comp") + + +class BreakTemplate(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg, node): + if isinstance(node, BreakTemplate) or has_no_lines(cfg, node) or with_instructions("RAISE_VARARGS")(cfg, node): + return None + return condense_mapping(cls, cfg, {"child": node}, "child") + + def to_indented_source(self, source): + return self.child.to_indented_source(source) + self.line("break") + + +class ContinueTemplate(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg, node): + if isinstance(node, ContinueTemplate) or has_no_lines(cfg, node): + return None + instruction = node.get_instructions()[-1].opname + if instruction in {"JUMP_ABSOLUTE", "JUMP_BACKWARD", "CONTINUE_LOOP"} and (node.get_instructions()[-1].starts_line is not None or node.get_instructions()[-2].starts_line is not None): + return condense_mapping(cls, cfg, {"child": node}, "child") + return None + + def to_indented_source(self, source): + return self.child.to_indented_source(source) + self.line("continue") + + +@register_template(0, 0) +class FixLoop(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: + # check that its a loop that we need to fix + # find the end of the loop + # find all nodes that belong to the loop + # find nodes in loop that go to end + # replace those edges with meta edges to the end + # find nodes in loop that go to header + # replace all but last of those edges with meta edge to end + + # a node is a loop header if there are back-edges to it + # a latching node is a node with a back-edge to the loop header + # a back-edge is an edge from any node that is dominated by this node + back_edges = [] + for predecessor in cfg.predecessors(node): + # A back edge exists if the predecessor is reachable from the node (node dominates predecessor) + if cfg.dominates(node, predecessor): + back_edges.append(predecessor) + + if not back_edges: + return None + + # Get all nodes encompassed by the loop excluding source node and initial false jump + loopnode = None + for succ in cfg.successors(node): + if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.Fall: + loopnode = succ + break + + dfs_edges = cfg.dfs_labeled_edges_no_loop(source=loopnode) + encompassed_nodes = [v for u, v, d in dfs_edges if d == "forward"] + + edges_to_remove = [] + + # Find the candidate end that break connects to + candidate_end = None + for succ in cfg.successors(node): + if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.FalseJump and cfg.out_degree(succ) <= 1: + candidate_end = succ + + # Candidate end is a buffer node + if cfg.in_degree(candidate_end) == 1 and all(x.opname in {"POP_TOP", "POP_BLOCK", "END_FOR", "RETURN_CONST", "LOAD_CONST", "RETURN_VALUE", "JUMP_BACKWARD"} for x in candidate_end.get_instructions()): + for ss in cfg.successors(candidate_end): + if cfg.get_edge_data(candidate_end, ss).get("kind") != EdgeKind.Exception: + candidate_end = ss + break + + if encompassed_nodes is not None: + for succ in encompassed_nodes: + if cfg.get_edge_data(succ, candidate_end) != None: + edges_to_remove.append((succ, candidate_end)) + + for pred, succ in edges_to_remove: + break_node = BreakTemplate.try_match(cfg, pred) + if break_node is not None: + cfg.remove_edge(break_node, succ) + + for candidate in back_edges: + cont_node = ContinueTemplate.try_match(cfg, candidate) + if cont_node is not None and cfg.in_degree(node) > 2: + cfg.remove_edge(cont_node, node) + + cfg.iterate() + return diff --git a/pylingual/control_flow_reconstruction/utils.py b/pylingual/control_flow_reconstruction/utils.py index 020d7c9..66367ea 100644 --- a/pylingual/control_flow_reconstruction/utils.py +++ b/pylingual/control_flow_reconstruction/utils.py @@ -134,6 +134,10 @@ def has_no_lines(cfg: CFG, node: ControlFlowTemplate | None) -> bool: return node is None or all(i.starts_line is None for i in node.get_instructions()) +def has_some_lines(cfg: CFG, node: ControlFlowTemplate | None) -> bool: + return node is None or any(i.starts_line is not None for i in node.get_instructions()) + + def run_is(n: int): def check_run(cfg: CFG, node: ControlFlowTemplate | None) -> bool: return cfg.run == n diff --git a/pylingual/main.py b/pylingual/main.py index ff8c76d..0826de3 100644 --- a/pylingual/main.py +++ b/pylingual/main.py @@ -130,11 +130,7 @@ def main(files: list[str], out_dir: Path | None, config_file: Path | None, versi pyc = result.original_pyc print_result(f"Equivalence Results for {pyc.pyc_path.name if pyc.pyc_path else repr(pyc)}", result.equivalence_results) except Exception: - import pdb - live.stop() - if hasattr(pdb, "xpm"): - pdb.xpm() # type: ignore logger.exception(f"Failed to decompile {pyc_path}") console.rule() diff --git a/pylingual/masking/global_masker.py b/pylingual/masking/global_masker.py index 188d85a..8a780e7 100644 --- a/pylingual/masking/global_masker.py +++ b/pylingual/masking/global_masker.py @@ -238,7 +238,7 @@ class Masker: if inst.is_jump_target: view = f"{inst.offset} {view}" # create list of offsets greater than or less than inst, for all jump origins to this inst - jumps_greater_or_less = [inst.offset < inst.offset for inst in inst.jumped_to_from_insts] + jumps_greater_or_less = [i.offset < inst.offset for i in inst.jumped_to_from_insts] if any(jumps_greater_or_less): # 1 in list view = f"^~> {view}" if not all(jumps_greater_or_less): # 0 in list diff --git a/test/Conditional.py b/test/Conditional.py index 19302fc..03394cc 100644 --- a/test/Conditional.py +++ b/test/Conditional.py @@ -302,3 +302,24 @@ def f_nofallthru_if_pass(): if a > b: pass print("end") + + +def g1_ifElseLoop(): + for a in range(3): + if a > b: + print(1) + + +def g2_ifElseLoop(): + for a in range(3): + if a > b: + print(1) + print(2) + + +def g3_ifElseLoop(): + for a in range(3): + if a > b: + print(1) + else: + print(2) diff --git a/test/TryExcept.py b/test/Exception.py similarity index 66% rename from test/TryExcept.py rename to test/Exception.py index e268fad..24224b1 100644 --- a/test/TryExcept.py +++ b/test/Exception.py @@ -1,4 +1,11 @@ -def a_TryExcept(): +def a0_bare_try_except(): + try: + print(1) + except: + print(2) + + +def a1_bare_try_except_fallthrough(): try: print(1) except: @@ -6,7 +13,8 @@ def a_TryExcept(): print(3) -def b_TryExceptBareNested(): +# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match +def b0_nested_try_except(): try: print(1) except: @@ -17,7 +25,7 @@ def b_TryExceptBareNested(): print(4) -def b1_TryExceptBareNestedFallthrough(): +def b1_nested_try_except_fallthrough(): try: print(1) except: @@ -29,7 +37,8 @@ def b1_TryExceptBareNestedFallthrough(): print(5) -def b2_TryExceptBareNestedEarlyFallthrough(): +# 3.13 Duplicate blocks +def b2_nested_try_except_early_fallthrough(): try: print(1) except: @@ -41,7 +50,7 @@ def b2_TryExceptBareNestedEarlyFallthrough(): print(5) -def b3_TryExceptBareNestedDoubleFallthrough(): +def b3_nested_try_except_double_fallthrough(): try: print(1) except: @@ -54,7 +63,8 @@ def b3_TryExceptBareNestedDoubleFallthrough(): print(6) -def c_TryExceptBareMultiNested(): +# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match +def c0_multi_except_nested(): try: print(1) except a: @@ -71,7 +81,8 @@ def c_TryExceptBareMultiNested(): print(7) -def c1_TryExceptBareMultiNestedFallthrough(): +# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match +def c1_multi_except_nested_fallthrough(): try: print(1) except a: @@ -89,7 +100,7 @@ def c1_TryExceptBareMultiNestedFallthrough(): print(8) -def c2_TryExceptBareMultiNestedFallthrough2(): +def c2_multi_except_nested_fallthrough2(): try: print(1) except a: @@ -108,7 +119,8 @@ def c2_TryExceptBareMultiNestedFallthrough2(): print(9) -def c3_TryExceptBareMultiNestedEarlyFallthrough(): +# 3.13 Duplicate blocks +def c3_multi_except_nested_early_fallthrough(): try: print(1) except a: @@ -126,7 +138,7 @@ def c3_TryExceptBareMultiNestedEarlyFallthrough(): print(8) -def c4_TryExceptBareMultiNestedAllFallthrough(): +def c4_multi_except_nested_all_fallthrough(): try: print(1) except a: @@ -146,7 +158,9 @@ def c4_TryExceptBareMultiNestedAllFallthrough(): print(10) -def d_TryExceptBareNestedNamed(): +# 3.10/3.11/3.12/3.13 Duplicate blocks causing templates to not match +# Discussed in issue 41 +def d0_named_except_nested(): try: print(1) except A as a: @@ -157,7 +171,7 @@ def d_TryExceptBareNestedNamed(): print(4) -def d1_TryExceptBareNestedNamedFallthrough(): +def d1_named_except_nested_fallthrough(): try: print(1) except A as a: @@ -169,7 +183,8 @@ def d1_TryExceptBareNestedNamedFallthrough(): print(5) -def d2_TryExceptBareNestedNamedEarlyFallthrough(): +# 3.13 Duplicate blocks +def d2_named_except_nested_early_fallthrough(): try: print(1) except A as a: @@ -181,7 +196,7 @@ def d2_TryExceptBareNestedNamedEarlyFallthrough(): print(5) -def d3_TryExceptBareNestedNamedDoubleFallthrough(): +def d3_named_except_nested_double_fallthrough(): try: print(1) except A as a: @@ -194,7 +209,7 @@ def d3_TryExceptBareNestedNamedDoubleFallthrough(): print(6) -def e_TryExceptElseBare(): +def e0_try_except_else(): try: print(1) except: @@ -204,7 +219,7 @@ def e_TryExceptElseBare(): print(4) -def f_TryExceptElseFinallyBare(): +def f0_try_except_else_finally(): try: print(1) except: @@ -216,7 +231,7 @@ def f_TryExceptElseFinallyBare(): print(5) -def g_TryExceptElseMulti(): +def g0_multi_except_with_else(): try: print(1) except a: @@ -228,7 +243,7 @@ def g_TryExceptElseMulti(): print(5) -def h_TryExceptElseMultiFallback(): +def h0_multi_except_fallback_with_else(): try: print(1) except a: @@ -242,7 +257,7 @@ def h_TryExceptElseMultiFallback(): print(6) -def i_TryExceptElseMultiNamedAndUnnamed(): +def i0_mixed_named_unnamed_except_with_else(): try: print(1) except A as a: @@ -256,7 +271,7 @@ def i_TryExceptElseMultiNamedAndUnnamed(): print(6) -def j_TryExceptElseNamed(): +def j0_named_except_with_else(): try: print(1) except A as a: @@ -268,7 +283,7 @@ def j_TryExceptElseNamed(): print(5) -def k_TryExceptFinallyBare(): +def k0_try_except_finally(): try: print(1) except: @@ -278,7 +293,7 @@ def k_TryExceptFinallyBare(): print(4) -def l_TryExceptFinallyBareSpecific(): +def l0_specific_except_finally(): try: print(1) except a: @@ -288,7 +303,7 @@ def l_TryExceptFinallyBareSpecific(): print(4) -def m_TryExceptMulti(): +def m0_multi_except(): try: print(1) except a: @@ -300,7 +315,7 @@ def m_TryExceptMulti(): print(5) -def n_TryExceptMultiFallback(): +def n0_multi_except_with_fallback(): try: print(1) except a: @@ -312,7 +327,7 @@ def n_TryExceptMultiFallback(): print(5) -def o_TryExceptMultiFallbackFinally(): +def o0_multi_except_fallback_finally(): try: print(1) except a: @@ -324,7 +339,7 @@ def o_TryExceptMultiFallbackFinally(): print(5) -def p_TryExceptMultiNamed(): +def p0_multi_named_except(): try: print(1) except A as a: @@ -336,7 +351,7 @@ def p_TryExceptMultiNamed(): print(5) -def q_TryExceptMultiNamedAndUnnamed(): +def q0_mixed_named_unnamed_except(): try: print(1) except A as a: @@ -348,7 +363,7 @@ def q_TryExceptMultiNamedAndUnnamed(): print(5) -def r_TryExceptMultiNamedAndUnnamedFinally(): +def r0_mixed_named_unnamed_except_finally(): try: print(1) except A as a: @@ -362,7 +377,7 @@ def r_TryExceptMultiNamedAndUnnamedFinally(): print(6) -def s_TryExceptMultiNamedFallback(): +def s0_named_except_fallback(): try: print(1) except A as a: @@ -372,7 +387,7 @@ def s_TryExceptMultiNamedFallback(): print(4) -def t_TryExceptMultiNamedFallbackFinally(): +def t0_named_except_fallback_finally(): try: print(1) except A as a: @@ -384,7 +399,7 @@ def t_TryExceptMultiNamedFallbackFinally(): print(5) -def u_TryExceptMultiNamedFinally(): +def u0_multi_named_except_finally(): try: print(1) except A as a: @@ -396,7 +411,7 @@ def u_TryExceptMultiNamedFinally(): print(5) -def v_TryExceptMultiFinally(): +def v0_multi_except_finally(): try: print(1) except a: @@ -408,7 +423,7 @@ def v_TryExceptMultiFinally(): print(5) -def w_TryExceptRaise(): +def w0_try_except_raise(): try: print(1) except: @@ -416,7 +431,7 @@ def w_TryExceptRaise(): raise Exc -def x_TryExceptRaiseMulti(): +def x0_multi_except_raise(): try: print(1) except a: @@ -427,7 +442,7 @@ def x_TryExceptRaiseMulti(): raise Exc -def y_TryExceptRaiseNamed(): +def y0_named_except_raise(): try: print(1) except A as a: @@ -435,7 +450,8 @@ def y_TryExceptRaiseNamed(): raise Exc -def z_TryExceptReturn(): +# 3.11 Try return getting left outside of TryExcept +def z0_try_except_return(): try: print(1) return 2 @@ -443,7 +459,8 @@ def z_TryExceptReturn(): print(2) -def z1_TryExceptReturn(): +# 3.11 Try return getting left outside of TryExcept +def z1_try_except_return_both(): try: print(1) return 2 @@ -452,7 +469,8 @@ def z1_TryExceptReturn(): return 3 -def aa_TryExceptReturnMulti(): +# 3.11 Try return getting left outside of TryExcept +def aa0_multi_except_return(): try: print(1) return 2 @@ -462,7 +480,10 @@ def aa_TryExceptReturnMulti(): print(3) -def aa1_TryExceptReturnMulti(): +# 3.6/3.7/3.8 ExceptExc abandons tail node. +# Could be fixed (?) but breaks other test cases +# 3.11 Try return getting left outside of TryExcept +def aa1_multi_except_return_both(): try: print(1) return 2 @@ -473,7 +494,8 @@ def aa1_TryExceptReturnMulti(): print(3) -def ab_TryExceptReturnNamed(): +# 3.11 Try return getting left outside of TryExcept +def ab0_named_except_raise_return(): try: print(1) return 2 @@ -485,14 +507,33 @@ def ab_TryExceptReturnNamed(): raise Exc -def TryEmptryFinally(): +# 3.8 Double natural edge graph error (?) +# 3.11 Try return getting left outside of TryExcept +def ab1_named_except_return(): + try: + print(1) + return 2 + except A as a: + print(2) + return 3 + + +def ab2_named_except_return(): + try: + return a + except Exception as a: + return b + + +# 3.11/3.12/3.13 No template match +def ac0_empty_try_finally(): try: pass finally: print(1) -def TryMultiple(): +def ad0_multiple_try_blocks(): try: print(1) except: @@ -504,7 +545,7 @@ def TryMultiple(): print(4) -def TryExceptElseTry(): +def ae0_try_except_else_nested_try(): try: print(1) except: @@ -516,7 +557,9 @@ def TryExceptElseTry(): print(4) -def TryFinallyNestedExcept(): +# 3.9 Duplicate blocks (?) +# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) +def af0_try_finally_nested_except(): try: print(1) finally: @@ -526,35 +569,40 @@ def TryFinallyNestedExcept(): print(3) -def TryExceptTuple(): +def ag0_try_except_tuple(): try: print(1) except (A, B): print(2) -def TryFinallyReturn(): +# 3.9 Difficult template ambiguity between Try/TryFinally +# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) +def ah0_try_finally_return(): try: print(1) finally: return 2 -def TryReturnFinally(): +# 3.11/3.12/3.13 No template match (?) +def ai0_try_return_finally(): try: return 1 finally: print(2) -def TryReturnFinallyReturn(): +# 3.9/3.10 Difficult template ambiguity between Try/TryFinally +# 3.11/3.12/3.13 No template match (?) +def aj0_try_return_finally_return(): try: return 1 finally: return 2 -def TryExceptRaise(): +def ak0_try_except_raise_return(): try: print(1) return 2 @@ -562,8 +610,8 @@ def TryExceptRaise(): raise Exception() -""" -def TryExceptReturnFinally(): +# 3.8/3.9/3.10 No template match +def al0_try_except_return_finally(): try: raise Exception() except: @@ -571,28 +619,19 @@ def TryExceptReturnFinally(): return 2 finally: print(3) -""" -""" -def TryFinallyRaise(): + +# 3.8/3.9/3.10 No template match +# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) +def am0_try_finally_raise(): try: print(1) return 2 finally: raise Exception() -""" -def ab1_TryExceptReturnNamed(): - try: - print(1) - return 2 - except A as a: - print(2) - return 3 - - -def ac_TryFinallyBareFallthrough(): +def an0_try_finally_fallthrough(): try: print(1) finally: @@ -600,7 +639,7 @@ def ac_TryFinallyBareFallthrough(): print(3) -def ad_TryFinallyBare(): +def ao0_try_finally_simple(): try: print(1) finally: diff --git a/test/Loop.py b/test/Loop.py index 323272e..20117a9 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -1,31 +1,78 @@ -# FOR LOOP TESTS - - -def a_for_over_list(): +def a0_for_over_list(): for x in [1, 2, 3]: print("for over list") -def b_for_over_tuples(): +def a1_for_over_list_nofallthru(): + for x in [1, 2, 3]: + print("for over list") + print("end") + + +def b0_for_over_tuples(): for a, b in [(1, 2), (3, 4)]: print("tuples") -def c_for_else(): +def b1_for_over_tuples_nofallthru(): + for a, b in [(1, 2), (3, 4)]: + print("tuples") + print("end") + + +# 3.6/3.7 No else template +def c0_for_else(): for i in range(3): print("for body") else: print("for else") -def d_for_with_break(): +# 3.6/3.7 No else template +def c1_for_else_nofallthru(): + for i in range(3): + print("for body") + else: + print("for else") + print("end") + + +# 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9 Naive break detection, an unexpected buffer block to end +def d0_for_with_break(): for x in range(10): if x == 5: print("breaking") break -def e_for_with_continue(): +# 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end +def d1_for_with_break_nofallthru(): + for x in range(10): + if x == 5: + print("breaking") + break + print("end") + + +# Help to implement break +def d2_for_without_break(): + for x in range(10): + if x == 5: + print("not breaking") + print("end") + + +# Help to implement break +def d3_for_return(): + for x in range(10): + if x == 5: + print("not breaking") + return + print("end") + + +def e0_for_with_continue(): for x in range(5): if x % 2 == 0: print("continuing") @@ -33,13 +80,29 @@ def e_for_with_continue(): print("after continue") -def f_nested_for_loops(): +def e1_for_with_continue_nofallthru(): + for x in range(5): + if x % 2 == 0: + print("continuing") + continue + print("after continue") + print("end") + + +def f0_nested_for_loops(): for i in range(2): for j in range(3): print(f"nested {i},{j}") -def g_for_with_try_except(): +def f1_nested_for_loops_nofallthru(): + for i in range(2): + for j in range(3): + print(f"nested {i},{j}") + print("end") + + +def g0_for_with_try_except(): for x in range(2): try: print("try block") @@ -47,13 +110,29 @@ def g_for_with_try_except(): print("except block") -def h_for_with_with_statement(): +def g1_for_with_try_except_nofallthru(): + for x in range(2): + try: + print("try block") + except Exception: + print("except block") + print("end") + + +def h0_for_with_with_statement(): for _ in range(1): with a: print("inside with") -def i_for_with_function_call_iterable(): +def h1_for_with_with_statement_nofallthru(): + for _ in range(1): + with a: + print("inside with") + print("end") + + +def i0_for_with_function_call_iterable(): def get_items(): return [1, 2, 3] @@ -61,147 +140,8 @@ def i_for_with_function_call_iterable(): print(f"item: {item}") -def j_for_with_empty_body_ellipsis(): - for _ in range(3): - ... - - -def k_while_true_with_break(): - while True: - print("while true") - break - - -def l_while_with_else(): - i = 0 - while i < 3: - print(f"looping {i}") - i += 1 - else: - print("while else") - - -def m_while_with_continue(): - i = 0 - while i < 5: - i += 1 - if i % 2 == 0: - print("continue") - continue - print("after continue") - - -def n_while_with_break(): - i = 0 - while True: - print("break in while") - break - - -def o_nested_while_loops(): - i = 0 - while i < 2: - j = 0 - while j < 2: - print(f"nested while {i},{j}") - j += 1 - i += 1 - - -def p_while_with_try_except(): - while True: - try: - print("try in while") - except: - print("except in while") - - -def q_while_with_with_statement(): - while True: - with a: - print("inside while with") - - -def r_for_inside_while(): - while True: - for x in [1, 2]: - print("for in while") - - -def s_while_inside_for(): - for _ in range(1): - while True: - print("while in for") - break - - -def t_while_with_empty_body_ellipsis(): - while True: - ... - - -def a_nofallthru_for_over_list(): - for x in [1, 2, 3]: - print("for over list") - print("end") - - -def b_nofallthru_for_over_tuples(): - for a, b in [(1, 2), (3, 4)]: - print("tuples") - print("end") - - -def c_nofallthru_for_else(): - for i in range(3): - print("for body") - else: - print("for else") - print("end") - - -def d_nofallthru_for_with_break(): - for x in range(10): - if x == 5: - print("breaking") - break - print("end") - - -def e_nofallthru_for_with_continue(): - for x in range(5): - if x % 2 == 0: - print("continuing") - continue - print("after continue") - print("end") - - -def f_nofallthru_nested_for_loops(): - for i in range(2): - for j in range(3): - print(f"nested {i},{j}") - print("end") - - -def g_nofallthru_for_with_try_except(): - for x in range(2): - try: - print("try block") - except Exception: - print("except block") - print("end") - - -def h_nofallthru_for_with_with_statement(): - for _ in range(1): - with a: - print("inside with") - print("end") - - -def i_nofallthru_for_with_function_call_iterable(): - def g_nofallthruet_items(): +def i1_for_with_function_call_iterable_nofallthru(): + def get_items(): return [1, 2, 3] for item in get_items(): @@ -209,20 +149,54 @@ def i_nofallthru_for_with_function_call_iterable(): print("end") -def j_nofallthru_for_with_empty_body_ellipsis(): +def j0_for_with_empty_body_ellipsis(): + for _ in range(3): + ... + + +def j1_for_with_empty_body_ellipsis_nofallthru(): for _ in range(3): ... print("end") -def k_nofallthru_while_true_with_break(): +# 3.6/3.7 Naive break detection, no back edge +# 3.9/3.11 No while loop detection, self false_jump edge & naive break detection +def k0_while_true_with_break(): + x = 0 while True: print("while true") - break + x += 1 + if x >= 1: + break + + +# 3.6/3.7 Naive break detection, no back edge +# 3.9/3.11 No while loop detection, self false_jump edge & naive break detection +def k1_while_true_with_break_nofallthru(): + x = 0 + while True: + print("while true") + x += 1 + if x >= 1: + break print("end") -def l_nofallthru_while_with_else(): +# 3.6/3.7 No else template +# 3.11 No while loop detection, self false_jump edge +def l0_while_with_else(): + i = 0 + while i < 3: + print(f"looping {i}") + i += 1 + else: + print("while else") + + +# 3.6/3.7 No else template +# 3.11 No while loop detection, self false_jump edge +def l1_while_with_else_nofallthru(): i = 0 while i < 3: print(f"looping {i}") @@ -232,7 +206,19 @@ def l_nofallthru_while_with_else(): print("end") -def m_nofallthru_while_with_continue(): +# 3.11 No continue +def m0_while_with_continue(): + i = 0 + while i < 5: + i += 1 + if i % 2 == 0: + print("continue") + continue + print("after continue") + + +# 3.11 No continue +def m1_while_with_continue_nofallthru(): i = 0 while i < 5: i += 1 @@ -243,7 +229,16 @@ def m_nofallthru_while_with_continue(): print("end") -def n_nofallthru_while_with_break(): +# 3.6/3.7 Naive break detection, no back edge +def n0_while_with_break(): + i = 0 + while True: + print("break in while") + break + + +# 3.6/3.7 Naive break detection, no back edge +def n1_while_with_break_nofallthru(): i = 0 while True: print("break in while") @@ -251,7 +246,19 @@ def n_nofallthru_while_with_break(): print("end") -def o_nofallthru_nested_while_loops(): +# 3.11 While template broke +def o0_nested_while_loops(): + i = 0 + while i < 2: + j = 0 + while j < 2: + print(f"nested while {i},{j}") + j += 1 + i += 1 + + +# 3.11 While template broke +def o1_nested_while_loops_nofallthru(): i = 0 while i < 2: j = 0 @@ -262,7 +269,19 @@ def o_nofallthru_nested_while_loops(): print("end") -def p_nofallthru_while_with_try_except(): +# 3.6/3.7 While template broke (?) +# 3.9 Disconnected with MetaTemplate[end] (?) +def p0_while_with_try_except(): + while True: + try: + print("try in while") + except: + print("except in while") + + +# 3.6/3.7 While template broke (?) +# 3.9 Disconnected with MetaTemplate[end] (?) +def p1_while_with_try_except_nofallthru(): while True: try: print("try in while") @@ -271,21 +290,46 @@ def p_nofallthru_while_with_try_except(): print("end") -def q_nofallthru_while_with_with_statement(): +# 3.6/3.7 While template broke (?) abandoning nodes +def q0_while_with_with_statement(): + while True: + with a: + print("inside while with") + + +# 3.6/3.7 While template broke (?) abandoning nodes +def q1_while_with_with_statement_nofallthru(): while True: with a: print("inside while with") print("end") -def r_nofallthru_for_inside_while(): +# 3.6/3.7 While template broke +def r0_for_inside_while(): + while True: + for x in [1, 2]: + print("for in while") + + +# 3.6/3.7 While template broke +def r1_for_inside_while_nofallthru(): while True: for x in [1, 2]: print("for in while") print("end") -def s_nofallthru_while_inside_for(): +# 3.6/3.7 While template broke +def s0_while_inside_for(): + for _ in range(1): + while True: + print("while in for") + break + + +# 3.6/3.7 While template broke +def s1_while_inside_for_nofallthru(): for _ in range(1): while True: print("while in for") @@ -293,7 +337,151 @@ def s_nofallthru_while_inside_for(): print("end") -def t_nofallthru_while_with_empty_body_ellipsis(): +# 3.6/3.7 While template broke +def t0_while_with_empty_body_ellipsis(): + while True: + ... + + +# 3.6/3.7 While template broke +def t1_while_with_empty_body_ellipsis_nofallthru(): while True: ... print("end") + + +def u0_break_in_nested_for(): + for i in range(3): + for j in range(3): + if i == 1 and j == 1: + print("Breaking inner loop") + break + print(f"i={i}, j={j}") + + +def u1_break_in_nested_for_nofallthru(): + for i in range(3): + for j in range(3): + if i == 1 and j == 1: + print("Breaking inner loop") + break + print(f"i={i}, j={j}") + print("end") + + +def v0_continue_in_nested_for(): + for i in range(3): + for j in range(3): + if j == 1: + continue + print(f"Processing i={i}, j={j}") + + +def v1_continue_in_nested_for_nofallthru(): + for i in range(3): + for j in range(3): + if j == 1: + continue + print(f"Processing i={i}, j={j}") + print("end") + + +# 3.13 if statement putting code in the else block +def w0_break_with_else(): + for i in range(5): + if i == 3: + print("Breaking before else") + break + else: + print("This won't execute due to break") + + +# 3.13 if statement putting code in the else block +def w1_break_with_else_nofallthru(): + for i in range(5): + if i == 3: + print("Breaking before else") + break + else: + print("This won't execute due to break") + print("end") + + +# 3.6/3.7 No continue detection +def x0_continue_with_else(): + for i in range(3): + if i == 1: + continue + print(f"Processing {i}") + else: + print("Else clause still executes after continue") + + +# 3.6/3.7 No continue detection +def x1_continue_with_else_nofallthru(): + for i in range(3): + if i == 1: + continue + print(f"Processing {i}") + else: + print("Else clause still executes after continue") + print("end") + + +# 3.9/3.11 Naive break detection, break statement is further up +def y0_break_in_try_except(): + for i in range(5): + try: + if i == 3: + break + print(f"Value: {i}") + except: + print("Exception occurred") + + +# 3.9 Naive break detection, break statement is further up +def y1_break_in_try_except_nofallthru(): + for i in range(5): + try: + if i == 3: + break + print(f"Value: {i}") + except: + print("Exception occurred") + print("end") + + +# 3.9 Naive break detection, break statement is further up +def y2_return_in_try_except_nofallthru(): + for i in range(5): + try: + if i == 3: + print(f"Value: {i}") + else: + break + except: + print("Exception occurred") + print("end") + + +# 3.6/3.9 No continue detection +def z0_continue_in_try_except(): + for i in range(5): + try: + if i == 2: + continue + print(f"Value: {i}") + except: + print("Exception occurred") + + +# 3.6/3.9/3.11 No continue detection +def z1_continue_in_try_except_nofallthru(): + for i in range(5): + try: + if i == 2: + continue + print(f"Value: {i}") + except: + print("Exception occurred") + print("end") diff --git a/test/with.py b/test/With.py similarity index 54% rename from test/with.py rename to test/With.py index e6359fa..6e3c603 100644 --- a/test/with.py +++ b/test/With.py @@ -1,165 +1,81 @@ -def bare_with(): +def a0_bare_with(): with a: print(1) -def bare_with_fallthrough(): +def a1_bare_with_fallthrough(): with a: print(1) print(2) -## Known to fail on 3.10 -def multi_with(): +def b0_multi_with(): with a, b: print(1) -def multi_with_fallthrough(): +def b1_multi_with_fallthrough(): with a, b: print(1) print(2) -def with_as(): +def c0_with_as(): with a as c: print(1) -def with_as_fallthrough(): +def c1_with_as_fallthrough(): with a as c: print(1) print(2) -def multi_with_as(): +def d0_multi_with_as(): with a, b as c: print(1) -def multi_with_as_fallthrough(): +def d1_multi_with_as_fallthrough(): with a, b as c: print(1) print(2) -def with_multi_as(): +def e0_with_multi_as(): with a as b, c: print(1) -def with_multi_as_fallthrough(): +def e1_with_multi_as_fallthrough(): with a as b, c: print(1) print(2) -def multi_with_multi_as(): +def f0_multi_with_multi_as(): with a as b, c as d: print(1) -# Known to fail on 3.10 -def multi_with_multi_as_fallthrough(): +def f1_multi_with_multi_as_fallthrough(): with a as b, c as d: print(1) print(2) -def multi_with_multi_as_alt(): +def g0_multi_with_multi_as_alt(): with a, b as c, d: print(1) -# Known to fail on 3.10 -def multi_with_multi_as_fallthrough_alt(): +def g1_multi_with_multi_as_fallthrough_alt(): with a, b as c, d: print(1) print(2) -async def bare_async_with(): - async with a: - print(1) - - -async def bare_async_with_fallthrough(): - async with a: - print(1) - print(2) - - -## Known to fail on 3.10 -async def multi_async_with(): - async with a, b: - print(1) - - -async def multi_async_with_fallthrough(): - async with a, b: - print(1) - print(2) - - -async def with_as(): - async with a as c: - print(1) - - -async def with_as_fallthrough(): - async with a as c: - print(1) - print(2) - - -async def multi_async_with_as(): - async with a, b as c: - print(1) - - -async def multi_async_with_as_fallthrough(): - async with a, b as c: - print(1) - print(2) - - -async def with_multi_as(): - async with a as b, c: - print(1) - - -async def with_multi_as_fallthrough(): - async with a as b, c: - print(1) - print(2) - - -async def multi_async_with_multi_as(): - async with a as b, c as d: - print(1) - - -# Known to fail on 3.10 -async def multi_async_with_multi_as_fallthrough(): - async with a as b, c as d: - print(1) - print(2) - - -async def multi_async_with_multi_as_alt(): - async with a, b as c, d: - print(1) - - -# Known to fail on 3.10 -async def multi_async_with_multi_as_fallthrough_alt(): - async with a, b as c, d: - print(1) - print(2) - - -def try_with_except(): - # With statement with outer exception handler +def h0_try_with_except(): try: with a: print(1) @@ -168,15 +84,90 @@ def try_with_except(): print(3) -def with_return(): - # With statement with return +def i0_with_return(): with a: return 1 print(1) -def with_raise(): - # With statement with raise +def j0_with_raise(): with a: raise Exc print(1) + + +async def k0_bare_async_with(): + async with a: + print(1) + + +async def k1_bare_async_with_fallthrough(): + async with a: + print(1) + print(2) + + +async def l0_multi_async_with(): + async with a, b: + print(1) + + +async def l1_multi_async_with_fallthrough(): + async with a, b: + print(1) + print(2) + + +async def m0_async_with_as(): + async with a as c: + print(1) + + +async def m1_async_with_as_fallthrough(): + async with a as c: + print(1) + print(2) + + +async def n0_multi_async_with_as(): + async with a, b as c: + print(1) + + +async def n1_multi_async_with_as_fallthrough(): + async with a, b as c: + print(1) + print(2) + + +async def o0_async_with_multi_as(): + async with a as b, c: + print(1) + + +async def o1_async_with_multi_as_fallthrough(): + async with a as b, c: + print(1) + print(2) + + +async def p0_multi_async_with_multi_as(): + async with a as b, c as d: + print(1) + + +async def p1_multi_async_with_multi_as_fallthrough(): + async with a as b, c as d: + print(1) + print(2) + + +async def q0_multi_async_with_multi_as_alt(): + async with a, b as c, d: + print(1) + + +async def q1_multi_async_with_multi_as_fallthrough_alt(): + async with a, b as c, d: + print(1) + print(2)