From b2bfeccc0f6806cfe6a0d0942466b238f6042f72 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 10 Jul 2025 17:18:14 -0500 Subject: [PATCH 01/55] Start of loops --- .../templates/Loop.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index 3f56a0c..c935c2e 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -1,3 +1,5 @@ +from __future__ import annotations +from typing import TYPE_CHECKING from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, @@ -5,9 +7,11 @@ from ..utils import ( defer_source_to, starting_instructions, to_indented_source, - make_try_match, + make_try_match, ) +if TYPE_CHECKING: + from pylingual.control_flow_reconstruction.cfg import CFG @register_template(0, 1) class ForLoop(ControlFlowTemplate): @@ -58,3 +62,15 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): ) to_indented_source = defer_source_to("comp") + +class FixLoop(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: + # check that its a loop that we need to fix + # find the end of the loop + # find all nodes that belong to the loop + # find nodes in loop that go to end + # replace those edges with meta edges to the end + # find nodes in loop that go to header + # replace all but last of those edges with meta edge to end + \ No newline at end of file From ca704a89f4a3e3c757fe61a94ecf150c9c4dec98 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 11 Jul 2025 15:46:02 -0500 Subject: [PATCH 02/55] Renaming TryExcept test cases --- test/{TryExcept.py => Exception.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{TryExcept.py => Exception.py} (100%) diff --git a/test/TryExcept.py b/test/Exception.py similarity index 100% rename from test/TryExcept.py rename to test/Exception.py From fbbb3d1c877464d36ce1b041b0cbcf73d1834c37 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 11 Jul 2025 15:46:24 -0500 Subject: [PATCH 03/55] Alphabetizing With test cases + explanations on fails --- test/{with.py => With.py} | 205 ++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 119 deletions(-) rename test/{with.py => With.py} (51%) diff --git a/test/with.py b/test/With.py similarity index 51% rename from test/with.py rename to test/With.py index e6359fa..0d3fd32 100644 --- a/test/with.py +++ b/test/With.py @@ -1,165 +1,73 @@ -def bare_with(): +def a_bare_with(): with a: print(1) - -def bare_with_fallthrough(): +def a1_bare_with_fallthrough(): with a: print(1) print(2) - -## Known to fail on 3.10 -def multi_with(): +# Fails in 3.10, duplicate blocks explained further in issue 32 +def b_multi_with(): with a, b: print(1) - -def multi_with_fallthrough(): +def b1_multi_with_fallthrough(): with a, b: print(1) print(2) - -def with_as(): +def c_with_as(): with a as c: print(1) - -def with_as_fallthrough(): +def c1_with_as_fallthrough(): with a as c: print(1) print(2) - -def multi_with_as(): +# Fails in 3.10, same issue as b +def d_multi_with_as(): with a, b as c: print(1) - -def multi_with_as_fallthrough(): +def d1_multi_with_as_fallthrough(): with a, b as c: print(1) print(2) - -def with_multi_as(): +# Fails in 3.10, same issue as b +def e_with_multi_as(): with a as b, c: print(1) - -def with_multi_as_fallthrough(): +def e1_with_multi_as_fallthrough(): with a as b, c: print(1) print(2) - -def multi_with_multi_as(): +# Fails in 3.10, same issue as b +def f_multi_with_multi_as(): with a as b, c as d: print(1) - -# Known to fail on 3.10 -def multi_with_multi_as_fallthrough(): +def f1_multi_with_multi_as_fallthrough(): with a as b, c as d: print(1) print(2) - -def multi_with_multi_as_alt(): +# Fails in 3.10, same issue as b +def g_multi_with_multi_as_alt(): with a, b as c, d: print(1) - -# Known to fail on 3.10 -def multi_with_multi_as_fallthrough_alt(): +def g1_multi_with_multi_as_fallthrough_alt(): with a, b as c, d: print(1) print(2) - -async def bare_async_with(): - async with a: - print(1) - - -async def bare_async_with_fallthrough(): - async with a: - print(1) - print(2) - - -## Known to fail on 3.10 -async def multi_async_with(): - async with a, b: - print(1) - - -async def multi_async_with_fallthrough(): - async with a, b: - print(1) - print(2) - - -async def with_as(): - async with a as c: - print(1) - - -async def with_as_fallthrough(): - async with a as c: - print(1) - print(2) - - -async def multi_async_with_as(): - async with a, b as c: - print(1) - - -async def multi_async_with_as_fallthrough(): - async with a, b as c: - print(1) - print(2) - - -async def with_multi_as(): - async with a as b, c: - print(1) - - -async def with_multi_as_fallthrough(): - async with a as b, c: - print(1) - print(2) - - -async def multi_async_with_multi_as(): - async with a as b, c as d: - print(1) - - -# Known to fail on 3.10 -async def multi_async_with_multi_as_fallthrough(): - async with a as b, c as d: - print(1) - print(2) - - -async def multi_async_with_multi_as_alt(): - async with a, b as c, d: - print(1) - - -# Known to fail on 3.10 -async def multi_async_with_multi_as_fallthrough_alt(): - async with a, b as c, d: - print(1) - print(2) - - -def try_with_except(): - # With statement with outer exception handler +# Fails in 3.13, unexpected JUMP_BACKWARD_NO_INTERRUPT messes up the template +def h_try_with_except(): try: with a: print(1) @@ -167,16 +75,75 @@ def try_with_except(): print(2) print(3) - -def with_return(): - # With statement with return +def i_with_return(): with a: return 1 print(1) - -def with_raise(): - # With statement with raise +def j_with_raise(): with a: raise Exc print(1) + +async def k_bare_async_with(): + async with a: + print(1) + +async def k1_bare_async_with_fallthrough(): + async with a: + print(1) + print(2) + +async def l_multi_async_with(): + async with a, b: + print(1) + +async def l1_multi_async_with_fallthrough(): + async with a, b: + print(1) + print(2) + +async def m_async_with_as(): + async with a as c: + print(1) + +async def m1_async_with_as_fallthrough(): + async with a as c: + print(1) + print(2) + +async def n_multi_async_with_as(): + async with a, b as c: + print(1) + +async def n1_multi_async_with_as_fallthrough(): + async with a, b as c: + print(1) + print(2) + +async def o_async_with_multi_as(): + async with a as b, c: + print(1) + +async def o1_async_with_multi_as_fallthrough(): + async with a as b, c: + print(1) + print(2) + +async def p_multi_async_with_multi_as(): + async with a as b, c as d: + print(1) + +async def p1_multi_async_with_multi_as_fallthrough(): + async with a as b, c as d: + print(1) + print(2) + +async def q_multi_async_with_multi_as_alt(): + async with a, b as c, d: + print(1) + +async def q1_multi_async_with_multi_as_fallthrough_alt(): + async with a, b as c, d: + print(1) + print(2) \ No newline at end of file From b8e0c934fba3d194663be147c929d819d9f27826 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 11 Jul 2025 18:50:03 -0500 Subject: [PATCH 04/55] Loop test case adjustments --- test/Loop.py | 268 ++++++++++++++++++++++----------------------------- 1 file changed, 113 insertions(+), 155 deletions(-) diff --git a/test/Loop.py b/test/Loop.py index 323272e..dd56973 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -1,15 +1,20 @@ -# FOR LOOP TESTS - - def a_for_over_list(): for x in [1, 2, 3]: print("for over list") +def a1_for_over_list_nofallthru(): + for x in [1, 2, 3]: + print("for over list") + print("end") def b_for_over_tuples(): for a, b in [(1, 2), (3, 4)]: print("tuples") +def b1_for_over_tuples_nofallthru(): + for a, b in [(1, 2), (3, 4)]: + print("tuples") + print("end") def c_for_else(): for i in range(3): @@ -17,6 +22,12 @@ def c_for_else(): else: print("for else") +def c1_for_else_nofallthru(): + for i in range(3): + print("for body") + else: + print("for else") + print("end") def d_for_with_break(): for x in range(10): @@ -24,6 +35,12 @@ def d_for_with_break(): print("breaking") break +def d1_for_with_break_nofallthru(): + for x in range(10): + if x == 5: + print("breaking") + break + print("end") def e_for_with_continue(): for x in range(5): @@ -32,12 +49,24 @@ def e_for_with_continue(): continue print("after continue") +def e1_for_with_continue_nofallthru(): + for x in range(5): + if x % 2 == 0: + print("continuing") + continue + print("after continue") + print("end") def f_nested_for_loops(): for i in range(2): for j in range(3): print(f"nested {i},{j}") +def f1_nested_for_loops_nofallthru(): + for i in range(2): + for j in range(3): + print(f"nested {i},{j}") + print("end") def g_for_with_try_except(): for x in range(2): @@ -46,12 +75,24 @@ def g_for_with_try_except(): except Exception: print("except block") +def g1_for_with_try_except_nofallthru(): + for x in range(2): + try: + print("try block") + except Exception: + print("except block") + print("end") def h_for_with_with_statement(): for _ in range(1): with a: print("inside with") +def h1_for_with_with_statement_nofallthru(): + for _ in range(1): + with a: + print("inside with") + print("end") def i_for_with_function_call_iterable(): def get_items(): @@ -60,17 +101,33 @@ def i_for_with_function_call_iterable(): for item in get_items(): print(f"item: {item}") +def i1_for_with_function_call_iterable_nofallthru(): + def get_items(): + return [1, 2, 3] + + for item in get_items(): + print(f"item: {item}") + print("end") def j_for_with_empty_body_ellipsis(): for _ in range(3): ... +def j1_for_with_empty_body_ellipsis_nofallthru(): + for _ in range(3): + ... + print("end") def k_while_true_with_break(): while True: print("while true") break +def k1_while_true_with_break_nofallthru(): + while True: + print("while true") + break + print("end") def l_while_with_else(): i = 0 @@ -80,6 +137,14 @@ def l_while_with_else(): else: print("while else") +def l1_while_with_else_nofallthru(): + i = 0 + while i < 3: + print(f"looping {i}") + i += 1 + else: + print("while else") + print("end") def m_while_with_continue(): i = 0 @@ -90,6 +155,15 @@ def m_while_with_continue(): continue print("after continue") +def m1_while_with_continue_nofallthru(): + i = 0 + while i < 5: + i += 1 + if i % 2 == 0: + print("continue") + continue + print("after continue") + print("end") def n_while_with_break(): i = 0 @@ -97,6 +171,12 @@ def n_while_with_break(): print("break in while") break +def n1_while_with_break_nofallthru(): + i = 0 + while True: + print("break in while") + break + print("end") def o_nested_while_loops(): i = 0 @@ -107,151 +187,7 @@ def o_nested_while_loops(): j += 1 i += 1 - -def p_while_with_try_except(): - while True: - try: - print("try in while") - except: - print("except in while") - - -def q_while_with_with_statement(): - while True: - with a: - print("inside while with") - - -def r_for_inside_while(): - while True: - for x in [1, 2]: - print("for in while") - - -def s_while_inside_for(): - for _ in range(1): - while True: - print("while in for") - break - - -def t_while_with_empty_body_ellipsis(): - while True: - ... - - -def a_nofallthru_for_over_list(): - for x in [1, 2, 3]: - print("for over list") - print("end") - - -def b_nofallthru_for_over_tuples(): - for a, b in [(1, 2), (3, 4)]: - print("tuples") - print("end") - - -def c_nofallthru_for_else(): - for i in range(3): - print("for body") - else: - print("for else") - print("end") - - -def d_nofallthru_for_with_break(): - for x in range(10): - if x == 5: - print("breaking") - break - print("end") - - -def e_nofallthru_for_with_continue(): - for x in range(5): - if x % 2 == 0: - print("continuing") - continue - print("after continue") - print("end") - - -def f_nofallthru_nested_for_loops(): - for i in range(2): - for j in range(3): - print(f"nested {i},{j}") - print("end") - - -def g_nofallthru_for_with_try_except(): - for x in range(2): - try: - print("try block") - except Exception: - print("except block") - print("end") - - -def h_nofallthru_for_with_with_statement(): - for _ in range(1): - with a: - print("inside with") - print("end") - - -def i_nofallthru_for_with_function_call_iterable(): - def g_nofallthruet_items(): - return [1, 2, 3] - - for item in get_items(): - print(f"item: {item}") - print("end") - - -def j_nofallthru_for_with_empty_body_ellipsis(): - for _ in range(3): - ... - print("end") - - -def k_nofallthru_while_true_with_break(): - while True: - print("while true") - break - print("end") - - -def l_nofallthru_while_with_else(): - i = 0 - while i < 3: - print(f"looping {i}") - i += 1 - else: - print("while else") - print("end") - - -def m_nofallthru_while_with_continue(): - i = 0 - while i < 5: - i += 1 - if i % 2 == 0: - print("continue") - continue - print("after continue") - print("end") - - -def n_nofallthru_while_with_break(): - i = 0 - while True: - print("break in while") - break - print("end") - - -def o_nofallthru_nested_while_loops(): +def o1_nested_while_loops_nofallthru(): i = 0 while i < 2: j = 0 @@ -261,8 +197,14 @@ def o_nofallthru_nested_while_loops(): i += 1 print("end") +def p_while_with_try_except(): + while True: + try: + print("try in while") + except: + print("except in while") -def p_nofallthru_while_with_try_except(): +def p1_while_with_try_except_nofallthru(): while True: try: print("try in while") @@ -270,30 +212,46 @@ def p_nofallthru_while_with_try_except(): print("except in while") print("end") +def q_while_with_with_statement(): + while True: + with a: + print("inside while with") -def q_nofallthru_while_with_with_statement(): +def q1_while_with_with_statement_nofallthru(): while True: with a: print("inside while with") print("end") +def r_for_inside_while(): + while True: + for x in [1, 2]: + print("for in while") -def r_nofallthru_for_inside_while(): +def r1_for_inside_while_nofallthru(): while True: for x in [1, 2]: print("for in while") print("end") +def s_while_inside_for(): + for _ in range(1): + while True: + print("while in for") + break -def s_nofallthru_while_inside_for(): +def s1_while_inside_for_nofallthru(): for _ in range(1): while True: print("while in for") break print("end") - -def t_nofallthru_while_with_empty_body_ellipsis(): +def t_while_with_empty_body_ellipsis(): while True: ... - print("end") + +def t1_while_with_empty_body_ellipsis_nofallthru(): + while True: + ... + print("end") \ No newline at end of file From f681d4d05f9631b8849a498b02e72099de58aa38 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 11 Jul 2025 19:24:19 -0500 Subject: [PATCH 05/55] Updating test cases & documentation --- test/Exception.py | 213 +++++++++++++++++++++------------------------- test/Loop.py | 171 ++++++++++++++++++++++++++++++++----- test/With.py | 34 ++++---- 3 files changed, 262 insertions(+), 156 deletions(-) diff --git a/test/Exception.py b/test/Exception.py index e268fad..06bae00 100644 --- a/test/Exception.py +++ b/test/Exception.py @@ -1,12 +1,18 @@ -def a_TryExcept(): +def a0_bare_try_except(): + try: + print(1) + except: + print(2) + +def a1_bare_try_except_fallthrough(): try: print(1) except: print(2) print(3) - -def b_TryExceptBareNested(): +# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match +def b0_nested_try_except(): try: print(1) except: @@ -16,8 +22,7 @@ def b_TryExceptBareNested(): except: print(4) - -def b1_TryExceptBareNestedFallthrough(): +def b1_nested_try_except_fallthrough(): try: print(1) except: @@ -28,8 +33,8 @@ def b1_TryExceptBareNestedFallthrough(): print(4) print(5) - -def b2_TryExceptBareNestedEarlyFallthrough(): +# 3.13 Duplicate blocks +def b2_nested_try_except_early_fallthrough(): try: print(1) except: @@ -40,8 +45,7 @@ def b2_TryExceptBareNestedEarlyFallthrough(): print(4) print(5) - -def b3_TryExceptBareNestedDoubleFallthrough(): +def b3_nested_try_except_double_fallthrough(): try: print(1) except: @@ -53,8 +57,8 @@ def b3_TryExceptBareNestedDoubleFallthrough(): print(5) print(6) - -def c_TryExceptBareMultiNested(): +# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match +def c0_multi_except_nested(): try: print(1) except a: @@ -70,8 +74,8 @@ def c_TryExceptBareMultiNested(): except: print(7) - -def c1_TryExceptBareMultiNestedFallthrough(): +# 3.11/3.12/3.13 Duplicate blocks causing blocks to not match +def c1_multi_except_nested_fallthrough(): try: print(1) except a: @@ -88,8 +92,7 @@ def c1_TryExceptBareMultiNestedFallthrough(): except: print(8) - -def c2_TryExceptBareMultiNestedFallthrough2(): +def c2_multi_except_nested_fallthrough2(): try: print(1) except a: @@ -107,8 +110,8 @@ def c2_TryExceptBareMultiNestedFallthrough2(): print(8) print(9) - -def c3_TryExceptBareMultiNestedEarlyFallthrough(): +# 3.13 Duplicate blocks +def c3_multi_except_nested_early_fallthrough(): try: print(1) except a: @@ -125,8 +128,7 @@ def c3_TryExceptBareMultiNestedEarlyFallthrough(): print(7) print(8) - -def c4_TryExceptBareMultiNestedAllFallthrough(): +def c4_multi_except_nested_all_fallthrough(): try: print(1) except a: @@ -145,8 +147,9 @@ def c4_TryExceptBareMultiNestedAllFallthrough(): print(9) print(10) - -def d_TryExceptBareNestedNamed(): +# 3.10/3.11/3.12/3.13 Duplicate blocks causing templates to not match +# Discussed in issue 41 +def d0_named_except_nested(): try: print(1) except A as a: @@ -156,8 +159,7 @@ def d_TryExceptBareNestedNamed(): except: print(4) - -def d1_TryExceptBareNestedNamedFallthrough(): +def d1_named_except_nested_fallthrough(): try: print(1) except A as a: @@ -168,8 +170,8 @@ def d1_TryExceptBareNestedNamedFallthrough(): print(4) print(5) - -def d2_TryExceptBareNestedNamedEarlyFallthrough(): +# 3.13 Duplicate blocks +def d2_named_except_nested_early_fallthrough(): try: print(1) except A as a: @@ -180,8 +182,7 @@ def d2_TryExceptBareNestedNamedEarlyFallthrough(): print(4) print(5) - -def d3_TryExceptBareNestedNamedDoubleFallthrough(): +def d3_named_except_nested_double_fallthrough(): try: print(1) except A as a: @@ -193,8 +194,7 @@ def d3_TryExceptBareNestedNamedDoubleFallthrough(): print(5) print(6) - -def e_TryExceptElseBare(): +def e0_try_except_else(): try: print(1) except: @@ -203,8 +203,7 @@ def e_TryExceptElseBare(): print(3) print(4) - -def f_TryExceptElseFinallyBare(): +def f0_try_except_else_finally(): try: print(1) except: @@ -215,8 +214,7 @@ def f_TryExceptElseFinallyBare(): print(4) print(5) - -def g_TryExceptElseMulti(): +def g0_multi_except_with_else(): try: print(1) except a: @@ -227,8 +225,7 @@ def g_TryExceptElseMulti(): print(4) print(5) - -def h_TryExceptElseMultiFallback(): +def h0_multi_except_fallback_with_else(): try: print(1) except a: @@ -241,8 +238,7 @@ def h_TryExceptElseMultiFallback(): print(5) print(6) - -def i_TryExceptElseMultiNamedAndUnnamed(): +def i0_mixed_named_unnamed_except_with_else(): try: print(1) except A as a: @@ -255,8 +251,7 @@ def i_TryExceptElseMultiNamedAndUnnamed(): print(5) print(6) - -def j_TryExceptElseNamed(): +def j0_named_except_with_else(): try: print(1) except A as a: @@ -267,8 +262,7 @@ def j_TryExceptElseNamed(): print(4) print(5) - -def k_TryExceptFinallyBare(): +def k0_try_except_finally(): try: print(1) except: @@ -277,8 +271,7 @@ def k_TryExceptFinallyBare(): print(3) print(4) - -def l_TryExceptFinallyBareSpecific(): +def l0_specific_except_finally(): try: print(1) except a: @@ -287,8 +280,7 @@ def l_TryExceptFinallyBareSpecific(): print(3) print(4) - -def m_TryExceptMulti(): +def m0_multi_except(): try: print(1) except a: @@ -299,8 +291,7 @@ def m_TryExceptMulti(): print(4) print(5) - -def n_TryExceptMultiFallback(): +def n0_multi_except_with_fallback(): try: print(1) except a: @@ -311,8 +302,7 @@ def n_TryExceptMultiFallback(): print(4) print(5) - -def o_TryExceptMultiFallbackFinally(): +def o0_multi_except_fallback_finally(): try: print(1) except a: @@ -323,8 +313,7 @@ def o_TryExceptMultiFallbackFinally(): print(4) print(5) - -def p_TryExceptMultiNamed(): +def p0_multi_named_except(): try: print(1) except A as a: @@ -335,8 +324,7 @@ def p_TryExceptMultiNamed(): print(4) print(5) - -def q_TryExceptMultiNamedAndUnnamed(): +def q0_mixed_named_unnamed_except(): try: print(1) except A as a: @@ -347,8 +335,7 @@ def q_TryExceptMultiNamedAndUnnamed(): print(4) print(5) - -def r_TryExceptMultiNamedAndUnnamedFinally(): +def r0_mixed_named_unnamed_except_finally(): try: print(1) except A as a: @@ -361,8 +348,7 @@ def r_TryExceptMultiNamedAndUnnamedFinally(): print(5) print(6) - -def s_TryExceptMultiNamedFallback(): +def s0_named_except_fallback(): try: print(1) except A as a: @@ -371,8 +357,7 @@ def s_TryExceptMultiNamedFallback(): print(3) print(4) - -def t_TryExceptMultiNamedFallbackFinally(): +def t0_named_except_fallback_finally(): try: print(1) except A as a: @@ -383,8 +368,7 @@ def t_TryExceptMultiNamedFallbackFinally(): print(4) print(5) - -def u_TryExceptMultiNamedFinally(): +def u0_multi_named_except_finally(): try: print(1) except A as a: @@ -395,8 +379,7 @@ def u_TryExceptMultiNamedFinally(): print(4) print(5) - -def v_TryExceptMultiFinally(): +def v0_multi_except_finally(): try: print(1) except a: @@ -407,16 +390,14 @@ def v_TryExceptMultiFinally(): print(4) print(5) - -def w_TryExceptRaise(): +def w0_try_except_raise(): try: print(1) except: print(2) raise Exc - -def x_TryExceptRaiseMulti(): +def x0_multi_except_raise(): try: print(1) except a: @@ -426,24 +407,23 @@ def x_TryExceptRaiseMulti(): print(3) raise Exc - -def y_TryExceptRaiseNamed(): +def y0_named_except_raise(): try: print(1) except A as a: print(2) raise Exc - -def z_TryExceptReturn(): +# 3.11 Try return getting left outside of TryExcept +def z0_try_except_return(): try: print(1) return 2 except: print(2) - -def z1_TryExceptReturn(): +# 3.11 Try return getting left outside of TryExcept +def z1_try_except_return_both(): try: print(1) return 2 @@ -451,8 +431,8 @@ def z1_TryExceptReturn(): print(2) return 3 - -def aa_TryExceptReturnMulti(): +# 3.11 Try return getting left outside of TryExcept +def aa0_multi_except_return(): try: print(1) return 2 @@ -461,8 +441,10 @@ def aa_TryExceptReturnMulti(): except b: print(3) - -def aa1_TryExceptReturnMulti(): +# 3.6/3.7/3.8 ExceptExc abandons tail node. +# Could be fixed (?) but breaks other test cases +# 3.11 Try return getting left outside of TryExcept +def aa1_multi_except_return_both(): try: print(1) return 2 @@ -472,8 +454,8 @@ def aa1_TryExceptReturnMulti(): except b: print(3) - -def ab_TryExceptReturnNamed(): +# 3.11 Try return getting left outside of TryExcept +def ab0_named_except_raise_return(): try: print(1) return 2 @@ -484,15 +466,24 @@ def ab_TryExceptReturnNamed(): print(3) raise Exc +# 3.8 Double natural edge graph error (?) +# 3.11 Try return getting left outside of TryExcept +def ab1_named_except_return(): + try: + print(1) + return 2 + except A as a: + print(2) + return 3 -def TryEmptryFinally(): +# 3.11/3.12/3.13 No template match +def ac0_empty_try_finally(): try: pass finally: print(1) - -def TryMultiple(): +def ad0_multiple_try_blocks(): try: print(1) except: @@ -503,8 +494,8 @@ def TryMultiple(): except: print(4) - -def TryExceptElseTry(): +# 3.10/3.11 Try matching before TryElse +def ae0_try_except_else_nested_try(): try: print(1) except: @@ -515,8 +506,9 @@ def TryExceptElseTry(): except: print(4) - -def TryFinallyNestedExcept(): +# 3.9 Duplicate blocks (?) +# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) +def af0_try_finally_nested_except(): try: print(1) finally: @@ -525,45 +517,44 @@ def TryFinallyNestedExcept(): except: print(3) - -def TryExceptTuple(): +def ag0_try_except_tuple(): try: print(1) except (A, B): print(2) - -def TryFinallyReturn(): +# 3.9 Difficult template ambiguity between Try/TryFinally +# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) +def ah0_try_finally_return(): try: print(1) finally: return 2 - -def TryReturnFinally(): +# 3.11/3.12/3.13 No template match (?) +def ai0_try_return_finally(): try: return 1 finally: print(2) - -def TryReturnFinallyReturn(): +# 3.9/3.10 Difficult template ambiguity between Try/TryFinally +# 3.11/3.12/3.13 No template match (?) +def aj0_try_return_finally_return(): try: return 1 finally: return 2 - -def TryExceptRaise(): +def ak0_try_except_raise_return(): try: print(1) return 2 except: raise Exception() - -""" -def TryExceptReturnFinally(): +# 3.8/3.9/3.10 No template match +def al0_try_except_return_finally(): try: raise Exception() except: @@ -571,37 +562,25 @@ def TryExceptReturnFinally(): return 2 finally: print(3) -""" -""" -def TryFinallyRaise(): +# 3.8/3.9/3.10 No template match +# 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) +def am0_try_finally_raise(): try: print(1) return 2 finally: raise Exception() -""" - -def ab1_TryExceptReturnNamed(): - try: - print(1) - return 2 - except A as a: - print(2) - return 3 - - -def ac_TryFinallyBareFallthrough(): +def an0_try_finally_fallthrough(): try: print(1) finally: print(2) print(3) - -def ad_TryFinallyBare(): +def ao0_try_finally_simple(): try: print(1) finally: - print(2) + print(2) \ No newline at end of file diff --git a/test/Loop.py b/test/Loop.py index dd56973..135e0a0 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -1,4 +1,4 @@ -def a_for_over_list(): +def a0_for_over_list(): for x in [1, 2, 3]: print("for over list") @@ -7,7 +7,7 @@ def a1_for_over_list_nofallthru(): print("for over list") print("end") -def b_for_over_tuples(): +def b0_for_over_tuples(): for a, b in [(1, 2), (3, 4)]: print("tuples") @@ -16,7 +16,7 @@ def b1_for_over_tuples_nofallthru(): print("tuples") print("end") -def c_for_else(): +def c0_for_else(): for i in range(3): print("for body") else: @@ -29,12 +29,14 @@ def c1_for_else_nofallthru(): print("for else") print("end") -def d_for_with_break(): +# Fails due to no break +def d0_for_with_break(): for x in range(10): if x == 5: print("breaking") break +# Fails due to no break def d1_for_with_break_nofallthru(): for x in range(10): if x == 5: @@ -42,7 +44,22 @@ def d1_for_with_break_nofallthru(): break print("end") -def e_for_with_continue(): +# Help to implement break +def d2_for_without_break(): + for x in range(10): + if x == 5: + print("not breaking") + print("end") + +# Help to implement break +def d3_for_return(): + for x in range(10): + if x == 5: + print("not breaking") + return + print("end") + +def e0_for_with_continue(): for x in range(5): if x % 2 == 0: print("continuing") @@ -57,7 +74,7 @@ def e1_for_with_continue_nofallthru(): print("after continue") print("end") -def f_nested_for_loops(): +def f0_nested_for_loops(): for i in range(2): for j in range(3): print(f"nested {i},{j}") @@ -68,7 +85,7 @@ def f1_nested_for_loops_nofallthru(): print(f"nested {i},{j}") print("end") -def g_for_with_try_except(): +def g0_for_with_try_except(): for x in range(2): try: print("try block") @@ -83,7 +100,7 @@ def g1_for_with_try_except_nofallthru(): print("except block") print("end") -def h_for_with_with_statement(): +def h0_for_with_with_statement(): for _ in range(1): with a: print("inside with") @@ -94,7 +111,7 @@ def h1_for_with_with_statement_nofallthru(): print("inside with") print("end") -def i_for_with_function_call_iterable(): +def i0_for_with_function_call_iterable(): def get_items(): return [1, 2, 3] @@ -109,7 +126,7 @@ def i1_for_with_function_call_iterable_nofallthru(): print(f"item: {item}") print("end") -def j_for_with_empty_body_ellipsis(): +def j0_for_with_empty_body_ellipsis(): for _ in range(3): ... @@ -118,18 +135,24 @@ def j1_for_with_empty_body_ellipsis_nofallthru(): ... print("end") -def k_while_true_with_break(): +def k0_while_true_with_break(): + x = 0 while True: print("while true") - break + x += 1 + if x >= 1: + break def k1_while_true_with_break_nofallthru(): + x = 0 while True: print("while true") - break + x += 1 + if x >= 1: + break print("end") -def l_while_with_else(): +def l0_while_with_else(): i = 0 while i < 3: print(f"looping {i}") @@ -146,7 +169,7 @@ def l1_while_with_else_nofallthru(): print("while else") print("end") -def m_while_with_continue(): +def m0_while_with_continue(): i = 0 while i < 5: i += 1 @@ -165,7 +188,7 @@ def m1_while_with_continue_nofallthru(): print("after continue") print("end") -def n_while_with_break(): +def n0_while_with_break(): i = 0 while True: print("break in while") @@ -178,7 +201,7 @@ def n1_while_with_break_nofallthru(): break print("end") -def o_nested_while_loops(): +def o0_nested_while_loops(): i = 0 while i < 2: j = 0 @@ -197,7 +220,7 @@ def o1_nested_while_loops_nofallthru(): i += 1 print("end") -def p_while_with_try_except(): +def p0_while_with_try_except(): while True: try: print("try in while") @@ -212,7 +235,7 @@ def p1_while_with_try_except_nofallthru(): print("except in while") print("end") -def q_while_with_with_statement(): +def q0_while_with_with_statement(): while True: with a: print("inside while with") @@ -223,7 +246,7 @@ def q1_while_with_with_statement_nofallthru(): print("inside while with") print("end") -def r_for_inside_while(): +def r0_for_inside_while(): while True: for x in [1, 2]: print("for in while") @@ -234,7 +257,7 @@ def r1_for_inside_while_nofallthru(): print("for in while") print("end") -def s_while_inside_for(): +def s0_while_inside_for(): for _ in range(1): while True: print("while in for") @@ -247,11 +270,115 @@ def s1_while_inside_for_nofallthru(): break print("end") -def t_while_with_empty_body_ellipsis(): +def t0_while_with_empty_body_ellipsis(): while True: ... def t1_while_with_empty_body_ellipsis_nofallthru(): while True: ... + print("end") + +def u0_break_in_nested_for(): + for i in range(3): + for j in range(3): + if i == 1 and j == 1: + print("Breaking inner loop") + break + print(f"i={i}, j={j}") + +def u1_break_in_nested_for_nofallthru(): + for i in range(3): + for j in range(3): + if i == 1 and j == 1: + print("Breaking inner loop") + break + print(f"i={i}, j={j}") + print("end") + +def v0_continue_in_nested_for(): + for i in range(3): + for j in range(3): + if j == 1: + continue + print(f"Processing i={i}, j={j}") + +def v1_continue_in_nested_for_nofallthru(): + for i in range(3): + for j in range(3): + if j == 1: + continue + print(f"Processing i={i}, j={j}") + print("end") + +def w0_break_with_else(): + for i in range(5): + if i == 3: + print("Breaking before else") + break + else: + print("This won't execute due to break") + +def w1_break_with_else_nofallthru(): + for i in range(5): + if i == 3: + print("Breaking before else") + break + else: + print("This won't execute due to break") + print("end") + +def x0_continue_with_else(): + for i in range(3): + if i == 1: + continue + print(f"Processing {i}") + else: + print("Else clause still executes after continue") + +def x1_continue_with_else_nofallthru(): + for i in range(3): + if i == 1: + continue + print(f"Processing {i}") + else: + print("Else clause still executes after continue") + print("end") + +def y0_break_in_try_except(): + for i in range(5): + try: + if i == 3: + break + print(f"Value: {i}") + except: + print("Exception occurred") + +def y1_break_in_try_except_nofallthru(): + for i in range(5): + try: + if i == 3: + break + print(f"Value: {i}") + except: + print("Exception occurred") + print("end") + +def z0_continue_in_try_except(): + for i in range(5): + try: + if i == 2: + continue + print(f"Value: {i}") + except: + print("Exception occurred") + +def z1_continue_in_try_except_nofallthru(): + for i in range(5): + try: + if i == 2: + continue + print(f"Value: {i}") + except: + print("Exception occurred") print("end") \ No newline at end of file diff --git a/test/With.py b/test/With.py index 0d3fd32..491f463 100644 --- a/test/With.py +++ b/test/With.py @@ -1,4 +1,4 @@ -def a_bare_with(): +def a0_bare_with(): with a: print(1) @@ -8,7 +8,7 @@ def a1_bare_with_fallthrough(): print(2) # Fails in 3.10, duplicate blocks explained further in issue 32 -def b_multi_with(): +def b0_multi_with(): with a, b: print(1) @@ -17,7 +17,7 @@ def b1_multi_with_fallthrough(): print(1) print(2) -def c_with_as(): +def c0_with_as(): with a as c: print(1) @@ -27,7 +27,7 @@ def c1_with_as_fallthrough(): print(2) # Fails in 3.10, same issue as b -def d_multi_with_as(): +def d0_multi_with_as(): with a, b as c: print(1) @@ -37,7 +37,7 @@ def d1_multi_with_as_fallthrough(): print(2) # Fails in 3.10, same issue as b -def e_with_multi_as(): +def e0_with_multi_as(): with a as b, c: print(1) @@ -47,7 +47,7 @@ def e1_with_multi_as_fallthrough(): print(2) # Fails in 3.10, same issue as b -def f_multi_with_multi_as(): +def f0_multi_with_multi_as(): with a as b, c as d: print(1) @@ -57,7 +57,7 @@ def f1_multi_with_multi_as_fallthrough(): print(2) # Fails in 3.10, same issue as b -def g_multi_with_multi_as_alt(): +def g0_multi_with_multi_as_alt(): with a, b as c, d: print(1) @@ -67,7 +67,7 @@ def g1_multi_with_multi_as_fallthrough_alt(): print(2) # Fails in 3.13, unexpected JUMP_BACKWARD_NO_INTERRUPT messes up the template -def h_try_with_except(): +def h0_try_with_except(): try: with a: print(1) @@ -75,17 +75,17 @@ def h_try_with_except(): print(2) print(3) -def i_with_return(): +def i0_with_return(): with a: return 1 print(1) -def j_with_raise(): +def j0_with_raise(): with a: raise Exc print(1) -async def k_bare_async_with(): +async def k0_bare_async_with(): async with a: print(1) @@ -94,7 +94,7 @@ async def k1_bare_async_with_fallthrough(): print(1) print(2) -async def l_multi_async_with(): +async def l0_multi_async_with(): async with a, b: print(1) @@ -103,7 +103,7 @@ async def l1_multi_async_with_fallthrough(): print(1) print(2) -async def m_async_with_as(): +async def m0_async_with_as(): async with a as c: print(1) @@ -112,7 +112,7 @@ async def m1_async_with_as_fallthrough(): print(1) print(2) -async def n_multi_async_with_as(): +async def n0_multi_async_with_as(): async with a, b as c: print(1) @@ -121,7 +121,7 @@ async def n1_multi_async_with_as_fallthrough(): print(1) print(2) -async def o_async_with_multi_as(): +async def o0_async_with_multi_as(): async with a as b, c: print(1) @@ -130,7 +130,7 @@ async def o1_async_with_multi_as_fallthrough(): print(1) print(2) -async def p_multi_async_with_multi_as(): +async def p0_multi_async_with_multi_as(): async with a as b, c as d: print(1) @@ -139,7 +139,7 @@ async def p1_multi_async_with_multi_as_fallthrough(): print(1) print(2) -async def q_multi_async_with_multi_as_alt(): +async def q0_multi_async_with_multi_as_alt(): async with a, b as c, d: print(1) From 4a055c59cad4feeb6dbb8215069672d215a7adbe Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:30:19 -0500 Subject: [PATCH 06/55] Fixes test case ae0 --- pylingual/control_flow_reconstruction/templates/Exception.py | 4 ++-- test/Exception.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Exception.py b/pylingual/control_flow_reconstruction/templates/Exception.py index 79be386..baca490 100644 --- a/pylingual/control_flow_reconstruction/templates/Exception.py +++ b/pylingual/control_flow_reconstruction/templates/Exception.py @@ -39,7 +39,7 @@ class Except3_11(ControlFlowTemplate): return x -@register_template(0, 0, *versions_from(3, 11)) +@register_template(0, 1, *versions_from(3, 11)) class Try3_11(ControlFlowTemplate): template = T( try_header=N("try_body"), @@ -324,7 +324,7 @@ class Except3_9(ControlFlowTemplate): return node -@register_template(0, 0, (3, 9), (3, 10)) +@register_template(0, 1, (3, 9), (3, 10)) class Try3_9(ControlFlowTemplate): template = T( try_header=~N("try_body"), diff --git a/test/Exception.py b/test/Exception.py index 06bae00..41b5dc3 100644 --- a/test/Exception.py +++ b/test/Exception.py @@ -494,7 +494,6 @@ def ad0_multiple_try_blocks(): except: print(4) -# 3.10/3.11 Try matching before TryElse def ae0_try_except_else_nested_try(): try: print(1) From 738279d812a71255878cb7b07ea3c29b208bb9ef Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Mon, 14 Jul 2025 17:58:56 -0500 Subject: [PATCH 07/55] Update Exception test cases --- test/Exception.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/Exception.py b/test/Exception.py index 41b5dc3..9f7be5b 100644 --- a/test/Exception.py +++ b/test/Exception.py @@ -476,6 +476,13 @@ def ab1_named_except_return(): print(2) return 3 +# 3.12 NamedExc not matching +def ab2_named_except_return(): + try: + return a + except Exception as a: + return b + # 3.11/3.12/3.13 No template match def ac0_empty_try_finally(): try: From e241c52d4ddddc1d6b75f290337b640de9b692e2 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Mon, 14 Jul 2025 17:48:03 -0500 Subject: [PATCH 08/55] Fixed 3.13 Try Except ab2 not matching --- .../templates/Exception.py | 16 +--------------- test/Exception.py | 1 - 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Exception.py b/pylingual/control_flow_reconstruction/templates/Exception.py index baca490..260a7ff 100644 --- a/pylingual/control_flow_reconstruction/templates/Exception.py +++ b/pylingual/control_flow_reconstruction/templates/Exception.py @@ -185,7 +185,7 @@ class ExceptExc3_11(Except3_11): except_header=N("except_body", "no_match", "reraise").with_cond(ending_instructions("CHECK_EXC_MATCH", "POP_JUMP_FORWARD_IF_FALSE"), ending_instructions("CHECK_EXC_MATCH", "POP_JUMP_IF_FALSE")), except_body=N("except_footer.", None, "reraise").of_subtemplate(ExcBody3_11).with_in_deg(1), no_match=N("tail?", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11), - except_footer=~N("tail.").with_in_deg(1).with_cond(starting_instructions("POP_EXCEPT")), + except_footer=~N("tail.").with_in_deg(1).with_cond(starting_instructions("SWAP", "POP_EXCEPT"), starting_instructions("POP_EXCEPT")), reraise=reraise, tail=N.tail(), ) @@ -310,20 +310,6 @@ class Except3_9(ControlFlowTemplate): return node -class Except3_9(ControlFlowTemplate): - @classmethod - @override - def try_match(cls, cfg, node) -> ControlFlowTemplate | None: - if [x.opname for x in node.get_instructions()] == ["RERAISE"]: - return node - if x := ExceptExc3_9.try_match(cfg, node): - return x - if x := BareExcept3_9.try_match(cfg, node): - return x - if isinstance(node, Except3_9): - return node - - @register_template(0, 1, (3, 9), (3, 10)) class Try3_9(ControlFlowTemplate): template = T( diff --git a/test/Exception.py b/test/Exception.py index 9f7be5b..7a88433 100644 --- a/test/Exception.py +++ b/test/Exception.py @@ -476,7 +476,6 @@ def ab1_named_except_return(): print(2) return 3 -# 3.12 NamedExc not matching def ab2_named_except_return(): try: return a From 8a748502236fe5bf1a506558347acfbc0367f24d Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Tue, 15 Jul 2025 15:33:45 -0500 Subject: [PATCH 09/55] Break template --- .../templates/Loop.py | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index c935c2e..03d5d4e 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -4,6 +4,7 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, N, + condense_mapping, defer_source_to, starting_instructions, to_indented_source, @@ -63,6 +64,15 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("comp") +class BreakTemplate(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg, node): + return condense_mapping(cls, cfg, {'child': node}, 'child') + + def to_indented_source(self, source): + return self.child.to_indented_source(source) + self.line('break') + +@register_template(0, 4) class FixLoop(ControlFlowTemplate): @classmethod def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: @@ -73,4 +83,38 @@ class FixLoop(ControlFlowTemplate): # replace those edges with meta edges to the end # find nodes in loop that go to header # replace all but last of those edges with meta edge to end - \ No newline at end of file + + # a node is a loop header if there are back-edges to it + # a latching node is a node with a back-edge to the loop header + # a back-edge is an edge from any node that is dominated by this node + back_edges = [] + for predecessor in cfg.predecessors(node): + # A back edge exists if the predecessor is reachable from the node (node dominates predecessor) + if cfg.dominates(node, predecessor): + back_edges.append(predecessor) + if cfg.has_edge(node, node): + return None + + if not back_edges: + return None + + nodes = [] + edges_to_remove = [] + + for successor in list(cfg.successors(node)): + if cfg.in_degree(successor) != 1: + for predecessor in list(cfg.predecessors(successor)): + if predecessor != node: + nodes.append(predecessor) + edges_to_remove.append((predecessor, successor)) + + valid = [] + for pred, succ in edges_to_remove: + if cfg.get_edge_data(pred, succ).get("kind") != EdgeKind.Exception: + cfg.remove_edge(pred, succ) + valid.append((pred, succ)) + + for pred in set(x for x, _ in valid): + BreakTemplate.try_match(cfg, pred) + cfg.iterate() + return \ No newline at end of file From fe2ad4bb1a591676d2ff56b280681f0dda6f7b24 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 17 Jul 2025 10:25:48 -0500 Subject: [PATCH 10/55] Break statement --- pylingual/control_flow_reconstruction/cfg.py | 53 ++++++++++++++++ .../templates/Loop.py | 63 +++++++++++++++---- 2 files changed, 103 insertions(+), 13 deletions(-) diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index 1e6c21d..5151806 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -75,6 +75,59 @@ class CFG(DiGraph_CFT): self._create_dominator_tree() return nx.dfs_postorder_nodes(self, source=self.start, sort_neighbors=lambda nodes: sorted(nodes, key=lambda x: x.offset, reverse=True)) + def is_loop_header(self, node): + # Check all predecessors + for predecessor in self.predecessors(node): + # A back edge exists if the predecessor is dominated by this node + if self.dominates(node, predecessor): + return True + return False + + def dfs_labeled_edges_no_loop(self, source=None, depth_limit=None, *, sort_neighbors=None): + if source is None: + # edges for all components + nodes = self + else: + # edges for components with source + nodes = [source] + if depth_limit is None: + depth_limit = len(self) + + get_children = ( + self.neighbors + if sort_neighbors is None + else lambda n: iter(sort_neighbors(self.neighbors(n))) + ) + + visited = set() + for start in nodes: + if start in visited: + continue + yield start, start, "forward" + visited.add(start) + stack = [(start, get_children(start))] + depth_now = 1 + while stack: + parent, children = stack[-1] + for child in children: + if child in visited or self.is_loop_header(child): + yield parent, child, "nontree" + else: + yield parent, child, "forward" + visited.add(child) + if depth_now < depth_limit: + stack.append((child, iter(get_children(child)))) + depth_now += 1 + break + else: + yield parent, child, "reverse-depth_limit" + else: + stack.pop() + depth_now -= 1 + if stack: + yield stack[-1][0], parent, "reverse" + yield start, start, "reverse" + def apply_graphs(self): graphs = self.iteration_graphs.pop() if self.iteration_graphs: diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index 03d5d4e..e3d5e66 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -4,6 +4,7 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, N, + without_top_level_instructions, condense_mapping, defer_source_to, starting_instructions, @@ -67,12 +68,22 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): class BreakTemplate(ControlFlowTemplate): @classmethod def try_match(cls, cfg, node): + if isinstance(node, BreakTemplate): + return None return condense_mapping(cls, cfg, {'child': node}, 'child') def to_indented_source(self, source): return self.child.to_indented_source(source) + self.line('break') -@register_template(0, 4) +class ContinueTemplate(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg, node): + return condense_mapping(cls, cfg, {'child': node}, 'child') + + def to_indented_source(self, source): + return self.child.to_indented_source(source) + self.line('continue') + +@register_template(0, 0) class FixLoop(ControlFlowTemplate): @classmethod def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: @@ -89,31 +100,57 @@ class FixLoop(ControlFlowTemplate): # a back-edge is an edge from any node that is dominated by this node back_edges = [] for predecessor in cfg.predecessors(node): + # A back edge exists if the predecessor is reachable from the node (node dominates predecessor) if cfg.dominates(node, predecessor): back_edges.append(predecessor) if cfg.has_edge(node, node): return None + if not back_edges: return None - nodes = [] - edges_to_remove = [] + # Get all nodes encompassed by the loop excluding source node and initial false jump + loopnode = None + for succ in cfg.successors(node): + if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.Fall: + loopnode = succ + break - for successor in list(cfg.successors(node)): - if cfg.in_degree(successor) != 1: - for predecessor in list(cfg.predecessors(successor)): - if predecessor != node: - nodes.append(predecessor) - edges_to_remove.append((predecessor, successor)) + dfs_edges = cfg.dfs_labeled_edges_no_loop(source=loopnode) + encompassed_nodes = [ + v for u, v, d in dfs_edges + if d == "forward" and v != node + ] + edges_to_remove = [] + + # Find the candidate end that break connect to + candidate_end = None + for succ in cfg.successors(node): + if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.FalseJump and cfg.out_degree(succ) <= 1: + candidate_end = succ + + # Candidate end is a buffer node + if cfg.in_degree(candidate_end) == 1: + for ss in cfg.successors(candidate_end): + + # If the successor has only one predecessor and one successor, it is a buffer node + if cfg.out_degree(ss) <= 1: + candidate_end = ss + + if encompassed_nodes is not None: + for succ in encompassed_nodes: + if cfg.get_edge_data(succ, candidate_end) != None: + edges_to_remove.append((succ, candidate_end)) valid = [] for pred, succ in edges_to_remove: - if cfg.get_edge_data(pred, succ).get("kind") != EdgeKind.Exception: - cfg.remove_edge(pred, succ) - valid.append((pred, succ)) - + if not (cfg.get_edge_data(pred, succ).get("kind") == EdgeKind.Exception or cfg.get_edge_data(pred, succ).get("kind") == EdgeKind.FalseJump): + if without_top_level_instructions("RAISE_VARARGS")(cfg, pred): + cfg.remove_edge(pred, succ) + valid.append((pred, succ)) + for pred in set(x for x, _ in valid): BreakTemplate.try_match(cfg, pred) cfg.iterate() From 6e38442b95a713fbbbd87dc57eee7e821bcca063 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 17 Jul 2025 10:26:07 -0500 Subject: [PATCH 11/55] Slight fixes to 3.11 While loops --- .../templates/Loop.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index e3d5e66..f91c52c 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -46,6 +46,24 @@ class SelfLoop(ControlFlowTemplate): {loop_body} """ +@register_template(0, 2) +class TrueSelfLoop(ControlFlowTemplate): + template = T( + loop_body=~N("tail.", "loop_body"), + tail=N.tail()) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + }, + "loop_body" + ) + + @to_indented_source + def to_indented_source(): + """ + {loop_body} + """ @register_template(0, 3) class InlinedComprehensionTemplate(ControlFlowTemplate): From 22b512b2d2885254fe06e1f1cc68541a0fef34c8 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:01:38 -0500 Subject: [PATCH 12/55] Documentation on failing test cases --- test/Loop.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/test/Loop.py b/test/Loop.py index 135e0a0..26e6a8b 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -16,12 +16,14 @@ def b1_for_over_tuples_nofallthru(): print("tuples") print("end") +# 3.6 No else template def c0_for_else(): for i in range(3): print("for body") else: print("for else") +# 3.6 No else template def c1_for_else_nofallthru(): for i in range(3): print("for body") @@ -29,14 +31,15 @@ def c1_for_else_nofallthru(): print("for else") print("end") -# Fails due to no break +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9 Naive break detection, an unexpected buffer block to end def d0_for_with_break(): for x in range(10): if x == 5: print("breaking") break -# Fails due to no break +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end def d1_for_with_break_nofallthru(): for x in range(10): if x == 5: @@ -135,6 +138,8 @@ def j1_for_with_empty_body_ellipsis_nofallthru(): ... print("end") +# 3.6 Naive break detection, no back edge +# 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k0_while_true_with_break(): x = 0 while True: @@ -143,6 +148,8 @@ def k0_while_true_with_break(): if x >= 1: break +# 3.6 Naive break detection, no back edge +# 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k1_while_true_with_break_nofallthru(): x = 0 while True: @@ -152,6 +159,8 @@ def k1_while_true_with_break_nofallthru(): break print("end") +# 3.6 No else template +# 3.11 No while loop detection, self false_jump edge def l0_while_with_else(): i = 0 while i < 3: @@ -160,6 +169,8 @@ def l0_while_with_else(): else: print("while else") +# 3.6 No else template +# 3.11 No while loop detection, self false_jump edge def l1_while_with_else_nofallthru(): i = 0 while i < 3: @@ -169,6 +180,7 @@ def l1_while_with_else_nofallthru(): print("while else") print("end") +# 3.11 No continue def m0_while_with_continue(): i = 0 while i < 5: @@ -178,6 +190,7 @@ def m0_while_with_continue(): continue print("after continue") +# 3.11 No continue def m1_while_with_continue_nofallthru(): i = 0 while i < 5: @@ -188,12 +201,14 @@ def m1_while_with_continue_nofallthru(): print("after continue") print("end") +# 3.6 Naive break detection, no back edge def n0_while_with_break(): i = 0 while True: print("break in while") break +# 3.6 Naive break detection, no back edge def n1_while_with_break_nofallthru(): i = 0 while True: @@ -201,6 +216,7 @@ def n1_while_with_break_nofallthru(): break print("end") +# 3.11 While template broke def o0_nested_while_loops(): i = 0 while i < 2: @@ -210,6 +226,7 @@ def o0_nested_while_loops(): j += 1 i += 1 +# 3.11 While template broke def o1_nested_while_loops_nofallthru(): i = 0 while i < 2: @@ -220,6 +237,8 @@ def o1_nested_while_loops_nofallthru(): i += 1 print("end") +# 3.6 While template broke (?) +# 3.9 Disconnected with MetaTemplate[end] (?) def p0_while_with_try_except(): while True: try: @@ -227,6 +246,8 @@ def p0_while_with_try_except(): except: print("except in while") +# 3.6 While template broke (?) +# 3.9 Disconnected with MetaTemplate[end] (?) def p1_while_with_try_except_nofallthru(): while True: try: @@ -235,34 +256,40 @@ def p1_while_with_try_except_nofallthru(): print("except in while") print("end") +# 3.6 While template broke (?) abandoning nodes def q0_while_with_with_statement(): while True: with a: print("inside while with") +# 3.6 While template broke (?) abandoning nodes def q1_while_with_with_statement_nofallthru(): while True: with a: print("inside while with") print("end") +# 3.6 While template broke def r0_for_inside_while(): while True: for x in [1, 2]: print("for in while") +# 3.6 While template broke def r1_for_inside_while_nofallthru(): while True: for x in [1, 2]: print("for in while") print("end") +# 3.6 While template broke def s0_while_inside_for(): for _ in range(1): while True: print("while in for") break +# 3.6 While template broke def s1_while_inside_for_nofallthru(): for _ in range(1): while True: @@ -270,10 +297,12 @@ def s1_while_inside_for_nofallthru(): break print("end") +# 3.6 While template broke def t0_while_with_empty_body_ellipsis(): while True: ... +# 3.6 While template broke def t1_while_with_empty_body_ellipsis_nofallthru(): while True: ... @@ -311,6 +340,8 @@ def v1_continue_in_nested_for_nofallthru(): print(f"Processing i={i}, j={j}") print("end") +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9/3.11 Naive break detection, an unexpected buffer block to end def w0_break_with_else(): for i in range(5): if i == 3: @@ -319,6 +350,8 @@ def w0_break_with_else(): else: print("This won't execute due to break") +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9/3.11 Naive break detection, an unexpected buffer block to end def w1_break_with_else_nofallthru(): for i in range(5): if i == 3: @@ -328,6 +361,7 @@ def w1_break_with_else_nofallthru(): print("This won't execute due to break") print("end") +# 3.6 No continue detection def x0_continue_with_else(): for i in range(3): if i == 1: @@ -336,6 +370,7 @@ def x0_continue_with_else(): else: print("Else clause still executes after continue") +# 3.6 No continue detection def x1_continue_with_else_nofallthru(): for i in range(3): if i == 1: @@ -345,6 +380,8 @@ def x1_continue_with_else_nofallthru(): print("Else clause still executes after continue") print("end") +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9/3.11 Naive break detection, break statement is further up def y0_break_in_try_except(): for i in range(5): try: @@ -354,6 +391,8 @@ def y0_break_in_try_except(): except: print("Exception occurred") +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9/3.11 Naive break detection, break statement is further up def y1_break_in_try_except_nofallthru(): for i in range(5): try: @@ -364,6 +403,7 @@ def y1_break_in_try_except_nofallthru(): print("Exception occurred") print("end") +# 3.6/3.9/3.11 No continue detection def z0_continue_in_try_except(): for i in range(5): try: @@ -373,6 +413,7 @@ def z0_continue_in_try_except(): except: print("Exception occurred") +# 3.6/3.9/3.11 No continue detection def z1_continue_in_try_except_nofallthru(): for i in range(5): try: From bafc3b787815e95d278672c03ac71a0d9167bd29 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:02:45 -0500 Subject: [PATCH 13/55] Fixed 3.6 Loops y0 y1 y2 test cases --- .../templates/Conditional.py | 5 +++-- test/Loop.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Conditional.py b/pylingual/control_flow_reconstruction/templates/Conditional.py index f163981..f0f06a8 100644 --- a/pylingual/control_flow_reconstruction/templates/Conditional.py +++ b/pylingual/control_flow_reconstruction/templates/Conditional.py @@ -1,13 +1,14 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import T, N, defer_source_to, run_is, has_no_lines, with_instructions, has_instval, starting_instructions, to_indented_source, make_try_match, without_top_level_instructions +from .Loop import BreakTemplate @register_template(1, 40) class IfElse(ControlFlowTemplate): template = T( if_header=~N("if_body", "else_body").with_cond(without_top_level_instructions("WITH_EXCEPT_START", "CHECK_EXC_MATCH", "FOR_ITER")), - if_body=~N("tail.").with_in_deg(1), - else_body=~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1), + if_body=N.tail().with_in_deg(1).of_type(BreakTemplate) | ~N("tail.").with_in_deg(1), + else_body=N.tail().with_in_deg(1).of_type(BreakTemplate) | ~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1), tail=N.tail(), ) diff --git a/test/Loop.py b/test/Loop.py index 26e6a8b..0340277 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -380,7 +380,6 @@ def x1_continue_with_else_nofallthru(): print("Else clause still executes after continue") print("end") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end # 3.9/3.11 Naive break detection, break statement is further up def y0_break_in_try_except(): for i in range(5): @@ -391,7 +390,6 @@ def y0_break_in_try_except(): except: print("Exception occurred") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end # 3.9/3.11 Naive break detection, break statement is further up def y1_break_in_try_except_nofallthru(): for i in range(5): @@ -403,6 +401,18 @@ def y1_break_in_try_except_nofallthru(): print("Exception occurred") print("end") +# 3.9/3.11 Naive break detection, break statement is further up +def y2_return_in_try_except_nofallthru(): + for i in range(5): + try: + if i == 3: + print(f"Value: {i}") + else: + break + except: + print("Exception occurred") + print("end") + # 3.6/3.9/3.11 No continue detection def z0_continue_in_try_except(): for i in range(5): From 2b56f6685f46cb62d5d270ce552d45936e97a064 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 16:56:55 -0500 Subject: [PATCH 14/55] Graph fix Co-Authored-By: caandt --- pylingual/control_flow_reconstruction/cfg.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index 1e6c21d..c45a893 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -90,9 +90,11 @@ class CFG(DiGraph_CFT): def _layout_nodes(self): relabeled = nx.convert_node_labels_to_integers(self, label_attribute="template") # type: ignore - root = next(i for i in relabeled.nodes if relabeled.nodes[i]["template"] == self.start) - for i, pos in nx.nx_pydot.pydot_layout(relabeled, prog="dot", root=root).items(): - relabeled.nodes[i]["template"]._pos = [pos] + try: + root = next(i for i in relabeled.nodes if relabeled.nodes[i]["template"] == self.start) + for i, pos in nx.nx_pydot.pydot_layout(relabeled, prog="dot", root=root).items(): + relabeled.nodes[i]["template"]._pos = [pos] + except: pass def node_by_offset(self, offset: int): return next(x for x in self.nodes if x.offset == offset) @@ -122,7 +124,7 @@ class CFG(DiGraph_CFT): nodes = {} for node, data in self.nodes.data(): - nodes[node] = pydot.Node(str(hash(node)), label=repr(node).replace("\n", "\\l").replace("\t", "| ") + "\\l", fontname="Noto Sans", labeljust="l", shape="box", pos=node.pos()) + nodes[node] = pydot.Node(str(hash(node)), label=repr(node).replace("\n", "\\l").replace("\t", "| ").replace('"', '') + "\\l", fontname="Noto Sans", labeljust="l", shape="box", pos=node.pos()) dot.add_node(nodes[node]) for a, b, data in self.edges.data(): dot.add_edge(pydot.Edge(nodes[a], nodes[b], **data, label=data["kind"].value, color=data["kind"].color(), fontname="Noto Sans", labeljust="l")) From 5e12db92372ef7744c7d9090b518b6ff6b98a6ff Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 18:43:26 -0500 Subject: [PATCH 15/55] Disregard initial natural node from Break template --- pylingual/control_flow_reconstruction/templates/Loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index f91c52c..ea852fb 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -140,7 +140,7 @@ class FixLoop(ControlFlowTemplate): encompassed_nodes = [ v for u, v, d in dfs_edges if d == "forward" and v != node - ] + ][1:] edges_to_remove = [] # Find the candidate end that break connect to From e884e05389982e4299ee7efe4d02fc4aa2b14c01 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 21:01:50 -0500 Subject: [PATCH 16/55] Updated conditions for encompassed nodes --- .../control_flow_reconstruction/templates/Loop.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index ea852fb..f2d26f5 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -140,7 +140,7 @@ class FixLoop(ControlFlowTemplate): encompassed_nodes = [ v for u, v, d in dfs_edges if d == "forward" and v != node - ][1:] + ] edges_to_remove = [] # Find the candidate end that break connect to @@ -160,7 +160,13 @@ class FixLoop(ControlFlowTemplate): if encompassed_nodes is not None: for succ in encompassed_nodes: if cfg.get_edge_data(succ, candidate_end) != None: - edges_to_remove.append((succ, candidate_end)) + for parent in cfg.predecessors(succ): + if parent in encompassed_nodes: + continue + else: + break + else: + edges_to_remove.append((succ, candidate_end)) valid = [] for pred, succ in edges_to_remove: From fb825b44687f1878f4de4c99010f749a73898adb Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 16:57:08 -0500 Subject: [PATCH 17/55] RemoveUnreachable template Co-Authored-By: caandt --- .../templates/Block.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 83397a4..1294aee 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -4,6 +4,8 @@ from typing import TYPE_CHECKING, override from itertools import chain from pylingual.editable_bytecode import Inst +import networkx as nx + from ..cft import ControlFlowTemplate, EdgeKind, SourceContext, SourceLine, register_template, EdgeCategory, out_edge_dict, MetaTemplate, indent_str from ..utils import E, N, T, defer_source_to, remove_nodes @@ -37,6 +39,21 @@ class EndTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("body") +@register_template(3, 0) +class RemoveUnreachable(ControlFlowTemplate): + @override + @classmethod + def try_match(cls, cfg, node) -> ControlFlowTemplate | None: + if node is not cfg.start: + return None + + valid = list(nx.dfs_preorder_nodes(cfg, source=cfg.start)) + invalid = [n for n in cfg.nodes if n not in valid] + if invalid: + cfg.remove_nodes_from(invalid) + return node + + @register_template(0, 20) @register_template(2, 20) class BlockTemplate(ControlFlowTemplate): From ecf215677e5b442b10a677d8b8571a3d6edb40e6 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 21:07:17 -0500 Subject: [PATCH 18/55] JumpTemplate to allow other templates to match in versions 3.12+ --- .../templates/Block.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 1294aee..86c7321 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -7,7 +7,7 @@ from pylingual.editable_bytecode import Inst import networkx as nx from ..cft import ControlFlowTemplate, EdgeKind, SourceContext, SourceLine, register_template, EdgeCategory, out_edge_dict, MetaTemplate, indent_str -from ..utils import E, N, T, defer_source_to, remove_nodes +from ..utils import E, N, T, defer_source_to, remove_nodes, without_instructions, exact_instructions, make_try_match if TYPE_CHECKING: from pylingual.control_flow_reconstruction.cfg import CFG @@ -54,6 +54,27 @@ class RemoveUnreachable(ControlFlowTemplate): return node +@register_template(0, 0) +class JumpTemplate(ControlFlowTemplate): + template = T( + body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), + jump=N("tail", "block.").with_in_deg(1).with_cond(exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), exact_instructions("POP_JUMP_IF_TRUE")), + block=~N.tail(), + tail=N.tail(), + ) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + EdgeKind.TrueJump: "block", + }, + "body", + "jump", + ) + + to_indented_source = defer_source_to("body") + + @register_template(0, 20) @register_template(2, 20) class BlockTemplate(ControlFlowTemplate): From 4905d6966ca4e54c5a017c28cbe2580f7cb618d8 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 21:12:43 -0500 Subject: [PATCH 19/55] Fixing With statement test cases --- .../templates/Block.py | 21 +++++++++++++++++++ test/With.py | 6 ------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 86c7321..2734ec4 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -75,6 +75,27 @@ class JumpTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("body") +@register_template(0, 0) +class JumpTemplate(ControlFlowTemplate): + template = T( + body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), + jump=N("tail", "block?").with_in_deg(1).with_cond(exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), exact_instructions("POP_JUMP_IF_TRUE")), + block=~N.tail(), + tail=N.tail(), + ) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + EdgeKind.TrueJump: "block", + }, + "body", + "jump", + ) + + to_indented_source = defer_source_to("body") + + @register_template(0, 20) @register_template(2, 20) class BlockTemplate(ControlFlowTemplate): diff --git a/test/With.py b/test/With.py index 491f463..d4b4fc0 100644 --- a/test/With.py +++ b/test/With.py @@ -7,7 +7,6 @@ def a1_bare_with_fallthrough(): print(1) print(2) -# Fails in 3.10, duplicate blocks explained further in issue 32 def b0_multi_with(): with a, b: print(1) @@ -26,7 +25,6 @@ def c1_with_as_fallthrough(): print(1) print(2) -# Fails in 3.10, same issue as b def d0_multi_with_as(): with a, b as c: print(1) @@ -36,7 +34,6 @@ def d1_multi_with_as_fallthrough(): print(1) print(2) -# Fails in 3.10, same issue as b def e0_with_multi_as(): with a as b, c: print(1) @@ -46,7 +43,6 @@ def e1_with_multi_as_fallthrough(): print(1) print(2) -# Fails in 3.10, same issue as b def f0_multi_with_multi_as(): with a as b, c as d: print(1) @@ -56,7 +52,6 @@ def f1_multi_with_multi_as_fallthrough(): print(1) print(2) -# Fails in 3.10, same issue as b def g0_multi_with_multi_as_alt(): with a, b as c, d: print(1) @@ -66,7 +61,6 @@ def g1_multi_with_multi_as_fallthrough_alt(): print(1) print(2) -# Fails in 3.13, unexpected JUMP_BACKWARD_NO_INTERRUPT messes up the template def h0_try_with_except(): try: with a: From eaaa6137cd391e0e3014eb12f95136b5102a7668 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 21:14:13 -0500 Subject: [PATCH 20/55] Update Loop.py errors --- test/Loop.py | 56 +++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/test/Loop.py b/test/Loop.py index 0340277..8c730bf 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -16,14 +16,14 @@ def b1_for_over_tuples_nofallthru(): print("tuples") print("end") -# 3.6 No else template +# 3.6/3.7 No else template def c0_for_else(): for i in range(3): print("for body") else: print("for else") -# 3.6 No else template +# 3.6/3.7 No else template def c1_for_else_nofallthru(): for i in range(3): print("for body") @@ -31,7 +31,7 @@ def c1_for_else_nofallthru(): print("for else") print("end") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end # 3.9 Naive break detection, an unexpected buffer block to end def d0_for_with_break(): for x in range(10): @@ -39,7 +39,7 @@ def d0_for_with_break(): print("breaking") break -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end def d1_for_with_break_nofallthru(): for x in range(10): if x == 5: @@ -138,7 +138,7 @@ def j1_for_with_empty_body_ellipsis_nofallthru(): ... print("end") -# 3.6 Naive break detection, no back edge +# 3.6/3.7 Naive break detection, no back edge # 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k0_while_true_with_break(): x = 0 @@ -148,7 +148,7 @@ def k0_while_true_with_break(): if x >= 1: break -# 3.6 Naive break detection, no back edge +# 3.6/3.7 Naive break detection, no back edge # 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k1_while_true_with_break_nofallthru(): x = 0 @@ -159,7 +159,7 @@ def k1_while_true_with_break_nofallthru(): break print("end") -# 3.6 No else template +# 3.6/3.7 No else template # 3.11 No while loop detection, self false_jump edge def l0_while_with_else(): i = 0 @@ -169,7 +169,7 @@ def l0_while_with_else(): else: print("while else") -# 3.6 No else template +# 3.6/3.7 No else template # 3.11 No while loop detection, self false_jump edge def l1_while_with_else_nofallthru(): i = 0 @@ -201,14 +201,14 @@ def m1_while_with_continue_nofallthru(): print("after continue") print("end") -# 3.6 Naive break detection, no back edge +# 3.6/3.7 Naive break detection, no back edge def n0_while_with_break(): i = 0 while True: print("break in while") break -# 3.6 Naive break detection, no back edge +# 3.6/3.7 Naive break detection, no back edge def n1_while_with_break_nofallthru(): i = 0 while True: @@ -237,7 +237,7 @@ def o1_nested_while_loops_nofallthru(): i += 1 print("end") -# 3.6 While template broke (?) +# 3.6/3.7 While template broke (?) # 3.9 Disconnected with MetaTemplate[end] (?) def p0_while_with_try_except(): while True: @@ -246,7 +246,7 @@ def p0_while_with_try_except(): except: print("except in while") -# 3.6 While template broke (?) +# 3.6/3.7 While template broke (?) # 3.9 Disconnected with MetaTemplate[end] (?) def p1_while_with_try_except_nofallthru(): while True: @@ -256,40 +256,40 @@ def p1_while_with_try_except_nofallthru(): print("except in while") print("end") -# 3.6 While template broke (?) abandoning nodes +# 3.6/3.7 While template broke (?) abandoning nodes def q0_while_with_with_statement(): while True: with a: print("inside while with") -# 3.6 While template broke (?) abandoning nodes +# 3.6/3.7 While template broke (?) abandoning nodes def q1_while_with_with_statement_nofallthru(): while True: with a: print("inside while with") print("end") -# 3.6 While template broke +# 3.6/3.7 While template broke def r0_for_inside_while(): while True: for x in [1, 2]: print("for in while") -# 3.6 While template broke +# 3.6/3.7 While template broke def r1_for_inside_while_nofallthru(): while True: for x in [1, 2]: print("for in while") print("end") -# 3.6 While template broke +# 3.6/3.7 While template broke def s0_while_inside_for(): for _ in range(1): while True: print("while in for") break -# 3.6 While template broke +# 3.6/3.7 While template broke def s1_while_inside_for_nofallthru(): for _ in range(1): while True: @@ -297,12 +297,12 @@ def s1_while_inside_for_nofallthru(): break print("end") -# 3.6 While template broke +# 3.6/3.7 While template broke def t0_while_with_empty_body_ellipsis(): while True: ... -# 3.6 While template broke +# 3.6/3.7 While template broke def t1_while_with_empty_body_ellipsis_nofallthru(): while True: ... @@ -340,8 +340,7 @@ def v1_continue_in_nested_for_nofallthru(): print(f"Processing i={i}, j={j}") print("end") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end -# 3.9/3.11 Naive break detection, an unexpected buffer block to end +# 3.13 if statement putting code in the else block def w0_break_with_else(): for i in range(5): if i == 3: @@ -350,8 +349,7 @@ def w0_break_with_else(): else: print("This won't execute due to break") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end -# 3.9/3.11 Naive break detection, an unexpected buffer block to end +# 3.13 if statement putting code in the else block def w1_break_with_else_nofallthru(): for i in range(5): if i == 3: @@ -361,7 +359,7 @@ def w1_break_with_else_nofallthru(): print("This won't execute due to break") print("end") -# 3.6 No continue detection +# 3.6/3.7 No continue detection def x0_continue_with_else(): for i in range(3): if i == 1: @@ -370,7 +368,7 @@ def x0_continue_with_else(): else: print("Else clause still executes after continue") -# 3.6 No continue detection +# 3.6/3.7 No continue detection def x1_continue_with_else_nofallthru(): for i in range(3): if i == 1: @@ -390,7 +388,7 @@ def y0_break_in_try_except(): except: print("Exception occurred") -# 3.9/3.11 Naive break detection, break statement is further up +# 3.9 Naive break detection, break statement is further up def y1_break_in_try_except_nofallthru(): for i in range(5): try: @@ -401,7 +399,7 @@ def y1_break_in_try_except_nofallthru(): print("Exception occurred") print("end") -# 3.9/3.11 Naive break detection, break statement is further up +# 3.9 Naive break detection, break statement is further up def y2_return_in_try_except_nofallthru(): for i in range(5): try: @@ -413,7 +411,7 @@ def y2_return_in_try_except_nofallthru(): print("Exception occurred") print("end") -# 3.6/3.9/3.11 No continue detection +# 3.6/3.9 No continue detection def z0_continue_in_try_except(): for i in range(5): try: From baaca8572ff8ecae56c5cec664485e121e6fe267 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Sun, 20 Jul 2025 09:20:44 -0500 Subject: [PATCH 21/55] Refining Breaks/Continues --- pylingual/control_flow_reconstruction/cfg.py | 2 +- .../templates/Loop.py | 55 +++++++++---------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index 5151806..41091f5 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -110,7 +110,7 @@ class CFG(DiGraph_CFT): while stack: parent, children = stack[-1] for child in children: - if child in visited or self.is_loop_header(child): + if child in visited or self.is_loop_header(child) or not all(p in visited for p in self.predecessors(child)): yield parent, child, "nontree" else: yield parent, child, "forward" diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index f2d26f5..0745c94 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -4,7 +4,9 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, N, - without_top_level_instructions, + with_instructions, + exact_instructions, + has_no_lines, condense_mapping, defer_source_to, starting_instructions, @@ -86,7 +88,7 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): class BreakTemplate(ControlFlowTemplate): @classmethod def try_match(cls, cfg, node): - if isinstance(node, BreakTemplate): + if isinstance(node, BreakTemplate) or has_no_lines(cfg, node) or with_instructions("RAISE_VARARGS")(cfg, node): return None return condense_mapping(cls, cfg, {'child': node}, 'child') @@ -96,7 +98,12 @@ class BreakTemplate(ControlFlowTemplate): class ContinueTemplate(ControlFlowTemplate): @classmethod def try_match(cls, cfg, node): - return condense_mapping(cls, cfg, {'child': node}, 'child') + if isinstance(node, ContinueTemplate) or has_no_lines(cfg, node): + return None + instruction = node.get_instructions()[-1].opname + if instruction in {"JUMP_ABSOLUTE", "JUMP_BACKWARD", "CONTINUE_LOOP"} and (node.get_instructions()[-1].starts_line is not None or node.get_instructions()[-2].starts_line is not None): + return condense_mapping(cls, cfg, {'child': node}, 'child') + return None def to_indented_source(self, source): return self.child.to_indented_source(source) + self.line('continue') @@ -122,9 +129,6 @@ class FixLoop(ControlFlowTemplate): # A back edge exists if the predecessor is reachable from the node (node dominates predecessor) if cfg.dominates(node, predecessor): back_edges.append(predecessor) - if cfg.has_edge(node, node): - return None - if not back_edges: return None @@ -137,45 +141,38 @@ class FixLoop(ControlFlowTemplate): break dfs_edges = cfg.dfs_labeled_edges_no_loop(source=loopnode) - encompassed_nodes = [ - v for u, v, d in dfs_edges - if d == "forward" and v != node - ] + encompassed_nodes = [v for u, v, d in dfs_edges if d == "forward"] + edges_to_remove = [] - # Find the candidate end that break connect to + # Find the candidate end that break connects to candidate_end = None for succ in cfg.successors(node): if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.FalseJump and cfg.out_degree(succ) <= 1: candidate_end = succ # Candidate end is a buffer node - if cfg.in_degree(candidate_end) == 1: + if cfg.in_degree(candidate_end) == 1 and any(exact_instructions(*op)(cfg, candidate_end) for op in [ + ("POP_BLOCK",), ("END_FOR",), ("END_FOR", "POP_TOP"), ("LOAD_CONST", "RETURN_VALUE")]): for ss in cfg.successors(candidate_end): - - # If the successor has only one predecessor and one successor, it is a buffer node if cfg.out_degree(ss) <= 1: candidate_end = ss - + break + if encompassed_nodes is not None: for succ in encompassed_nodes: if cfg.get_edge_data(succ, candidate_end) != None: - for parent in cfg.predecessors(succ): - if parent in encompassed_nodes: - continue - else: - break - else: - edges_to_remove.append((succ, candidate_end)) + edges_to_remove.append((succ, candidate_end)) - valid = [] for pred, succ in edges_to_remove: - if not (cfg.get_edge_data(pred, succ).get("kind") == EdgeKind.Exception or cfg.get_edge_data(pred, succ).get("kind") == EdgeKind.FalseJump): - if without_top_level_instructions("RAISE_VARARGS")(cfg, pred): - cfg.remove_edge(pred, succ) - valid.append((pred, succ)) + break_node = BreakTemplate.try_match(cfg, pred) + if break_node is not None: + cfg.remove_edge(break_node, succ) + + for candidate in back_edges: + cont_node = ContinueTemplate.try_match(cfg, candidate) + if cont_node is not None and cfg.in_degree(node) > 2: + cfg.remove_edge(cont_node, node) - for pred in set(x for x, _ in valid): - BreakTemplate.try_match(cfg, pred) cfg.iterate() return \ No newline at end of file From bd2b0596d70546b9c680339c99ac0c109cff6544 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 10 Jul 2025 17:18:14 -0500 Subject: [PATCH 22/55] Start of loops --- .../templates/Loop.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index 3f56a0c..c935c2e 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -1,3 +1,5 @@ +from __future__ import annotations +from typing import TYPE_CHECKING from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, @@ -5,9 +7,11 @@ from ..utils import ( defer_source_to, starting_instructions, to_indented_source, - make_try_match, + make_try_match, ) +if TYPE_CHECKING: + from pylingual.control_flow_reconstruction.cfg import CFG @register_template(0, 1) class ForLoop(ControlFlowTemplate): @@ -58,3 +62,15 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): ) to_indented_source = defer_source_to("comp") + +class FixLoop(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: + # check that its a loop that we need to fix + # find the end of the loop + # find all nodes that belong to the loop + # find nodes in loop that go to end + # replace those edges with meta edges to the end + # find nodes in loop that go to header + # replace all but last of those edges with meta edge to end + \ No newline at end of file From 12091be3a30906b23c31ba68513f89f7b1394b74 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Tue, 15 Jul 2025 15:33:45 -0500 Subject: [PATCH 23/55] Break template --- .../templates/Loop.py | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index c935c2e..03d5d4e 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -4,6 +4,7 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, N, + condense_mapping, defer_source_to, starting_instructions, to_indented_source, @@ -63,6 +64,15 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("comp") +class BreakTemplate(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg, node): + return condense_mapping(cls, cfg, {'child': node}, 'child') + + def to_indented_source(self, source): + return self.child.to_indented_source(source) + self.line('break') + +@register_template(0, 4) class FixLoop(ControlFlowTemplate): @classmethod def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: @@ -73,4 +83,38 @@ class FixLoop(ControlFlowTemplate): # replace those edges with meta edges to the end # find nodes in loop that go to header # replace all but last of those edges with meta edge to end - \ No newline at end of file + + # a node is a loop header if there are back-edges to it + # a latching node is a node with a back-edge to the loop header + # a back-edge is an edge from any node that is dominated by this node + back_edges = [] + for predecessor in cfg.predecessors(node): + # A back edge exists if the predecessor is reachable from the node (node dominates predecessor) + if cfg.dominates(node, predecessor): + back_edges.append(predecessor) + if cfg.has_edge(node, node): + return None + + if not back_edges: + return None + + nodes = [] + edges_to_remove = [] + + for successor in list(cfg.successors(node)): + if cfg.in_degree(successor) != 1: + for predecessor in list(cfg.predecessors(successor)): + if predecessor != node: + nodes.append(predecessor) + edges_to_remove.append((predecessor, successor)) + + valid = [] + for pred, succ in edges_to_remove: + if cfg.get_edge_data(pred, succ).get("kind") != EdgeKind.Exception: + cfg.remove_edge(pred, succ) + valid.append((pred, succ)) + + for pred in set(x for x, _ in valid): + BreakTemplate.try_match(cfg, pred) + cfg.iterate() + return \ No newline at end of file From c342cb0038ae75b6a80d3927ecc393ae1d7a9880 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 17 Jul 2025 10:25:48 -0500 Subject: [PATCH 24/55] Break statement --- pylingual/control_flow_reconstruction/cfg.py | 53 ++++++++++++++++ .../templates/Loop.py | 63 +++++++++++++++---- 2 files changed, 103 insertions(+), 13 deletions(-) diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index 1e6c21d..5151806 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -75,6 +75,59 @@ class CFG(DiGraph_CFT): self._create_dominator_tree() return nx.dfs_postorder_nodes(self, source=self.start, sort_neighbors=lambda nodes: sorted(nodes, key=lambda x: x.offset, reverse=True)) + def is_loop_header(self, node): + # Check all predecessors + for predecessor in self.predecessors(node): + # A back edge exists if the predecessor is dominated by this node + if self.dominates(node, predecessor): + return True + return False + + def dfs_labeled_edges_no_loop(self, source=None, depth_limit=None, *, sort_neighbors=None): + if source is None: + # edges for all components + nodes = self + else: + # edges for components with source + nodes = [source] + if depth_limit is None: + depth_limit = len(self) + + get_children = ( + self.neighbors + if sort_neighbors is None + else lambda n: iter(sort_neighbors(self.neighbors(n))) + ) + + visited = set() + for start in nodes: + if start in visited: + continue + yield start, start, "forward" + visited.add(start) + stack = [(start, get_children(start))] + depth_now = 1 + while stack: + parent, children = stack[-1] + for child in children: + if child in visited or self.is_loop_header(child): + yield parent, child, "nontree" + else: + yield parent, child, "forward" + visited.add(child) + if depth_now < depth_limit: + stack.append((child, iter(get_children(child)))) + depth_now += 1 + break + else: + yield parent, child, "reverse-depth_limit" + else: + stack.pop() + depth_now -= 1 + if stack: + yield stack[-1][0], parent, "reverse" + yield start, start, "reverse" + def apply_graphs(self): graphs = self.iteration_graphs.pop() if self.iteration_graphs: diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index 03d5d4e..e3d5e66 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -4,6 +4,7 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, N, + without_top_level_instructions, condense_mapping, defer_source_to, starting_instructions, @@ -67,12 +68,22 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): class BreakTemplate(ControlFlowTemplate): @classmethod def try_match(cls, cfg, node): + if isinstance(node, BreakTemplate): + return None return condense_mapping(cls, cfg, {'child': node}, 'child') def to_indented_source(self, source): return self.child.to_indented_source(source) + self.line('break') -@register_template(0, 4) +class ContinueTemplate(ControlFlowTemplate): + @classmethod + def try_match(cls, cfg, node): + return condense_mapping(cls, cfg, {'child': node}, 'child') + + def to_indented_source(self, source): + return self.child.to_indented_source(source) + self.line('continue') + +@register_template(0, 0) class FixLoop(ControlFlowTemplate): @classmethod def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: @@ -89,31 +100,57 @@ class FixLoop(ControlFlowTemplate): # a back-edge is an edge from any node that is dominated by this node back_edges = [] for predecessor in cfg.predecessors(node): + # A back edge exists if the predecessor is reachable from the node (node dominates predecessor) if cfg.dominates(node, predecessor): back_edges.append(predecessor) if cfg.has_edge(node, node): return None + if not back_edges: return None - nodes = [] - edges_to_remove = [] + # Get all nodes encompassed by the loop excluding source node and initial false jump + loopnode = None + for succ in cfg.successors(node): + if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.Fall: + loopnode = succ + break - for successor in list(cfg.successors(node)): - if cfg.in_degree(successor) != 1: - for predecessor in list(cfg.predecessors(successor)): - if predecessor != node: - nodes.append(predecessor) - edges_to_remove.append((predecessor, successor)) + dfs_edges = cfg.dfs_labeled_edges_no_loop(source=loopnode) + encompassed_nodes = [ + v for u, v, d in dfs_edges + if d == "forward" and v != node + ] + edges_to_remove = [] + + # Find the candidate end that break connect to + candidate_end = None + for succ in cfg.successors(node): + if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.FalseJump and cfg.out_degree(succ) <= 1: + candidate_end = succ + + # Candidate end is a buffer node + if cfg.in_degree(candidate_end) == 1: + for ss in cfg.successors(candidate_end): + + # If the successor has only one predecessor and one successor, it is a buffer node + if cfg.out_degree(ss) <= 1: + candidate_end = ss + + if encompassed_nodes is not None: + for succ in encompassed_nodes: + if cfg.get_edge_data(succ, candidate_end) != None: + edges_to_remove.append((succ, candidate_end)) valid = [] for pred, succ in edges_to_remove: - if cfg.get_edge_data(pred, succ).get("kind") != EdgeKind.Exception: - cfg.remove_edge(pred, succ) - valid.append((pred, succ)) - + if not (cfg.get_edge_data(pred, succ).get("kind") == EdgeKind.Exception or cfg.get_edge_data(pred, succ).get("kind") == EdgeKind.FalseJump): + if without_top_level_instructions("RAISE_VARARGS")(cfg, pred): + cfg.remove_edge(pred, succ) + valid.append((pred, succ)) + for pred in set(x for x, _ in valid): BreakTemplate.try_match(cfg, pred) cfg.iterate() From 70ca897103c115f3c0f3ceea4395a50842bc9ca4 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 17 Jul 2025 10:26:07 -0500 Subject: [PATCH 25/55] Slight fixes to 3.11 While loops --- .../templates/Loop.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index e3d5e66..f91c52c 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -46,6 +46,24 @@ class SelfLoop(ControlFlowTemplate): {loop_body} """ +@register_template(0, 2) +class TrueSelfLoop(ControlFlowTemplate): + template = T( + loop_body=~N("tail.", "loop_body"), + tail=N.tail()) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + }, + "loop_body" + ) + + @to_indented_source + def to_indented_source(): + """ + {loop_body} + """ @register_template(0, 3) class InlinedComprehensionTemplate(ControlFlowTemplate): From 14c02432ba39ea7f4e293b6bdc2b66574a5c7524 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:01:38 -0500 Subject: [PATCH 26/55] Documentation on failing test cases --- test/Loop.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/test/Loop.py b/test/Loop.py index 135e0a0..26e6a8b 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -16,12 +16,14 @@ def b1_for_over_tuples_nofallthru(): print("tuples") print("end") +# 3.6 No else template def c0_for_else(): for i in range(3): print("for body") else: print("for else") +# 3.6 No else template def c1_for_else_nofallthru(): for i in range(3): print("for body") @@ -29,14 +31,15 @@ def c1_for_else_nofallthru(): print("for else") print("end") -# Fails due to no break +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9 Naive break detection, an unexpected buffer block to end def d0_for_with_break(): for x in range(10): if x == 5: print("breaking") break -# Fails due to no break +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end def d1_for_with_break_nofallthru(): for x in range(10): if x == 5: @@ -135,6 +138,8 @@ def j1_for_with_empty_body_ellipsis_nofallthru(): ... print("end") +# 3.6 Naive break detection, no back edge +# 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k0_while_true_with_break(): x = 0 while True: @@ -143,6 +148,8 @@ def k0_while_true_with_break(): if x >= 1: break +# 3.6 Naive break detection, no back edge +# 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k1_while_true_with_break_nofallthru(): x = 0 while True: @@ -152,6 +159,8 @@ def k1_while_true_with_break_nofallthru(): break print("end") +# 3.6 No else template +# 3.11 No while loop detection, self false_jump edge def l0_while_with_else(): i = 0 while i < 3: @@ -160,6 +169,8 @@ def l0_while_with_else(): else: print("while else") +# 3.6 No else template +# 3.11 No while loop detection, self false_jump edge def l1_while_with_else_nofallthru(): i = 0 while i < 3: @@ -169,6 +180,7 @@ def l1_while_with_else_nofallthru(): print("while else") print("end") +# 3.11 No continue def m0_while_with_continue(): i = 0 while i < 5: @@ -178,6 +190,7 @@ def m0_while_with_continue(): continue print("after continue") +# 3.11 No continue def m1_while_with_continue_nofallthru(): i = 0 while i < 5: @@ -188,12 +201,14 @@ def m1_while_with_continue_nofallthru(): print("after continue") print("end") +# 3.6 Naive break detection, no back edge def n0_while_with_break(): i = 0 while True: print("break in while") break +# 3.6 Naive break detection, no back edge def n1_while_with_break_nofallthru(): i = 0 while True: @@ -201,6 +216,7 @@ def n1_while_with_break_nofallthru(): break print("end") +# 3.11 While template broke def o0_nested_while_loops(): i = 0 while i < 2: @@ -210,6 +226,7 @@ def o0_nested_while_loops(): j += 1 i += 1 +# 3.11 While template broke def o1_nested_while_loops_nofallthru(): i = 0 while i < 2: @@ -220,6 +237,8 @@ def o1_nested_while_loops_nofallthru(): i += 1 print("end") +# 3.6 While template broke (?) +# 3.9 Disconnected with MetaTemplate[end] (?) def p0_while_with_try_except(): while True: try: @@ -227,6 +246,8 @@ def p0_while_with_try_except(): except: print("except in while") +# 3.6 While template broke (?) +# 3.9 Disconnected with MetaTemplate[end] (?) def p1_while_with_try_except_nofallthru(): while True: try: @@ -235,34 +256,40 @@ def p1_while_with_try_except_nofallthru(): print("except in while") print("end") +# 3.6 While template broke (?) abandoning nodes def q0_while_with_with_statement(): while True: with a: print("inside while with") +# 3.6 While template broke (?) abandoning nodes def q1_while_with_with_statement_nofallthru(): while True: with a: print("inside while with") print("end") +# 3.6 While template broke def r0_for_inside_while(): while True: for x in [1, 2]: print("for in while") +# 3.6 While template broke def r1_for_inside_while_nofallthru(): while True: for x in [1, 2]: print("for in while") print("end") +# 3.6 While template broke def s0_while_inside_for(): for _ in range(1): while True: print("while in for") break +# 3.6 While template broke def s1_while_inside_for_nofallthru(): for _ in range(1): while True: @@ -270,10 +297,12 @@ def s1_while_inside_for_nofallthru(): break print("end") +# 3.6 While template broke def t0_while_with_empty_body_ellipsis(): while True: ... +# 3.6 While template broke def t1_while_with_empty_body_ellipsis_nofallthru(): while True: ... @@ -311,6 +340,8 @@ def v1_continue_in_nested_for_nofallthru(): print(f"Processing i={i}, j={j}") print("end") +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9/3.11 Naive break detection, an unexpected buffer block to end def w0_break_with_else(): for i in range(5): if i == 3: @@ -319,6 +350,8 @@ def w0_break_with_else(): else: print("This won't execute due to break") +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9/3.11 Naive break detection, an unexpected buffer block to end def w1_break_with_else_nofallthru(): for i in range(5): if i == 3: @@ -328,6 +361,7 @@ def w1_break_with_else_nofallthru(): print("This won't execute due to break") print("end") +# 3.6 No continue detection def x0_continue_with_else(): for i in range(3): if i == 1: @@ -336,6 +370,7 @@ def x0_continue_with_else(): else: print("Else clause still executes after continue") +# 3.6 No continue detection def x1_continue_with_else_nofallthru(): for i in range(3): if i == 1: @@ -345,6 +380,8 @@ def x1_continue_with_else_nofallthru(): print("Else clause still executes after continue") print("end") +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9/3.11 Naive break detection, break statement is further up def y0_break_in_try_except(): for i in range(5): try: @@ -354,6 +391,8 @@ def y0_break_in_try_except(): except: print("Exception occurred") +# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.9/3.11 Naive break detection, break statement is further up def y1_break_in_try_except_nofallthru(): for i in range(5): try: @@ -364,6 +403,7 @@ def y1_break_in_try_except_nofallthru(): print("Exception occurred") print("end") +# 3.6/3.9/3.11 No continue detection def z0_continue_in_try_except(): for i in range(5): try: @@ -373,6 +413,7 @@ def z0_continue_in_try_except(): except: print("Exception occurred") +# 3.6/3.9/3.11 No continue detection def z1_continue_in_try_except_nofallthru(): for i in range(5): try: From ea94f6d43c5e1b4e8100918ac1d0acbf493b302a Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:02:45 -0500 Subject: [PATCH 27/55] Fixed 3.6 Loops y0 y1 y2 test cases --- .../templates/Conditional.py | 5 +++-- test/Loop.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Conditional.py b/pylingual/control_flow_reconstruction/templates/Conditional.py index f163981..f0f06a8 100644 --- a/pylingual/control_flow_reconstruction/templates/Conditional.py +++ b/pylingual/control_flow_reconstruction/templates/Conditional.py @@ -1,13 +1,14 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import T, N, defer_source_to, run_is, has_no_lines, with_instructions, has_instval, starting_instructions, to_indented_source, make_try_match, without_top_level_instructions +from .Loop import BreakTemplate @register_template(1, 40) class IfElse(ControlFlowTemplate): template = T( if_header=~N("if_body", "else_body").with_cond(without_top_level_instructions("WITH_EXCEPT_START", "CHECK_EXC_MATCH", "FOR_ITER")), - if_body=~N("tail.").with_in_deg(1), - else_body=~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1), + if_body=N.tail().with_in_deg(1).of_type(BreakTemplate) | ~N("tail.").with_in_deg(1), + else_body=N.tail().with_in_deg(1).of_type(BreakTemplate) | ~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1), tail=N.tail(), ) diff --git a/test/Loop.py b/test/Loop.py index 26e6a8b..0340277 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -380,7 +380,6 @@ def x1_continue_with_else_nofallthru(): print("Else clause still executes after continue") print("end") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end # 3.9/3.11 Naive break detection, break statement is further up def y0_break_in_try_except(): for i in range(5): @@ -391,7 +390,6 @@ def y0_break_in_try_except(): except: print("Exception occurred") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end # 3.9/3.11 Naive break detection, break statement is further up def y1_break_in_try_except_nofallthru(): for i in range(5): @@ -403,6 +401,18 @@ def y1_break_in_try_except_nofallthru(): print("Exception occurred") print("end") +# 3.9/3.11 Naive break detection, break statement is further up +def y2_return_in_try_except_nofallthru(): + for i in range(5): + try: + if i == 3: + print(f"Value: {i}") + else: + break + except: + print("Exception occurred") + print("end") + # 3.6/3.9/3.11 No continue detection def z0_continue_in_try_except(): for i in range(5): From 96695c0eed93bed055c0891a86a21069d3170775 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 18:43:26 -0500 Subject: [PATCH 28/55] Disregard initial natural node from Break template --- pylingual/control_flow_reconstruction/templates/Loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index f91c52c..ea852fb 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -140,7 +140,7 @@ class FixLoop(ControlFlowTemplate): encompassed_nodes = [ v for u, v, d in dfs_edges if d == "forward" and v != node - ] + ][1:] edges_to_remove = [] # Find the candidate end that break connect to From 9ce0d4443f09a65fed9ab87d4e96886fa8fa1c3a Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 21:01:50 -0500 Subject: [PATCH 29/55] Updated conditions for encompassed nodes --- .../control_flow_reconstruction/templates/Loop.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index ea852fb..f2d26f5 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -140,7 +140,7 @@ class FixLoop(ControlFlowTemplate): encompassed_nodes = [ v for u, v, d in dfs_edges if d == "forward" and v != node - ][1:] + ] edges_to_remove = [] # Find the candidate end that break connect to @@ -160,7 +160,13 @@ class FixLoop(ControlFlowTemplate): if encompassed_nodes is not None: for succ in encompassed_nodes: if cfg.get_edge_data(succ, candidate_end) != None: - edges_to_remove.append((succ, candidate_end)) + for parent in cfg.predecessors(succ): + if parent in encompassed_nodes: + continue + else: + break + else: + edges_to_remove.append((succ, candidate_end)) valid = [] for pred, succ in edges_to_remove: From 6f5c5208846e9916741b331ddd931b8bf610c7fb Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 21:14:13 -0500 Subject: [PATCH 30/55] Update Loop.py errors --- test/Loop.py | 56 +++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/test/Loop.py b/test/Loop.py index 0340277..8c730bf 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -16,14 +16,14 @@ def b1_for_over_tuples_nofallthru(): print("tuples") print("end") -# 3.6 No else template +# 3.6/3.7 No else template def c0_for_else(): for i in range(3): print("for body") else: print("for else") -# 3.6 No else template +# 3.6/3.7 No else template def c1_for_else_nofallthru(): for i in range(3): print("for body") @@ -31,7 +31,7 @@ def c1_for_else_nofallthru(): print("for else") print("end") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end # 3.9 Naive break detection, an unexpected buffer block to end def d0_for_with_break(): for x in range(10): @@ -39,7 +39,7 @@ def d0_for_with_break(): print("breaking") break -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end +# 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end def d1_for_with_break_nofallthru(): for x in range(10): if x == 5: @@ -138,7 +138,7 @@ def j1_for_with_empty_body_ellipsis_nofallthru(): ... print("end") -# 3.6 Naive break detection, no back edge +# 3.6/3.7 Naive break detection, no back edge # 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k0_while_true_with_break(): x = 0 @@ -148,7 +148,7 @@ def k0_while_true_with_break(): if x >= 1: break -# 3.6 Naive break detection, no back edge +# 3.6/3.7 Naive break detection, no back edge # 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k1_while_true_with_break_nofallthru(): x = 0 @@ -159,7 +159,7 @@ def k1_while_true_with_break_nofallthru(): break print("end") -# 3.6 No else template +# 3.6/3.7 No else template # 3.11 No while loop detection, self false_jump edge def l0_while_with_else(): i = 0 @@ -169,7 +169,7 @@ def l0_while_with_else(): else: print("while else") -# 3.6 No else template +# 3.6/3.7 No else template # 3.11 No while loop detection, self false_jump edge def l1_while_with_else_nofallthru(): i = 0 @@ -201,14 +201,14 @@ def m1_while_with_continue_nofallthru(): print("after continue") print("end") -# 3.6 Naive break detection, no back edge +# 3.6/3.7 Naive break detection, no back edge def n0_while_with_break(): i = 0 while True: print("break in while") break -# 3.6 Naive break detection, no back edge +# 3.6/3.7 Naive break detection, no back edge def n1_while_with_break_nofallthru(): i = 0 while True: @@ -237,7 +237,7 @@ def o1_nested_while_loops_nofallthru(): i += 1 print("end") -# 3.6 While template broke (?) +# 3.6/3.7 While template broke (?) # 3.9 Disconnected with MetaTemplate[end] (?) def p0_while_with_try_except(): while True: @@ -246,7 +246,7 @@ def p0_while_with_try_except(): except: print("except in while") -# 3.6 While template broke (?) +# 3.6/3.7 While template broke (?) # 3.9 Disconnected with MetaTemplate[end] (?) def p1_while_with_try_except_nofallthru(): while True: @@ -256,40 +256,40 @@ def p1_while_with_try_except_nofallthru(): print("except in while") print("end") -# 3.6 While template broke (?) abandoning nodes +# 3.6/3.7 While template broke (?) abandoning nodes def q0_while_with_with_statement(): while True: with a: print("inside while with") -# 3.6 While template broke (?) abandoning nodes +# 3.6/3.7 While template broke (?) abandoning nodes def q1_while_with_with_statement_nofallthru(): while True: with a: print("inside while with") print("end") -# 3.6 While template broke +# 3.6/3.7 While template broke def r0_for_inside_while(): while True: for x in [1, 2]: print("for in while") -# 3.6 While template broke +# 3.6/3.7 While template broke def r1_for_inside_while_nofallthru(): while True: for x in [1, 2]: print("for in while") print("end") -# 3.6 While template broke +# 3.6/3.7 While template broke def s0_while_inside_for(): for _ in range(1): while True: print("while in for") break -# 3.6 While template broke +# 3.6/3.7 While template broke def s1_while_inside_for_nofallthru(): for _ in range(1): while True: @@ -297,12 +297,12 @@ def s1_while_inside_for_nofallthru(): break print("end") -# 3.6 While template broke +# 3.6/3.7 While template broke def t0_while_with_empty_body_ellipsis(): while True: ... -# 3.6 While template broke +# 3.6/3.7 While template broke def t1_while_with_empty_body_ellipsis_nofallthru(): while True: ... @@ -340,8 +340,7 @@ def v1_continue_in_nested_for_nofallthru(): print(f"Processing i={i}, j={j}") print("end") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end -# 3.9/3.11 Naive break detection, an unexpected buffer block to end +# 3.13 if statement putting code in the else block def w0_break_with_else(): for i in range(5): if i == 3: @@ -350,8 +349,7 @@ def w0_break_with_else(): else: print("This won't execute due to break") -# 3.6 Naive break detection, an unexpected buffer POP_BLOCK to end -# 3.9/3.11 Naive break detection, an unexpected buffer block to end +# 3.13 if statement putting code in the else block def w1_break_with_else_nofallthru(): for i in range(5): if i == 3: @@ -361,7 +359,7 @@ def w1_break_with_else_nofallthru(): print("This won't execute due to break") print("end") -# 3.6 No continue detection +# 3.6/3.7 No continue detection def x0_continue_with_else(): for i in range(3): if i == 1: @@ -370,7 +368,7 @@ def x0_continue_with_else(): else: print("Else clause still executes after continue") -# 3.6 No continue detection +# 3.6/3.7 No continue detection def x1_continue_with_else_nofallthru(): for i in range(3): if i == 1: @@ -390,7 +388,7 @@ def y0_break_in_try_except(): except: print("Exception occurred") -# 3.9/3.11 Naive break detection, break statement is further up +# 3.9 Naive break detection, break statement is further up def y1_break_in_try_except_nofallthru(): for i in range(5): try: @@ -401,7 +399,7 @@ def y1_break_in_try_except_nofallthru(): print("Exception occurred") print("end") -# 3.9/3.11 Naive break detection, break statement is further up +# 3.9 Naive break detection, break statement is further up def y2_return_in_try_except_nofallthru(): for i in range(5): try: @@ -413,7 +411,7 @@ def y2_return_in_try_except_nofallthru(): print("Exception occurred") print("end") -# 3.6/3.9/3.11 No continue detection +# 3.6/3.9 No continue detection def z0_continue_in_try_except(): for i in range(5): try: From 5ca5f36b46dcd9f5102710e74d069d4634d81e34 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Sun, 20 Jul 2025 09:20:44 -0500 Subject: [PATCH 31/55] Refining Breaks/Continues --- pylingual/control_flow_reconstruction/cfg.py | 2 +- .../templates/Loop.py | 55 +++++++++---------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index 5151806..41091f5 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -110,7 +110,7 @@ class CFG(DiGraph_CFT): while stack: parent, children = stack[-1] for child in children: - if child in visited or self.is_loop_header(child): + if child in visited or self.is_loop_header(child) or not all(p in visited for p in self.predecessors(child)): yield parent, child, "nontree" else: yield parent, child, "forward" diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index f2d26f5..0745c94 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -4,7 +4,9 @@ from ..cft import ControlFlowTemplate, EdgeKind, register_template from ..utils import ( T, N, - without_top_level_instructions, + with_instructions, + exact_instructions, + has_no_lines, condense_mapping, defer_source_to, starting_instructions, @@ -86,7 +88,7 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): class BreakTemplate(ControlFlowTemplate): @classmethod def try_match(cls, cfg, node): - if isinstance(node, BreakTemplate): + if isinstance(node, BreakTemplate) or has_no_lines(cfg, node) or with_instructions("RAISE_VARARGS")(cfg, node): return None return condense_mapping(cls, cfg, {'child': node}, 'child') @@ -96,7 +98,12 @@ class BreakTemplate(ControlFlowTemplate): class ContinueTemplate(ControlFlowTemplate): @classmethod def try_match(cls, cfg, node): - return condense_mapping(cls, cfg, {'child': node}, 'child') + if isinstance(node, ContinueTemplate) or has_no_lines(cfg, node): + return None + instruction = node.get_instructions()[-1].opname + if instruction in {"JUMP_ABSOLUTE", "JUMP_BACKWARD", "CONTINUE_LOOP"} and (node.get_instructions()[-1].starts_line is not None or node.get_instructions()[-2].starts_line is not None): + return condense_mapping(cls, cfg, {'child': node}, 'child') + return None def to_indented_source(self, source): return self.child.to_indented_source(source) + self.line('continue') @@ -122,9 +129,6 @@ class FixLoop(ControlFlowTemplate): # A back edge exists if the predecessor is reachable from the node (node dominates predecessor) if cfg.dominates(node, predecessor): back_edges.append(predecessor) - if cfg.has_edge(node, node): - return None - if not back_edges: return None @@ -137,45 +141,38 @@ class FixLoop(ControlFlowTemplate): break dfs_edges = cfg.dfs_labeled_edges_no_loop(source=loopnode) - encompassed_nodes = [ - v for u, v, d in dfs_edges - if d == "forward" and v != node - ] + encompassed_nodes = [v for u, v, d in dfs_edges if d == "forward"] + edges_to_remove = [] - # Find the candidate end that break connect to + # Find the candidate end that break connects to candidate_end = None for succ in cfg.successors(node): if cfg.get_edge_data(node, succ).get("kind") == EdgeKind.FalseJump and cfg.out_degree(succ) <= 1: candidate_end = succ # Candidate end is a buffer node - if cfg.in_degree(candidate_end) == 1: + if cfg.in_degree(candidate_end) == 1 and any(exact_instructions(*op)(cfg, candidate_end) for op in [ + ("POP_BLOCK",), ("END_FOR",), ("END_FOR", "POP_TOP"), ("LOAD_CONST", "RETURN_VALUE")]): for ss in cfg.successors(candidate_end): - - # If the successor has only one predecessor and one successor, it is a buffer node if cfg.out_degree(ss) <= 1: candidate_end = ss - + break + if encompassed_nodes is not None: for succ in encompassed_nodes: if cfg.get_edge_data(succ, candidate_end) != None: - for parent in cfg.predecessors(succ): - if parent in encompassed_nodes: - continue - else: - break - else: - edges_to_remove.append((succ, candidate_end)) + edges_to_remove.append((succ, candidate_end)) - valid = [] for pred, succ in edges_to_remove: - if not (cfg.get_edge_data(pred, succ).get("kind") == EdgeKind.Exception or cfg.get_edge_data(pred, succ).get("kind") == EdgeKind.FalseJump): - if without_top_level_instructions("RAISE_VARARGS")(cfg, pred): - cfg.remove_edge(pred, succ) - valid.append((pred, succ)) + break_node = BreakTemplate.try_match(cfg, pred) + if break_node is not None: + cfg.remove_edge(break_node, succ) + + for candidate in back_edges: + cont_node = ContinueTemplate.try_match(cfg, candidate) + if cont_node is not None and cfg.in_degree(node) > 2: + cfg.remove_edge(cont_node, node) - for pred in set(x for x, _ in valid): - BreakTemplate.try_match(cfg, pred) cfg.iterate() return \ No newline at end of file From da349c2fd7bebe08d8722a4eca63c0fdb76f6383 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Sun, 20 Jul 2025 17:48:53 -0500 Subject: [PATCH 32/55] Duplicate JumpTemplate oops --- .../templates/Block.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 2734ec4..93cead8 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -54,27 +54,6 @@ class RemoveUnreachable(ControlFlowTemplate): return node -@register_template(0, 0) -class JumpTemplate(ControlFlowTemplate): - template = T( - body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), - jump=N("tail", "block.").with_in_deg(1).with_cond(exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), exact_instructions("POP_JUMP_IF_TRUE")), - block=~N.tail(), - tail=N.tail(), - ) - - try_match = make_try_match( - { - EdgeKind.Fall: "tail", - EdgeKind.TrueJump: "block", - }, - "body", - "jump", - ) - - to_indented_source = defer_source_to("body") - - @register_template(0, 0) class JumpTemplate(ControlFlowTemplate): template = T( From d32dcaadfcf57f51f7ec9b5ddc829ce2d9f51d24 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:58:13 -0500 Subject: [PATCH 33/55] Underlying issues with TryExceptElse and TryExcept --- pylingual/control_flow_reconstruction/templates/Exception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Exception.py b/pylingual/control_flow_reconstruction/templates/Exception.py index 260a7ff..854c49e 100644 --- a/pylingual/control_flow_reconstruction/templates/Exception.py +++ b/pylingual/control_flow_reconstruction/templates/Exception.py @@ -39,7 +39,7 @@ class Except3_11(ControlFlowTemplate): return x -@register_template(0, 1, *versions_from(3, 11)) +@register_template(0, 0, *versions_from(3, 11)) class Try3_11(ControlFlowTemplate): template = T( try_header=N("try_body"), @@ -310,7 +310,7 @@ class Except3_9(ControlFlowTemplate): return node -@register_template(0, 1, (3, 9), (3, 10)) +@register_template(0, 0, (3, 9), (3, 10)) class Try3_9(ControlFlowTemplate): template = T( try_header=~N("try_body"), From 0a360902032850e21b0f8d2ae1f167825c8c91a0 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:58:38 -0500 Subject: [PATCH 34/55] 3.12/3.13 IfElse in a loop template --- .../templates/Conditional.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pylingual/control_flow_reconstruction/templates/Conditional.py b/pylingual/control_flow_reconstruction/templates/Conditional.py index f0f06a8..41989cc 100644 --- a/pylingual/control_flow_reconstruction/templates/Conditional.py +++ b/pylingual/control_flow_reconstruction/templates/Conditional.py @@ -24,6 +24,28 @@ class IfElse(ControlFlowTemplate): """ +@register_template(1, 39, (3, 12), (3, 13)) +class IfElseLoop(ControlFlowTemplate): + template = T( + if_header=~N("else_body", "if_body").with_cond(without_top_level_instructions("WITH_EXCEPT_START", "CHECK_EXC_MATCH", "FOR_ITER")), + if_body=~N("tail.").with_in_deg(1), + else_body=~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1).with_cond(has_no_lines), + for_iter=N.tail().with_cond(with_instructions("FOR_ITER")), + tail=N.tail(), + ) + + try_match = make_try_match({EdgeKind.Fall: "tail"}, "if_header", "if_body", "else_body") + + @to_indented_source + def to_indented_source(): + """ + {if_header} + {if_body} + {else_body?else:} + {else_body} + """ + + @register_template(1, 41) @register_template(2, 41) class IfThen(ControlFlowTemplate): From 10df2f4b6ddcb583009fa806be5dcef00539c695 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 16:57:08 -0500 Subject: [PATCH 35/55] RemoveUnreachable template Co-Authored-By: caandt --- .../templates/Block.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 83397a4..1294aee 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -4,6 +4,8 @@ from typing import TYPE_CHECKING, override from itertools import chain from pylingual.editable_bytecode import Inst +import networkx as nx + from ..cft import ControlFlowTemplate, EdgeKind, SourceContext, SourceLine, register_template, EdgeCategory, out_edge_dict, MetaTemplate, indent_str from ..utils import E, N, T, defer_source_to, remove_nodes @@ -37,6 +39,21 @@ class EndTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("body") +@register_template(3, 0) +class RemoveUnreachable(ControlFlowTemplate): + @override + @classmethod + def try_match(cls, cfg, node) -> ControlFlowTemplate | None: + if node is not cfg.start: + return None + + valid = list(nx.dfs_preorder_nodes(cfg, source=cfg.start)) + invalid = [n for n in cfg.nodes if n not in valid] + if invalid: + cfg.remove_nodes_from(invalid) + return node + + @register_template(0, 20) @register_template(2, 20) class BlockTemplate(ControlFlowTemplate): From 288322cdcaa7f4636a0e70f5bea368c50f0ec99c Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 21:07:17 -0500 Subject: [PATCH 36/55] JumpTemplate to allow other templates to match in versions 3.12+ --- .../templates/Block.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 1294aee..86c7321 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -7,7 +7,7 @@ from pylingual.editable_bytecode import Inst import networkx as nx from ..cft import ControlFlowTemplate, EdgeKind, SourceContext, SourceLine, register_template, EdgeCategory, out_edge_dict, MetaTemplate, indent_str -from ..utils import E, N, T, defer_source_to, remove_nodes +from ..utils import E, N, T, defer_source_to, remove_nodes, without_instructions, exact_instructions, make_try_match if TYPE_CHECKING: from pylingual.control_flow_reconstruction.cfg import CFG @@ -54,6 +54,27 @@ class RemoveUnreachable(ControlFlowTemplate): return node +@register_template(0, 0) +class JumpTemplate(ControlFlowTemplate): + template = T( + body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), + jump=N("tail", "block.").with_in_deg(1).with_cond(exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), exact_instructions("POP_JUMP_IF_TRUE")), + block=~N.tail(), + tail=N.tail(), + ) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + EdgeKind.TrueJump: "block", + }, + "body", + "jump", + ) + + to_indented_source = defer_source_to("body") + + @register_template(0, 20) @register_template(2, 20) class BlockTemplate(ControlFlowTemplate): From 01637976ab818980424bd37dd7fc95c671117a5f Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Fri, 18 Jul 2025 21:12:43 -0500 Subject: [PATCH 37/55] Fixing With statement test cases --- .../templates/Block.py | 21 +++++++++++++++++++ test/With.py | 6 ------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 86c7321..2734ec4 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -75,6 +75,27 @@ class JumpTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("body") +@register_template(0, 0) +class JumpTemplate(ControlFlowTemplate): + template = T( + body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), + jump=N("tail", "block?").with_in_deg(1).with_cond(exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), exact_instructions("POP_JUMP_IF_TRUE")), + block=~N.tail(), + tail=N.tail(), + ) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + EdgeKind.TrueJump: "block", + }, + "body", + "jump", + ) + + to_indented_source = defer_source_to("body") + + @register_template(0, 20) @register_template(2, 20) class BlockTemplate(ControlFlowTemplate): diff --git a/test/With.py b/test/With.py index 491f463..d4b4fc0 100644 --- a/test/With.py +++ b/test/With.py @@ -7,7 +7,6 @@ def a1_bare_with_fallthrough(): print(1) print(2) -# Fails in 3.10, duplicate blocks explained further in issue 32 def b0_multi_with(): with a, b: print(1) @@ -26,7 +25,6 @@ def c1_with_as_fallthrough(): print(1) print(2) -# Fails in 3.10, same issue as b def d0_multi_with_as(): with a, b as c: print(1) @@ -36,7 +34,6 @@ def d1_multi_with_as_fallthrough(): print(1) print(2) -# Fails in 3.10, same issue as b def e0_with_multi_as(): with a as b, c: print(1) @@ -46,7 +43,6 @@ def e1_with_multi_as_fallthrough(): print(1) print(2) -# Fails in 3.10, same issue as b def f0_multi_with_multi_as(): with a as b, c as d: print(1) @@ -56,7 +52,6 @@ def f1_multi_with_multi_as_fallthrough(): print(1) print(2) -# Fails in 3.10, same issue as b def g0_multi_with_multi_as_alt(): with a, b as c, d: print(1) @@ -66,7 +61,6 @@ def g1_multi_with_multi_as_fallthrough_alt(): print(1) print(2) -# Fails in 3.13, unexpected JUMP_BACKWARD_NO_INTERRUPT messes up the template def h0_try_with_except(): try: with a: From 7a70fa35f7a9f69259b1a8c2a2d85e9e5aa758c5 Mon Sep 17 00:00:00 2001 From: Xinlong Hu Date: Sun, 20 Jul 2025 17:48:53 -0500 Subject: [PATCH 38/55] Duplicate JumpTemplate oops --- .../templates/Block.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 2734ec4..93cead8 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -54,27 +54,6 @@ class RemoveUnreachable(ControlFlowTemplate): return node -@register_template(0, 0) -class JumpTemplate(ControlFlowTemplate): - template = T( - body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), - jump=N("tail", "block.").with_in_deg(1).with_cond(exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), exact_instructions("POP_JUMP_IF_TRUE")), - block=~N.tail(), - tail=N.tail(), - ) - - try_match = make_try_match( - { - EdgeKind.Fall: "tail", - EdgeKind.TrueJump: "block", - }, - "body", - "jump", - ) - - to_indented_source = defer_source_to("body") - - @register_template(0, 0) class JumpTemplate(ControlFlowTemplate): template = T( From bae8492b09c79b225ae3b784fbd60bd521316795 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:58:13 -0500 Subject: [PATCH 39/55] Underlying issues with TryExceptElse and TryExcept --- pylingual/control_flow_reconstruction/templates/Exception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Exception.py b/pylingual/control_flow_reconstruction/templates/Exception.py index 260a7ff..854c49e 100644 --- a/pylingual/control_flow_reconstruction/templates/Exception.py +++ b/pylingual/control_flow_reconstruction/templates/Exception.py @@ -39,7 +39,7 @@ class Except3_11(ControlFlowTemplate): return x -@register_template(0, 1, *versions_from(3, 11)) +@register_template(0, 0, *versions_from(3, 11)) class Try3_11(ControlFlowTemplate): template = T( try_header=N("try_body"), @@ -310,7 +310,7 @@ class Except3_9(ControlFlowTemplate): return node -@register_template(0, 1, (3, 9), (3, 10)) +@register_template(0, 0, (3, 9), (3, 10)) class Try3_9(ControlFlowTemplate): template = T( try_header=~N("try_body"), From 10a57851fa8a616ad6cd55a7f81df644c52594a4 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:58:38 -0500 Subject: [PATCH 40/55] 3.12/3.13 IfElse in a loop template --- .../templates/Conditional.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pylingual/control_flow_reconstruction/templates/Conditional.py b/pylingual/control_flow_reconstruction/templates/Conditional.py index f0f06a8..41989cc 100644 --- a/pylingual/control_flow_reconstruction/templates/Conditional.py +++ b/pylingual/control_flow_reconstruction/templates/Conditional.py @@ -24,6 +24,28 @@ class IfElse(ControlFlowTemplate): """ +@register_template(1, 39, (3, 12), (3, 13)) +class IfElseLoop(ControlFlowTemplate): + template = T( + if_header=~N("else_body", "if_body").with_cond(without_top_level_instructions("WITH_EXCEPT_START", "CHECK_EXC_MATCH", "FOR_ITER")), + if_body=~N("tail.").with_in_deg(1), + else_body=~N("tail.").with_cond(without_top_level_instructions("RERAISE", "END_FINALLY")).with_in_deg(1).with_cond(has_no_lines), + for_iter=N.tail().with_cond(with_instructions("FOR_ITER")), + tail=N.tail(), + ) + + try_match = make_try_match({EdgeKind.Fall: "tail"}, "if_header", "if_body", "else_body") + + @to_indented_source + def to_indented_source(): + """ + {if_header} + {if_body} + {else_body?else:} + {else_body} + """ + + @register_template(1, 41) @register_template(2, 41) class IfThen(ControlFlowTemplate): From e0cfc70147a29ee2271f9eb61b35bad7c4d64606 Mon Sep 17 00:00:00 2001 From: caandt Date: Mon, 21 Jul 2025 16:47:37 -0500 Subject: [PATCH 41/55] format --- pylingual/control_flow_reconstruction/cfg.py | 6 +- .../templates/Loop.py | 38 ++++++----- test/Exception.py | 65 +++++++++++++++++-- test/Loop.py | 60 ++++++++++++++++- test/With.py | 32 ++++++++- 5 files changed, 169 insertions(+), 32 deletions(-) diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index 41091f5..a7b368e 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -93,11 +93,7 @@ class CFG(DiGraph_CFT): if depth_limit is None: depth_limit = len(self) - get_children = ( - self.neighbors - if sort_neighbors is None - else lambda n: iter(sort_neighbors(self.neighbors(n))) - ) + get_children = self.neighbors if sort_neighbors is None else lambda n: iter(sort_neighbors(self.neighbors(n))) visited = set() for start in nodes: diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index 0745c94..817aa7a 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -11,12 +11,13 @@ from ..utils import ( defer_source_to, starting_instructions, to_indented_source, - make_try_match, + make_try_match, ) if TYPE_CHECKING: from pylingual.control_flow_reconstruction.cfg import CFG + @register_template(0, 1) class ForLoop(ControlFlowTemplate): template = T( @@ -48,17 +49,16 @@ class SelfLoop(ControlFlowTemplate): {loop_body} """ + @register_template(0, 2) class TrueSelfLoop(ControlFlowTemplate): - template = T( - loop_body=~N("tail.", "loop_body"), - tail=N.tail()) + template = T(loop_body=~N("tail.", "loop_body"), tail=N.tail()) try_match = make_try_match( { EdgeKind.Fall: "tail", - }, - "loop_body" + }, + "loop_body", ) @to_indented_source @@ -67,6 +67,7 @@ class TrueSelfLoop(ControlFlowTemplate): {loop_body} """ + @register_template(0, 3) class InlinedComprehensionTemplate(ControlFlowTemplate): template = T( @@ -85,15 +86,17 @@ class InlinedComprehensionTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("comp") + class BreakTemplate(ControlFlowTemplate): @classmethod def try_match(cls, cfg, node): if isinstance(node, BreakTemplate) or has_no_lines(cfg, node) or with_instructions("RAISE_VARARGS")(cfg, node): return None - return condense_mapping(cls, cfg, {'child': node}, 'child') + return condense_mapping(cls, cfg, {"child": node}, "child") def to_indented_source(self, source): - return self.child.to_indented_source(source) + self.line('break') + return self.child.to_indented_source(source) + self.line("break") + class ContinueTemplate(ControlFlowTemplate): @classmethod @@ -102,11 +105,12 @@ class ContinueTemplate(ControlFlowTemplate): return None instruction = node.get_instructions()[-1].opname if instruction in {"JUMP_ABSOLUTE", "JUMP_BACKWARD", "CONTINUE_LOOP"} and (node.get_instructions()[-1].starts_line is not None or node.get_instructions()[-2].starts_line is not None): - return condense_mapping(cls, cfg, {'child': node}, 'child') + return condense_mapping(cls, cfg, {"child": node}, "child") return None def to_indented_source(self, source): - return self.child.to_indented_source(source) + self.line('continue') + return self.child.to_indented_source(source) + self.line("continue") + @register_template(0, 0) class FixLoop(ControlFlowTemplate): @@ -125,14 +129,13 @@ class FixLoop(ControlFlowTemplate): # a back-edge is an edge from any node that is dominated by this node back_edges = [] for predecessor in cfg.predecessors(node): - # A back edge exists if the predecessor is reachable from the node (node dominates predecessor) if cfg.dominates(node, predecessor): back_edges.append(predecessor) if not back_edges: return None - + # Get all nodes encompassed by the loop excluding source node and initial false jump loopnode = None for succ in cfg.successors(node): @@ -142,9 +145,9 @@ class FixLoop(ControlFlowTemplate): dfs_edges = cfg.dfs_labeled_edges_no_loop(source=loopnode) encompassed_nodes = [v for u, v, d in dfs_edges if d == "forward"] - + edges_to_remove = [] - + # Find the candidate end that break connects to candidate_end = None for succ in cfg.successors(node): @@ -152,13 +155,12 @@ class FixLoop(ControlFlowTemplate): candidate_end = succ # Candidate end is a buffer node - if cfg.in_degree(candidate_end) == 1 and any(exact_instructions(*op)(cfg, candidate_end) for op in [ - ("POP_BLOCK",), ("END_FOR",), ("END_FOR", "POP_TOP"), ("LOAD_CONST", "RETURN_VALUE")]): + if cfg.in_degree(candidate_end) == 1 and any(exact_instructions(*op)(cfg, candidate_end) for op in [("POP_BLOCK",), ("END_FOR",), ("END_FOR", "POP_TOP"), ("LOAD_CONST", "RETURN_VALUE")]): for ss in cfg.successors(candidate_end): if cfg.out_degree(ss) <= 1: candidate_end = ss break - + if encompassed_nodes is not None: for succ in encompassed_nodes: if cfg.get_edge_data(succ, candidate_end) != None: @@ -175,4 +177,4 @@ class FixLoop(ControlFlowTemplate): cfg.remove_edge(cont_node, node) cfg.iterate() - return \ No newline at end of file + return diff --git a/test/Exception.py b/test/Exception.py index 7a88433..24224b1 100644 --- a/test/Exception.py +++ b/test/Exception.py @@ -4,6 +4,7 @@ def a0_bare_try_except(): except: print(2) + def a1_bare_try_except_fallthrough(): try: print(1) @@ -11,6 +12,7 @@ def a1_bare_try_except_fallthrough(): print(2) print(3) + # 3.11/3.12/3.13 Duplicate blocks causing blocks to not match def b0_nested_try_except(): try: @@ -22,6 +24,7 @@ def b0_nested_try_except(): except: print(4) + def b1_nested_try_except_fallthrough(): try: print(1) @@ -33,7 +36,8 @@ def b1_nested_try_except_fallthrough(): print(4) print(5) -# 3.13 Duplicate blocks + +# 3.13 Duplicate blocks def b2_nested_try_except_early_fallthrough(): try: print(1) @@ -45,6 +49,7 @@ def b2_nested_try_except_early_fallthrough(): print(4) print(5) + def b3_nested_try_except_double_fallthrough(): try: print(1) @@ -57,6 +62,7 @@ def b3_nested_try_except_double_fallthrough(): print(5) print(6) + # 3.11/3.12/3.13 Duplicate blocks causing blocks to not match def c0_multi_except_nested(): try: @@ -74,6 +80,7 @@ def c0_multi_except_nested(): except: print(7) + # 3.11/3.12/3.13 Duplicate blocks causing blocks to not match def c1_multi_except_nested_fallthrough(): try: @@ -92,6 +99,7 @@ def c1_multi_except_nested_fallthrough(): except: print(8) + def c2_multi_except_nested_fallthrough2(): try: print(1) @@ -110,7 +118,8 @@ def c2_multi_except_nested_fallthrough2(): print(8) print(9) -# 3.13 Duplicate blocks + +# 3.13 Duplicate blocks def c3_multi_except_nested_early_fallthrough(): try: print(1) @@ -128,6 +137,7 @@ def c3_multi_except_nested_early_fallthrough(): print(7) print(8) + def c4_multi_except_nested_all_fallthrough(): try: print(1) @@ -147,6 +157,7 @@ def c4_multi_except_nested_all_fallthrough(): print(9) print(10) + # 3.10/3.11/3.12/3.13 Duplicate blocks causing templates to not match # Discussed in issue 41 def d0_named_except_nested(): @@ -159,6 +170,7 @@ def d0_named_except_nested(): except: print(4) + def d1_named_except_nested_fallthrough(): try: print(1) @@ -170,7 +182,8 @@ def d1_named_except_nested_fallthrough(): print(4) print(5) -# 3.13 Duplicate blocks + +# 3.13 Duplicate blocks def d2_named_except_nested_early_fallthrough(): try: print(1) @@ -182,6 +195,7 @@ def d2_named_except_nested_early_fallthrough(): print(4) print(5) + def d3_named_except_nested_double_fallthrough(): try: print(1) @@ -194,6 +208,7 @@ def d3_named_except_nested_double_fallthrough(): print(5) print(6) + def e0_try_except_else(): try: print(1) @@ -203,6 +218,7 @@ def e0_try_except_else(): print(3) print(4) + def f0_try_except_else_finally(): try: print(1) @@ -214,6 +230,7 @@ def f0_try_except_else_finally(): print(4) print(5) + def g0_multi_except_with_else(): try: print(1) @@ -225,6 +242,7 @@ def g0_multi_except_with_else(): print(4) print(5) + def h0_multi_except_fallback_with_else(): try: print(1) @@ -238,6 +256,7 @@ def h0_multi_except_fallback_with_else(): print(5) print(6) + def i0_mixed_named_unnamed_except_with_else(): try: print(1) @@ -251,6 +270,7 @@ def i0_mixed_named_unnamed_except_with_else(): print(5) print(6) + def j0_named_except_with_else(): try: print(1) @@ -262,6 +282,7 @@ def j0_named_except_with_else(): print(4) print(5) + def k0_try_except_finally(): try: print(1) @@ -271,6 +292,7 @@ def k0_try_except_finally(): print(3) print(4) + def l0_specific_except_finally(): try: print(1) @@ -280,6 +302,7 @@ def l0_specific_except_finally(): print(3) print(4) + def m0_multi_except(): try: print(1) @@ -291,6 +314,7 @@ def m0_multi_except(): print(4) print(5) + def n0_multi_except_with_fallback(): try: print(1) @@ -302,6 +326,7 @@ def n0_multi_except_with_fallback(): print(4) print(5) + def o0_multi_except_fallback_finally(): try: print(1) @@ -313,6 +338,7 @@ def o0_multi_except_fallback_finally(): print(4) print(5) + def p0_multi_named_except(): try: print(1) @@ -324,6 +350,7 @@ def p0_multi_named_except(): print(4) print(5) + def q0_mixed_named_unnamed_except(): try: print(1) @@ -335,6 +362,7 @@ def q0_mixed_named_unnamed_except(): print(4) print(5) + def r0_mixed_named_unnamed_except_finally(): try: print(1) @@ -348,6 +376,7 @@ def r0_mixed_named_unnamed_except_finally(): print(5) print(6) + def s0_named_except_fallback(): try: print(1) @@ -357,6 +386,7 @@ def s0_named_except_fallback(): print(3) print(4) + def t0_named_except_fallback_finally(): try: print(1) @@ -368,6 +398,7 @@ def t0_named_except_fallback_finally(): print(4) print(5) + def u0_multi_named_except_finally(): try: print(1) @@ -379,6 +410,7 @@ def u0_multi_named_except_finally(): print(4) print(5) + def v0_multi_except_finally(): try: print(1) @@ -390,6 +422,7 @@ def v0_multi_except_finally(): print(4) print(5) + def w0_try_except_raise(): try: print(1) @@ -397,6 +430,7 @@ def w0_try_except_raise(): print(2) raise Exc + def x0_multi_except_raise(): try: print(1) @@ -407,6 +441,7 @@ def x0_multi_except_raise(): print(3) raise Exc + def y0_named_except_raise(): try: print(1) @@ -414,6 +449,7 @@ def y0_named_except_raise(): print(2) raise Exc + # 3.11 Try return getting left outside of TryExcept def z0_try_except_return(): try: @@ -422,6 +458,7 @@ def z0_try_except_return(): except: print(2) + # 3.11 Try return getting left outside of TryExcept def z1_try_except_return_both(): try: @@ -431,6 +468,7 @@ def z1_try_except_return_both(): print(2) return 3 + # 3.11 Try return getting left outside of TryExcept def aa0_multi_except_return(): try: @@ -441,7 +479,8 @@ def aa0_multi_except_return(): except b: print(3) -# 3.6/3.7/3.8 ExceptExc abandons tail node. + +# 3.6/3.7/3.8 ExceptExc abandons tail node. # Could be fixed (?) but breaks other test cases # 3.11 Try return getting left outside of TryExcept def aa1_multi_except_return_both(): @@ -454,6 +493,7 @@ def aa1_multi_except_return_both(): except b: print(3) + # 3.11 Try return getting left outside of TryExcept def ab0_named_except_raise_return(): try: @@ -466,6 +506,7 @@ def ab0_named_except_raise_return(): print(3) raise Exc + # 3.8 Double natural edge graph error (?) # 3.11 Try return getting left outside of TryExcept def ab1_named_except_return(): @@ -476,12 +517,14 @@ def ab1_named_except_return(): print(2) return 3 + def ab2_named_except_return(): try: return a except Exception as a: return b + # 3.11/3.12/3.13 No template match def ac0_empty_try_finally(): try: @@ -489,6 +532,7 @@ def ac0_empty_try_finally(): finally: print(1) + def ad0_multiple_try_blocks(): try: print(1) @@ -500,6 +544,7 @@ def ad0_multiple_try_blocks(): except: print(4) + def ae0_try_except_else_nested_try(): try: print(1) @@ -511,6 +556,7 @@ def ae0_try_except_else_nested_try(): except: print(4) + # 3.9 Duplicate blocks (?) # 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) def af0_try_finally_nested_except(): @@ -522,12 +568,14 @@ def af0_try_finally_nested_except(): except: print(3) + def ag0_try_except_tuple(): try: print(1) except (A, B): print(2) + # 3.9 Difficult template ambiguity between Try/TryFinally # 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) def ah0_try_finally_return(): @@ -536,6 +584,7 @@ def ah0_try_finally_return(): finally: return 2 + # 3.11/3.12/3.13 No template match (?) def ai0_try_return_finally(): try: @@ -543,6 +592,7 @@ def ai0_try_return_finally(): finally: print(2) + # 3.9/3.10 Difficult template ambiguity between Try/TryFinally # 3.11/3.12/3.13 No template match (?) def aj0_try_return_finally_return(): @@ -551,6 +601,7 @@ def aj0_try_return_finally_return(): finally: return 2 + def ak0_try_except_raise_return(): try: print(1) @@ -558,6 +609,7 @@ def ak0_try_except_raise_return(): except: raise Exception() + # 3.8/3.9/3.10 No template match def al0_try_except_return_finally(): try: @@ -568,6 +620,7 @@ def al0_try_except_return_finally(): finally: print(3) + # 3.8/3.9/3.10 No template match # 3.11/3.12/3.13 Matching priority TryElse TryFinally (?) def am0_try_finally_raise(): @@ -577,6 +630,7 @@ def am0_try_finally_raise(): finally: raise Exception() + def an0_try_finally_fallthrough(): try: print(1) @@ -584,8 +638,9 @@ def an0_try_finally_fallthrough(): print(2) print(3) + def ao0_try_finally_simple(): try: print(1) finally: - print(2) \ No newline at end of file + print(2) diff --git a/test/Loop.py b/test/Loop.py index 8c730bf..20117a9 100644 --- a/test/Loop.py +++ b/test/Loop.py @@ -2,20 +2,24 @@ def a0_for_over_list(): for x in [1, 2, 3]: print("for over list") + def a1_for_over_list_nofallthru(): for x in [1, 2, 3]: print("for over list") print("end") + def b0_for_over_tuples(): for a, b in [(1, 2), (3, 4)]: print("tuples") + def b1_for_over_tuples_nofallthru(): for a, b in [(1, 2), (3, 4)]: print("tuples") print("end") + # 3.6/3.7 No else template def c0_for_else(): for i in range(3): @@ -23,6 +27,7 @@ def c0_for_else(): else: print("for else") + # 3.6/3.7 No else template def c1_for_else_nofallthru(): for i in range(3): @@ -31,6 +36,7 @@ def c1_for_else_nofallthru(): print("for else") print("end") + # 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end # 3.9 Naive break detection, an unexpected buffer block to end def d0_for_with_break(): @@ -39,6 +45,7 @@ def d0_for_with_break(): print("breaking") break + # 3.6/3.7 Naive break detection, an unexpected buffer POP_BLOCK to end def d1_for_with_break_nofallthru(): for x in range(10): @@ -47,6 +54,7 @@ def d1_for_with_break_nofallthru(): break print("end") + # Help to implement break def d2_for_without_break(): for x in range(10): @@ -54,6 +62,7 @@ def d2_for_without_break(): print("not breaking") print("end") + # Help to implement break def d3_for_return(): for x in range(10): @@ -62,6 +71,7 @@ def d3_for_return(): return print("end") + def e0_for_with_continue(): for x in range(5): if x % 2 == 0: @@ -69,6 +79,7 @@ def e0_for_with_continue(): continue print("after continue") + def e1_for_with_continue_nofallthru(): for x in range(5): if x % 2 == 0: @@ -77,17 +88,20 @@ def e1_for_with_continue_nofallthru(): print("after continue") print("end") + def f0_nested_for_loops(): for i in range(2): for j in range(3): print(f"nested {i},{j}") + def f1_nested_for_loops_nofallthru(): for i in range(2): for j in range(3): print(f"nested {i},{j}") print("end") + def g0_for_with_try_except(): for x in range(2): try: @@ -95,6 +109,7 @@ def g0_for_with_try_except(): except Exception: print("except block") + def g1_for_with_try_except_nofallthru(): for x in range(2): try: @@ -103,17 +118,20 @@ def g1_for_with_try_except_nofallthru(): print("except block") print("end") + def h0_for_with_with_statement(): for _ in range(1): with a: print("inside with") + def h1_for_with_with_statement_nofallthru(): for _ in range(1): with a: print("inside with") print("end") + def i0_for_with_function_call_iterable(): def get_items(): return [1, 2, 3] @@ -121,6 +139,7 @@ def i0_for_with_function_call_iterable(): for item in get_items(): print(f"item: {item}") + def i1_for_with_function_call_iterable_nofallthru(): def get_items(): return [1, 2, 3] @@ -129,15 +148,18 @@ def i1_for_with_function_call_iterable_nofallthru(): print(f"item: {item}") print("end") + def j0_for_with_empty_body_ellipsis(): for _ in range(3): ... + def j1_for_with_empty_body_ellipsis_nofallthru(): for _ in range(3): ... print("end") + # 3.6/3.7 Naive break detection, no back edge # 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k0_while_true_with_break(): @@ -148,6 +170,7 @@ def k0_while_true_with_break(): if x >= 1: break + # 3.6/3.7 Naive break detection, no back edge # 3.9/3.11 No while loop detection, self false_jump edge & naive break detection def k1_while_true_with_break_nofallthru(): @@ -159,6 +182,7 @@ def k1_while_true_with_break_nofallthru(): break print("end") + # 3.6/3.7 No else template # 3.11 No while loop detection, self false_jump edge def l0_while_with_else(): @@ -169,6 +193,7 @@ def l0_while_with_else(): else: print("while else") + # 3.6/3.7 No else template # 3.11 No while loop detection, self false_jump edge def l1_while_with_else_nofallthru(): @@ -180,6 +205,7 @@ def l1_while_with_else_nofallthru(): print("while else") print("end") + # 3.11 No continue def m0_while_with_continue(): i = 0 @@ -190,6 +216,7 @@ def m0_while_with_continue(): continue print("after continue") + # 3.11 No continue def m1_while_with_continue_nofallthru(): i = 0 @@ -201,6 +228,7 @@ def m1_while_with_continue_nofallthru(): print("after continue") print("end") + # 3.6/3.7 Naive break detection, no back edge def n0_while_with_break(): i = 0 @@ -208,6 +236,7 @@ def n0_while_with_break(): print("break in while") break + # 3.6/3.7 Naive break detection, no back edge def n1_while_with_break_nofallthru(): i = 0 @@ -216,6 +245,7 @@ def n1_while_with_break_nofallthru(): break print("end") + # 3.11 While template broke def o0_nested_while_loops(): i = 0 @@ -226,6 +256,7 @@ def o0_nested_while_loops(): j += 1 i += 1 + # 3.11 While template broke def o1_nested_while_loops_nofallthru(): i = 0 @@ -237,8 +268,9 @@ def o1_nested_while_loops_nofallthru(): i += 1 print("end") + # 3.6/3.7 While template broke (?) -# 3.9 Disconnected with MetaTemplate[end] (?) +# 3.9 Disconnected with MetaTemplate[end] (?) def p0_while_with_try_except(): while True: try: @@ -246,8 +278,9 @@ def p0_while_with_try_except(): except: print("except in while") + # 3.6/3.7 While template broke (?) -# 3.9 Disconnected with MetaTemplate[end] (?) +# 3.9 Disconnected with MetaTemplate[end] (?) def p1_while_with_try_except_nofallthru(): while True: try: @@ -256,12 +289,14 @@ def p1_while_with_try_except_nofallthru(): print("except in while") print("end") + # 3.6/3.7 While template broke (?) abandoning nodes def q0_while_with_with_statement(): while True: with a: print("inside while with") + # 3.6/3.7 While template broke (?) abandoning nodes def q1_while_with_with_statement_nofallthru(): while True: @@ -269,12 +304,14 @@ def q1_while_with_with_statement_nofallthru(): print("inside while with") print("end") + # 3.6/3.7 While template broke def r0_for_inside_while(): while True: for x in [1, 2]: print("for in while") + # 3.6/3.7 While template broke def r1_for_inside_while_nofallthru(): while True: @@ -282,6 +319,7 @@ def r1_for_inside_while_nofallthru(): print("for in while") print("end") + # 3.6/3.7 While template broke def s0_while_inside_for(): for _ in range(1): @@ -289,6 +327,7 @@ def s0_while_inside_for(): print("while in for") break + # 3.6/3.7 While template broke def s1_while_inside_for_nofallthru(): for _ in range(1): @@ -297,17 +336,20 @@ def s1_while_inside_for_nofallthru(): break print("end") + # 3.6/3.7 While template broke def t0_while_with_empty_body_ellipsis(): while True: ... + # 3.6/3.7 While template broke def t1_while_with_empty_body_ellipsis_nofallthru(): while True: ... print("end") + def u0_break_in_nested_for(): for i in range(3): for j in range(3): @@ -316,6 +358,7 @@ def u0_break_in_nested_for(): break print(f"i={i}, j={j}") + def u1_break_in_nested_for_nofallthru(): for i in range(3): for j in range(3): @@ -325,6 +368,7 @@ def u1_break_in_nested_for_nofallthru(): print(f"i={i}, j={j}") print("end") + def v0_continue_in_nested_for(): for i in range(3): for j in range(3): @@ -332,6 +376,7 @@ def v0_continue_in_nested_for(): continue print(f"Processing i={i}, j={j}") + def v1_continue_in_nested_for_nofallthru(): for i in range(3): for j in range(3): @@ -340,6 +385,7 @@ def v1_continue_in_nested_for_nofallthru(): print(f"Processing i={i}, j={j}") print("end") + # 3.13 if statement putting code in the else block def w0_break_with_else(): for i in range(5): @@ -349,6 +395,7 @@ def w0_break_with_else(): else: print("This won't execute due to break") + # 3.13 if statement putting code in the else block def w1_break_with_else_nofallthru(): for i in range(5): @@ -359,6 +406,7 @@ def w1_break_with_else_nofallthru(): print("This won't execute due to break") print("end") + # 3.6/3.7 No continue detection def x0_continue_with_else(): for i in range(3): @@ -368,6 +416,7 @@ def x0_continue_with_else(): else: print("Else clause still executes after continue") + # 3.6/3.7 No continue detection def x1_continue_with_else_nofallthru(): for i in range(3): @@ -378,6 +427,7 @@ def x1_continue_with_else_nofallthru(): print("Else clause still executes after continue") print("end") + # 3.9/3.11 Naive break detection, break statement is further up def y0_break_in_try_except(): for i in range(5): @@ -388,6 +438,7 @@ def y0_break_in_try_except(): except: print("Exception occurred") + # 3.9 Naive break detection, break statement is further up def y1_break_in_try_except_nofallthru(): for i in range(5): @@ -399,6 +450,7 @@ def y1_break_in_try_except_nofallthru(): print("Exception occurred") print("end") + # 3.9 Naive break detection, break statement is further up def y2_return_in_try_except_nofallthru(): for i in range(5): @@ -411,6 +463,7 @@ def y2_return_in_try_except_nofallthru(): print("Exception occurred") print("end") + # 3.6/3.9 No continue detection def z0_continue_in_try_except(): for i in range(5): @@ -421,6 +474,7 @@ def z0_continue_in_try_except(): except: print("Exception occurred") + # 3.6/3.9/3.11 No continue detection def z1_continue_in_try_except_nofallthru(): for i in range(5): @@ -430,4 +484,4 @@ def z1_continue_in_try_except_nofallthru(): print(f"Value: {i}") except: print("Exception occurred") - print("end") \ No newline at end of file + print("end") diff --git a/test/With.py b/test/With.py index d4b4fc0..6e3c603 100644 --- a/test/With.py +++ b/test/With.py @@ -2,65 +2,79 @@ def a0_bare_with(): with a: print(1) + def a1_bare_with_fallthrough(): with a: print(1) print(2) + def b0_multi_with(): with a, b: print(1) + def b1_multi_with_fallthrough(): with a, b: print(1) print(2) + def c0_with_as(): with a as c: print(1) + def c1_with_as_fallthrough(): with a as c: print(1) print(2) + def d0_multi_with_as(): with a, b as c: print(1) + def d1_multi_with_as_fallthrough(): with a, b as c: print(1) print(2) + def e0_with_multi_as(): with a as b, c: print(1) + def e1_with_multi_as_fallthrough(): with a as b, c: print(1) print(2) + def f0_multi_with_multi_as(): with a as b, c as d: print(1) + def f1_multi_with_multi_as_fallthrough(): with a as b, c as d: print(1) print(2) + def g0_multi_with_multi_as_alt(): with a, b as c, d: print(1) + def g1_multi_with_multi_as_fallthrough_alt(): with a, b as c, d: print(1) print(2) + def h0_try_with_except(): try: with a: @@ -69,75 +83,91 @@ def h0_try_with_except(): print(2) print(3) + def i0_with_return(): with a: return 1 print(1) + def j0_with_raise(): with a: raise Exc print(1) + async def k0_bare_async_with(): async with a: print(1) + async def k1_bare_async_with_fallthrough(): async with a: print(1) print(2) + async def l0_multi_async_with(): async with a, b: print(1) + async def l1_multi_async_with_fallthrough(): async with a, b: print(1) print(2) + async def m0_async_with_as(): async with a as c: print(1) + async def m1_async_with_as_fallthrough(): async with a as c: print(1) print(2) + async def n0_multi_async_with_as(): async with a, b as c: print(1) + async def n1_multi_async_with_as_fallthrough(): async with a, b as c: print(1) print(2) + async def o0_async_with_multi_as(): async with a as b, c: print(1) + async def o1_async_with_multi_as_fallthrough(): async with a as b, c: print(1) print(2) + async def p0_multi_async_with_multi_as(): async with a as b, c as d: print(1) + async def p1_multi_async_with_multi_as_fallthrough(): async with a as b, c as d: print(1) print(2) + async def q0_multi_async_with_multi_as_alt(): async with a, b as c, d: print(1) + async def q1_multi_async_with_multi_as_fallthrough_alt(): async with a, b as c, d: print(1) - print(2) \ No newline at end of file + print(2) From e2522df6393e103af4808323181aacc97126f60c Mon Sep 17 00:00:00 2001 From: caandt Date: Mon, 21 Jul 2025 16:55:53 -0500 Subject: [PATCH 42/55] nit --- pylingual/control_flow_reconstruction/cfg.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index ebfcb1c..a7b368e 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -139,11 +139,9 @@ class CFG(DiGraph_CFT): def _layout_nodes(self): relabeled = nx.convert_node_labels_to_integers(self, label_attribute="template") # type: ignore - try: - root = next(i for i in relabeled.nodes if relabeled.nodes[i]["template"] == self.start) - for i, pos in nx.nx_pydot.pydot_layout(relabeled, prog="dot", root=root).items(): - relabeled.nodes[i]["template"]._pos = [pos] - except: pass + root = next(i for i in relabeled.nodes if relabeled.nodes[i]["template"] == self.start) + for i, pos in nx.nx_pydot.pydot_layout(relabeled, prog="dot", root=root).items(): + relabeled.nodes[i]["template"]._pos = [pos] def node_by_offset(self, offset: int): return next(x for x in self.nodes if x.offset == offset) @@ -173,7 +171,7 @@ class CFG(DiGraph_CFT): nodes = {} for node, data in self.nodes.data(): - nodes[node] = pydot.Node(str(hash(node)), label=repr(node).replace("\n", "\\l").replace("\t", "| ").replace('"', '') + "\\l", fontname="Noto Sans", labeljust="l", shape="box", pos=node.pos()) + nodes[node] = pydot.Node(str(hash(node)), label=repr(node).replace("\n", "\\l").replace("\t", "| ") + "\\l", fontname="Noto Sans", labeljust="l", shape="box", pos=node.pos()) dot.add_node(nodes[node]) for a, b, data in self.edges.data(): dot.add_edge(pydot.Edge(nodes[a], nodes[b], **data, label=data["kind"].value, color=data["kind"].color(), fontname="Noto Sans", labeljust="l")) From 9fd60fe989e738149e1c9133c314b72bff698939 Mon Sep 17 00:00:00 2001 From: Joel Flores Date: Thu, 24 Jul 2025 14:05:14 -0500 Subject: [PATCH 43/55] fix model view --- pylingual/masking/global_masker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylingual/masking/global_masker.py b/pylingual/masking/global_masker.py index 188d85a..8a780e7 100644 --- a/pylingual/masking/global_masker.py +++ b/pylingual/masking/global_masker.py @@ -238,7 +238,7 @@ class Masker: if inst.is_jump_target: view = f"{inst.offset} {view}" # create list of offsets greater than or less than inst, for all jump origins to this inst - jumps_greater_or_less = [inst.offset < inst.offset for inst in inst.jumped_to_from_insts] + jumps_greater_or_less = [i.offset < inst.offset for i in inst.jumped_to_from_insts] if any(jumps_greater_or_less): # 1 in list view = f"^~> {view}" if not all(jumps_greater_or_less): # 0 in list From 06fffdf8c70eefc8d0335d6e3ce73e9f16ec91a2 Mon Sep 17 00:00:00 2001 From: Joel Flores Date: Thu, 24 Jul 2025 15:10:57 -0500 Subject: [PATCH 44/55] update dataset generation --- dev_scripts/dataset_generation/bytecode2csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_scripts/dataset_generation/bytecode2csv.py b/dev_scripts/dataset_generation/bytecode2csv.py index 130751b..1ab0c77 100644 --- a/dev_scripts/dataset_generation/bytecode2csv.py +++ b/dev_scripts/dataset_generation/bytecode2csv.py @@ -115,7 +115,7 @@ def bytecode2csv(py_path: pathlib.Path, pyc_path: pathlib.Path) -> tuple[list, l pyc = PYCFile(str(pyc_path.resolve())) if pyc.version == (3, 10): pyc.replace_duplicated_returns10(py_path.read_text().split("\n")) - elif pyc.version == (3, 12): + elif pyc.version >= (3, 12): pyc.replace_duplicated_returns12(py_path.read_text().split("\n")) global_masker = create_global_masker(pyc) From 69f28e59cb1dd20419c85feb5579e062e588fda4 Mon Sep 17 00:00:00 2001 From: Joel Flores Date: Thu, 24 Jul 2025 17:50:37 -0500 Subject: [PATCH 45/55] add token for statement translation training --- dev_scripts/statement/tokenizer/special_tokens_map.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev_scripts/statement/tokenizer/special_tokens_map.json b/dev_scripts/statement/tokenizer/special_tokens_map.json index 5d7a5d0..47193ed 100644 --- a/dev_scripts/statement/tokenizer/special_tokens_map.json +++ b/dev_scripts/statement/tokenizer/special_tokens_map.json @@ -1085,6 +1085,13 @@ "normalized": false, "single_word": false }, + { + "content": "FORMAT_WITH_SPEC", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, { "content": "GEN_START", "lstrip": false, From 6c1bc8c5398b5322dc20e252f21f989c175258e3 Mon Sep 17 00:00:00 2001 From: drospierski24 Date: Wed, 23 Jul 2025 10:44:50 -0500 Subject: [PATCH 46/55] IfElseLoop test cases for fixes made on 7/21 --- test/Conditional.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/Conditional.py b/test/Conditional.py index 19302fc..1362445 100644 --- a/test/Conditional.py +++ b/test/Conditional.py @@ -302,3 +302,22 @@ def f_nofallthru_if_pass(): if a > b: pass print("end") + + +def g1_ifElseLoop(): + for a in range(3): + if a> b: + print(1) + +def g2_ifElseLoop(): + for a in range(3): + if a> b: + print(1) + print(2) + +def g3_ifElseLoop(): + for a in range(3): + if a> b: + print(1) + else: + print(2) From a6ae04dd2ec1f0c582f5f3022a1822a065b9c108 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 24 Jul 2025 11:30:11 -0500 Subject: [PATCH 47/55] JumpTemplate --- .../templates/Block.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 93cead8..7e096bd 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -7,7 +7,7 @@ from pylingual.editable_bytecode import Inst import networkx as nx from ..cft import ControlFlowTemplate, EdgeKind, SourceContext, SourceLine, register_template, EdgeCategory, out_edge_dict, MetaTemplate, indent_str -from ..utils import E, N, T, defer_source_to, remove_nodes, without_instructions, exact_instructions, make_try_match +from ..utils import E, N, T, defer_source_to, remove_nodes, without_instructions, has_no_lines, exact_instructions, make_try_match if TYPE_CHECKING: from pylingual.control_flow_reconstruction.cfg import CFG @@ -54,12 +54,19 @@ class RemoveUnreachable(ControlFlowTemplate): return node -@register_template(0, 0) +@register_template(0, 0, (3, 13)) class JumpTemplate(ControlFlowTemplate): template = T( body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), - jump=N("tail", "block?").with_in_deg(1).with_cond(exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), exact_instructions("POP_JUMP_IF_TRUE")), - block=~N.tail(), + jump=N("tail", "block?").with_in_deg(1).with_cond( + exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), + exact_instructions("POP_JUMP_IF_TRUE"), + exact_instructions("JUMP_FORWARD"), + exact_instructions("JUMP_BACKWARD"), + exact_instructions("POP_JUMP_IF_NOT_NONE"), + exact_instructions("POP_JUMP_IF_NONE"), + exact_instructions("POP_JUMP_IF_FALSE")), + block=N.tail(), tail=N.tail(), ) @@ -74,6 +81,24 @@ class JumpTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("body") +@register_template(0, 0, (3, 13)) +class NopTemplate(ControlFlowTemplate): + template = T( + body=~N("nop", None).with_cond(without_instructions("CLEANUP_THROW")), + nop=N("tail", None).with_in_deg(1).with_cond(exact_instructions("NOP")).with_cond(has_no_lines), + tail=N.tail(), + ) + + try_match = make_try_match( + { + EdgeKind.Fall: "tail", + }, + "body", + "nop", + ) + + to_indented_source = defer_source_to("body") + @register_template(0, 20) @register_template(2, 20) From 498e80e61eabffe941de8bf59cd608a7e770d0ea Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 24 Jul 2025 11:30:35 -0500 Subject: [PATCH 48/55] 3.11 TryExcept adjustments --- .../templates/Exception.py | 84 +++++++++++++++++-- .../control_flow_reconstruction/utils.py | 4 + 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Exception.py b/pylingual/control_flow_reconstruction/templates/Exception.py index 854c49e..1956553 100644 --- a/pylingual/control_flow_reconstruction/templates/Exception.py +++ b/pylingual/control_flow_reconstruction/templates/Exception.py @@ -13,6 +13,8 @@ from ..utils import ( with_instructions, without_instructions, ending_instructions, + has_no_lines, + has_some_lines, exact_instructions, no_back_edges, without_top_level_instructions, @@ -39,8 +41,8 @@ class Except3_11(ControlFlowTemplate): return x -@register_template(0, 0, *versions_from(3, 11)) -class Try3_11(ControlFlowTemplate): +@register_template(0, 0, *versions_from(3, 12)) +class Try3_12(ControlFlowTemplate): template = T( try_header=N("try_body"), try_body=N("tail.", None, "except_body"), @@ -71,8 +73,8 @@ class Try3_11(ControlFlowTemplate): """ -@register_template(0, 0, *versions_from(3, 11)) -class TryElse3_11(ControlFlowTemplate): +@register_template(0, 0, *versions_from(3, 12)) +class TryElse3_12(ControlFlowTemplate): template = T( try_header=N("try_body"), try_body=N("try_else.", None, "except_body"), @@ -105,6 +107,76 @@ class TryElse3_11(ControlFlowTemplate): else: {try_else} """ + + +@register_template(0, 0, (3, 11)) +class Try3_11(ControlFlowTemplate): + template = T( + try_header=N("try_body"), + try_body=N("try_else.", None, "except_body"), + except_body=N("tail.", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11), + try_else=~N("tail.").with_in_deg(1).with_cond(has_no_lines), + reraise=reraise, + tail=N.tail(), + ) + + try_match = revert_on_fail( + make_try_match( + { + EdgeKind.Fall: "tail", + }, + "try_header", + "try_else", + "try_body", + "except_body", + "reraise", + ) + ) + + @to_indented_source + def to_indented_source(): + """ + {try_header} + try: + {try_body} + {except_body} + """ + + +@register_template(0, 0, (3, 11)) +class TryElse3_11(ControlFlowTemplate): + template = T( + try_header=N("try_body"), + try_body=N("try_else.", None, "except_body"), + except_body=N("tail.", None, "reraise").with_in_deg(1).of_subtemplate(Except3_11), + try_else=~N("tail.").with_in_deg(1).with_cond(has_some_lines), + reraise=reraise, + tail=N.tail(), + ) + + try_match = revert_on_fail( + make_try_match( + { + EdgeKind.Fall: "tail", + }, + "try_header", + "try_body", + "except_body", + "try_else", + "reraise", + ) + ) + + @to_indented_source + def to_indented_source(): + """ + {try_header} + try: + {try_body} + {except_body} + else: + {try_else} + """ class BareExcept3_11(Except3_11): @@ -225,7 +297,7 @@ class TryFinally3_11(ControlFlowTemplate): tail=N.tail(), ) template2 = T( - try_except=N("finally_body", None, "fail_body").of_type(Try3_11, TryElse3_11), + try_except=N("finally_body", None, "fail_body").of_type(Try3_11, TryElse3_11, Try3_12, TryElse3_12), finally_body=~N("tail.").with_in_deg(1).with_cond(no_back_edges), fail_body=N(E.exc("reraise")).with_cond(ending_instructions("POP_TOP", "RERAISE")), reraise=reraise, @@ -280,7 +352,7 @@ class TryFinally3_11(ControlFlowTemplate): def to_indented_source(self, source: SourceContext) -> list[SourceLine]: header = source[self.try_header] body = source[self.try_body, 1] - if isinstance(self.try_header, (Try3_11, TryElse3_11)) and self.members["try_body"] is None: + if isinstance(self.try_header, (Try3_11, TryElse3_11, Try3_12, TryElse3_12)) and self.members["try_body"] is None: s = header else: s = chain(header, self.line("try:"), body) diff --git a/pylingual/control_flow_reconstruction/utils.py b/pylingual/control_flow_reconstruction/utils.py index 020d7c9..66367ea 100644 --- a/pylingual/control_flow_reconstruction/utils.py +++ b/pylingual/control_flow_reconstruction/utils.py @@ -134,6 +134,10 @@ def has_no_lines(cfg: CFG, node: ControlFlowTemplate | None) -> bool: return node is None or all(i.starts_line is None for i in node.get_instructions()) +def has_some_lines(cfg: CFG, node: ControlFlowTemplate | None) -> bool: + return node is None or any(i.starts_line is not None for i in node.get_instructions()) + + def run_is(n: int): def check_run(cfg: CFG, node: ControlFlowTemplate | None) -> bool: return cfg.run == n From 79f715531db93ffa1c236872005d01d157f13261 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 24 Jul 2025 13:58:46 -0500 Subject: [PATCH 49/55] 3.12/3.13 Fixing Continue/Break test cases --- pylingual/control_flow_reconstruction/templates/Loop.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index 817aa7a..3faeca4 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -155,12 +155,12 @@ class FixLoop(ControlFlowTemplate): candidate_end = succ # Candidate end is a buffer node - if cfg.in_degree(candidate_end) == 1 and any(exact_instructions(*op)(cfg, candidate_end) for op in [("POP_BLOCK",), ("END_FOR",), ("END_FOR", "POP_TOP"), ("LOAD_CONST", "RETURN_VALUE")]): + if cfg.in_degree(candidate_end) == 1 and all(x.opname in {"POP_TOP", "POP_BLOCK", "END_FOR", "RETURN_CONST", "LOAD_CONST", "RETURN_VALUE", "JUMP_BACKWARD"} for x in candidate_end.get_instructions()): for ss in cfg.successors(candidate_end): - if cfg.out_degree(ss) <= 1: + if cfg.get_edge_data(candidate_end, ss).get("kind") != EdgeKind.Exception: candidate_end = ss break - + if encompassed_nodes is not None: for succ in encompassed_nodes: if cfg.get_edge_data(succ, candidate_end) != None: From 1278aea78f5a6766bbbbcd7151be3a1e311d8674 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 24 Jul 2025 13:59:06 -0500 Subject: [PATCH 50/55] 3.11 Expanding NopTemplate --- pylingual/control_flow_reconstruction/templates/Block.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 7e096bd..0398879 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -7,7 +7,7 @@ from pylingual.editable_bytecode import Inst import networkx as nx from ..cft import ControlFlowTemplate, EdgeKind, SourceContext, SourceLine, register_template, EdgeCategory, out_edge_dict, MetaTemplate, indent_str -from ..utils import E, N, T, defer_source_to, remove_nodes, without_instructions, has_no_lines, exact_instructions, make_try_match +from ..utils import E, N, T, defer_source_to, remove_nodes, versions_from, without_instructions, has_no_lines, exact_instructions, make_try_match if TYPE_CHECKING: from pylingual.control_flow_reconstruction.cfg import CFG @@ -81,7 +81,7 @@ class JumpTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("body") -@register_template(0, 0, (3, 13)) +@register_template(0, 0, *versions_from(3, 11)) class NopTemplate(ControlFlowTemplate): template = T( body=~N("nop", None).with_cond(without_instructions("CLEANUP_THROW")), From 909657ffbee5b9978b7f2c2cbbc78b60b0ae8c1d Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 24 Jul 2025 13:59:31 -0500 Subject: [PATCH 51/55] 3.12/3.13 Loosening Generator3_12 template --- pylingual/control_flow_reconstruction/templates/Generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Generator.py b/pylingual/control_flow_reconstruction/templates/Generator.py index 87be641..9027460 100644 --- a/pylingual/control_flow_reconstruction/templates/Generator.py +++ b/pylingual/control_flow_reconstruction/templates/Generator.py @@ -1,5 +1,5 @@ from ..cft import ControlFlowTemplate, EdgeKind, MetaTemplate, register_template -from ..utils import E, T, N, defer_source_to, exact_instructions, no_back_edges, to_indented_source, make_try_match +from ..utils import E, T, N, defer_source_to, ending_instructions, exact_instructions, no_back_edges, to_indented_source, make_try_match @register_template(0, 0) @@ -38,7 +38,7 @@ class AwaitWith3_12(ControlFlowTemplate): @register_template(0, 0) class Generator3_12(ControlFlowTemplate): template = T( - entry=N("body").with_cond(exact_instructions("RETURN_GENERATOR", "POP_TOP")), + entry=N("body").with_cond(ending_instructions("RETURN_GENERATOR", "POP_TOP")), body=N(E.exc("gen_cleanup"), E.meta("end?")), gen_cleanup=N(E.meta("end")).with_cond(exact_instructions("CALL_INTRINSIC_1", "RERAISE")), end=N().of_type(MetaTemplate), From aa28e2f4d2e7dff08cbe7b75172e840854f5b650 Mon Sep 17 00:00:00 2001 From: Xinlong Hu <118075581+XinlongCS@users.noreply.github.com> Date: Thu, 24 Jul 2025 14:57:52 -0500 Subject: [PATCH 52/55] Update cflow.py --- dev_scripts/cflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_scripts/cflow.py b/dev_scripts/cflow.py index 8df7538..72e84ad 100644 --- a/dev_scripts/cflow.py +++ b/dev_scripts/cflow.py @@ -36,7 +36,7 @@ class Result(Enum): def edit_pyc_lines(pyc: PYCFile, src_lines: list[str]): if pyc.version == (3, 10): pyc.replace_duplicated_returns10(src_lines) - elif pyc.version == (3, 12): + elif pyc.version >= (3, 12): pyc.replace_duplicated_returns12(src_lines) seen_lines = set() # multiple instructions can start the same lno, but the segmentation model will only assign the lno to the first one From a0e5f4f1439f8176c16e8a70eba8383f017b5f80 Mon Sep 17 00:00:00 2001 From: Joel Flores Date: Fri, 25 Jul 2025 11:44:55 -0500 Subject: [PATCH 53/55] get rid of pdbs --- pylingual/control_flow_reconstruction/structure.py | 13 +++---------- pylingual/main.py | 4 ---- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/pylingual/control_flow_reconstruction/structure.py b/pylingual/control_flow_reconstruction/structure.py index b7d1ea8..98a26eb 100644 --- a/pylingual/control_flow_reconstruction/structure.py +++ b/pylingual/control_flow_reconstruction/structure.py @@ -1,5 +1,3 @@ -import pdb - from pylingual.editable_bytecode import EditableBytecode from pylingual.editable_bytecode.control_flow_graph import bytecode_to_control_flow_graph @@ -27,13 +25,8 @@ def structure_control_flow(cfg: nx.DiGraph, bytecode: EditableBytecode) -> Contr cfg = CFG.from_graph(cfg, bytecode) runs = get_template_runs(bytecode.version[:2]) - try: - while len(cfg) > 1: - if not iteration(cfg, runs): - return MetaTemplate("\x1b[31mirreducible cflow\x1b[0m", bytecode.codeobj) - except Exception: - if hasattr(pdb, "xpm"): - pdb.xpm() # type: ignore - raise + while len(cfg) > 1: + if not iteration(cfg, runs): + return MetaTemplate("\x1b[31mirreducible cflow\x1b[0m", bytecode.codeobj) return next(iter(cfg.nodes)) diff --git a/pylingual/main.py b/pylingual/main.py index ff8c76d..0826de3 100644 --- a/pylingual/main.py +++ b/pylingual/main.py @@ -130,11 +130,7 @@ def main(files: list[str], out_dir: Path | None, config_file: Path | None, versi pyc = result.original_pyc print_result(f"Equivalence Results for {pyc.pyc_path.name if pyc.pyc_path else repr(pyc)}", result.equivalence_results) except Exception: - import pdb - live.stop() - if hasattr(pdb, "xpm"): - pdb.xpm() # type: ignore logger.exception(f"Failed to decompile {pyc_path}") console.rule() From 814e99cb2d093b7959fca8f8d9a0ce85bdabab6a Mon Sep 17 00:00:00 2001 From: caandt Date: Fri, 25 Jul 2025 16:25:53 -0500 Subject: [PATCH 54/55] cdg fallback --- pylingual/control_flow_reconstruction/cfg.py | 14 +++++ .../templates/CDG.py | 51 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 pylingual/control_flow_reconstruction/templates/CDG.py diff --git a/pylingual/control_flow_reconstruction/cfg.py b/pylingual/control_flow_reconstruction/cfg.py index a7b368e..2ae8494 100644 --- a/pylingual/control_flow_reconstruction/cfg.py +++ b/pylingual/control_flow_reconstruction/cfg.py @@ -1,5 +1,6 @@ from __future__ import annotations +import itertools from typing import TYPE_CHECKING from pathlib import Path @@ -179,3 +180,16 @@ class CFG(DiGraph_CFT): dot.write(out, prog=["neato", "-n"], format=CFG.graph_format) else: self.iteration_graphs[-1].append(dot.to_string()) + + def cdg(self) -> CFG: + pdt = nx.create_empty_copy(self) + pdt.add_edges_from((B, A) for A, B in nx.immediate_dominators(self.reverse(), self.end).items()) + pdt.remove_edge(self.end, self.end) + pdr = nx.transitive_closure_dag(pdt) + postdominates = lambda A, B: pdr.has_edge(A, B) or A == B + control_dependent = lambda A, B: 0 < sum(postdominates(A, succ) for succ in self.successors(B)) < self.out_degree(B) + cdg = nx.create_empty_copy(self) + cdg.add_edges_from((B, A, {"kind": EdgeKind.Fall}) for A, B in itertools.product(self.nodes, self.nodes) if A != B and control_dependent(A, B)) + cdg.remove_node(self.end) + cdg.add_edges_from(((self.start, n) for n in cdg.nodes if cdg.in_degree(n) == 0 and n != self.start), kind=EdgeKind.Fall) + return cdg diff --git a/pylingual/control_flow_reconstruction/templates/CDG.py b/pylingual/control_flow_reconstruction/templates/CDG.py new file mode 100644 index 0000000..5cd96f9 --- /dev/null +++ b/pylingual/control_flow_reconstruction/templates/CDG.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, override + +import networkx as nx + +from ..cft import ControlFlowTemplate, register_template + +if TYPE_CHECKING: + from pylingual.control_flow_reconstruction.cfg import CFG + + +# it's better than nothing +@register_template(101, 0) +class CDG(ControlFlowTemplate): + def __init__(self, cfg: CFG): + self.cdg = cfg.cdg() + self.start = cfg.start + self.blame = cfg.start.blame + self.header_lines = self.line("# irreducible cflow, using cdg fallback", meta=True) + + @override + @classmethod + def try_match(cls, cfg: CFG, node: ControlFlowTemplate) -> ControlFlowTemplate | None: + cdg = CDG(cfg) + + if cfg.visualize == cfg._visualize: + cfg.remove_edges_from(list(cfg.edges)) + cfg.add_edges_from(cdg.cdg.edges(data=True)) + cfg.remove_node(cfg.end) + cfg.layout_nodes() + cfg.visualize() + + cfg.clear() + cfg.add_node(cdg) + return cdg + + @override + def get_instructions(self): + return [] + + @override + def to_indented_source(self, source): + cdg = self.cdg + for p, n in nx.dfs_edges(cdg, self.start): + cdg.nodes[n]["indent"] = cdg.nodes[p].get("indent", -1) + 1 + cdg.remove_node(self.start) + src = [] + for n in sorted(cdg.nodes, key=lambda x: x.offset): + src.extend(source[n, cdg.nodes[n].get("indent", 0)]) + return src From da1f44caff8efe481d00e5229f37b798aeb4aebc Mon Sep 17 00:00:00 2001 From: caandt Date: Fri, 25 Jul 2025 16:26:03 -0500 Subject: [PATCH 55/55] format --- .../control_flow_reconstruction/templates/Block.py | 14 +++++++++----- .../templates/Exception.py | 2 +- .../control_flow_reconstruction/templates/Loop.py | 2 +- test/Conditional.py | 10 ++++++---- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pylingual/control_flow_reconstruction/templates/Block.py b/pylingual/control_flow_reconstruction/templates/Block.py index 0398879..1788a43 100644 --- a/pylingual/control_flow_reconstruction/templates/Block.py +++ b/pylingual/control_flow_reconstruction/templates/Block.py @@ -58,14 +58,17 @@ class RemoveUnreachable(ControlFlowTemplate): class JumpTemplate(ControlFlowTemplate): template = T( body=~N("jump", None).with_cond(without_instructions("CLEANUP_THROW")), - jump=N("tail", "block?").with_in_deg(1).with_cond( + jump=N("tail", "block?") + .with_in_deg(1) + .with_cond( exact_instructions("JUMP_BACKWARD_NO_INTERRUPT"), - exact_instructions("POP_JUMP_IF_TRUE"), + exact_instructions("POP_JUMP_IF_TRUE"), exact_instructions("JUMP_FORWARD"), exact_instructions("JUMP_BACKWARD"), - exact_instructions("POP_JUMP_IF_NOT_NONE"), - exact_instructions("POP_JUMP_IF_NONE"), - exact_instructions("POP_JUMP_IF_FALSE")), + exact_instructions("POP_JUMP_IF_NOT_NONE"), + exact_instructions("POP_JUMP_IF_NONE"), + exact_instructions("POP_JUMP_IF_FALSE"), + ), block=N.tail(), tail=N.tail(), ) @@ -81,6 +84,7 @@ class JumpTemplate(ControlFlowTemplate): to_indented_source = defer_source_to("body") + @register_template(0, 0, *versions_from(3, 11)) class NopTemplate(ControlFlowTemplate): template = T( diff --git a/pylingual/control_flow_reconstruction/templates/Exception.py b/pylingual/control_flow_reconstruction/templates/Exception.py index 1956553..7aa5619 100644 --- a/pylingual/control_flow_reconstruction/templates/Exception.py +++ b/pylingual/control_flow_reconstruction/templates/Exception.py @@ -107,7 +107,7 @@ class TryElse3_12(ControlFlowTemplate): else: {try_else} """ - + @register_template(0, 0, (3, 11)) class Try3_11(ControlFlowTemplate): diff --git a/pylingual/control_flow_reconstruction/templates/Loop.py b/pylingual/control_flow_reconstruction/templates/Loop.py index 3faeca4..fbdf78c 100644 --- a/pylingual/control_flow_reconstruction/templates/Loop.py +++ b/pylingual/control_flow_reconstruction/templates/Loop.py @@ -160,7 +160,7 @@ class FixLoop(ControlFlowTemplate): if cfg.get_edge_data(candidate_end, ss).get("kind") != EdgeKind.Exception: candidate_end = ss break - + if encompassed_nodes is not None: for succ in encompassed_nodes: if cfg.get_edge_data(succ, candidate_end) != None: diff --git a/test/Conditional.py b/test/Conditional.py index 1362445..03394cc 100644 --- a/test/Conditional.py +++ b/test/Conditional.py @@ -306,18 +306,20 @@ def f_nofallthru_if_pass(): def g1_ifElseLoop(): for a in range(3): - if a> b: + if a > b: print(1) - + + def g2_ifElseLoop(): for a in range(3): - if a> b: + if a > b: print(1) print(2) + def g3_ifElseLoop(): for a in range(3): - if a> b: + if a > b: print(1) else: print(2)