Merge branch 'cflow-refactor' of https://github.com/XinlongCS/pylingual into cflow-refactor

dev_scripts/cflow.py

@@ -83,7 +83,7 @@ def run(file: Path, out_dir: Path, version: PythonVersion, print=False):
         result = compare_pyc(in_pyc, out_pyc)
         if print:
             print_result(f"Equivalance results for {file}", result)
-        return Result.Success if all(x.success for x in result) else Result.Failure, [(x.success, str(x)) for x in result], file, out_dir
+        return Result.Success if all(x.success for x in result) else Result.Failure, [(x.success, x) for x in result], file, out_dir
     except (CompileError, SyntaxError) as e:
         return Result.CompileError, e, file, out_dir
     except Exception:
@@ -95,7 +95,7 @@ class NoPool:
     imap_unordered = map
 
 
-def print_results(a: Path, b: Path, result: Result, results: list[tuple[bool, str]] | Exception):
+def print_results(a: Path, b: Path, result: Result, results: list[tuple[bool, TestResult]] | Exception):
     a_text = a.read_text()
     b_text = b.read_text()
     console = rich.console.Console(highlight=False)
@@ -109,10 +109,37 @@ def print_results(a: Path, b: Path, result: Result, results: list[tuple[bool, st
         console.print(results)
     elif isinstance(results, list) and results:
         for success, name in results:
-            console.print(name, style="" if success else "red bold underline")
+            console.print(str(name), style="" if success else "red bold underline")
     else:
         console.print(result, style="red bold underline")
 
+def equivalence_report_json(
+    infilename: Path, result: 'Result',
+    results: list[tuple[bool, TestResult]] | Exception) -> dict:
+    report = []
+
+    if isinstance(results, Exception):
+        report.append({
+            "file": str(infilename),
+            "name": result.name,
+            "status": "error"
+        })
+    elif isinstance(results, list) and results:
+        for success, result in results:
+            report.append({
+            "file": str(infilename),
+                "name": result.name_a,
+                "status": "true" if success else "false"
+            })
+    else:
+        report.append({
+            "file": str(infilename),
+            "name": "error",
+            "status": "error"
+        })
+
+    return {"equivalence_report": report}
+
 
 def get_unused(a: Path, _=True):
     if not _ and not a.exists():
@@ -133,8 +160,9 @@ def get_unused(a: Path, _=True):
 @click.option("-p", "--processes", type=int, default=os.cpu_count(), help="Number of processes")
 @click.option("-d", "--prefix", type=Path, default=Path("/tmp/cflow_test"), help="Base dir for all output")
 @click.option("-g", "--graph", is_flag=False, flag_value="graph", help="Enable CFG visualization")
+@click.option("-j", "--jsonout", is_flag=True, flag_value="jsonout", help="Enable json output")
 @click.option("-f", "--graph-format", default="jpg", help="Output format supported by pydot")
-def main(input: Path, output: str, version: PythonVersion, graph: str | None, prefix: Path, processes: int, graph_format: str):
+def main(input: Path, output: str, version: PythonVersion, graph: str | None, jsonout: bool, prefix: Path, processes: int, graph_format: str):
     warnings.filterwarnings("ignore")
     print = rich.get_console().print
     progress_columns = [
@@ -159,8 +187,11 @@ def main(input: Path, output: str, version: PythonVersion, graph: str | None, pr
             out = TemporaryDirectory()
         with out as o:
             o = Path(o)
-            result, eqr, _, _ = run(input, o, version)
-            print_results(o / input.stem / "a.py", o / input.stem / "b.py", result, eqr)
+            result, eqr, infilename, _ = run(input, o, version)
+            if jsonout:
+                print(json.dumps(equivalence_report_json(infilename, result, eqr)))
+            else:
+                print_results(o / input.stem / "a.py", o / input.stem / "b.py", result, eqr)
     else:
         if not output:
             out_dir = get_unused(prefix / str(version) / input.stem)

pylingual/control_flow_reconstruction/templates/Exception.py

@@ -11,6 +11,7 @@ from ..utils import (
     condense_mapping,
     defer_source_to,
     with_instructions,
+    without_instructions,
     ending_instructions,
     exact_instructions,
     no_back_edges,
@@ -294,6 +295,21 @@ class TryFinally3_11(ControlFlowTemplate):
 
         return list(chain(s, self.line("finally:"), in_finally, after))
 
+      
+
+class Except3_9(ControlFlowTemplate):
+    @classmethod
+    @override
+    def try_match(cls, cfg, node) -> ControlFlowTemplate | None:
+        if [x.opname for x in node.get_instructions()] == ["RERAISE"]:
+            return node
+        if x := ExceptExc3_9.try_match(cfg, node):
+            return x
+        if x := BareExcept3_9.try_match(cfg, node):
+            return x
+        if isinstance(node, Except3_9):
+            return node
+        
 
 class Except3_9(ControlFlowTemplate):
     @classmethod
@@ -631,8 +647,8 @@ class TryElse3_6(ControlFlowTemplate):
     template = T(
         try_header=~N("try_body").with_cond(exact_instructions("SETUP_EXCEPT"), exact_instructions("SETUP_FINALLY")),
         try_body=N("try_footer.", None, "except_body"),
-        try_footer=~N("else_body").with_in_deg(1),
-        except_body=~N("tail.").with_in_deg(1).of_subtemplate(Except3_6),
+        try_footer=~N("else_body").with_in_deg(1),  
+        except_body=~N("tail.").with_in_deg(1).of_subtemplate(Except3_6).with_cond(without_instructions("RETURN_VALUE")),
         else_body=~N("tail.").with_in_deg(1),
         tail=N.tail(),
     )
@@ -661,6 +677,36 @@ class TryElse3_6(ControlFlowTemplate):
             {else_body}
         """
 
+@register_template(0, 0, (3, 6), (3, 7), (3, 8))
+class ReturnFinally3_6(ControlFlowTemplate):
+    template = T(
+        try_header=~N("try_body").with_cond(exact_instructions("SETUP_FINALLY")),
+        try_body=N(None, None, "fail_body").with_cond(with_instructions("LOAD_CONST","RETURN_VALUE")),
+        fail_body=~N("tail."),
+        tail=N.tail(),
+    )
+
+    try_match = revert_on_fail(
+        make_try_match(
+            {
+                EdgeKind.Fall: "tail",
+            },
+            "try_header",
+            "try_body",
+            "fail_body",
+        )
+    )
+
+    @to_indented_source
+    def to_indented_source():
+        """
+        {try_header}
+        try:
+            {try_body}
+        finally:
+            {fail_body}
+        """
+
 
 class BareExcept3_6(Except3_6):
     template = T(
@@ -688,16 +734,17 @@ class TryFinally3_6(ControlFlowTemplate):
         try_header=N("try_body"),
         try_body=N("finally_body", None, "fail_body"),
         finally_body=~N("fail_body").with_in_deg(1).with_cond(no_back_edges),
-        fail_body=N("tail.").with_cond(with_instructions("POP_TOP", "END_FINALLY"), with_instructions("DELETE_SUBSCR", "END_FINALLY")),
+        fail_body=N("tail.").with_cond(with_instructions("POP_TOP", "END_FINALLY"), with_instructions("LOAD_CONST", "RETURN_VALUE"), with_instructions("DELETE_SUBSCR", "END_FINALLY")),
         tail=N.tail(),
     )
     template2 = T(
-        try_except=N("finally_tail", None, "fail_body").of_type(TryElse3_6, Try3_6),
+        try_except=N("finally_tail", None, "fail_body").of_type(TryElse3_6, Try3_6, ReturnFinally3_6),
         finally_tail=N("finally_body", None, "fail_body"),
         finally_body=~N("fail_body").with_in_deg(1).with_cond(no_back_edges),
-        fail_body=N("tail.").with_cond(with_instructions("POP_TOP", "END_FINALLY")),
+        fail_body=N("tail.").with_cond(with_instructions("POP_TOP", "END_FINALLY"),with_instructions("LOAD_CONST", "RETURN_VALUE")),
         tail=N.tail(),
     )
+    
 
     cutoff: int
 
@@ -732,3 +779,5 @@ class TryFinally3_6(ControlFlowTemplate):
             after = []
 
         return list(chain(header, self.line("try:"), body, self.line("finally:"), in_finally, after))
+
+      

test/Conditional.py

@@ -0,0 +1,266 @@
+def a_if():
+    if a > b:
+        print(1)
+
+def b_elif():
+    if a > b:
+        print(1)
+    elif a == b:
+        print(2)
+
+def c_else():
+    if a > b:
+        print(1)
+    else:
+        print(2)
+
+def d_elif_else():
+    if a > b:
+        print(1)
+    elif a == b:
+        print(2)
+    else:
+        print(3)
+
+def e_oneline():
+    print(1) if a > b else print(2) if a == b else print(3)
+
+def a1_if_and():
+    if a > b and c > d:
+        print(1)
+
+def b1_elif_and():
+    if a > b and c > d:
+        print(1)
+    elif a == b:
+        print(2)
+
+def c1_else_and():
+    if a > b and c > d:
+        print(1)
+    else:
+        print(2)
+
+def d1_elif_else_and():
+    if a > b and c > d:
+        print(1)
+    elif a == b:
+        print(2)
+    else:
+        print(3)
+
+def a2_if_or():
+    if a > b or c > d:
+        print(1)
+
+def b2_elif_or():
+    if a > b or c > d:
+        print(1)
+    elif a == b:
+        print(2)
+
+def c2_else_or():
+    if a > b or c > d:
+        print(1)
+    else:
+        print(2)
+
+def d2_elif_else_or():
+    if a > b or c > d:
+        print(1)
+    elif a == b:
+        print(2)
+    else:
+        print(3)
+
+def a3_if_not():
+    if not a > b:
+        print(1)
+
+def b3_elif_not():
+    if not a > b:
+        print(1)
+    elif a == b:
+        print(2)
+
+def c3_else_not():
+    if not a > b:
+        print(1)
+    else:
+        print(2)
+
+def d3_elif_else_not():
+    if not a > b:
+        print(1)
+    elif a == b:
+        print(2)
+    else:
+        print(3)
+
+def e_nested_if():
+    if a > b:
+        print(1)
+        if a == b:
+            print(2)
+        else:
+            print(3)
+    else:
+        print(4)
+
+def e_nested_else():
+    if a > b:
+        print(1)
+    else:
+        print(2)
+        if a == b:
+            print(3)
+        else:
+            print(4)
+
+def f_if_pass():
+    if a > b:
+        pass
+
+
+def a_nofallthru_if():
+    if a > b:
+        print(1)
+    print("end")
+
+def b_nofallthru_elif():
+    if a > b:
+        print(1)
+    elif a == b:
+        print(2)
+    print("end")
+
+def c_nofallthru_else():
+    if a > b:
+        print(1)
+    else:
+        print(2)
+    print("end")
+
+def d_nofallthru_elif_else():
+    if a > b:
+        print(1)
+    elif a == b:
+        print(2)
+    else:
+        print(3)
+    print("end")
+
+def e_nofallthru_oneline():
+    print(1) if a > b else print(2) if a == b else print(3)
+    print("end")
+
+def a_nofallthru1_if_and():
+    if a > b and c > d:
+        print(1)
+    print("end")
+
+def b_nofallthru1_elif_and():
+    if a > b and c > d:
+        print(1)
+    elif a == b:
+        print(2)
+    print("end")
+
+def c_nofallthru1_else_and():
+    if a > b and c > d:
+        print(1)
+    else:
+        print(2)
+    print("end")
+
+def d_nofallthru1_elif_else_and():
+    if a > b and c > d:
+        print(1)
+    elif a == b:
+        print(2)
+    else:
+        print(3)
+    print("end")
+
+def a_nofallthru2_if_or():
+    if a > b or c > d:
+        print(1)
+    print("end")
+
+def b_nofallthru2_elif_or():
+    if a > b or c > d:
+        print(1)
+    elif a == b:
+        print(2)
+    print("end")
+
+def c_nofallthru2_else_or():
+    if a > b or c > d:
+        print(1)
+    else:
+        print(2)
+    print("end")
+
+def d_nofallthru2_elif_else_or():
+    if a > b or c > d:
+        print(1)
+    elif a == b:
+        print(2)
+    else:
+        print(3)
+    print("end")
+
+def a_nofallthru3_if_not():
+    if not a > b:
+        print(1)
+    print("end")
+
+def b_nofallthru3_elif_not():
+    if not a > b:
+        print(1)
+    elif a == b:
+        print(2)
+    print("end")
+
+def c_nofallthru3_else_not():
+    if not a > b:
+        print(1)
+    else:
+        print(2)
+    print("end")
+
+def d_nofallthru3_elif_else_not():
+    if not a > b:
+        print(1)
+    elif a == b:
+        print(2)
+    else:
+        print(3)
+    print("end")
+
+def e_nofallthru_nested_if():
+    if a > b:
+        print(1)
+        if a == b:
+            print(2)
+        else:
+            print(3)
+    else:
+        print(4)
+    print("end")
+
+def e_nofallthru_nested_else():
+    if a > b:
+        print(1)
+    else:
+        print(2)
+        if a == b:
+            print(3)
+        else:
+            print(4)
+    print("end")
+
+def f_nofallthru_if_pass():
+    if a > b:
+        pass
+    print("end")

test/Generator.py

@@ -0,0 +1,66 @@
+import asyncio
+
+# Test 1: Simple async generator
+async def gen1():
+    yield 1
+    yield 2
+
+# Test 2: Async generator with await
+async def gen2():
+    await asyncio.sleep(0)
+    yield "done"
+
+# Test 3: Async generator using a loop
+async def gen3():
+    for i in range(3):
+        yield i
+
+# Test 4: Async generator with async for loop (consuming another async generator)
+async def gen4():
+    async for x in gen3():
+        yield x
+
+# Test 5: Async generator with async with
+class DummyContext:
+    async def __aenter__(self): return self
+    async def __aexit__(self, exc_type, exc, tb): pass
+
+async def gen5():
+    async with DummyContext():
+        yield "inside context"
+
+# Test 6: Async generator that returns (implicitly ends)
+async def gen6():
+    yield "hello"
+    return  # Ends the generator
+
+# Test 7: Nested yield and await
+async def gen7():
+    yield await asyncio.sleep(0, result="nested")
+
+# Test 8: Function using 'yield' but not 'async def' (should be a regular generator)
+def regular_gen():
+    yield "normal generator"
+
+# Test 9: Coroutine consuming async generator
+async def consume_gen():
+    async for item in gen2():
+        pass
+
+# Test 10: Calling an async generator (should return an async generator object)
+g = gen1()
+assert hasattr(g, '__anext__')  # Just to test that it's an async generator object
+
+# Test 11: Async generator with try/finally
+async def gen11():
+    try:
+        yield "try"
+    finally:
+        await asyncio.sleep(0)
+
+# Test 12: Async generator with exception handling
+async def gen12():
+    try:
+        raise ValueError("fail")
+    except ValueError:
+        yield "handled"

test/Loop.py

@@ -0,0 +1,260 @@
+# FOR LOOP TESTS
+
+def a_for_over_list():
+    for x in [1, 2, 3]:
+        print("for over list")
+
+def b_for_over_tuples():
+    for a, b in [(1, 2), (3, 4)]:
+        print("tuples")
+
+def c_for_else():
+    for i in range(3):
+        print("for body")
+    else:
+        print("for else")
+
+def d_for_with_break():
+    for x in range(10):
+        if x == 5:
+            print("breaking")
+            break
+
+def e_for_with_continue():
+    for x in range(5):
+        if x % 2 == 0:
+            print("continuing")
+            continue
+        print("after continue")
+
+def f_nested_for_loops():
+    for i in range(2):
+        for j in range(3):
+            print(f"nested {i},{j}")
+
+def g_for_with_try_except():
+    for x in range(2):
+        try:
+            print("try block")
+        except Exception:
+            print("except block")
+
+def h_for_with_with_statement():
+    for _ in range(1):
+        with a:
+            print("inside with")
+
+def i_for_with_function_call_iterable():
+    def get_items():
+        return [1, 2, 3]
+    for item in get_items():
+        print(f"item: {item}")
+
+def j_for_with_empty_body_ellipsis():
+    for _ in range(3): ...
+
+
+def k_while_true_with_break():
+    while True:
+        print("while true")
+        break
+
+def l_while_with_else():
+    i = 0
+    while i < 3:
+        print(f"looping {i}")
+        i += 1
+    else:
+        print("while else")
+
+def m_while_with_continue():
+    i = 0
+    while i < 5:
+        i += 1
+        if i % 2 == 0:
+            print("continue")
+            continue
+        print("after continue")
+
+def n_while_with_break():
+    i = 0
+    while True:
+        print("break in while")
+        break
+
+def o_nested_while_loops():
+    i = 0
+    while i < 2:
+        j = 0
+        while j < 2:
+            print(f"nested while {i},{j}")
+            j += 1
+        i += 1
+
+def p_while_with_try_except():
+    while True:
+        try:
+            print("try in while")
+        except:
+            print("except in while")
+
+def q_while_with_with_statement():
+    while True:
+        with a:
+            print("inside while with")
+
+def r_for_inside_while():
+    while True:
+        for x in [1, 2]:
+            print("for in while")
+
+def s_while_inside_for():
+    for _ in range(1):
+        while True:
+            print("while in for")
+            break
+
+def t_while_with_empty_body_ellipsis():
+    while True: ...
+
+
+
+
+
+
+def a_nofallthru_for_over_list():
+    for x in [1, 2, 3]:
+        print("for over list")
+    print("end")
+
+def b_nofallthru_for_over_tuples():
+    for a, b in [(1, 2), (3, 4)]:
+        print("tuples")
+    print("end")
+
+def c_nofallthru_for_else():
+    for i in range(3):
+        print("for body")
+    else:
+        print("for else")
+    print("end")
+
+def d_nofallthru_for_with_break():
+    for x in range(10):
+        if x == 5:
+            print("breaking")
+            break
+    print("end")
+
+def e_nofallthru_for_with_continue():
+    for x in range(5):
+        if x % 2 == 0:
+            print("continuing")
+            continue
+        print("after continue")
+    print("end")
+
+def f_nofallthru_nested_for_loops():
+    for i in range(2):
+        for j in range(3):
+            print(f"nested {i},{j}")
+    print("end")
+
+def g_nofallthru_for_with_try_except():
+    for x in range(2):
+        try:
+            print("try block")
+        except Exception:
+            print("except block")
+    print("end")
+
+def h_nofallthru_for_with_with_statement():
+    for _ in range(1):
+        with a:
+            print("inside with")
+    print("end")
+
+def i_nofallthru_for_with_function_call_iterable():
+    def g_nofallthruet_items():
+        return [1, 2, 3]
+    for item in get_items():
+        print(f"item: {item}")
+    print("end")
+
+def j_nofallthru_for_with_empty_body_ellipsis():
+    for _ in range(3): ...
+    print("end")
+
+
+def k_nofallthru_while_true_with_break():
+    while True:
+        print("while true")
+        break
+    print("end")
+
+def l_nofallthru_while_with_else():
+    i = 0
+    while i < 3:
+        print(f"looping {i}")
+        i += 1
+    else:
+        print("while else")
+    print("end")
+
+def m_nofallthru_while_with_continue():
+    i = 0
+    while i < 5:
+        i += 1
+        if i % 2 == 0:
+            print("continue")
+            continue
+        print("after continue")
+    print("end")
+
+def n_nofallthru_while_with_break():
+    i = 0
+    while True:
+        print("break in while")
+        break
+    print("end")
+
+def o_nofallthru_nested_while_loops():
+    i = 0
+    while i < 2:
+        j = 0
+        while j < 2:
+            print(f"nested while {i},{j}")
+            j += 1
+        i += 1
+    print("end")
+
+def p_nofallthru_while_with_try_except():
+    while True:
+        try:
+            print("try in while")
+        except:
+            print("except in while")
+    print("end")
+
+def q_nofallthru_while_with_with_statement():
+    while True:
+        with a:
+            print("inside while with")
+    print("end")
+
+def r_nofallthru_for_inside_while():
+    while True:
+        for x in [1, 2]:
+            print("for in while")
+    print("end")
+
+def s_nofallthru_while_inside_for():
+    for _ in range(1):
+        while True:
+            print("while in for")
+            break
+    print("end")
+
+def t_nofallthru_while_with_empty_body_ellipsis():
+    while True: ...
+    print("end")

test/TryExcept.py

@@ -256,7 +256,7 @@ def k_TryExceptFinallyBare():
     finally:
         print(3)
     print(4)
-
+    
 def l_TryExceptFinallyBareSpecific():
    try:
        print(1)
@@ -383,6 +383,7 @@ def w_TryExceptRaise():
         print(2)
         raise Exc
 
+
 def x_TryExceptRaiseMulti():
     try:
         print(1)
@@ -440,6 +441,99 @@ def ab_TryExceptReturnNamed():
         return 2
     except A as a:
         print(2)
+        raise Exc
+    except b:
+        print(3)
+        raise Exc
+
+def TryEmptryFinally():
+    try:
+        pass
+    finally:
+        print(1)
+
+def TryMultiple():
+    try:
+        print(1)
+    except:
+        print(2)
+
+    try:
+        print(3)
+    except:
+        print(4)
+
+def TryExceptElseTry():
+    try:
+        print(1)
+    except:
+        print(2)
+    else:
+        try:
+            print(3)
+        except:
+            print(4)
+
+def TryFinallyNestedExcept():
+    try:
+        print(1)
+    finally:
+        try:
+            print(2)
+        except:
+            print(3)
+
+def TryExceptTuple():
+    try:
+        print(1)
+    except (A, B):
+        print(2)
+
+def TryFinallyReturn():
+    try:
+        print(1)
+    finally:
+        return 2
+
+def TryReturnFinally():
+    try:
+        return 1
+    finally:
+        print(2)
+
+def TryReturnFinallyReturn():
+    try:
+        return 1
+    finally:
+        return 2
+    
+def TryExceptRaise():
+    try:
+        print(1)
+        return 2
+    except:
+        raise Exception()
+    
+'''
+def TryExceptReturnFinally():
+    try:
+        raise Exception()
+    except:
+        print(1)
+        return 2
+    finally:
+        print(3)
+'''
+    
+'''
+def TryFinallyRaise():
+    try:
+        print(1)
+        return 2
+    finally:
+        raise Exception()
+'''
+
 
 def ab1_TryExceptReturnNamed():
     try:
@@ -460,4 +554,5 @@ def ad_TryFinallyBare():
     try:
         print(1)
     finally:
-        print(2)
+        print(2)
+