commit 0f9ced15287d1c705bdde54c47b43e602b10e810 Author: Joel Flores Date: Fri Mar 7 16:44:23 2025 -0600 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f4eb6f4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +dataset/ +venv/ +*.pyc +.idea/ +.python-version +poetry.toml +test/ +.DS_Store +results/ +.vscode/ +errors/ +logs/ +segmentation_test_cases/ +__pycache__/ +.env +mise.toml +dist/ +decompiled_*/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8460380 --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +# PyLingual - Python Decompiler for 3.6+ + +PyLingual is a CPython bytecode decompiler supporting all released Python versions since 3.6. For information about the design and implementation of PyLingual, please refer to our [research paper](https://www.computer.org/csdl/proceedings-article/sp/2025/223600a052/21B7QZB86cg). + +PyLingual can be run through our [web service](https://pylingual.io) or run locally. + +This codebase is optimized for readability and future extension, so there may initially be some control flow accuracy regression compared to the version hosted on the web service. + +## Requirements + +- Python 3.11 or higher + +### Compiling bytecode + +Some parts of PyLingual require the ability to compile bytecode in a different Python version (equivalence check and model training). For this, you will need the following: + +- [pyenv](https://github.com/pyenv/pyenv) with all Python versions you want to compile to +- Unix-like operating system (pyenv does not support Windows) + +## Setup + +Install from source, using [Poetry](https://python-poetry.org/): + +```sh +git clone https://github.com/syssec-utd/pylingual +cd pylingual +python -m venv venv +source venv/bin/activate +pip install poetry +poetry install +``` + +## Usage + +``` +Usage: pylingual [OPTIONS] [FILES]... + + End to end pipeline to decompile Python bytecode into source code. + +Options: + -o, --out-dir PATH The directory to export results to. + -c, --config-file PATH Config file for model information. + -v, --version VERSION Python version of the .pyc, default is auto + detection. + -k, --top-k INT Maximum number of additional segmentations to + consider. + -q, --quiet Suppress console output. + --trust-lnotab Use the lnotab for segmentation instead of the + segmentation model. + --init-pyenv Install pyenv before decompiling. + -h, --help Show this message and exit. +``` + +## Demo + +![demo gif](demo.gif) + +## Support + +If you have any issues for installing and using PyLingual, please create an issue or send your message via our support email at pylingual@gmail.com. diff --git a/demo.gif b/demo.gif new file mode 100644 index 0000000..cc6a38d Binary files /dev/null and b/demo.gif differ diff --git a/model_training/README.md b/model_training/README.md new file mode 100644 index 0000000..9cfc0ce --- /dev/null +++ b/model_training/README.md @@ -0,0 +1,48 @@ +# Model Training + +PyLingual's accuracy is dependent on having accurate segmentation and statement models [^1]. The segmentation model divides a list of bytecode instructions into groups for each source instruction. The statement model transforms each group of instructions into source code. The instructions for training these models is as follows: + +## Dataset generation + +First install [pyenv](https://github.com/pyenv/pyenv) and the required Python versions for the dataset. Create a dataset JSON file based off the sample (`sample_jsons/py36-sample-data.json`). + +The dataset directory should be structured like so, with only one `.py` file per directory: + +``` +dataset +├── 0 +│   └── file.py +├── 1 +│   └── file.py +... +├── 999 +│   └── file.py +└── 1000 + └── file.py +``` + +The names of the inner directories and files do not matter. Then create the dataset: + +``` +python prepare_dataset.py +``` + +## Segmentation model + +Create a segmentation model JSON file based off the sample (`sample_jsons/py36-sample-segmentation.json`). Then train the model: + +``` +python train_models.py --segmentation +``` + +## Statement model + +Create a statement model JSON file based off the sample (`sample_jsons/py36-sample-statement.json`). Then train the model: + +``` +python train_models.py --statement +``` + +Once models are trained, update `../pylingual/decompiler_config.yaml` or create a separate config file by replacing the old models with the newly trained ones. + +[^1]: [pylingual models](https://huggingface.co/syssec-utd). diff --git a/model_training/dataset_generation/DatasetDescription.py b/model_training/dataset_generation/DatasetDescription.py new file mode 100644 index 0000000..5513270 --- /dev/null +++ b/model_training/dataset_generation/DatasetDescription.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +import pathlib + +from typing import Tuple, List + + +@dataclass +class DataRequest: + name: str + source_path: pathlib.Path + num_train: int + num_test: int + num_valid: int + + @property + def total_files(self): + return self.num_train + self.num_test + self.num_valid + + def __post_init__(self): + self.source_path = pathlib.Path(self.source_path) + if not self.source_path.exists(): + raise FileNotFoundError(f"{self.source_path} for DataRequest {self.name} does not exist") + + if self.num_train < 0: + raise ValueError(f"Training sample count for DataRequest {self.name} must be non-negative") + if self.num_test < 0: + raise ValueError(f"Testing sample count for DataRequest {self.name} must be non-negative") + if self.num_valid < 0: + raise ValueError(f"Validation sample count for DataRequest {self.name} must be non-negative") + + +@dataclass +class DatasetDescription: + name: str + version: Tuple[int, int] + save_to_dir: pathlib.Path + huggingface_user: str + data_requests: List[DataRequest] + + @property + def code_dir(self): + return self.save_to_dir / self.name / "code" + + @property + def csv_dir(self): + return self.save_to_dir / self.name / "csv" + + def __post_init__(self): + self.save_to_dir = pathlib.Path(self.save_to_dir) + self.version = tuple(self.version) diff --git a/model_training/dataset_generation/__init__.py b/model_training/dataset_generation/__init__.py new file mode 100644 index 0000000..af9b642 --- /dev/null +++ b/model_training/dataset_generation/__init__.py @@ -0,0 +1,3 @@ +from .create_code_dataset import transfer_and_compile_file + +__all__ = ["transfer_and_compile_file"] diff --git a/model_training/dataset_generation/bytecode2csv.py b/model_training/dataset_generation/bytecode2csv.py new file mode 100644 index 0000000..130751b --- /dev/null +++ b/model_training/dataset_generation/bytecode2csv.py @@ -0,0 +1,216 @@ +import csv +import itertools +import logging +import multiprocessing +import pathlib +import re +import signal +from typing import Callable, Tuple + +import tqdm +from pylingual.editable_bytecode import PYCFile + +from pylingual.masking.ast_masker import DUMMY_DECORATOR +from pylingual.masking.model_disasm import fix_jump_targets +from .DatasetDescription import DataRequest +from pylingual.masking.model_disasm import create_global_masker, mask_source + +bytecode_separator = " " +source_seperator = " " +CSV_SGMT_HEADER = ["source", "bytecode", "boundary", "file"] +CSV_STMT_HEADER = ["source", "bytecode", "file"] + + +def create_csv_dataset(code_dataset_path: pathlib.Path, csv_dataset_path: pathlib.Path, data_requests: list[DataRequest], logger: logging.Logger = None): + progress_bar = tqdm.tqdm(total=sum([request.total_files for request in data_requests])) + for split in ("train", "test", "valid"): + if logger: + logger.info(f"Converting the {split} split to CSV...") + write_csvs(code_dataset_path / split, csv_dataset_path / split, logger, progress_bar=progress_bar) + + +def write_csvs(source_path: pathlib.Path, csv_output_path: pathlib.Path, logger: logging.Logger = None, max_csv_rows: int = 30000, progress_bar: tqdm.tqdm = None): + # validate output directory + if csv_output_path.exists(): + if not csv_output_path.is_dir(): + raise OSError("CSV output path is not a directory") + else: + csv_output_path.mkdir(parents=True) + + ##### csv write wrappers to preserve csv row limit + + def csv_writer(file_prefix: str, csv_header: list) -> Callable: + out_dir = csv_output_path.joinpath(file_prefix) + out_dir.mkdir(exist_ok=True) + + for csv_idx in itertools.count(): + new_path = out_dir.joinpath(f"{file_prefix}_{csv_idx}.csv") + new_path.touch() + if logger: + logger.info(f"Creating new csv {new_path.resolve()}...") + with new_path.open(mode="w") as csv_file: + writer = csv.writer(csv_file) + writer.writerow(csv_header) + for writer in itertools.repeat(writer, max_csv_rows): + yield writer.writerow + + segmentation_writer = csv_writer("segmentation", CSV_SGMT_HEADER) + statement_writer = csv_writer("statement", CSV_STMT_HEADER) + + # create dirs + code_dirs = (child for child in source_path.iterdir() if child.is_dir()) + + def bytecode2csv_args(): + for dir in code_dirs: + py_path = next(dir.glob("*.py"), None) + pyc_path = next(dir.glob("*.pyc"), None) + if None in (py_path, pyc_path): + logging.debug(f"PY or PYC file not found in {dir}") + continue + else: + yield (py_path, pyc_path) + + num_fails = 0 + with multiprocessing.Pool() as pool: + for result in pool.imap_unordered(bytecode2csv_exception_wrapper, bytecode2csv_args()): + if isinstance(result, Exception): + num_fails += 1 + logger.debug(f"DIR: {dir}\nERR: {result}\nTYPE ERR: {type(result)}\n") + continue + + (segmentation_rows, statement_rows) = result + for row, writerow in zip(segmentation_rows, segmentation_writer): + writerow(row) + for row, writerow in zip(statement_rows, statement_writer): + writerow(row) + + if progress_bar: + progress_bar.update() + progress_bar.set_postfix({"num_fails": num_fails}) + + logger.info(f"NUMBER OF FAILS !!! {num_fails}") + + +def timeout_handler(signum, frame): + raise TimeoutError() + + +def bytecode2csv_exception_wrapper(paths=Tuple[pathlib.Path, pathlib.Path]) -> Tuple[list, list] | Exception: + signal.signal(signal.SIGALRM, timeout_handler) + try: + signal.alarm(30) # set 30 second timeout + results = bytecode2csv(*paths) + signal.alarm(0) # success; disable timer + return results + except Exception as error: + signal.alarm(0) # disable timer in case another exception triggered the fail + return Exception(f"{type(error)}: {error} in file {paths}") + + +def bytecode2csv(py_path: pathlib.Path, pyc_path: pathlib.Path) -> tuple[list, list]: + """Creates segmentation and statement csv rows for given bytecode and source file""" + segmentation_rows = [] + statement_rows = [] + + pyc = PYCFile(str(pyc_path.resolve())) + if pyc.version == (3, 10): + pyc.replace_duplicated_returns10(py_path.read_text().split("\n")) + elif pyc.version == (3, 12): + pyc.replace_duplicated_returns12(py_path.read_text().split("\n")) + global_masker = create_global_masker(pyc) + + masked_source_text = mask_source(py_path, global_masker, pyc.version) + masked_source_lines = masked_source_text.split("\n") + + # filter out dummy decorators added in <= 3.7 + dummy_lnos = [] + if pyc.version <= (3, 7): + # remove dummy decorators from bytecode' + pyc._patch_dummy_decorator(dummy_decorator_name=DUMMY_DECORATOR) + try: # if no functions are in source, then dummy will not exist + dummy_decorator_line = f"@{global_masker.mask(DUMMY_DECORATOR)}" + except KeyError: + dummy_decorator_line = None + dummy_lnos = [lno + 1 for lno, source in enumerate(masked_source_lines) if source.strip() == dummy_decorator_line] + + seen_lines = set() + + # create rows for each bytecode + for bc in pyc.iter_bytecodes(): + # we ignore comprehensions, hoisted later + if bc.is_comprehension: + continue + + # attempt to filter lines + lno_insts = bc.get_lno_insts(previously_seen_lines=seen_lines) + + # create line num : model disasm view of insts + lno_model_view_insts = {lno: [global_masker.get_model_view(inst) for inst in line_insts] for lno, line_insts in lno_insts.items()} + seen_lines.update(lno_model_view_insts.keys()) + + # segment source + if pyc.version <= (3, 7): + segmented_source_lines = [] + for line_num in lno_model_view_insts: + if not line_num: + segmented_source_lines.append("") + elif line_num in dummy_lnos: + segmented_source_lines.append(masked_source_lines[line_num].strip()) + else: + segmented_source_lines.append(masked_source_lines[line_num - 1].strip()) + else: + segmented_source_lines = [masked_source_lines[line_num - 1].strip() if line_num else "" for line_num in lno_model_view_insts.keys()] # -1 to convert from line num to index in array + + model_disasm_text = bytecode_separator.join(val for val in itertools.chain(*lno_model_view_insts.values())) + + if len(segmented_source_lines) != len(lno_model_view_insts): + raise ValueError("Length mismatch between segmented source and segmented bytecodes") + + # create bytecode segmentation + boundaries = [] + for bc_line in lno_model_view_insts.values(): + if len(bc_line) == 1: + bounds = "B" + elif len(bc_line) >= 2: + bounds = "B" + "I" * (len(bc_line) - 2) + "E" + else: + raise ValueError("Unexpected amount of bytecodes segmented into a line") + boundaries.extend(list(bounds)) + + # append rows + segmentation_rows.append([source_seperator.join(segmented_source_lines), model_disasm_text, boundaries, str(py_path)]) + for segmented_source, bytecodes in zip(segmented_source_lines, lno_model_view_insts.values()): + # skip empty lines + if not segmented_source or segmented_source == "None": + continue + # skip fillers + if segmented_source in ("pass", "...") and ("RETURN_VALUE" in bytecodes or "RETURN_CONST , None" in bytecodes): + continue + # skip string-only lines that aren't docstrings + if (segmented_source.startswith("'") or segmented_source.startswith('"')) and not any("__doc__" in b for b in bytecodes): + continue + if segmented_source.startswith("elif "): + segmented_source = segmented_source[2:] + + joined_bytecode = bytecode_separator.join(bytecodes) + + # DUCT-TAPE; skip samples where model has to guess masks + source_masks = set(re.findall(r"", segmented_source)) + bytecode_masks = set(re.findall(r"", joined_bytecode)) + if not source_masks <= bytecode_masks: + continue + + # normalize source mask order for statements + # replace mask values to start at 0 and count up + mask_regex = re.compile(r"(?<=)") + masks = mask_regex.findall(joined_bytecode) + mask_order = [x for i, x in enumerate(masks) if masks.index(x) == i] + normalized_mask_bytecode = mask_regex.sub(lambda x: str(mask_order.index(x.group(0))), joined_bytecode) + normalized_mask_source = mask_regex.sub(lambda x: str(mask_order.index(x.group(0))), segmented_source) + + # normalize jump targets + normalized_mask_bytecode = fix_jump_targets(normalized_mask_bytecode) + + statement_rows.append([normalized_mask_source, normalized_mask_bytecode, str(py_path)]) + + return (segmentation_rows, statement_rows) diff --git a/model_training/dataset_generation/create_code_dataset.py b/model_training/dataset_generation/create_code_dataset.py new file mode 100644 index 0000000..1afc03b --- /dev/null +++ b/model_training/dataset_generation/create_code_dataset.py @@ -0,0 +1,114 @@ +import itertools +import logging +import multiprocessing +import pathlib +import random +from typing import List, Optional, Set, Tuple + +import tqdm + +from .DatasetDescription import DataRequest +from pylingual.utils.generate_bytecode import compile_version +from .normalize_source import normalize_source +from pylingual.masking.ast_masker import add_dummy_decorators + + +def transfer_and_compile_file( + original_file: pathlib.Path, + destination_file: pathlib.Path, + version: Tuple[int, int], +) -> Optional[Exception]: + # copy over normalized source file + try: + normalized_source = normalize_source(original_file.read_text(), version=version, replace_docstrings=True) + + if version[:2] <= (3, 7): + normalized_source = add_dummy_decorators(normalized_source) + except Exception as err: + return err + + destination_file.parent.mkdir(parents=True, exist_ok=True) + destination_file.write_text(normalized_source) + + # compile the copied file with the given version + try: + compile_version( + destination_file.resolve(), + destination_file.with_suffix(".pyc").resolve(), + version, + ) + except Exception as err: + return err + + return None + + +def star_transfer_and_compile_file(args) -> Optional[Exception]: + return transfer_and_compile_file(*args) + + +# samples num_files files from the given directory +# expects the directory to have the structure +# source_dir -> identifier -> file.py +def sample_directory_splits( + data_request: DataRequest, +) -> Tuple[List[pathlib.Path], List[pathlib.Path], List[pathlib.Path]]: + all_files: Set[pathlib.Path] = set() + for identifier in data_request.source_path.iterdir(): + source_file = next(identifier.glob("*.py"), None) # get the first python file from the identifier + if source_file is not None: + all_files.add(source_file) + + # sample batches until we have enough files to satisfy the data requests + # this avoids running expensive tests on unsampled files + clean_sample: Set[pathlib.Path] = set() + while len(clean_sample) < data_request.total_files: + remaining_files = data_request.total_files - len(clean_sample) + sample_batch = random.sample(list(all_files), k=remaining_files) + # add the acceptable files to the sample and remove them from the population + to_add = set(candidate for candidate in sample_batch if candidate is not None) + clean_sample.update(to_add) + all_files -= to_add + + full_sample = iter(clean_sample) + + train = list(itertools.islice(full_sample, data_request.num_train)) + test = list(itertools.islice(full_sample, data_request.num_test)) + valid = list(itertools.islice(full_sample, data_request.num_valid)) + + return train, test, valid + + +def prepare_single_directory_transfer_args(data_request: DataRequest, target_dir: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]: + train, test, valid = sample_directory_splits(data_request) + + transfer_args = [] + for split_name, split_files in zip(("train", "test", "valid"), (train, test, valid)): + for source_file in split_files: + target_file = target_dir / split_name / f"{data_request.name}-{source_file.parent.name}" / source_file.name + transfer_args.append((source_file, target_file)) + + return transfer_args + + +# takes a dict of {: (num_train, num_test, num_valid)} and a target directory +# makes train, test, and split directories in the target directory with the normalized source files +def create_code_dataset( + data_requests: List[DataRequest], + target_dir: pathlib.Path, + version: Tuple[int, int], + logger: logging.Logger, +): + with multiprocessing.Pool() as pool: + # prepare a list of file transfers to execute + logger.info(f"Sampling {', '.join(str(req.source_path.resolve()) for req in data_requests)}...") + transfer_arg_lists = pool.starmap( + prepare_single_directory_transfer_args, + zip(data_requests, itertools.repeat(target_dir)), + ) + # execute the file transfers + versioned_transfer_arg_lists = [(source_file, target_file, version) for (source_file, target_file) in itertools.chain(*transfer_arg_lists)] + logger.info(f"Normalizing and Compiling {len(versioned_transfer_arg_lists)} files...") + for error in tqdm.tqdm(pool.imap_unordered(star_transfer_and_compile_file, versioned_transfer_arg_lists), total=len(versioned_transfer_arg_lists)): + if error is not None: + logger.debug(error) diff --git a/model_training/dataset_generation/normalize_source.py b/model_training/dataset_generation/normalize_source.py new file mode 100644 index 0000000..f9b684c --- /dev/null +++ b/model_training/dataset_generation/normalize_source.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import ast +import pathlib +import sys +from typing import Tuple + + +def version_str_to_tuple(version_str: str) -> tuple[int, int]: + # a version string is a string like 3.9.2 + versions = [int(version) for version in version_str.split(".")] + return tuple(versions[:2]) + + +# must be run in python 3.9 or later for ast.unparse() support +# version defaults to whatever version this script is running in; needs to be set explicitly for backwards compatibility +# ast only supports versions 3.4 and later +def normalize_source( + source: str, + version: Tuple[int, int] = sys.version_info[0:2], + replace_docstrings=False, +) -> str: + """ + Parse the source code into an AST, then convert back to source. + This has the following normalizing effects: + 1. whitespace is set according to the PEP standard + 2. each statement is on exactly one line + 3. # comments are removed (note: docstrings are not removed) + + :param str source: The source code to normalize + :param tuple version: The (Major, Minor) version of python to parse with; must be at least (3, 4); defaults to + same version as this script + :param bool replace_docstrings: Replace all docstrings with 'pass' + """ + tree = ast.parse(source, feature_version=version) + if replace_docstrings: + for node in ast.walk(tree): + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Str): + node.value.s = "pass" + return ast.unparse(tree) + + +def normalize_source_file( + source_file_path: str, + cleaned_suffix: str = "-cleaned", + version: tuple[int, int] = sys.version_info[0:2], +): + """ + Normalizes the source code in a given file, then saves it to a '-cleaned' version in the same directory + + :param str source_file_path: The absolute or relative path to the source .py file + :param str cleaned_suffix: The suffix to add to the cleaned file, typically left as default + :param tuple version: The (Major, Minor) version of python to parse with; must be at least (3, 4); defaults to + same version as this script + """ + + # add the cleaned_suffix to the output_path + input_path = pathlib.Path(source_file_path).resolve() + output_path = input_path.with_stem(f"{input_path.stem}{cleaned_suffix}") + + with open(input_path, "r") as source_file: + normalized_source = normalize_source(source_file.read(), version=version) + + with open(output_path, "w") as cleaned_file: + cleaned_file.write(normalized_source) + + return output_path diff --git a/model_training/dataset_generation/upload_raw_dataset.py b/model_training/dataset_generation/upload_raw_dataset.py new file mode 100644 index 0000000..e930d9e --- /dev/null +++ b/model_training/dataset_generation/upload_raw_dataset.py @@ -0,0 +1,60 @@ +from io import BytesIO +from typing import Dict, List, Literal + +from datasets import load_dataset +from huggingface_hub import HfApi + +from .DatasetDescription import DatasetDescription + +LOCAL_DATASET = Dict[Literal["train", "test", "valid"], List[str]] + + +def upload_single_dataset(data_files: LOCAL_DATASET, dataset_name: str, dataset_card: str): + local_datasets = load_dataset("csv", data_files=data_files) + local_datasets.push_to_hub(dataset_name, private=True) + + dataset_card_with_stats = dataset_card + f"\n\nDataset Statistics:\n\n```\n{local_datasets}\n```" + + api = HfApi() + api.upload_file( + path_or_fileobj=BytesIO(bytes(dataset_card_with_stats, "utf-8")), + path_in_repo="README.md", + repo_id=dataset_name, + repo_type="dataset", + ) + + +def upload_dataset_to_huggingface(dataset_description: DatasetDescription): + formatted_data_requests = "\n".join(f"{str(req.source_path.resolve())}: (train: {req.num_train}, test: {req.num_test}, valid: {req.num_valid})" for req in dataset_description.data_requests) + dataset_card = f""" +# {dataset_description.name} + +Created by the Syssec team @ UTD + +Dataset Composition: + +``` +{formatted_data_requests} +``` + +Python version: `{".".join(map(str, dataset_description.version))}` +""" + + splits: List[Literal["train", "test", "valid"]] = [ + "train", + "test", + "valid", + ] + + # collect data files + segmentation_data_files: LOCAL_DATASET = {} + statement_data_files: LOCAL_DATASET = {} + for split in splits: + segmentation_data_files[split] = [str(path.resolve()) for path in (dataset_description.csv_dir / split / "segmentation").glob("*.csv")] + statement_data_files[split] = [str(path.resolve()) for path in (dataset_description.csv_dir / split / "statement").glob("*.csv")] + + # upload datasets + segmentation_dataset_name = f"{dataset_description.huggingface_user}/segmentation-{dataset_description.name}" + upload_single_dataset(segmentation_data_files, segmentation_dataset_name, dataset_card) + statement_dataset_name = f"{dataset_description.huggingface_user}/statement-{dataset_description.name}" + upload_single_dataset(statement_data_files, statement_dataset_name, dataset_card) diff --git a/model_training/prepare_dataset.py b/model_training/prepare_dataset.py new file mode 100644 index 0000000..90fcf69 --- /dev/null +++ b/model_training/prepare_dataset.py @@ -0,0 +1,66 @@ +import json +import logging +import pathlib +from typing import Union +import click + +from dataset_generation.bytecode2csv import create_csv_dataset +from dataset_generation.create_code_dataset import create_code_dataset +from dataset_generation.DatasetDescription import DataRequest, DatasetDescription +from dataset_generation.upload_raw_dataset import upload_dataset_to_huggingface +from pylingual.utils.get_logger import get_logger + + +def get_dataset_description_from_arg_json(json_path: str, logger: Union[logging.Logger, None] = None) -> DatasetDescription: + json_file_path = pathlib.Path(json_path) + + if not json_file_path.exists(): + raise FileNotFoundError(f"{json_file_path} does not exist") + + if logger: + logger.info(f"Loading dataset description from {json_file_path}...") + + with json_file_path.open() as json_file: + dataset_description_dict = json.load(json_file) + + dataset_description_dict["data_requests"] = [DataRequest(**d) for d in dataset_description_dict["data_requests"]] + return DatasetDescription(**dataset_description_dict) + + +@click.command(help="Samples, splits, processes, and uploads a given dataset described by JSON.") +@click.argument("json_path", type=str) +def main(json_path: str): + logger = get_logger("prepare-dataset") + + dataset_description = get_dataset_description_from_arg_json(json_path, logger) + logger.debug(dataset_description) + + if dataset_description.code_dir.exists(): + raise FileExistsError(f"{dataset_description.code_dir} already exists! The dataset name is probably already taken.") + + logger.info("Creating code dataset...") + if not (dataset_description.data_requests and dataset_description.code_dir and dataset_description.version): + logger.error("Dataset description is missing required fields") + exit(1) + create_code_dataset( + dataset_description.data_requests, + dataset_description.code_dir, + dataset_description.version, + logger, + ) + + # create csv dataset + logger.info("Converting code dataset to csv...") + create_csv_dataset( + dataset_description.code_dir, + dataset_description.csv_dir, + dataset_description.data_requests, + logger, + ) + + logger.info(f"Uploading {dataset_description.name} to HuggingFace...") + upload_dataset_to_huggingface(dataset_description) + + +if __name__ == "__main__": + main() diff --git a/model_training/sample_jsons/py36-sample-data.json b/model_training/sample_jsons/py36-sample-data.json new file mode 100644 index 0000000..9486176 --- /dev/null +++ b/model_training/sample_jsons/py36-sample-data.json @@ -0,0 +1,24 @@ +{ + "name": "sample_dataset_name", + "version": [3, 6], + "save_to_dir": "./save_dir/", + "huggingface_user": "sample_user", + + "data_requests": + [ + { + "name": "dataset", + "source_path": "./dataset", + "num_train": 200, + "num_test": 200, + "num_valid": 200 + }, + { + "name": "dataset2", + "source_path": "./dataset2", + "num_train": 200, + "num_test": 200, + "num_valid": 200 + } + ] +} diff --git a/model_training/sample_jsons/py36-sample-segmentation.json b/model_training/sample_jsons/py36-sample-segmentation.json new file mode 100644 index 0000000..e2a81cc --- /dev/null +++ b/model_training/sample_jsons/py36-sample-segmentation.json @@ -0,0 +1,18 @@ +{ + "base_repo_name": "sample_user/sample_segmenter_name", + "dataset_repo_name": "sample_user/segmentation-sample_dataset_name", + "pretrained_mlm_repo_name": "", + "cache_dir": "./cache-dir/", + "max_token_length": 512, + "dataset_percentage": 100, + "mlm_training_parameters": { + "batch_size": 48, + "epochs": 2, + "learning_rate": 5e-5 + }, + "segmentation_training_parameters": { + "batch_size": 48, + "epochs": 2, + "learning_rate": 2e-5 + } +} diff --git a/model_training/sample_jsons/py36-sample-statement.json b/model_training/sample_jsons/py36-sample-statement.json new file mode 100644 index 0000000..ceb1d8c --- /dev/null +++ b/model_training/sample_jsons/py36-sample-statement.json @@ -0,0 +1,16 @@ +{ + "base_repo_name": "sample_user/sample_name", + "dataset_repo_name": "sample_user/statement-sample_dataset_name", + "tokenizer_repo_name": "sample_user/sample_name-tok", + "pretrained_seq2seq_repo_name": "Salesforce/codet5-base", + "cache_dir": "./cache-dir/", + "max_token_length": 256, + "dataset_percentage": 100, + "do_eval": true, + "fp16": true, + "statement_training_parameters": { + "batch_size": 24, + "epochs": 2, + "learning_rate": 2e-5 + } +} diff --git a/model_training/segmentation/SegmentationConfiguration.py b/model_training/segmentation/SegmentationConfiguration.py new file mode 100644 index 0000000..cad70c5 --- /dev/null +++ b/model_training/segmentation/SegmentationConfiguration.py @@ -0,0 +1,74 @@ +import json +import logging +import pathlib +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class TrainingParameters: + batch_size: int + epochs: int + learning_rate: float + + +@dataclass +class SegmentationConfiguration: + base_repo_name: str + dataset_repo_name: str + pretrained_mlm_repo_name: str + cache_dir: pathlib.Path + max_token_length: int + dataset_percentage: int + mlm_training_parameters: TrainingParameters + segmentation_training_parameters: TrainingParameters + + @property + def tokenizer_repo_name(self): + return self.base_repo_name + "-tokenizer" + + @property + def tokenizer_json_path(self): + return self.cache_dir / "tokenizers" / self.tokenizer_repo_name / "tokenizer.json" + + @property + def tokenized_dataset_repo_name(self): + return self.dataset_repo_name + "-tokenized" + + @property + def mlm_repo_name(self): + return self.base_repo_name + "-mlm" + + @property + def mlm_dir(self): + return self.cache_dir / "models" / self.mlm_repo_name + + @property + def segmenter_repo_name(self): + return self.base_repo_name + "-segmenter" + + @property + def segmenter_dir(self): + return self.cache_dir / "models" / self.segmenter_repo_name + + @property + def dataset_dir(self): + return self.cache_dir / "datasets" / self.dataset_repo_name + + def __post_init__(self): + self.cache_dir = pathlib.Path(self.cache_dir) + + +def parse_segmentation_config_json(json_file_path: pathlib.Path, logger: Optional[logging.Logger] = None) -> SegmentationConfiguration: + if not json_file_path.exists(): + raise FileNotFoundError(f"{json_file_path} does not exist") + + if logger: + logger.info(f"Loading model description from {json_file_path}...") + + with json_file_path.open() as json_file: + segmentation_config_dict = json.load(json_file) + + segmentation_config_dict["mlm_training_parameters"] = TrainingParameters(**segmentation_config_dict["mlm_training_parameters"]) + segmentation_config_dict["segmentation_training_parameters"] = TrainingParameters(**segmentation_config_dict["segmentation_training_parameters"]) + return SegmentationConfiguration(**segmentation_config_dict) diff --git a/model_training/segmentation/tokenize_seg.py b/model_training/segmentation/tokenize_seg.py new file mode 100644 index 0000000..aaba7e1 --- /dev/null +++ b/model_training/segmentation/tokenize_seg.py @@ -0,0 +1,152 @@ +import ast +import functools +import os +import pathlib +import click + +from datasets import load_dataset +from huggingface_hub import hf_hub_download +from SegmentationConfiguration import SegmentationConfiguration, parse_segmentation_config_json +from pylingual.segmentation.sliding_window import sliding_window +from transformers import PreTrainedTokenizerFast + +bytecode_separator = " " + + +def load_tokenizer(tokenizer_repo_name: str, cache_dir: pathlib.Path) -> PreTrainedTokenizerFast: + tokenizer_dir = cache_dir / "tokenizers" / tokenizer_repo_name + + tokenizer_file = hf_hub_download(repo_id=tokenizer_repo_name, filename="tokenizer.json", token=True, cache_dir=str(tokenizer_dir)) + tokenizer = PreTrainedTokenizerFast( + tokenizer_file=tokenizer_file, + unk_token="[UNK]", + pad_token="[PAD]", + cls_token="[CLS]", + sep_token="[SEP]", + mask_token="[MASK]", + ) + + return tokenizer + + +# we need to make sure we align all the labels with the proper words. +def align_labels_with_tokens(labels, word_ids): + label_names = ["B", "I", "E"] + id2label = {str(i): label for i, label in enumerate(label_names)} + label2id = {v: k for k, v in id2label.items()} + + new_labels = [] + current_word = None + for word_id in word_ids: + if word_id != current_word: + # Start of a new word! + current_word = word_id + label = -100 if word_id is None else int(label2id[labels[word_id]]) + new_labels.append(label) + elif word_id is None: + # Special token + new_labels.append(-100) + else: + # Same word as previous token + label = int(label2id[labels[word_id]]) + new_labels.append(label) + return new_labels + + +# the process function used for tokenize the dataset +def tokenize_and_align_labels(tokenizer: PreTrainedTokenizerFast, max_length: int, examples): + MAX_WINDOW_LENGTH = 512 + STEP_SIZE = 128 + + # parse the strings into lists to better work with the bytecode and boundaries + parsed_bc = [(codeobj.split(" "), ast.literal_eval(bounds)) for codeobj, bounds in zip(examples["bytecode"], examples["boundary"])] + + codeobj_tokens = [] + + # count the tokens for each bytecode instruction in a codeobj + for codeobj, bounds in parsed_bc: + token_list = [] + + for bc, bounds in zip(codeobj, bounds): + token_list.append(((bc, bounds), len(tokenizer(bc)[0]))) + + codeobj_tokens.append(token_list) + + windows = [sliding_window(codeobj, MAX_WINDOW_LENGTH, STEP_SIZE) for codeobj in codeobj_tokens] + + # remake examples using our windows + examples["boundary"] = [] + examples["bytecode"] = [] + + # go through each window + for window in windows: + for item in window: + # where we will temporarily store our bytecode and bounds + bytecode = [] + bounds = [] + + for bc in item[0]: + bytecode.append(bc[0]) + bounds.append(bc[1]) + + # append it into examples + examples["bytecode"].append(bytecode_separator.join(bytecode)) + examples["boundary"].append(str(bounds)) + + tokenized_inputs = tokenizer( + examples["bytecode"], + truncation=True, + max_length=max_length, + ) + + all_labels = examples["boundary"] + new_labels = [] + for i, labels in enumerate(all_labels): + labels = labels.replace("'", "").strip("][").split(", ") + word_ids = tokenized_inputs.word_ids(i) + labels_len = len(labels) + max_word_id = word_ids[-2] + # for those data might cause error due to the incorrect tokenization, we fix the data exceed-length issue and + # leave them here as some noisy data. + if max_word_id >= labels_len: + new_labels.append([-100] * max_word_id) + else: + new_labels.append(align_labels_with_tokens(labels, word_ids)) + + tokenized_inputs["labels"] = new_labels + + return tokenized_inputs + + +def tokenize_segmentation_dataset(config: SegmentationConfiguration): + raw_dataset = load_dataset(config.dataset_repo_name, token=True, cache_dir=str(config.dataset_dir)) + + tokenizer = load_tokenizer(config.tokenizer_repo_name, config.cache_dir) + prepped_tokenize_and_align_labels = functools.partial(tokenize_and_align_labels, tokenizer, config.max_token_length) + + # tokenize input dataset + column_names = raw_dataset["train"].column_names + tokenized_datasets = raw_dataset.map( + prepped_tokenize_and_align_labels, + batched=True, + remove_columns=column_names, + num_proc=os.cpu_count(), + desc="Tokenizing datasets", + ) + + tokenized_datasets.push_to_hub( + config.tokenized_dataset_repo_name, + private=True, + ) + + +@click.command(help="Script to tokenize the segmentation dataset given a segmentation json.") +@click.argument("json_path", type=str) +def main(json_path: str): + json_file_path = pathlib.Path(json_path) + segmentation_config = parse_segmentation_config_json(json_file_path) + tokenize_segmentation_dataset(segmentation_config) + + +if __name__ == "__main__": + main() diff --git a/model_training/segmentation/train_mlm.py b/model_training/segmentation/train_mlm.py new file mode 100644 index 0000000..989416b --- /dev/null +++ b/model_training/segmentation/train_mlm.py @@ -0,0 +1,195 @@ +import logging +import os +import pathlib +import click + +from datasets import load_dataset +from huggingface_hub import hf_hub_download, repo_exists +from SegmentationConfiguration import SegmentationConfiguration, parse_segmentation_config_json +from transformers import AutoModelForMaskedLM, DataCollatorForLanguageModeling, PreTrainedTokenizerFast, RobertaConfig, RobertaForMaskedLM, Trainer, TrainingArguments + +from pylingual.segmentation.sliding_window import sliding_window + +bytecode_separator = " " + + +def load_tokenizer(tokenizer_repo_name: str, cache_dir: pathlib.Path) -> PreTrainedTokenizerFast: + tokenizer_dir = cache_dir / "tokenizers" / tokenizer_repo_name + + tokenizer_file = hf_hub_download( + repo_id=tokenizer_repo_name, + filename="tokenizer.json", + token=True, + cache_dir=str(tokenizer_dir), + ) + tokenizer = PreTrainedTokenizerFast( + tokenizer_file=tokenizer_file, + unk_token="[UNK]", + pad_token="[PAD]", + cls_token="[CLS]", + sep_token="[SEP]", + mask_token="[MASK]", + ) + + return tokenizer + + +def load_tokenized_train_dataset( + dataset_repo_name: str, + tokenizer: PreTrainedTokenizerFast, + max_length: int, + cache_dir: pathlib.Path, +): + dataset_dir = cache_dir / "datasets" / dataset_repo_name + raw_dataset = load_dataset(dataset_repo_name, token=True, cache_dir=dataset_dir, split="train") + + # tokenize the input data + column_names = raw_dataset.column_names + + def tokenize(examples): + # sliding window compatibility + MAX_WINDOW_LENGTH = 512 + STEP_SIZE = 128 + + # parse the strings into lists to better work with the bytecode and boundaries + parsed_bc = [codeobj.split(" ") for codeobj in examples["bytecode"]] + + codeobj_tokens = [] + + # count the tokens for each bytecode instruction in a codeobj + for codeobj in parsed_bc: + token_list = [] + + for bytecode in codeobj: + token_list.append((bytecode, len(tokenizer(bytecode)[0]))) + + codeobj_tokens.append(token_list) + + windows = [sliding_window(codeobj, MAX_WINDOW_LENGTH, STEP_SIZE) for codeobj in codeobj_tokens] + + # remake examples using our windows + examples["bytecode"] = [] + + # go through each window + for window in windows: + for item in window: + # where we will temporarily store our bytecode and bounds + bytecode = [] + + for bc in item[0]: + bytecode.append(bc) + + # append to examples + examples["bytecode"].append(bytecode_separator.join(bytecode)) + + return tokenizer(examples["bytecode"], max_length=max_length, truncation=True) + + tokenized_dataset = raw_dataset.map( + tokenize, + batched=True, + remove_columns=column_names, + num_proc=os.cpu_count(), + desc="Tokenizing datasets", + ) + + return tokenized_dataset + + +def load_pretrained_mlm( + pretrained_mlm_repo_name: str, + tokenizer_embedding_length: int, + cache_dir: pathlib.Path, +) -> AutoModelForMaskedLM: + # load a basic pretrained BERT model + pretrained_mlm_dir = cache_dir / "models" / pretrained_mlm_repo_name + model = AutoModelForMaskedLM.from_pretrained(pretrained_mlm_repo_name, cache_dir=str(pretrained_mlm_dir)) + + # resize token embeddings to fit the model + model.resize_token_embeddings(tokenizer_embedding_length) + + return model + + +def initialize_untrained_mlm( + tokenizer_embedding_length: int, + max_token_length: int, +) -> RobertaForMaskedLM: + # initialize untrained RoBERTa model + # most configuration options set to match https://huggingface.co/microsoft/codebert-base/blob/main/config.json for direct comparison + model_config = RobertaConfig( + max_position_embeddings=max_token_length, # INPUT LENGTH LIMIT + vocab_size=tokenizer_embedding_length, + layer_norm_eps=1e-05, + type_vocab_size=1, + ) + model = RobertaForMaskedLM(model_config) + + return model + + +def train_mlm(config: SegmentationConfiguration): + if repo_exists(config.base_repo_name): + logging.error(f"{config.base_repo_name} has already exists") + exit(1) + + using_pretrained_model = bool(config.pretrained_mlm_repo_name) + # train model, for now the configuration comes from a regular T5 translation model. + training_args = TrainingArguments( + output_dir=str(config.mlm_dir), + num_train_epochs=config.mlm_training_parameters.epochs, + per_device_train_batch_size=config.mlm_training_parameters.batch_size, + save_steps=1000, + save_total_limit=5, + prediction_loss_only=True, + push_to_hub=True, + hub_model_id=config.mlm_repo_name, + hub_private_repo=True, + ddp_backend="nccl", + ddp_find_unused_parameters=using_pretrained_model, # only look for unused parameters in pretrained models + remove_unused_columns=False, + ) + + tokenizer = load_tokenizer(config.tokenizer_repo_name, config.cache_dir) + + # Set DataCollator for MLM task, set the probability of masking. + data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=True, mlm_probability=0.15) + + if using_pretrained_model: + pretrained_mlm = load_pretrained_mlm(config.pretrained_mlm_repo_name, len(tokenizer), config.cache_dir) + else: + pretrained_mlm = initialize_untrained_mlm(len(tokenizer), config.max_token_length + 2) + + tokenized_training_data = load_tokenized_train_dataset(config.dataset_repo_name, tokenizer, config.max_token_length, config.cache_dir) + + # Hugging face trainer: a Trainer class to fine-tune pretrained models + trainer = Trainer( + model=pretrained_mlm, + args=training_args, + data_collator=data_collator, + train_dataset=tokenized_training_data, + ) + + # Training + trainer.train() + + if int(os.environ["LOCAL_RANK"]) == 0: + # Save the model + trainer.save_model(config.mlm_dir) + + trainer.push_to_hub( + finetuned_from=config.pretrained_mlm_repo_name, + dataset=config.dataset_repo_name, + commit_message=f"Trained on {config.dataset_repo_name} using {config.tokenizer_repo_name}", + ) + + +@click.command(help="Training script for the masked language model pretraining for the segmentation model given a segmentation json.") +@click.argument("json_path", type=str) +def main(json_path: str): + json_file_path = pathlib.Path(json_path) + segmentation_config = parse_segmentation_config_json(json_file_path) + train_mlm(segmentation_config) + + +if __name__ == "__main__": + main() diff --git a/model_training/segmentation/train_seg.py b/model_training/segmentation/train_seg.py new file mode 100644 index 0000000..941b39d --- /dev/null +++ b/model_training/segmentation/train_seg.py @@ -0,0 +1,155 @@ +import logging +import os +import pathlib +import click + +import evaluate +import numpy as np +from datasets import ReadInstruction, load_dataset +from huggingface_hub import hf_hub_download, repo_exists +from SegmentationConfiguration import SegmentationConfiguration, parse_segmentation_config_json +from transformers import AutoModelForTokenClassification, DataCollatorForTokenClassification, PreTrainedTokenizerFast, Trainer, TrainingArguments + +# two dictionaries, id2label and label2id, which contain the mappings from ID to label and vice versa. +label_names = ["B", "I", "E"] +id2label = {str(i): label for i, label in enumerate(label_names)} +label2id = {v: k for k, v in id2label.items()} + + +# compute_metrics: evaluate metric for training and evaluation. +def compute_metrics(eval_preds): + metric = evaluate.load("seqeval") + logits, labels = eval_preds + predictions = np.argmax(logits, axis=-1) + + # Remove ignored index (special tokens) and convert to labels + # noqa: E741 + true_labels = [[label_names[l] for l in label if l != -100] for label in labels] + true_predictions = [[label_names[p] for (p, l) in zip(prediction, label) if l != -100] for prediction, label in zip(predictions, labels)] + all_metrics = metric.compute(predictions=true_predictions, references=true_labels) + return { + "precision": all_metrics["overall_precision"], + "recall": all_metrics["overall_recall"], + "f1": all_metrics["overall_f1"], + "accuracy": all_metrics["overall_accuracy"], + } + + +def load_tokenizer(tokenizer_repo_name: str, cache_dir: pathlib.Path) -> PreTrainedTokenizerFast: + tokenizer_dir = cache_dir / "tokenizers" / tokenizer_repo_name + + tokenizer_file = hf_hub_download( + repo_id=tokenizer_repo_name, + filename="tokenizer.json", + token=True, + cache_dir=str(tokenizer_dir), + ) + tokenizer = PreTrainedTokenizerFast( + tokenizer_file=tokenizer_file, + unk_token="[UNK]", + pad_token="[PAD]", + cls_token="[CLS]", + sep_token="[SEP]", + mask_token="[MASK]", + ) + + return tokenizer + + +def load_tokenized_train_and_valid_dataset(dataset_repo_name: str, cache_dir: pathlib.Path, dataset_percentage: int = 100): + dataset_dir = cache_dir / "datasets" / dataset_repo_name + # Load the tokenized dataset + tokenized_train_dataset = load_dataset( + dataset_repo_name, + token=True, + cache_dir=str(dataset_dir), + split=ReadInstruction("train", to=dataset_percentage, unit="%"), + ) + + tokenized_validation_dataset = load_dataset( + dataset_repo_name, + token=True, + cache_dir=str(dataset_dir), + split="valid", + ) + + return tokenized_train_dataset, tokenized_validation_dataset + + +def train_segmentation_model(config: SegmentationConfiguration): + if repo_exists(config.base_repo_name): + logging.error(f"{config.base_repo_name} has already exists") + exit(1) + # training arguments. + training_args = TrainingArguments( + output_dir=str(config.segmenter_dir), + overwrite_output_dir=True, + eval_strategy="epoch", + logging_strategy="epoch", + save_strategy="epoch", + learning_rate=config.segmentation_training_parameters.learning_rate, + num_train_epochs=config.segmentation_training_parameters.epochs, + per_device_train_batch_size=config.segmentation_training_parameters.batch_size, + save_steps=1000, + weight_decay=0.01, + fp16=True, + push_to_hub=True, + hub_model_id=config.segmenter_repo_name, + hub_private_repo=True, + ddp_backend="nccl", + ddp_find_unused_parameters=True, + save_total_limit=5, + ) + + # load a basic pretrained BERT model + model = AutoModelForTokenClassification.from_pretrained( + pretrained_model_name_or_path=config.mlm_repo_name, + id2label=id2label, + label2id=label2id, + token=True, + ) + + # Set DataCollator for DataCollatorForTokenClassification + tokenizer = load_tokenizer(config.tokenizer_repo_name, config.cache_dir) + data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, max_length=config.max_token_length) + + ( + tokenized_train_dataset, + tokenized_validation_dataset, + ) = load_tokenized_train_and_valid_dataset(config.tokenized_dataset_repo_name, config.cache_dir, config.dataset_percentage) + + # Hugging face trainer: a Trainer class to fine-tune pretrained models + trainer = Trainer( + model=model, + args=training_args, + data_collator=data_collator, + train_dataset=tokenized_train_dataset, + eval_dataset=tokenized_validation_dataset, + compute_metrics=compute_metrics, + tokenizer=tokenizer, + ) + + # Training + trainer.train() + + if int(os.environ["LOCAL_RANK"]) == 0: + # Save the model + trainer.save_model(str(config.segmenter_dir)) + + trainer.push_to_hub( + finetuned_from=config.mlm_repo_name, + dataset=config.tokenized_dataset_repo_name, + commit_message=f"Trained on {config.tokenized_dataset_repo_name} using {config.mlm_repo_name}", + ) + + +@click.command(help="Training script for the segmentation model given a segmentation json.") +@click.argument("json_path", type=str) +def main(json_path: str): + json_file_path = pathlib.Path(json_path) + segmentation_config = parse_segmentation_config_json(json_file_path) + train_segmentation_model(segmentation_config) + + +if __name__ == "__main__": + main() diff --git a/model_training/segmentation/train_tokenizer.py b/model_training/segmentation/train_tokenizer.py new file mode 100644 index 0000000..ba74c80 --- /dev/null +++ b/model_training/segmentation/train_tokenizer.py @@ -0,0 +1,96 @@ +import logging +import pathlib +import click + +from datasets import ReadInstruction, load_dataset +from huggingface_hub import HfApi, create_repo, repo_exists +from SegmentationConfiguration import SegmentationConfiguration, parse_segmentation_config_json +from tokenizers import Tokenizer, decoders, models, normalizers, pre_tokenizers, processors, trainers + +special_tokens = ["[UNK]", "[PAD]", "[CLS]", "[SEP]", "[MASK]"] + + +def get_untrained_tokenizer() -> Tokenizer: + # WordPiece tokenization for BERT. + tokenizer = Tokenizer(models.WordPiece(unk_token="[UNK]")) + + # The normalizer recognizes the accented characters and strip them out. + tokenizer.normalizer = normalizers.Sequence([normalizers.NFD(), normalizers.StripAccents()]) + + # The pre-tokenizer splits on tokens. + tokenizer.pre_tokenizer = pre_tokenizers.Split("", "removed") + + return tokenizer + + +def post_training_configuration(tokenizer: Tokenizer): + cls_token_id = tokenizer.token_to_id("[CLS]") + sep_token_id = tokenizer.token_to_id("[SEP]") + + # Set decoder for the tokenizer + tokenizer.decoder = decoders.WordPiece(prefix="##") + + # For the TemplateProcessor, we have to specify how to treat a single sentence and a pair of sentences. + tokenizer.post_processor = processors.TemplateProcessing( + single="[CLS]:0 $A:0 [SEP]:0", + pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1", + special_tokens=[("[CLS]", cls_token_id), ("[SEP]", sep_token_id)], + ) + + +def save_and_upload_tokenizer( + tokenizer: Tokenizer, + tokenizer_json_path: pathlib.Path, + tokenizer_repo_name: str, + dataset_name: str, +): + # save the tokenizer locally + tokenizer_json_path.parent.mkdir(parents=True, exist_ok=True) + tokenizer.save(str(tokenizer_json_path.resolve())) + + # upload tokenizer to huggingface + api = HfApi() + create_repo(tokenizer_repo_name, exist_ok=True, private=True) + api.upload_file( + path_in_repo="tokenizer.json", + path_or_fileobj=str(tokenizer_json_path.resolve()), + repo_id=tokenizer_repo_name, + commit_message=f"Trained tokenizer using {dataset_name}", + ) + + +def train_tokenizer(config: SegmentationConfiguration): + if repo_exists(config.base_repo_name): + logging.error(f"{config.base_repo_name} has already exists") + exit(1) + + tokenizer = get_untrained_tokenizer() + + train_dataset = load_dataset( + config.dataset_repo_name, + token=True, + split=ReadInstruction("train", to=config.dataset_percentage, unit="%"), + )["bytecode"] + trainer = trainers.WordPieceTrainer(vocab_size=30000, special_tokens=special_tokens) + tokenizer.train_from_iterator(train_dataset, trainer=trainer) + + post_training_configuration(tokenizer) + + save_and_upload_tokenizer( + tokenizer, + config.tokenizer_json_path, + config.tokenizer_repo_name, + config.dataset_repo_name, + ) + + +@click.command(help="Training script for the bytecode tokenizer for the segmentation model given a segmentation json.") +@click.argument("json_path", type=str) +def main(json_path: str): + json_file_path = pathlib.Path(json_path) + segmentation_config = parse_segmentation_config_json(json_file_path) + train_tokenizer(segmentation_config) + + +if __name__ == "__main__": + main() diff --git a/model_training/statement/README.md b/model_training/statement/README.md new file mode 100644 index 0000000..e0c58a7 --- /dev/null +++ b/model_training/statement/README.md @@ -0,0 +1,18 @@ +# seq2seq + +- train_tokenizer_auto.py: + - trains the manual tokenizer + +- tokenize_seq2seq.py: + - tokenize the dataset for the seq2seq model + +- train_seq2seq.py: + - finetuning the pretrained model + - will create a sequence-to-sequence translation model + +- StatementConfiguration.py + - defines the JSON format for statement translation training + +# manual1 + +Contains JSONs mapping bytecode instructions and their configurations to use in training. diff --git a/model_training/statement/StatementConfiguration.py b/model_training/statement/StatementConfiguration.py new file mode 100644 index 0000000..a6c2b37 --- /dev/null +++ b/model_training/statement/StatementConfiguration.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass + +import pathlib +import json +import logging + + +@dataclass +class TrainingParameters: + batch_size: int + epochs: int + learning_rate: float + + +@dataclass +class StatementConfiguration: + base_repo_name: str + dataset_repo_name: str + tokenizer_repo_name: str + pretrained_seq2seq_repo_name: str + cache_dir: pathlib.Path + max_token_length: int + dataset_percentage: int + do_eval: bool + fp16: bool + statement_training_parameters: TrainingParameters + + @property + def tokenized_dataset_repo_name(self): + return self.dataset_repo_name + "-tokenized" + + @property + def statement_model_repo_name(self): + return self.base_repo_name + "-statement" + + @property + def statement_model_dir(self): + return self.cache_dir / "models" / self.statement_model_repo_name + + @property + def log_dir(self): + return self.statement_model_dir / "logs" + + def __post_init__(self): + self.cache_dir = pathlib.Path(self.cache_dir) + + +def parse_statement_config_json(json_file_path: pathlib.Path, logger: logging.Logger = None) -> StatementConfiguration: + if not json_file_path.exists(): + raise FileNotFoundError(f"{json_file_path} does not exist") + + if logger: + logger.info(f"Loading model description from {json_file_path}...") + + with json_file_path.open() as json_file: + statement_config_dict = json.load(json_file) + + statement_config_dict["statement_training_parameters"] = TrainingParameters(**statement_config_dict["statement_training_parameters"]) + return StatementConfiguration(**statement_config_dict) diff --git a/model_training/statement/__init__.py b/model_training/statement/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_training/statement/tokenize_seq2seq.py b/model_training/statement/tokenize_seq2seq.py new file mode 100644 index 0000000..1a17dcd --- /dev/null +++ b/model_training/statement/tokenize_seq2seq.py @@ -0,0 +1,51 @@ +import os +from typing import Any + +import click +import pathlib + +from datasets import load_dataset +from transformers import RobertaTokenizer + +from StatementConfiguration import StatementConfiguration, parse_statement_config_json + +import functools + + +def preprocess_function(tokenizer: RobertaTokenizer, max_token_length: int, input_key: str, examples: dict[str, Any]) -> dict[str, Any]: + """Set up Huggingface tokenizers for both inputs and targets""" + inputs = [ex if ex else "" for ex in examples[input_key]] + targets = [ex if ex else "" for ex in examples["source"]] + + return tokenizer(text=inputs, text_target=targets, max_length=max_token_length, truncation=True) + + +def tokenize_seq2seq_dataset(config: StatementConfiguration): + # ref: https://huggingface.co/Salesforce/codet5-base + tokenizer = RobertaTokenizer.from_pretrained(config.tokenizer_repo_name) + raw_datasets = load_dataset(config.dataset_repo_name, token=True) + + column_names = raw_datasets["train"].column_names + input_key = "bytecode" + prepped_preprocess_function = functools.partial(preprocess_function, tokenizer, config.max_token_length, input_key) + tokenized_datasets = raw_datasets.map( + prepped_preprocess_function, + batched=True, + remove_columns=column_names, + num_proc=os.cpu_count(), + desc="Tokenizing datasets", + ) + + tokenized_datasets.push_to_hub(config.tokenized_dataset_repo_name, private=True) + + +@click.command(help="Tokenization script for Statement Translation model given a statement json.") +@click.argument("json_path", type=str) +def main(json_path: str): + json_file_path = pathlib.Path(json_path) + statement_config = parse_statement_config_json(json_file_path) + tokenize_seq2seq_dataset(statement_config) + + +if __name__ == "__main__": + main() diff --git a/model_training/statement/tokenizer/special_tokens_map.json b/model_training/statement/tokenizer/special_tokens_map.json new file mode 100644 index 0000000..5d7a5d0 --- /dev/null +++ b/model_training/statement/tokenizer/special_tokens_map.json @@ -0,0 +1,5471 @@ +{ + "additional_special_tokens": [ + { + "content": "-=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "<<", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": ">>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": ":=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": ">=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "<=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "==", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "!=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "+=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "//=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "**=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "/=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "//", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "%=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "@=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "&=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "|=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "^=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": ">>=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "<<=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "*=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "()", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "):", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "~>>", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "**", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "E->", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "defaults", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "args:", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "vararg:", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "E-END", + "lstrip": true, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "~~>", + "lstrip": true, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": true, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": true, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "False", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "None", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "True", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "and", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "assert", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "async", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "await", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "break", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "class", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "continue", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "def", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "del", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "elif", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "else", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "else:", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "except", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "except:", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "finally", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "finally:", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "for", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "from", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "global", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "if", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "import", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "in", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "is", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "lambda", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "nonlocal", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "not", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "or", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "pass", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "raise", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "return", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "try", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "try:", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "while", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "with", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "yield", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "case", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "as", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "ASYNC_GEN_WRAP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BEFORE_ASYNC_WITH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BEFORE_WITH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BEGIN_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_ADD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_AND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_FLOOR_DIVIDE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_LSHIFT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_MATRIX_MULTIPLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_MODULO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_MULTIPLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_OP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_OR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_POWER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_RSHIFT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_SLICE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_SUBSCR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_SUBTRACT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_TRUE_DIVIDE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BINARY_XOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BREAK_LOOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_CONST_KEY_MAP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_LIST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_LIST_UNPACK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_MAP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_MAP_UNPACK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_MAP_UNPACK_WITH_CALL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_SET", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_SET_UNPACK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_SLICE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_STRING", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_TUPLE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_TUPLE_UNPACK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "BUILD_TUPLE_UNPACK_WITH_CALL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CACHE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CALL_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CALL_FUNCTION", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CALL_FUNCTION_EX", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CALL_FUNCTION_KW", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CALL_METHOD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CHECK_EG_MATCH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CHECK_EXC_MATCH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CLEANUP_THROW", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "COMPARE_OP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CONTAINS_OP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CONTINUE_LOOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "COPY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "COPY_DICT_WITHOUT_KEYS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "COPY_FREE_VARS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DELETE_ATTR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DELETE_DEREF", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DELETE_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DELETE_GLOBAL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DELETE_NAME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DELETE_SUBSCR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DICT_MERGE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DICT_UPDATE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DUP_TOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "DUP_TOP_TWO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "END_ASYNC_FOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "END_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "END_FOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "END_SEND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "EXTENDED_ARG", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "FOR_ITER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "FORMAT_VALUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "FORMAT_SIMPLE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "FORMAT_SPEC", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "GEN_START", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "GET_AITER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "GET_ANEXT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "GET_AWAITABLE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "GET_ITER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "GET_LEN", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "GET_YIELD_FROM_ITER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "IMPORT_FROM", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "IMPORT_NAME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "IMPORT_STAR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_ADD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_AND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_FLOOR_DIVIDE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_LSHIFT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_MATRIX_MULTIPLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_MODULO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_MULTIPLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_OR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_POWER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_RSHIFT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_SUBTRACT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_TRUE_DIVIDE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INPLACE_XOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "INTERPRETER_EXIT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "IS_OP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "JUMP_ABSOLUTE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "JUMP_BACKWARD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "JUMP_BACKWARD_NO_INTERRUPT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "JUMP_FORWARD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "JUMP_IF_FALSE_OR_POP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "JUMP_IF_NOT_EXC_MATCH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "JUMP_IF_TRUE_OR_POP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LIST_APPEND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LIST_EXTEND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LIST_TO_TUPLE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_ASSERTION_ERROR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_ATTR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_BUILD_CLASS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_CLASSDEREF", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_CLOSURE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_CONST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_FAST_LOAD_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_FROM_DICT_OR_DEREF", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_FROM_DICT_OR_GLOBAL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_FAST_AND_CLEAR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_FAST_CHECK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_GLOBAL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_LOCALS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_METHOD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_NAME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "LOAD_SUPER_ATTR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "MAKE_CELL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "MAKE_FUNCTION", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SET_FUNCTION_ATTRIBUTE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "MAP_ADD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "MATCH_CLASS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "MATCH_KEYS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "MATCH_MAPPING", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "MATCH_SEQUENCE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "NOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_BLOCK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_EXCEPT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_JUMP_FORWARD_IF_FALSE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_JUMP_FORWARD_IF_NONE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_JUMP_FORWARD_IF_NOT_NONE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_JUMP_FORWARD_IF_TRUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_JUMP_IF_FALSE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_JUMP_IF_NONE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_JUMP_IF_NOT_NONE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_JUMP_IF_TRUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "POP_TOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "PRECALL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "PREP_RERAISE_STAR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "PRINT_EXPR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "PUSH_EXC_INFO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "PUSH_NULL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "RAISE_VARARGS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "RERAISE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "RESERVED", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "RESUME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "RETURN_CONST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "RETURN_GENERATOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "RETURN_VALUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "CONVERT_VALUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "ROT_FOUR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "ROT_N", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "ROT_THREE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "ROT_TWO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "ENTER_EXECUTOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "TO_BOOL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SEND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SET_ADD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SET_UPDATE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SETUP_ANNOTATIONS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SETUP_ASYNC_WITH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SETUP_EXCEPT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SETUP_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SETUP_LOOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SETUP_WITH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_ATTR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_DEREF", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_FAST_LOAD_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_FAST_STORE_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_GLOBAL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_NAME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_SLICE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "STORE_SUBSCR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "SWAP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "UNARY_INVERT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "UNARY_NEGATIVE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "UNARY_NOT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "UNARY_POSITIVE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "UNPACK_EX", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "UNPACK_SEQUENCE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "WITH_CLEANUP_FINISH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "WITH_CLEANUP_START", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "WITH_EXCEPT_START", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "YIELD_FROM", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "YIELD_VALUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "match", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "type", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "HAVE_ARGUMENT", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "CALL_KW", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "CALL_INTRINSIC_1", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "CALL_INTRINSIC_2", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "JUMP_NO_INTERRUPT", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "nargs", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "vargs", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "compare", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "name", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "const", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + }, + { + "content": "local", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false + } + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "cls_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "sep_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/model_training/statement/tokenizer/tokenizer.json b/model_training/statement/tokenizer/tokenizer.json new file mode 100644 index 0000000..32cba79 --- /dev/null +++ b/model_training/statement/tokenizer/tokenizer.json @@ -0,0 +1,7775 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 100, + "content": "-=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 101, + "content": "<<", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 102, + "content": ">>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 103, + "content": ":=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 104, + "content": ">=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 105, + "content": "<=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 106, + "content": "==", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 107, + "content": "!=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 108, + "content": "+=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 109, + "content": "//=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 110, + "content": "**=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 111, + "content": "/=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 112, + "content": "//", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 113, + "content": "%=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 114, + "content": "@=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 115, + "content": "&=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 116, + "content": "|=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 117, + "content": "^=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 118, + "content": ">>=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 119, + "content": "<<=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 120, + "content": "*=", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 121, + "content": "()", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 122, + "content": "):", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 123, + "content": "~>>", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 124, + "content": "**", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 125, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 127, + "content": "E->", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 128, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 129, + "content": "defaults", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 130, + "content": "args:", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 131, + "content": "vararg:", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 132, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 133, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 134, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 135, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 136, + "content": "E-END", + "lstrip": true, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 137, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 138, + "content": "~~>", + "lstrip": true, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 139, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 140, + "content": "", + "lstrip": true, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 141, + "content": "", + "lstrip": true, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 142, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 143, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 144, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 145, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 146, + "content": "", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 147, + "content": "False", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 148, + "content": "None", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 149, + "content": "True", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 150, + "content": "and", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 151, + "content": "assert", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 152, + "content": "async", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 153, + "content": "await", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 154, + "content": "break", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 155, + "content": "class", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 156, + "content": "continue", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 157, + "content": "def", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 158, + "content": "del", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 159, + "content": "elif", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 160, + "content": "else", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 161, + "content": "else:", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 162, + "content": "except", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 163, + "content": "except:", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 164, + "content": "finally", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 165, + "content": "finally:", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 166, + "content": "for", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 167, + "content": "from", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 168, + "content": "global", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 169, + "content": "if", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 170, + "content": "import", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 171, + "content": "in", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 172, + "content": "is", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 173, + "content": "lambda", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 174, + "content": "nonlocal", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 175, + "content": "not", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 176, + "content": "or", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 177, + "content": "pass", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 178, + "content": "raise", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 179, + "content": "return", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 180, + "content": "try", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 181, + "content": "try:", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 182, + "content": "while", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 183, + "content": "with", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 184, + "content": "yield", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 185, + "content": "case", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 186, + "content": "as", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 187, + "content": "ASYNC_GEN_WRAP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 188, + "content": "BEFORE_ASYNC_WITH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 189, + "content": "BEFORE_WITH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 190, + "content": "BEGIN_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 191, + "content": "BINARY_ADD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 192, + "content": "BINARY_AND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 193, + "content": "BINARY_FLOOR_DIVIDE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 194, + "content": "BINARY_LSHIFT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 195, + "content": "BINARY_MATRIX_MULTIPLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 196, + "content": "BINARY_MODULO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 197, + "content": "BINARY_MULTIPLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 198, + "content": "BINARY_OP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 199, + "content": "BINARY_OR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 200, + "content": "BINARY_POWER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 201, + "content": "BINARY_RSHIFT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 202, + "content": "BINARY_SLICE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 203, + "content": "BINARY_SUBSCR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 204, + "content": "BINARY_SUBTRACT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 205, + "content": "BINARY_TRUE_DIVIDE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 206, + "content": "BINARY_XOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 207, + "content": "BREAK_LOOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 208, + "content": "BUILD_CONST_KEY_MAP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 209, + "content": "BUILD_LIST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 210, + "content": "BUILD_LIST_UNPACK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 211, + "content": "BUILD_MAP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 212, + "content": "BUILD_MAP_UNPACK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 213, + "content": "BUILD_MAP_UNPACK_WITH_CALL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 214, + "content": "BUILD_SET", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 215, + "content": "BUILD_SET_UNPACK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 216, + "content": "BUILD_SLICE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 217, + "content": "BUILD_STRING", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 218, + "content": "BUILD_TUPLE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 219, + "content": "BUILD_TUPLE_UNPACK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 220, + "content": "BUILD_TUPLE_UNPACK_WITH_CALL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 221, + "content": "CACHE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 222, + "content": "CALL_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 223, + "content": "CALL_FUNCTION", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 224, + "content": "CALL_FUNCTION_EX", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 225, + "content": "CALL_FUNCTION_KW", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 226, + "content": "CALL_METHOD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 227, + "content": "CHECK_EG_MATCH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 228, + "content": "CHECK_EXC_MATCH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 229, + "content": "CLEANUP_THROW", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 230, + "content": "COMPARE_OP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 231, + "content": "CONTAINS_OP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 232, + "content": "CONTINUE_LOOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 233, + "content": "COPY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 234, + "content": "COPY_DICT_WITHOUT_KEYS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 235, + "content": "COPY_FREE_VARS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 236, + "content": "DELETE_ATTR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 237, + "content": "DELETE_DEREF", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 238, + "content": "DELETE_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 239, + "content": "DELETE_GLOBAL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 240, + "content": "DELETE_NAME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 241, + "content": "DELETE_SUBSCR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 242, + "content": "DICT_MERGE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 243, + "content": "DICT_UPDATE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 244, + "content": "DUP_TOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 245, + "content": "DUP_TOP_TWO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 246, + "content": "END_ASYNC_FOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 247, + "content": "END_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 248, + "content": "END_FOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 249, + "content": "END_SEND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 250, + "content": "EXTENDED_ARG", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 251, + "content": "FOR_ITER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 252, + "content": "FORMAT_VALUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 253, + "content": "GEN_START", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 254, + "content": "GET_AITER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 255, + "content": "GET_ANEXT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 256, + "content": "GET_AWAITABLE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 257, + "content": "GET_ITER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 258, + "content": "GET_LEN", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 259, + "content": "GET_YIELD_FROM_ITER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 260, + "content": "IMPORT_FROM", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 261, + "content": "IMPORT_NAME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 262, + "content": "IMPORT_STAR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 263, + "content": "INPLACE_ADD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 264, + "content": "INPLACE_AND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 265, + "content": "INPLACE_FLOOR_DIVIDE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 266, + "content": "INPLACE_LSHIFT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 267, + "content": "INPLACE_MATRIX_MULTIPLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 268, + "content": "INPLACE_MODULO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 269, + "content": "INPLACE_MULTIPLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 270, + "content": "INPLACE_OR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 271, + "content": "INPLACE_POWER", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 272, + "content": "INPLACE_RSHIFT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 273, + "content": "INPLACE_SUBTRACT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 274, + "content": "INPLACE_TRUE_DIVIDE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 275, + "content": "INPLACE_XOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 276, + "content": "INTERPRETER_EXIT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 277, + "content": "IS_OP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 278, + "content": "JUMP_ABSOLUTE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 279, + "content": "JUMP_BACKWARD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 280, + "content": "JUMP_BACKWARD_NO_INTERRUPT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 281, + "content": "JUMP_FORWARD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 282, + "content": "JUMP_IF_FALSE_OR_POP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 283, + "content": "JUMP_IF_NOT_EXC_MATCH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 284, + "content": "JUMP_IF_TRUE_OR_POP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 285, + "content": "LIST_APPEND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 286, + "content": "LIST_EXTEND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 287, + "content": "LIST_TO_TUPLE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 288, + "content": "LOAD_ASSERTION_ERROR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 289, + "content": "LOAD_ATTR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 290, + "content": "LOAD_BUILD_CLASS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 291, + "content": "LOAD_CLASSDEREF", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 292, + "content": "LOAD_CLOSURE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 293, + "content": "LOAD_CONST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 294, + "content": "LOAD_DEREF", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 295, + "content": "LOAD_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 296, + "content": "LOAD_FAST_AND_CLEAR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 297, + "content": "LOAD_FAST_CHECK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 298, + "content": "LOAD_GLOBAL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 299, + "content": "LOAD_LOCALS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 300, + "content": "LOAD_METHOD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 301, + "content": "LOAD_NAME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 302, + "content": "LOAD_SUPER_ATTR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 303, + "content": "MAKE_CELL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 304, + "content": "MAKE_FUNCTION", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 305, + "content": "MAP_ADD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 306, + "content": "MATCH_CLASS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 307, + "content": "MATCH_KEYS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 308, + "content": "MATCH_MAPPING", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 309, + "content": "MATCH_SEQUENCE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 310, + "content": "NOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 311, + "content": "POP_BLOCK", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 312, + "content": "POP_EXCEPT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 313, + "content": "POP_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 314, + "content": "POP_JUMP_FORWARD_IF_FALSE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 315, + "content": "POP_JUMP_FORWARD_IF_NONE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 316, + "content": "POP_JUMP_FORWARD_IF_NOT_NONE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 317, + "content": "POP_JUMP_FORWARD_IF_TRUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 318, + "content": "POP_JUMP_IF_FALSE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 319, + "content": "POP_JUMP_IF_NONE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 320, + "content": "POP_JUMP_IF_NOT_NONE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 321, + "content": "POP_JUMP_IF_TRUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 322, + "content": "POP_TOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 323, + "content": "PRECALL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 324, + "content": "PREP_RERAISE_STAR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 325, + "content": "PRINT_EXPR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 326, + "content": "PUSH_EXC_INFO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 327, + "content": "PUSH_NULL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 328, + "content": "RAISE_VARARGS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 329, + "content": "RERAISE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 330, + "content": "RESERVED", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 331, + "content": "RESUME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 332, + "content": "RETURN_CONST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 333, + "content": "RETURN_GENERATOR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 334, + "content": "RETURN_VALUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 335, + "content": "ROT_FOUR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 336, + "content": "ROT_N", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 337, + "content": "ROT_THREE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 338, + "content": "ROT_TWO", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 339, + "content": "SEND", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 340, + "content": "SET_ADD", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 341, + "content": "SET_UPDATE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 342, + "content": "SETUP_ANNOTATIONS", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 343, + "content": "SETUP_ASYNC_WITH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 344, + "content": "SETUP_EXCEPT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 345, + "content": "SETUP_FINALLY", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 346, + "content": "SETUP_LOOP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 347, + "content": "SETUP_WITH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 348, + "content": "STORE_ATTR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 349, + "content": "STORE_DEREF", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 350, + "content": "STORE_FAST", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 351, + "content": "STORE_GLOBAL", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 352, + "content": "STORE_NAME", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 353, + "content": "STORE_SLICE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 354, + "content": "STORE_SUBSCR", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 355, + "content": "SWAP", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 356, + "content": "UNARY_INVERT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 357, + "content": "UNARY_NEGATIVE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 358, + "content": "UNARY_NOT", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 359, + "content": "UNARY_POSITIVE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 360, + "content": "UNPACK_EX", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 361, + "content": "UNPACK_SEQUENCE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 362, + "content": "WITH_CLEANUP_FINISH", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 363, + "content": "WITH_CLEANUP_START", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 364, + "content": "WITH_EXCEPT_START", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 365, + "content": "YIELD_FROM", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 366, + "content": "YIELD_VALUE", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 367, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 368, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 369, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 370, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 371, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 372, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 373, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 374, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 375, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 376, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 377, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 378, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 379, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 380, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 381, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 382, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 383, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 384, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 385, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 386, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 387, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 388, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 389, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 390, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 391, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 392, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 393, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 394, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 395, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 396, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 397, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 398, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 399, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 400, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 401, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 402, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 403, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 404, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 405, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 406, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 407, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 408, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 409, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 410, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 411, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 412, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 413, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 414, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 415, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 416, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 417, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 418, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 419, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 420, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 421, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 422, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 423, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 424, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 425, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 426, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 427, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 428, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 429, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 430, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 431, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 432, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 433, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 434, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 435, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 436, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 437, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 438, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 439, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 440, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 441, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 442, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 443, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 444, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 445, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 446, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 447, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 448, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 449, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 450, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 451, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 452, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 453, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 454, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 455, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 456, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 457, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 458, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 459, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 460, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 461, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 462, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 463, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 464, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 465, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 466, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 467, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 468, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 469, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 470, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 471, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 472, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 473, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 474, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 475, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 476, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 477, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 478, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 479, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 480, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 481, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 482, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 483, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 484, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 485, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 486, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 487, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 488, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 489, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 490, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 491, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 492, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 493, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 494, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 495, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 496, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 497, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 498, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 499, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 500, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 501, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 502, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 503, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 504, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 505, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 506, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 507, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 508, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 509, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 510, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 511, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 512, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 513, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 514, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 515, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 516, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 517, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 518, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 519, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 520, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 521, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 522, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 523, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 524, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 525, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 526, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 527, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 528, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 529, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 530, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 531, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 532, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 533, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 534, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 535, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 536, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 537, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 538, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 539, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 540, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 541, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 542, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 543, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 544, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 545, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 546, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 547, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 548, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 549, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 550, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 551, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 552, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 553, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 554, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 555, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 556, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 557, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 558, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 559, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 560, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 561, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 562, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 563, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 564, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 565, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 566, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 567, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 568, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 569, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 570, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 571, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 572, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 573, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 574, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 575, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 576, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 577, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 578, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 579, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 580, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 581, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 582, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 583, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 584, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 585, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 586, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 587, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 588, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 589, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 590, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 591, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 592, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 593, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 594, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 595, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 596, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 597, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 598, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 599, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 600, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 601, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 602, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 603, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 604, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 605, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 606, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 607, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 608, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 609, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 610, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 611, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 612, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 613, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 614, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 615, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 616, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 617, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 618, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 619, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 620, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 621, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 622, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 623, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 624, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 625, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 626, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 627, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 628, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 629, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 630, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 631, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 632, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 633, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 634, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 635, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 636, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 637, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 638, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 639, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 640, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 641, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 642, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 643, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 644, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 645, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 646, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 647, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 648, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 649, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 650, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 651, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 652, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 653, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 654, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 655, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 656, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 657, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 658, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 659, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 660, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 661, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 662, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 663, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 664, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 665, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 666, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 667, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 668, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 669, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 670, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 671, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 672, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 673, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 674, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 675, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 676, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 677, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 678, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 679, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 680, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 681, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 682, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 683, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 684, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 685, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 686, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 687, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 688, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 689, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 690, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 691, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 692, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 693, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 694, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 695, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 696, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 697, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 698, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 699, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 700, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 701, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 702, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 703, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 704, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 705, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 706, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 707, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 708, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 709, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 710, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 711, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 712, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 713, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 714, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 715, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 716, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 717, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 718, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 719, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 720, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 721, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 722, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 723, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 724, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 725, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 726, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 727, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 728, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 729, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 730, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 731, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 732, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 733, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 734, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 735, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 736, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 737, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 738, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 739, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 740, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 741, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 742, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 743, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 744, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 745, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 746, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 747, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 748, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 749, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 750, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 751, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 752, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 753, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 754, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 755, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 756, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 757, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 758, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 759, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 760, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 761, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 762, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 763, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 764, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 765, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 766, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 767, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 768, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 769, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 770, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 771, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 772, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 773, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 774, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 775, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 776, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 777, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 778, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 779, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 780, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 781, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 782, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 783, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 784, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 785, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 786, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 787, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 788, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 789, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 790, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 791, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 792, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 793, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 794, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 795, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 796, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 797, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 798, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 799, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 800, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 801, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 802, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 803, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 804, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 805, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 806, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 807, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 808, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 809, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 810, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 811, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 812, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 813, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 814, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 815, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 816, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 817, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 818, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 819, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 820, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 821, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 822, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 823, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 824, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 825, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 826, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 827, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 828, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 829, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 830, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 831, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 832, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 833, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 834, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 835, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 836, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 837, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 838, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 839, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 840, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 841, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 842, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 843, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 844, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 845, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 846, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 847, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 848, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 849, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 850, + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 851, + "content": "match", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 852, + "content": "type", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 853, + "content": "HAVE_ARGUMENT", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 854, + "content": "CALL_INTRINSIC_1", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 855, + "content": "CALL_INTRINSIC_2", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 856, + "content": "JUMP_NO_INTERRUPT", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 857, + "content": "nargs", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 858, + "content": "vargs", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 859, + "content": "compare", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 860, + "content": "name", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 861, + "content": "const", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + { + "id": 862, + "content": "local", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "post_processor": { + "type": "RobertaProcessing", + "sep": [ + "", + 2 + ], + "cls": [ + "", + 1 + ], + "trim_offsets": true, + "add_prefix_space": false + }, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": "", + "end_of_word_suffix": "", + "fuse_unk": false, + "byte_fallback": false, + "vocab": { + "": 0, + "": 1, + "": 2, + "": 3, + "": 4, + "!": 5, + "\"": 6, + "#": 7, + "$": 8, + "%": 9, + "&": 10, + "'": 11, + "(": 12, + ")": 13, + "*": 14, + "+": 15, + ",": 16, + "-": 17, + ".": 18, + "/": 19, + "0": 20, + "1": 21, + "2": 22, + "3": 23, + "4": 24, + "5": 25, + "6": 26, + "7": 27, + "8": 28, + "9": 29, + ":": 30, + ";": 31, + "<": 32, + "=": 33, + ">": 34, + "?": 35, + "@": 36, + "A": 37, + "B": 38, + "C": 39, + "D": 40, + "E": 41, + "F": 42, + "G": 43, + "H": 44, + "I": 45, + "J": 46, + "K": 47, + "L": 48, + "M": 49, + "N": 50, + "O": 51, + "P": 52, + "Q": 53, + "R": 54, + "S": 55, + "T": 56, + "U": 57, + "V": 58, + "W": 59, + "X": 60, + "Y": 61, + "Z": 62, + "[": 63, + "\\": 64, + "]": 65, + "^": 66, + "_": 67, + "`": 68, + "a": 69, + "b": 70, + "c": 71, + "d": 72, + "e": 73, + "f": 74, + "g": 75, + "h": 76, + "i": 77, + "j": 78, + "k": 79, + "l": 80, + "m": 81, + "n": 82, + "o": 83, + "p": 84, + "q": 85, + "r": 86, + "s": 87, + "t": 88, + "u": 89, + "v": 90, + "w": 91, + "x": 92, + "y": 93, + "z": 94, + "{": 95, + "|": 96, + "}": 97, + "~": 98, + "Ġ": 99, + "-=": 100, + "<<": 101, + ">>": 102, + ":=": 103, + ">=": 104, + "<=": 105, + "==": 106, + "!=": 107, + "+=": 108, + "//=": 109, + "**=": 110, + "/=": 111, + "//": 112, + "%=": 113, + "@=": 114, + "&=": 115, + "|=": 116, + "^=": 117, + ">>=": 118, + "<<=": 119, + "*=": 120, + "()": 121, + "):": 122, + "~>>": 123, + "**": 124, + "": 126, + "E->": 127, + "": 128, + "defaults": 129, + "args:": 130, + "vararg:": 131, + "": 132, + "": 133, + "": 134, + "": 135, + "E-END": 136, + "": 137, + "~~>": 138, + "": 139, + "": 140, + "": 141, + "": 142, + "": 143, + "": 144, + "": 145, + "": 146, + "False": 147, + "None": 148, + "True": 149, + "and": 150, + "assert": 151, + "async": 152, + "await": 153, + "break": 154, + "class": 155, + "continue": 156, + "def": 157, + "del": 158, + "elif": 159, + "else": 160, + "else:": 161, + "except": 162, + "except:": 163, + "finally": 164, + "finally:": 165, + "for": 166, + "from": 167, + "global": 168, + "if": 169, + "import": 170, + "in": 171, + "is": 172, + "lambda": 173, + "nonlocal": 174, + "not": 175, + "or": 176, + "pass": 177, + "raise": 178, + "return": 179, + "try": 180, + "try:": 181, + "while": 182, + "with": 183, + "yield": 184, + "case": 185, + "as": 186, + "ASYNC_GEN_WRAP": 187, + "BEFORE_ASYNC_WITH": 188, + "BEFORE_WITH": 189, + "BEGIN_FINALLY": 190, + "BINARY_ADD": 191, + "BINARY_AND": 192, + "BINARY_FLOOR_DIVIDE": 193, + "BINARY_LSHIFT": 194, + "BINARY_MATRIX_MULTIPLY": 195, + "BINARY_MODULO": 196, + "BINARY_MULTIPLY": 197, + "BINARY_OP": 198, + "BINARY_OR": 199, + "BINARY_POWER": 200, + "BINARY_RSHIFT": 201, + "BINARY_SLICE": 202, + "BINARY_SUBSCR": 203, + "BINARY_SUBTRACT": 204, + "BINARY_TRUE_DIVIDE": 205, + "BINARY_XOR": 206, + "BREAK_LOOP": 207, + "BUILD_CONST_KEY_MAP": 208, + "BUILD_LIST": 209, + "BUILD_LIST_UNPACK": 210, + "BUILD_MAP": 211, + "BUILD_MAP_UNPACK": 212, + "BUILD_MAP_UNPACK_WITH_CALL": 213, + "BUILD_SET": 214, + "BUILD_SET_UNPACK": 215, + "BUILD_SLICE": 216, + "BUILD_STRING": 217, + "BUILD_TUPLE": 218, + "BUILD_TUPLE_UNPACK": 219, + "BUILD_TUPLE_UNPACK_WITH_CALL": 220, + "CACHE": 221, + "CALL_FINALLY": 222, + "CALL_FUNCTION": 223, + "CALL_FUNCTION_EX": 224, + "CALL_FUNCTION_KW": 225, + "CALL_METHOD": 226, + "CHECK_EG_MATCH": 227, + "CHECK_EXC_MATCH": 228, + "CLEANUP_THROW": 229, + "COMPARE_OP": 230, + "CONTAINS_OP": 231, + "CONTINUE_LOOP": 232, + "COPY": 233, + "COPY_DICT_WITHOUT_KEYS": 234, + "COPY_FREE_VARS": 235, + "DELETE_ATTR": 236, + "DELETE_DEREF": 237, + "DELETE_FAST": 238, + "DELETE_GLOBAL": 239, + "DELETE_NAME": 240, + "DELETE_SUBSCR": 241, + "DICT_MERGE": 242, + "DICT_UPDATE": 243, + "DUP_TOP": 244, + "DUP_TOP_TWO": 245, + "END_ASYNC_FOR": 246, + "END_FINALLY": 247, + "END_FOR": 248, + "END_SEND": 249, + "EXTENDED_ARG": 250, + "FOR_ITER": 251, + "FORMAT_VALUE": 252, + "GEN_START": 253, + "GET_AITER": 254, + "GET_ANEXT": 255, + "GET_AWAITABLE": 256, + "GET_ITER": 257, + "GET_LEN": 258, + "GET_YIELD_FROM_ITER": 259, + "IMPORT_FROM": 260, + "IMPORT_NAME": 261, + "IMPORT_STAR": 262, + "INPLACE_ADD": 263, + "INPLACE_AND": 264, + "INPLACE_FLOOR_DIVIDE": 265, + "INPLACE_LSHIFT": 266, + "INPLACE_MATRIX_MULTIPLY": 267, + "INPLACE_MODULO": 268, + "INPLACE_MULTIPLY": 269, + "INPLACE_OR": 270, + "INPLACE_POWER": 271, + "INPLACE_RSHIFT": 272, + "INPLACE_SUBTRACT": 273, + "INPLACE_TRUE_DIVIDE": 274, + "INPLACE_XOR": 275, + "INTERPRETER_EXIT": 276, + "IS_OP": 277, + "JUMP_ABSOLUTE": 278, + "JUMP_BACKWARD": 279, + "JUMP_BACKWARD_NO_INTERRUPT": 280, + "JUMP_FORWARD": 281, + "JUMP_IF_FALSE_OR_POP": 282, + "JUMP_IF_NOT_EXC_MATCH": 283, + "JUMP_IF_TRUE_OR_POP": 284, + "LIST_APPEND": 285, + "LIST_EXTEND": 286, + "LIST_TO_TUPLE": 287, + "LOAD_ASSERTION_ERROR": 288, + "LOAD_ATTR": 289, + "LOAD_BUILD_CLASS": 290, + "LOAD_CLASSDEREF": 291, + "LOAD_CLOSURE": 292, + "LOAD_CONST": 293, + "LOAD_DEREF": 294, + "LOAD_FAST": 295, + "LOAD_FAST_AND_CLEAR": 296, + "LOAD_FAST_CHECK": 297, + "LOAD_GLOBAL": 298, + "LOAD_LOCALS": 299, + "LOAD_METHOD": 300, + "LOAD_NAME": 301, + "LOAD_SUPER_ATTR": 302, + "MAKE_CELL": 303, + "MAKE_FUNCTION": 304, + "MAP_ADD": 305, + "MATCH_CLASS": 306, + "MATCH_KEYS": 307, + "MATCH_MAPPING": 308, + "MATCH_SEQUENCE": 309, + "NOP": 310, + "POP_BLOCK": 311, + "POP_EXCEPT": 312, + "POP_FINALLY": 313, + "POP_JUMP_FORWARD_IF_FALSE": 314, + "POP_JUMP_FORWARD_IF_NONE": 315, + "POP_JUMP_FORWARD_IF_NOT_NONE": 316, + "POP_JUMP_FORWARD_IF_TRUE": 317, + "POP_JUMP_IF_FALSE": 318, + "POP_JUMP_IF_NONE": 319, + "POP_JUMP_IF_NOT_NONE": 320, + "POP_JUMP_IF_TRUE": 321, + "POP_TOP": 322, + "PRECALL": 323, + "PREP_RERAISE_STAR": 324, + "PRINT_EXPR": 325, + "PUSH_EXC_INFO": 326, + "PUSH_NULL": 327, + "RAISE_VARARGS": 328, + "RERAISE": 329, + "RESERVED": 330, + "RESUME": 331, + "RETURN_CONST": 332, + "RETURN_GENERATOR": 333, + "RETURN_VALUE": 334, + "ROT_FOUR": 335, + "ROT_N": 336, + "ROT_THREE": 337, + "ROT_TWO": 338, + "SEND": 339, + "SET_ADD": 340, + "SET_UPDATE": 341, + "SETUP_ANNOTATIONS": 342, + "SETUP_ASYNC_WITH": 343, + "SETUP_EXCEPT": 344, + "SETUP_FINALLY": 345, + "SETUP_LOOP": 346, + "SETUP_WITH": 347, + "STORE_ATTR": 348, + "STORE_DEREF": 349, + "STORE_FAST": 350, + "STORE_GLOBAL": 351, + "STORE_NAME": 352, + "STORE_SLICE": 353, + "STORE_SUBSCR": 354, + "SWAP": 355, + "UNARY_INVERT": 356, + "UNARY_NEGATIVE": 357, + "UNARY_NOT": 358, + "UNARY_POSITIVE": 359, + "UNPACK_EX": 360, + "UNPACK_SEQUENCE": 361, + "WITH_CLEANUP_FINISH": 362, + "WITH_CLEANUP_START": 363, + "WITH_EXCEPT_START": 364, + "YIELD_FROM": 365, + "YIELD_VALUE": 366, + "": 367, + "": 368, + "": 369, + "": 370, + "": 371, + "": 372, + "": 373, + "": 374, + "": 375, + "": 376, + "": 377, + "": 378, + "": 379, + "": 380, + "": 381, + "": 382, + "": 383, + "": 384, + "": 385, + "": 386, + "": 387, + "": 388, + "": 389, + "": 390, + "": 391, + "": 392, + "": 393, + "": 394, + "": 395, + "": 396, + "": 397, + "": 398, + "": 399, + "": 400, + "": 401, + "": 402, + "": 403, + "": 404, + "": 405, + "": 406, + "": 407, + "": 408, + "": 409, + "": 410, + "": 411, + "": 412, + "": 413, + "": 414, + "": 415, + "": 416, + "": 417, + "": 418, + "": 419, + "": 420, + "": 421, + "": 422, + "": 423, + "": 424, + "": 425, + "": 426, + "": 427, + "": 428, + "": 429, + "": 430, + "": 431, + "": 432, + "": 433, + "": 434, + "": 435, + "": 436, + "": 437, + "": 438, + "": 439, + "": 440, + "": 441, + "": 442, + "": 443, + "": 444, + "": 445, + "": 446, + "": 447, + "": 448, + "": 449, + "": 450, + "": 451, + "": 452, + "": 453, + "": 454, + "": 455, + "": 456, + "": 457, + "": 458, + "": 459, + "": 460, + "": 461, + "": 462, + "": 463, + "": 464, + "": 465, + "": 466, + "": 467, + "": 468, + "": 469, + "": 470, + "": 471, + "": 472, + "": 473, + "": 474, + "": 475, + "": 476, + "": 477, + "": 478, + "": 479, + "": 480, + "": 481, + "": 482, + "": 483, + "": 484, + "": 485, + "": 486, + "": 487, + "": 488, + "": 489, + "": 490, + "": 491, + "": 492, + "": 493, + "": 494, + "": 495, + "": 496, + "": 497, + "": 498, + "": 499, + "": 500, + "": 501, + "": 502, + "": 503, + "": 504, + "": 505, + "": 506, + "": 507, + "": 508, + "": 509, + "": 510, + "": 511, + "": 512, + "": 513, + "": 514, + "": 515, + "": 516, + "": 517, + "": 518, + "": 519, + "": 520, + "": 521, + "": 522, + "": 523, + "": 524, + "": 525, + "": 526, + "": 527, + "": 528, + "": 529, + "": 530, + "": 531, + "": 532, + "": 533, + "": 534, + "": 535, + "": 536, + "": 537, + "": 538, + "": 539, + "": 540, + "": 541, + "": 542, + "": 543, + "": 544, + "": 545, + "": 546, + "": 547, + "": 548, + "": 549, + "": 550, + "": 551, + "": 552, + "": 553, + "": 554, + "": 555, + "": 556, + "": 557, + "": 558, + "": 559, + "": 560, + "": 561, + "": 562, + "": 563, + "": 564, + "": 565, + "": 566, + "": 567, + "": 568, + "": 569, + "": 570, + "": 571, + "": 572, + "": 573, + "": 574, + "": 575, + "": 576, + "": 577, + "": 578, + "": 579, + "": 580, + "": 581, + "": 582, + "": 583, + "": 584, + "": 585, + "": 586, + "": 587, + "": 588, + "": 589, + "": 590, + "": 591, + "": 592, + "": 593, + "": 594, + "": 595, + "": 596, + "": 597, + "": 598, + "": 599, + "": 600, + "": 601, + "": 602, + "": 603, + "": 604, + "": 605, + "": 606, + "": 607, + "": 608, + "": 609, + "": 610, + "": 611, + "": 612, + "": 613, + "": 614, + "": 615, + "": 616, + "": 617, + "": 618, + "": 619, + "": 620, + "": 621, + "": 622, + "": 623, + "": 624, + "": 625, + "": 626, + "": 627, + "": 628, + "": 629, + "": 630, + "": 631, + "": 632, + "": 633, + "": 634, + "": 635, + "": 636, + "": 637, + "": 638, + "": 639, + "": 640, + "": 641, + "": 642, + "": 643, + "": 644, + "": 645, + "": 646, + "": 647, + "": 648, + "": 649, + "": 650, + "": 651, + "": 652, + "": 653, + "": 654, + "": 655, + "": 656, + "": 657, + "": 658, + "": 659, + "": 660, + "": 661, + "": 662, + "": 663, + "": 664, + "": 665, + "": 666, + "": 667, + "": 668, + "": 669, + "": 670, + "": 671, + "": 672, + "": 673, + "": 674, + "": 675, + "": 676, + "": 677, + "": 678, + "": 679, + "": 680, + "": 681, + "": 682, + "": 683, + "": 684, + "": 685, + "": 686, + "": 687, + "": 688, + "": 689, + "": 690, + "": 691, + "": 692, + "": 693, + "": 694, + "": 695, + "": 696, + "": 697, + "": 698, + "": 699, + "": 700, + "": 701, + "": 702, + "": 703, + "": 704, + "": 705, + "": 706, + "": 707, + "": 708, + "": 709, + "": 710, + "": 711, + "": 712, + "": 713, + "": 714, + "": 715, + "": 716, + "": 717, + "": 718, + "": 719, + "": 720, + "": 721, + "": 722, + "": 723, + "": 724, + "": 725, + "": 726, + "": 727, + "": 728, + "": 729, + "": 730, + "": 731, + "": 732, + "": 733, + "": 734, + "": 735, + "": 736, + "": 737, + "": 738, + "": 739, + "": 740, + "": 741, + "": 742, + "": 743, + "": 744, + "": 745, + "": 746, + "": 747, + "": 748, + "": 749, + "": 750, + "": 751, + "": 752, + "": 753, + "": 754, + "": 755, + "": 756, + "": 757, + "": 758, + "": 759, + "": 760, + "": 761, + "": 762, + "": 763, + "": 764, + "": 765, + "": 766, + "": 767, + "": 768, + "": 769, + "": 770, + "": 771, + "": 772, + "": 773, + "": 774, + "": 775, + "": 776, + "": 777, + "": 778, + "": 779, + "": 780, + "": 781, + "": 782, + "": 783, + "": 784, + "": 785, + "": 786, + "": 787, + "": 788, + "": 789, + "": 790, + "": 791, + "": 792, + "": 793, + "": 794, + "": 795, + "": 796, + "": 797, + "": 798, + "": 799, + "": 800, + "": 801, + "": 802, + "": 803, + "": 804, + "": 805, + "": 806, + "": 807, + "": 808, + "": 809, + "": 810, + "": 811, + "": 812, + "": 813, + "": 814, + "": 815, + "": 816, + "": 817, + "": 818, + "": 819, + "": 820, + "": 821, + "": 822, + "": 823, + "": 824, + "": 825, + "": 826, + "": 827, + "": 828, + "": 829, + "": 830, + "": 831, + "": 832, + "": 833, + "": 834, + "": 835, + "": 836, + "": 837, + "": 838, + "": 839, + "": 840, + "": 841, + "": 842, + "": 843, + "": 844, + "": 845, + "": 846, + "": 847, + "": 848, + "": 849, + "": 850, + "match": 851, + "type": 852, + "HAVE_ARGUMENT": 853, + "CALL_INTRINSIC_1": 854, + "CALL_INTRINSIC_2": 855, + "JUMP_NO_INTERRUPT": 856, + "nargs": 857, + "vargs": 858, + "compare": 859, + "name": 860, + "const": 861, + "local": 862 + }, + "merges": [] + } +} diff --git a/model_training/statement/tokenizer/tokenizer_config.json b/model_training/statement/tokenizer/tokenizer_config.json new file mode 100644 index 0000000..edcef60 --- /dev/null +++ b/model_training/statement/tokenizer/tokenizer_config.json @@ -0,0 +1,929 @@ +{ + "add_prefix_space": false, + "additional_special_tokens": [ + "", + "", + "", + "", + "", + "!", + "\"", + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "?", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\\", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + "Ġ", + "-=", + "<<", + ">>", + ":=", + ">=", + "<=", + "==", + "!=", + "+=", + "//=", + "**=", + "/=", + "//", + "%=", + "@=", + "&=", + "|=", + "^=", + ">>=", + "<<=", + "*=", + "()", + "):", + "~>>", + "**", + "", + "E->", + "", + "defaults", + "args:", + "vararg:", + "", + "", + "", + "", + "E-END", + "", + "~~>", + "", + "", + "", + "", + "", + "", + "", + "", + "False", + "None", + "True", + "and", + "assert", + "async", + "await", + "break", + "class", + "continue", + "def", + "del", + "elif", + "else", + "else:", + "except", + "except:", + "finally", + "finally:", + "for", + "from", + "global", + "if", + "import", + "in", + "is", + "lambda", + "nonlocal", + "not", + "or", + "pass", + "raise", + "return", + "try", + "try:", + "while", + "with", + "yield", + "case", + "as", + "ASYNC_GEN_WRAP", + "BEFORE_ASYNC_WITH", + "BEFORE_WITH", + "BEGIN_FINALLY", + "BINARY_ADD", + "BINARY_AND", + "BINARY_FLOOR_DIVIDE", + "BINARY_LSHIFT", + "BINARY_MATRIX_MULTIPLY", + "BINARY_MODULO", + "BINARY_MULTIPLY", + "BINARY_OP", + "BINARY_OR", + "BINARY_POWER", + "BINARY_RSHIFT", + "BINARY_SLICE", + "BINARY_SUBSCR", + "BINARY_SUBTRACT", + "BINARY_TRUE_DIVIDE", + "BINARY_XOR", + "BREAK_LOOP", + "BUILD_CONST_KEY_MAP", + "BUILD_LIST", + "BUILD_LIST_UNPACK", + "BUILD_MAP", + "BUILD_MAP_UNPACK", + "BUILD_MAP_UNPACK_WITH_CALL", + "BUILD_SET", + "BUILD_SET_UNPACK", + "BUILD_SLICE", + "BUILD_STRING", + "BUILD_TUPLE", + "BUILD_TUPLE_UNPACK", + "BUILD_TUPLE_UNPACK_WITH_CALL", + "CACHE", + "CALL_FINALLY", + "CALL_FUNCTION", + "CALL_FUNCTION_EX", + "CALL_FUNCTION_KW", + "CALL_METHOD", + "CHECK_EG_MATCH", + "CHECK_EXC_MATCH", + "CLEANUP_THROW", + "COMPARE_OP", + "CONTAINS_OP", + "CONTINUE_LOOP", + "COPY", + "COPY_DICT_WITHOUT_KEYS", + "COPY_FREE_VARS", + "DELETE_ATTR", + "DELETE_DEREF", + "DELETE_FAST", + "DELETE_GLOBAL", + "DELETE_NAME", + "DELETE_SUBSCR", + "DICT_MERGE", + "DICT_UPDATE", + "DUP_TOP", + "DUP_TOP_TWO", + "END_ASYNC_FOR", + "END_FINALLY", + "END_FOR", + "END_SEND", + "EXTENDED_ARG", + "FOR_ITER", + "FORMAT_VALUE", + "GEN_START", + "GET_AITER", + "GET_ANEXT", + "GET_AWAITABLE", + "GET_ITER", + "GET_LEN", + "GET_YIELD_FROM_ITER", + "IMPORT_FROM", + "IMPORT_NAME", + "IMPORT_STAR", + "INPLACE_ADD", + "INPLACE_AND", + "INPLACE_FLOOR_DIVIDE", + "INPLACE_LSHIFT", + "INPLACE_MATRIX_MULTIPLY", + "INPLACE_MODULO", + "INPLACE_MULTIPLY", + "INPLACE_OR", + "INPLACE_POWER", + "INPLACE_RSHIFT", + "INPLACE_SUBTRACT", + "INPLACE_TRUE_DIVIDE", + "INPLACE_XOR", + "INTERPRETER_EXIT", + "IS_OP", + "JUMP_ABSOLUTE", + "JUMP_BACKWARD", + "JUMP_BACKWARD_NO_INTERRUPT", + "JUMP_FORWARD", + "JUMP_IF_FALSE_OR_POP", + "JUMP_IF_NOT_EXC_MATCH", + "JUMP_IF_TRUE_OR_POP", + "LIST_APPEND", + "LIST_EXTEND", + "LIST_TO_TUPLE", + "LOAD_ASSERTION_ERROR", + "LOAD_ATTR", + "LOAD_BUILD_CLASS", + "LOAD_CLASSDEREF", + "LOAD_CLOSURE", + "LOAD_CONST", + "LOAD_DEREF", + "LOAD_FAST", + "LOAD_FAST_AND_CLEAR", + "LOAD_FAST_CHECK", + "LOAD_GLOBAL", + "LOAD_LOCALS", + "LOAD_METHOD", + "LOAD_NAME", + "LOAD_SUPER_ATTR", + "MAKE_CELL", + "MAKE_FUNCTION", + "MAP_ADD", + "MATCH_CLASS", + "MATCH_KEYS", + "MATCH_MAPPING", + "MATCH_SEQUENCE", + "NOP", + "POP_BLOCK", + "POP_EXCEPT", + "POP_FINALLY", + "POP_JUMP_FORWARD_IF_FALSE", + "POP_JUMP_FORWARD_IF_NONE", + "POP_JUMP_FORWARD_IF_NOT_NONE", + "POP_JUMP_FORWARD_IF_TRUE", + "POP_JUMP_IF_FALSE", + "POP_JUMP_IF_NONE", + "POP_JUMP_IF_NOT_NONE", + "POP_JUMP_IF_TRUE", + "POP_TOP", + "PRECALL", + "PREP_RERAISE_STAR", + "PRINT_EXPR", + "PUSH_EXC_INFO", + "PUSH_NULL", + "RAISE_VARARGS", + "RERAISE", + "RESERVED", + "RESUME", + "RETURN_CONST", + "RETURN_GENERATOR", + "RETURN_VALUE", + "ROT_FOUR", + "ROT_N", + "ROT_THREE", + "ROT_TWO", + "SEND", + "SET_ADD", + "SET_UPDATE", + "SETUP_ANNOTATIONS", + "SETUP_ASYNC_WITH", + "SETUP_EXCEPT", + "SETUP_FINALLY", + "SETUP_LOOP", + "SETUP_WITH", + "STORE_ATTR", + "STORE_DEREF", + "STORE_FAST", + "STORE_GLOBAL", + "STORE_NAME", + "STORE_SLICE", + "STORE_SUBSCR", + "SWAP", + "UNARY_INVERT", + "UNARY_NEGATIVE", + "UNARY_NOT", + "UNARY_POSITIVE", + "UNPACK_EX", + "UNPACK_SEQUENCE", + "WITH_CLEANUP_FINISH", + "WITH_CLEANUP_START", + "WITH_EXCEPT_START", + "YIELD_FROM", + "YIELD_VALUE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "match", + "type", + "HAVE_ARGUMENT", + "CALL_INTRINSIC_1", + "CALL_INTRINSIC_2", + "JUMP_NO_INTERRUPT", + "nargs", + "vargs", + "compare", + "name", + "const", + "local" + ], + "bos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "clean_up_tokenization_spaces": true, + "cls_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "errors": "replace", + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 512, + "pad_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "sep_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "tokenizer_class": "RobertaTokenizer", + "trim_offsets": true, + "unk_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/model_training/statement/train_seq2seq.py b/model_training/statement/train_seq2seq.py new file mode 100644 index 0000000..0127603 --- /dev/null +++ b/model_training/statement/train_seq2seq.py @@ -0,0 +1,93 @@ +import os +import pathlib +import time +from datetime import timedelta +import click + +from datasets import ReadInstruction, load_dataset +from StatementConfiguration import StatementConfiguration, parse_statement_config_json +from transformers import ( + DataCollatorForSeq2Seq, + RobertaTokenizer, + Seq2SeqTrainer, + Seq2SeqTrainingArguments, + T5ForConditionalGeneration, +) + + +def load_tokenized_train_dataset(dataset_repo_name: str, dataset_percentage: int): + # Load the tokenized dataset + tokenized_train_dataset = load_dataset( + dataset_repo_name, + token=True, + split=ReadInstruction("train", to=dataset_percentage, unit="%"), + ) + return tokenized_train_dataset + + +def train_statement_model(config: StatementConfiguration): + # load model, Salesforce/codet5-base is a pretrained model solving the code generation task. + tokenizer = RobertaTokenizer.from_pretrained(config.tokenizer_repo_name) + model = T5ForConditionalGeneration.from_pretrained(config.pretrained_seq2seq_repo_name) + data_collator = DataCollatorForSeq2Seq(tokenizer, model=model) + + model_dir = str(config.statement_model_dir) + model_repo_name = config.statement_model_repo_name + + train_args = Seq2SeqTrainingArguments( + output_dir=model_dir, + learning_rate=config.statement_training_parameters.learning_rate, + per_device_train_batch_size=config.statement_training_parameters.batch_size, + per_device_eval_batch_size=config.statement_training_parameters.batch_size, + weight_decay=0.01, + fp16=config.fp16, + logging_dir=str(config.log_dir), + report_to="tensorboard", + logging_strategy="steps", + logging_steps=1000, + save_strategy="steps", + save_steps=10000, + save_total_limit=2, + num_train_epochs=config.statement_training_parameters.epochs, + predict_with_generate=True, + push_to_hub=True, + hub_model_id=model_repo_name, + hub_private_repo=True, + ddp_backend="nccl", + ddp_find_unused_parameters=False, + ) + + tokenized_train_dataset = load_tokenized_train_dataset(config.tokenized_dataset_repo_name, config.dataset_percentage) + trainer = Seq2SeqTrainer( + model=model, + args=train_args, + data_collator=data_collator, + train_dataset=tokenized_train_dataset, + tokenizer=tokenizer, + ) + + start = time.time() + trainer.train() + duration = str(timedelta(seconds=time.time() - start)) + + if int(os.environ["LOCAL_RANK"]) == 0: + # upload the latest version of the model to the Model Hub on Huggingface + trainer.save_model(str(config.statement_model_dir)) + # this command returns the URL of the commit it just did + trainer.push_to_hub( + commit_message=duration, + finetuned_from=config.pretrained_seq2seq_repo_name, + dataset=config.tokenized_dataset_repo_name, + ) + + +@click.command(help="Training script for the statement translation model given a statement json.") +@click.argument("json_path", type=str) +def main(json_path: str): + json_file_path = pathlib.Path(json_path) + statement_config = parse_statement_config_json(json_file_path) + train_statement_model(statement_config) + + +if __name__ == "__main__": + main() diff --git a/model_training/statement/train_tokenizer_auto.py b/model_training/statement/train_tokenizer_auto.py new file mode 100644 index 0000000..54afea9 --- /dev/null +++ b/model_training/statement/train_tokenizer_auto.py @@ -0,0 +1,93 @@ +import logging +import pathlib +import click + +from datasets import ReadInstruction, load_dataset +from huggingface_hub import HfApi, repo_exists +from StatementConfiguration import StatementConfiguration, parse_statement_config_json +from tokenizers import Tokenizer +from transformers import AutoTokenizer + + +def get_untrained_tokenizer(tokenizer_repo_name: str) -> AutoTokenizer: + tokenizer_dir = pathlib.Path(__file__).parent / tokenizer_repo_name + tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir) + return tokenizer + + +def save_and_upload_tokenizer( + tokenizer: Tokenizer, + tokenizer_json_path: pathlib.Path, + tokenizer_repo_name: str, + dataset_name: str, +): + # Save the tokenizer locally + tokenizer.save_pretrained(str(tokenizer_json_path.parent.resolve())) + + # Upload files to Hugging Face Hub + api = HfApi() + api.create_repo(tokenizer_repo_name, exist_ok=True, private=True) + api.upload_file( + path_in_repo="tokenizer_config.json", + path_or_fileobj=str(tokenizer_json_path.parent / "tokenizer_config.json"), + repo_id=tokenizer_repo_name, + commit_message=f"Trained tokenizer using {dataset_name}", + ) + api.upload_file( + path_in_repo="vocab.json", + path_or_fileobj=str(tokenizer_json_path.parent / "vocab.json"), + repo_id=tokenizer_repo_name, + commit_message="Extracted vocabulary from tokenizer", + ) + api.upload_file( + path_in_repo="merges.txt", + path_or_fileobj=str(tokenizer_json_path.parent / "merges.txt"), + repo_id=tokenizer_repo_name, + commit_message="Extracted merges from tokenizer", + ) + api.upload_file( + path_in_repo="tokenizer.json", + path_or_fileobj=str(tokenizer_json_path.parent / "tokenizer.json"), + repo_id=tokenizer_repo_name, + commit_message="Extracted tokenizer", + ) + api.upload_file( + path_in_repo="special_tokens_map.json", + path_or_fileobj=str(tokenizer_json_path.parent / "special_tokens_map.json"), + repo_id=tokenizer_repo_name, + commit_message="Extracted special tokens map", + ) + + +def train_tokenizer(config: StatementConfiguration, tokenizer_json_path: pathlib.Path): + if repo_exists(config.base_repo_name): + logging.error(f"{config.base_repo_name} has already exists") + exit(1) + + tokenizer = get_untrained_tokenizer("tokenizer") + + train_dataset = load_dataset( + config.dataset_repo_name, + token=True, + split=ReadInstruction("train", to=config.dataset_percentage, unit="%"), + )["bytecode"] + + tokenizer = tokenizer.train_new_from_iterator(train_dataset, vocab_size=30000) + save_and_upload_tokenizer( + tokenizer, + tokenizer_json_path, + config.tokenizer_repo_name, + config.dataset_repo_name, + ) + + +@click.command(help="Training script for the bytecode tokenizer for the statement model given a statement json.") +@click.argument("json_path", type=str) +def main(json_path: str): + json_file_path = pathlib.Path(json_path) + statement_config = parse_statement_config_json(json_file_path) + train_tokenizer(statement_config, json_file_path) + + +if __name__ == "__main__": + main() diff --git a/model_training/train_models.py b/model_training/train_models.py new file mode 100644 index 0000000..9dae8c6 --- /dev/null +++ b/model_training/train_models.py @@ -0,0 +1,117 @@ +import logging +import os +import pathlib +import subprocess +import click + +from pylingual.utils.get_logger import get_logger + + +def train_segmentation(segmentation_config_path: pathlib.Path, logger: logging.Logger, nnodes: int = 1, nproc_per_node: int = 1, rdzv_port: int = 29400): + segmentation_root = pathlib.Path(__file__).parent / "segmentation" + + # train tokenizer + logger.info("training tokenizer...") + subprocess.run(["python", segmentation_root / "train_tokenizer.py", segmentation_config_path]) + + # train mlm (single gpu to avoid conflicts with local tokenized data) + logger.info("training masked language model...") + subprocess.run( + [ + "torchrun", + f"--nnodes={nnodes}", + f"--nproc-per-node={nproc_per_node}", + "--rdzv-backend=c10d", + f"--rdzv-endpoint=localhost:{rdzv_port}", + segmentation_root / "train_mlm.py", + segmentation_config_path, + ], + env=dict(os.environ, NCCL_P2P_DISABLE="1"), + ) + + # tokenize dataset + logger.info("tokenizing segmentation dataset...") + subprocess.run(["python", segmentation_root / "tokenize_seg.py", segmentation_config_path]) + + # train segmentation model (4 gpus) + logger.info("training segmentation model...") + subprocess.run( + [ + "torchrun", + f"--nnodes={nnodes}", + f"--nproc-per-node={nproc_per_node}", + "--rdzv-backend=c10d", + f"--rdzv-endpoint=localhost:{rdzv_port}", + segmentation_root / "train_seg.py", + segmentation_config_path, + ], + env=dict(os.environ, NCCL_P2P_DISABLE="1"), + ) + + +def train_statement(statement_config_path: pathlib.Path, logger: logging.Logger, nnodes: int = 1, nproc_per_node: int = 1, rdzv_port: int = 29400): + statement_root = pathlib.Path(__file__).parent / "statement" + + # manual tokenizer + subprocess.run(["python", statement_root / "train_tokenizer_auto.py", statement_config_path]) + + # tokenize statement dataset with salesforce tokenizer + logger.info("tokenizing statement dataset...") + subprocess.run(["python", statement_root / "tokenize_seq2seq.py", statement_config_path]) + + # train statement model (4 gpus) + logger.info("training statement model...") + subprocess.run( + [ + "torchrun", + f"--nnodes={nnodes}", + f"--nproc-per-node={nproc_per_node}", + "--rdzv-backend=c10d", + f"--rdzv-endpoint=localhost:{rdzv_port}", + statement_root / "train_seq2seq.py", + statement_config_path, + ], + env=dict(os.environ, NCCL_P2P_DISABLE="1"), + ) + + +@click.command(help="Full tokenization and training pipeline for the segmentation and statement translation models.") +@click.option("--segmentation", type=str, default=None, help="The path to the segmentation model description JSON file.") +@click.option("--statement", type=str, default=None, help="The path to the statement model description JSON file.") +@click.option("--nnodes", type=int, default=1, help="Torchrun nnodes arg") +@click.option("--nproc_per_node", type=int, default=1, help="Torchrun nproc_per_node arg") +@click.option("--rdzv_port", "-p", type=int, default=29400, help="Port to use for torchrun rendezvous endpoint") +def main(segmentation: str, statement: str, nnodes: int, nproc_per_node: int, rdzv_port: int): + logger = get_logger("train-models") + + ### LOAD JSON + logger.info("Training pipeline starting...") + logger.info("Loading dataset description JSON files...") + + ### CONFIG_PATHS + segmentation_config_path = pathlib.Path(segmentation).resolve() if segmentation is not None else None + statement_config_path = pathlib.Path(statement).resolve() if statement is not None else None + + logger.info("Dataset description JSON files loaded!") + + ### TRAIN SEGMENTATION + if segmentation_config_path is not None: + logger.info("Segmentation model training starting...") + train_segmentation(segmentation_config_path, logger, nnodes, nproc_per_node, rdzv_port) + logger.info("Segmentation model training complete!") + else: + logger.warning("Segmentation model configuration json path not provided in --segmentation; skipping segmentation model training...") + + ### TRAIN STATEMENT + if statement_config_path is not None: + logger.info("Statement model training starting...") + train_statement(statement_config_path, logger, nnodes, nproc_per_node, rdzv_port) + logger.info("Statement model training complete!") + else: + logger.warning("Statement model configuration json path not provided in --statement; skipping statement model training...") + + logger.info("Training pipeline complete!") + + +if __name__ == "__main__": + main() diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..0694adb --- /dev/null +++ b/poetry.lock @@ -0,0 +1,3044 @@ +# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. + +[[package]] +name = "accelerate" +version = "1.4.0" +description = "Accelerate" +optional = false +python-versions = ">=3.9.0" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "accelerate-1.4.0-py3-none-any.whl", hash = "sha256:f6e1e7dfaf9d799a20a1dc45efbf4b1546163eac133faa5acd0d89177c896e55"}, + {file = "accelerate-1.4.0.tar.gz", hash = "sha256:37d413e1b64cb8681ccd2908ae211cf73e13e6e636a2f598a96eccaa538773a5"}, +] + +[package.dependencies] +huggingface-hub = ">=0.21.0" +numpy = ">=1.17,<3.0.0" +packaging = ">=20.0" +psutil = "*" +pyyaml = "*" +safetensors = ">=0.4.3" +torch = ">=2.0.0" + +[package.extras] +deepspeed = ["deepspeed"] +dev = ["bitsandbytes", "black (>=23.1,<24.0)", "datasets", "diffusers", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest (>=7.2.0,<=8.0.0)", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.6.4,<0.7.0)", "scikit-learn", "scipy", "timm", "torchdata (>=0.8.0)", "torchpippy (>=0.2.0)", "tqdm", "transformers"] +quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.6.4,<0.7.0)"] +rich = ["rich"] +sagemaker = ["sagemaker"] +test-dev = ["bitsandbytes", "datasets", "diffusers", "evaluate", "scikit-learn", "scipy", "timm", "torchdata (>=0.8.0)", "torchpippy (>=0.2.0)", "tqdm", "transformers"] +test-prod = ["parameterized", "pytest (>=7.2.0,<=8.0.0)", "pytest-subtests", "pytest-xdist"] +test-trackers = ["comet-ml", "dvclive", "tensorboard", "wandb"] +testing = ["bitsandbytes", "datasets", "diffusers", "evaluate", "parameterized", "pytest (>=7.2.0,<=8.0.0)", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "timm", "torchdata (>=0.8.0)", "torchpippy (>=0.2.0)", "tqdm", "transformers"] + +[[package]] +name = "aiohappyeyeballs" +version = "2.5.0" +description = "Happy Eyeballs for asyncio" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "aiohappyeyeballs-2.5.0-py3-none-any.whl", hash = "sha256:0850b580748c7071db98bffff6d4c94028d0d3035acc20fd721a0ce7e8cac35d"}, + {file = "aiohappyeyeballs-2.5.0.tar.gz", hash = "sha256:18fde6204a76deeabc97c48bdd01d5801cfda5d6b9c8bbeb1aaaee9d648ca191"}, +] + +[[package]] +name = "aiohttp" +version = "3.11.13" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "aiohttp-3.11.13-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a4fe27dbbeec445e6e1291e61d61eb212ee9fed6e47998b27de71d70d3e8777d"}, + {file = "aiohttp-3.11.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9e64ca2dbea28807f8484c13f684a2f761e69ba2640ec49dacd342763cc265ef"}, + {file = "aiohttp-3.11.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9840be675de208d1f68f84d578eaa4d1a36eee70b16ae31ab933520c49ba1325"}, + {file = "aiohttp-3.11.13-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28a772757c9067e2aee8a6b2b425d0efaa628c264d6416d283694c3d86da7689"}, + {file = "aiohttp-3.11.13-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b88aca5adbf4625e11118df45acac29616b425833c3be7a05ef63a6a4017bfdb"}, + {file = "aiohttp-3.11.13-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce10ddfbe26ed5856d6902162f71b8fe08545380570a885b4ab56aecfdcb07f4"}, + {file = "aiohttp-3.11.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa48dac27f41b36735c807d1ab093a8386701bbf00eb6b89a0f69d9fa26b3671"}, + {file = "aiohttp-3.11.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:89ce611b1eac93ce2ade68f1470889e0173d606de20c85a012bfa24be96cf867"}, + {file = "aiohttp-3.11.13-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:78e4dd9c34ec7b8b121854eb5342bac8b02aa03075ae8618b6210a06bbb8a115"}, + {file = "aiohttp-3.11.13-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:66047eacbc73e6fe2462b77ce39fc170ab51235caf331e735eae91c95e6a11e4"}, + {file = "aiohttp-3.11.13-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5ad8f1c19fe277eeb8bc45741c6d60ddd11d705c12a4d8ee17546acff98e0802"}, + {file = "aiohttp-3.11.13-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:64815c6f02e8506b10113ddbc6b196f58dbef135751cc7c32136df27b736db09"}, + {file = "aiohttp-3.11.13-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:967b93f21b426f23ca37329230d5bd122f25516ae2f24a9cea95a30023ff8283"}, + {file = "aiohttp-3.11.13-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cf1f31f83d16ec344136359001c5e871915c6ab685a3d8dee38e2961b4c81730"}, + {file = "aiohttp-3.11.13-cp310-cp310-win32.whl", hash = "sha256:00c8ac69e259c60976aa2edae3f13d9991cf079aaa4d3cd5a49168ae3748dee3"}, + {file = "aiohttp-3.11.13-cp310-cp310-win_amd64.whl", hash = "sha256:90d571c98d19a8b6e793b34aa4df4cee1e8fe2862d65cc49185a3a3d0a1a3996"}, + {file = "aiohttp-3.11.13-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b35aab22419ba45f8fc290d0010898de7a6ad131e468ffa3922b1b0b24e9d2e"}, + {file = "aiohttp-3.11.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f81cba651db8795f688c589dd11a4fbb834f2e59bbf9bb50908be36e416dc760"}, + {file = "aiohttp-3.11.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f55d0f242c2d1fcdf802c8fabcff25a9d85550a4cf3a9cf5f2a6b5742c992839"}, + {file = "aiohttp-3.11.13-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4bea08a6aad9195ac9b1be6b0c7e8a702a9cec57ce6b713698b4a5afa9c2e33"}, + {file = "aiohttp-3.11.13-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6070bcf2173a7146bb9e4735b3c62b2accba459a6eae44deea0eb23e0035a23"}, + {file = "aiohttp-3.11.13-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:718d5deb678bc4b9d575bfe83a59270861417da071ab44542d0fcb6faa686636"}, + {file = "aiohttp-3.11.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f6b2c5b4a4d22b8fb2c92ac98e0747f5f195e8e9448bfb7404cd77e7bfa243f"}, + {file = "aiohttp-3.11.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:747ec46290107a490d21fe1ff4183bef8022b848cf9516970cb31de6d9460088"}, + {file = "aiohttp-3.11.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:01816f07c9cc9d80f858615b1365f8319d6a5fd079cd668cc58e15aafbc76a54"}, + {file = "aiohttp-3.11.13-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:a08ad95fcbd595803e0c4280671d808eb170a64ca3f2980dd38e7a72ed8d1fea"}, + {file = "aiohttp-3.11.13-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c97be90d70f7db3aa041d720bfb95f4869d6063fcdf2bb8333764d97e319b7d0"}, + {file = "aiohttp-3.11.13-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ab915a57c65f7a29353c8014ac4be685c8e4a19e792a79fe133a8e101111438e"}, + {file = "aiohttp-3.11.13-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:35cda4e07f5e058a723436c4d2b7ba2124ab4e0aa49e6325aed5896507a8a42e"}, + {file = "aiohttp-3.11.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:af55314407714fe77a68a9ccaab90fdb5deb57342585fd4a3a8102b6d4370080"}, + {file = "aiohttp-3.11.13-cp311-cp311-win32.whl", hash = "sha256:42d689a5c0a0c357018993e471893e939f555e302313d5c61dfc566c2cad6185"}, + {file = "aiohttp-3.11.13-cp311-cp311-win_amd64.whl", hash = "sha256:b73a2b139782a07658fbf170fe4bcdf70fc597fae5ffe75e5b67674c27434a9f"}, + {file = "aiohttp-3.11.13-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2eabb269dc3852537d57589b36d7f7362e57d1ece308842ef44d9830d2dc3c90"}, + {file = "aiohttp-3.11.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b77ee42addbb1c36d35aca55e8cc6d0958f8419e458bb70888d8c69a4ca833d"}, + {file = "aiohttp-3.11.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55789e93c5ed71832e7fac868167276beadf9877b85697020c46e9a75471f55f"}, + {file = "aiohttp-3.11.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c929f9a7249a11e4aa5c157091cfad7f49cc6b13f4eecf9b747104befd9f56f2"}, + {file = "aiohttp-3.11.13-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d33851d85537bbf0f6291ddc97926a754c8f041af759e0aa0230fe939168852b"}, + {file = "aiohttp-3.11.13-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9229d8613bd8401182868fe95688f7581673e1c18ff78855671a4b8284f47bcb"}, + {file = "aiohttp-3.11.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669dd33f028e54fe4c96576f406ebb242ba534dd3a981ce009961bf49960f117"}, + {file = "aiohttp-3.11.13-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c1b20a1ace54af7db1f95af85da530fe97407d9063b7aaf9ce6a32f44730778"}, + {file = "aiohttp-3.11.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5724cc77f4e648362ebbb49bdecb9e2b86d9b172c68a295263fa072e679ee69d"}, + {file = "aiohttp-3.11.13-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:aa36c35e94ecdb478246dd60db12aba57cfcd0abcad43c927a8876f25734d496"}, + {file = "aiohttp-3.11.13-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9b5b37c863ad5b0892cc7a4ceb1e435e5e6acd3f2f8d3e11fa56f08d3c67b820"}, + {file = "aiohttp-3.11.13-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e06cf4852ce8c4442a59bae5a3ea01162b8fcb49ab438d8548b8dc79375dad8a"}, + {file = "aiohttp-3.11.13-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5194143927e494616e335d074e77a5dac7cd353a04755330c9adc984ac5a628e"}, + {file = "aiohttp-3.11.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:afcb6b275c2d2ba5d8418bf30a9654fa978b4f819c2e8db6311b3525c86fe637"}, + {file = "aiohttp-3.11.13-cp312-cp312-win32.whl", hash = "sha256:7104d5b3943c6351d1ad7027d90bdd0ea002903e9f610735ac99df3b81f102ee"}, + {file = "aiohttp-3.11.13-cp312-cp312-win_amd64.whl", hash = "sha256:47dc018b1b220c48089b5b9382fbab94db35bef2fa192995be22cbad3c5730c8"}, + {file = "aiohttp-3.11.13-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9862d077b9ffa015dbe3ce6c081bdf35135948cb89116e26667dd183550833d1"}, + {file = "aiohttp-3.11.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbfef0666ae9e07abfa2c54c212ac18a1f63e13e0760a769f70b5717742f3ece"}, + {file = "aiohttp-3.11.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:93a1f7d857c4fcf7cabb1178058182c789b30d85de379e04f64c15b7e88d66fb"}, + {file = "aiohttp-3.11.13-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba40b7ae0f81c7029583a338853f6607b6d83a341a3dcde8bed1ea58a3af1df9"}, + {file = "aiohttp-3.11.13-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5b95787335c483cd5f29577f42bbe027a412c5431f2f80a749c80d040f7ca9f"}, + {file = "aiohttp-3.11.13-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7d474c5c1f0b9405c1565fafdc4429fa7d986ccbec7ce55bc6a330f36409cad"}, + {file = "aiohttp-3.11.13-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e83fb1991e9d8982b3b36aea1e7ad27ea0ce18c14d054c7a404d68b0319eebb"}, + {file = "aiohttp-3.11.13-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4586a68730bd2f2b04a83e83f79d271d8ed13763f64b75920f18a3a677b9a7f0"}, + {file = "aiohttp-3.11.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fe4eb0e7f50cdb99b26250d9328faef30b1175a5dbcfd6d0578d18456bac567"}, + {file = "aiohttp-3.11.13-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2a8a6bc19818ac3e5596310ace5aa50d918e1ebdcc204dc96e2f4d505d51740c"}, + {file = "aiohttp-3.11.13-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7f27eec42f6c3c1df09cfc1f6786308f8b525b8efaaf6d6bd76c1f52c6511f6a"}, + {file = "aiohttp-3.11.13-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2a4a13dfbb23977a51853b419141cd0a9b9573ab8d3a1455c6e63561387b52ff"}, + {file = "aiohttp-3.11.13-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:02876bf2f69b062584965507b07bc06903c2dc93c57a554b64e012d636952654"}, + {file = "aiohttp-3.11.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b992778d95b60a21c4d8d4a5f15aaab2bd3c3e16466a72d7f9bfd86e8cea0d4b"}, + {file = "aiohttp-3.11.13-cp313-cp313-win32.whl", hash = "sha256:507ab05d90586dacb4f26a001c3abf912eb719d05635cbfad930bdbeb469b36c"}, + {file = "aiohttp-3.11.13-cp313-cp313-win_amd64.whl", hash = "sha256:5ceb81a4db2decdfa087381b5fc5847aa448244f973e5da232610304e199e7b2"}, + {file = "aiohttp-3.11.13-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:51c3ff9c7a25f3cad5c09d9aacbc5aefb9267167c4652c1eb737989b554fe278"}, + {file = "aiohttp-3.11.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e271beb2b1dabec5cd84eb488bdabf9758d22ad13471e9c356be07ad139b3012"}, + {file = "aiohttp-3.11.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0e9eb7e5764abcb49f0e2bd8f5731849b8728efbf26d0cac8e81384c95acec3f"}, + {file = "aiohttp-3.11.13-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baae005092e3f200de02699314ac8933ec20abf998ec0be39448f6605bce93df"}, + {file = "aiohttp-3.11.13-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1982c98ac62c132d2b773d50e2fcc941eb0b8bad3ec078ce7e7877c4d5a2dce7"}, + {file = "aiohttp-3.11.13-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2b25b2eeb35707113b2d570cadc7c612a57f1c5d3e7bb2b13870fe284e08fc0"}, + {file = "aiohttp-3.11.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b27961d65639128336b7a7c3f0046dcc62a9443d5ef962e3c84170ac620cec47"}, + {file = "aiohttp-3.11.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a01fe9f1e05025eacdd97590895e2737b9f851d0eb2e017ae9574d9a4f0b6252"}, + {file = "aiohttp-3.11.13-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa1fb1b61881c8405829c50e9cc5c875bfdbf685edf57a76817dfb50643e4a1a"}, + {file = "aiohttp-3.11.13-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:25de43bb3cf83ad83efc8295af7310219af6dbe4c543c2e74988d8e9c8a2a917"}, + {file = "aiohttp-3.11.13-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:fe7065e2215e4bba63dc00db9ae654c1ba3950a5fff691475a32f511142fcddb"}, + {file = "aiohttp-3.11.13-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:7836587eef675a17d835ec3d98a8c9acdbeb2c1d72b0556f0edf4e855a25e9c1"}, + {file = "aiohttp-3.11.13-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:85fa0b18558eb1427090912bd456a01f71edab0872f4e0f9e4285571941e4090"}, + {file = "aiohttp-3.11.13-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a86dc177eb4c286c19d1823ac296299f59ed8106c9536d2b559f65836e0fb2c6"}, + {file = "aiohttp-3.11.13-cp39-cp39-win32.whl", hash = "sha256:684eea71ab6e8ade86b9021bb62af4bf0881f6be4e926b6b5455de74e420783a"}, + {file = "aiohttp-3.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:82c249f2bfa5ecbe4a1a7902c81c0fba52ed9ebd0176ab3047395d02ad96cfcb"}, + {file = "aiohttp-3.11.13.tar.gz", hash = "sha256:8ce789231404ca8fff7f693cdce398abf6d90fd5dae2b1847477196c243b1fbb"}, +] + +[package.dependencies] +aiohappyeyeballs = ">=2.3.0" +aiosignal = ">=1.1.2" +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +propcache = ">=0.2.0" +yarl = ">=1.17.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] + +[[package]] +name = "aiosignal" +version = "1.3.2" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5"}, + {file = "aiosignal-1.3.2.tar.gz", hash = "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "asttokens" +version = "3.0.0" +description = "Annotate AST trees with source code positions" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2"}, + {file = "asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7"}, +] + +[package.extras] +astroid = ["astroid (>=2,<4)"] +test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] + +[[package]] +name = "attrs" +version = "25.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "attrs-25.1.0-py3-none-any.whl", hash = "sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a"}, + {file = "attrs-25.1.0.tar.gz", hash = "sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e"}, +] + +[package.extras] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] + +[[package]] +name = "certifi" +version = "2025.1.31" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, + {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win32.whl", hash = "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win32.whl", hash = "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win32.whl", hash = "sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765"}, + {file = "charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85"}, + {file = "charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3"}, +] + +[[package]] +name = "click" +version = "8.1.8" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Windows\" or python_version >= \"3.12\" and platform_system == \"Windows\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "contourpy" +version = "1.3.1" +description = "Python library for calculating contours of 2D quadrilateral grids" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "contourpy-1.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a045f341a77b77e1c5de31e74e966537bba9f3c4099b35bf4c2e3939dd54cdab"}, + {file = "contourpy-1.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:500360b77259914f7805af7462e41f9cb7ca92ad38e9f94d6c8641b089338124"}, + {file = "contourpy-1.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2f926efda994cdf3c8d3fdb40b9962f86edbc4457e739277b961eced3d0b4c1"}, + {file = "contourpy-1.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adce39d67c0edf383647a3a007de0a45fd1b08dedaa5318404f1a73059c2512b"}, + {file = "contourpy-1.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abbb49fb7dac584e5abc6636b7b2a7227111c4f771005853e7d25176daaf8453"}, + {file = "contourpy-1.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0cffcbede75c059f535725c1680dfb17b6ba8753f0c74b14e6a9c68c29d7ea3"}, + {file = "contourpy-1.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ab29962927945d89d9b293eabd0d59aea28d887d4f3be6c22deaefbb938a7277"}, + {file = "contourpy-1.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:974d8145f8ca354498005b5b981165b74a195abfae9a8129df3e56771961d595"}, + {file = "contourpy-1.3.1-cp310-cp310-win32.whl", hash = "sha256:ac4578ac281983f63b400f7fe6c101bedc10651650eef012be1ccffcbacf3697"}, + {file = "contourpy-1.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:174e758c66bbc1c8576992cec9599ce8b6672b741b5d336b5c74e35ac382b18e"}, + {file = "contourpy-1.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3e8b974d8db2c5610fb4e76307e265de0edb655ae8169e8b21f41807ccbeec4b"}, + {file = "contourpy-1.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:20914c8c973f41456337652a6eeca26d2148aa96dd7ac323b74516988bea89fc"}, + {file = "contourpy-1.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19d40d37c1c3a4961b4619dd9d77b12124a453cc3d02bb31a07d58ef684d3d86"}, + {file = "contourpy-1.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:113231fe3825ebf6f15eaa8bc1f5b0ddc19d42b733345eae0934cb291beb88b6"}, + {file = "contourpy-1.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4dbbc03a40f916a8420e420d63e96a1258d3d1b58cbdfd8d1f07b49fcbd38e85"}, + {file = "contourpy-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a04ecd68acbd77fa2d39723ceca4c3197cb2969633836ced1bea14e219d077c"}, + {file = "contourpy-1.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c414fc1ed8ee1dbd5da626cf3710c6013d3d27456651d156711fa24f24bd1291"}, + {file = "contourpy-1.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:31c1b55c1f34f80557d3830d3dd93ba722ce7e33a0b472cba0ec3b6535684d8f"}, + {file = "contourpy-1.3.1-cp311-cp311-win32.whl", hash = "sha256:f611e628ef06670df83fce17805c344710ca5cde01edfdc72751311da8585375"}, + {file = "contourpy-1.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:b2bdca22a27e35f16794cf585832e542123296b4687f9fd96822db6bae17bfc9"}, + {file = "contourpy-1.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0ffa84be8e0bd33410b17189f7164c3589c229ce5db85798076a3fa136d0e509"}, + {file = "contourpy-1.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805617228ba7e2cbbfb6c503858e626ab528ac2a32a04a2fe88ffaf6b02c32bc"}, + {file = "contourpy-1.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade08d343436a94e633db932e7e8407fe7de8083967962b46bdfc1b0ced39454"}, + {file = "contourpy-1.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47734d7073fb4590b4a40122b35917cd77be5722d80683b249dac1de266aac80"}, + {file = "contourpy-1.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ba94a401342fc0f8b948e57d977557fbf4d515f03c67682dd5c6191cb2d16ec"}, + {file = "contourpy-1.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efa874e87e4a647fd2e4f514d5e91c7d493697127beb95e77d2f7561f6905bd9"}, + {file = "contourpy-1.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1bf98051f1045b15c87868dbaea84f92408337d4f81d0e449ee41920ea121d3b"}, + {file = "contourpy-1.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61332c87493b00091423e747ea78200659dc09bdf7fd69edd5e98cef5d3e9a8d"}, + {file = "contourpy-1.3.1-cp312-cp312-win32.whl", hash = "sha256:e914a8cb05ce5c809dd0fe350cfbb4e881bde5e2a38dc04e3afe1b3e58bd158e"}, + {file = "contourpy-1.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:08d9d449a61cf53033612cb368f3a1b26cd7835d9b8cd326647efe43bca7568d"}, + {file = "contourpy-1.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a761d9ccfc5e2ecd1bf05534eda382aa14c3e4f9205ba5b1684ecfe400716ef2"}, + {file = "contourpy-1.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:523a8ee12edfa36f6d2a49407f705a6ef4c5098de4f498619787e272de93f2d5"}, + {file = "contourpy-1.3.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece6df05e2c41bd46776fbc712e0996f7c94e0d0543af1656956d150c4ca7c81"}, + {file = "contourpy-1.3.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:573abb30e0e05bf31ed067d2f82500ecfdaec15627a59d63ea2d95714790f5c2"}, + {file = "contourpy-1.3.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9fa36448e6a3a1a9a2ba23c02012c43ed88905ec80163f2ffe2421c7192a5d7"}, + {file = "contourpy-1.3.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ea9924d28fc5586bf0b42d15f590b10c224117e74409dd7a0be3b62b74a501c"}, + {file = "contourpy-1.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5b75aa69cb4d6f137b36f7eb2ace9280cfb60c55dc5f61c731fdf6f037f958a3"}, + {file = "contourpy-1.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:041b640d4ec01922083645a94bb3b2e777e6b626788f4095cf21abbe266413c1"}, + {file = "contourpy-1.3.1-cp313-cp313-win32.whl", hash = "sha256:36987a15e8ace5f58d4d5da9dca82d498c2bbb28dff6e5d04fbfcc35a9cb3a82"}, + {file = "contourpy-1.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:a7895f46d47671fa7ceec40f31fae721da51ad34bdca0bee83e38870b1f47ffd"}, + {file = "contourpy-1.3.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9ddeb796389dadcd884c7eb07bd14ef12408aaae358f0e2ae24114d797eede30"}, + {file = "contourpy-1.3.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:19c1555a6801c2f084c7ddc1c6e11f02eb6a6016ca1318dd5452ba3f613a1751"}, + {file = "contourpy-1.3.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:841ad858cff65c2c04bf93875e384ccb82b654574a6d7f30453a04f04af71342"}, + {file = "contourpy-1.3.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4318af1c925fb9a4fb190559ef3eec206845f63e80fb603d47f2d6d67683901c"}, + {file = "contourpy-1.3.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:14c102b0eab282427b662cb590f2e9340a9d91a1c297f48729431f2dcd16e14f"}, + {file = "contourpy-1.3.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05e806338bfeaa006acbdeba0ad681a10be63b26e1b17317bfac3c5d98f36cda"}, + {file = "contourpy-1.3.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4d76d5993a34ef3df5181ba3c92fabb93f1eaa5729504fb03423fcd9f3177242"}, + {file = "contourpy-1.3.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:89785bb2a1980c1bd87f0cb1517a71cde374776a5f150936b82580ae6ead44a1"}, + {file = "contourpy-1.3.1-cp313-cp313t-win32.whl", hash = "sha256:8eb96e79b9f3dcadbad2a3891672f81cdcab7f95b27f28f1c67d75f045b6b4f1"}, + {file = "contourpy-1.3.1-cp313-cp313t-win_amd64.whl", hash = "sha256:287ccc248c9e0d0566934e7d606201abd74761b5703d804ff3df8935f523d546"}, + {file = "contourpy-1.3.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b457d6430833cee8e4b8e9b6f07aa1c161e5e0d52e118dc102c8f9bd7dd060d6"}, + {file = "contourpy-1.3.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb76c1a154b83991a3cbbf0dfeb26ec2833ad56f95540b442c73950af2013750"}, + {file = "contourpy-1.3.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:44a29502ca9c7b5ba389e620d44f2fbe792b1fb5734e8b931ad307071ec58c53"}, + {file = "contourpy-1.3.1.tar.gz", hash = "sha256:dfd97abd83335045a913e3bcc4a09c0ceadbe66580cf573fe961f4a825efa699"}, +] + +[package.dependencies] +numpy = ">=1.23" + +[package.extras] +bokeh = ["bokeh", "selenium"] +docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] +mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.11.1)", "types-Pillow"] +test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] +test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] + +[[package]] +name = "cycler" +version = "0.12.1" +description = "Composable style cycles" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, + {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, +] + +[package.extras] +docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] +tests = ["pytest", "pytest-cov", "pytest-xdist"] + +[[package]] +name = "datasets" +version = "3.3.2" +description = "HuggingFace community-driven open-source library of datasets" +optional = false +python-versions = ">=3.9.0" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "datasets-3.3.2-py3-none-any.whl", hash = "sha256:fdaf3d5d70242621210b044e9b9b15a56e908bfc3e9d077bcf5605ac390f70bd"}, + {file = "datasets-3.3.2.tar.gz", hash = "sha256:20901a97da870fb80b407ccc45f034a7ac99accd07da897ed42f11641bdb8c6e"}, +] + +[package.dependencies] +aiohttp = "*" +dill = ">=0.3.0,<0.3.9" +filelock = "*" +fsspec = {version = ">=2023.1.0,<=2024.12.0", extras = ["http"]} +huggingface-hub = ">=0.24.0" +multiprocess = "<0.70.17" +numpy = ">=1.17" +packaging = "*" +pandas = "*" +pyarrow = ">=15.0.0" +pyyaml = ">=5.1" +requests = ">=2.32.2" +tqdm = ">=4.66.3" +xxhash = "*" + +[package.extras] +audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0)"] +benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] +dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] +docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] +jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] +quality = ["ruff (>=0.3.0)"] +s3 = ["s3fs"] +tensorflow = ["tensorflow (>=2.6.0)"] +tensorflow-gpu = ["tensorflow (>=2.6.0)"] +tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] +tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] +torch = ["torch"] +vision = ["Pillow (>=9.4.0)"] + +[[package]] +name = "dill" +version = "0.3.8" +description = "serialize all of Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, + {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] +profile = ["gprof2dot (>=2022.7.29)"] + +[[package]] +name = "filelock" +version = "3.17.0" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "filelock-3.17.0-py3-none-any.whl", hash = "sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338"}, + {file = "filelock-3.17.0.tar.gz", hash = "sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e"}, +] + +[package.extras] +docs = ["furo (>=2024.8.6)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.10)", "diff-cover (>=9.2.1)", "pytest (>=8.3.4)", "pytest-asyncio (>=0.25.2)", "pytest-cov (>=6)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.28.1)"] +typing = ["typing-extensions (>=4.12.2)"] + +[[package]] +name = "fonttools" +version = "4.56.0" +description = "Tools to manipulate font files" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "fonttools-4.56.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:331954d002dbf5e704c7f3756028e21db07097c19722569983ba4d74df014000"}, + {file = "fonttools-4.56.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8d1613abd5af2f93c05867b3a3759a56e8bf97eb79b1da76b2bc10892f96ff16"}, + {file = "fonttools-4.56.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:705837eae384fe21cee5e5746fd4f4b2f06f87544fa60f60740007e0aa600311"}, + {file = "fonttools-4.56.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc871904a53a9d4d908673c6faa15689874af1c7c5ac403a8e12d967ebd0c0dc"}, + {file = "fonttools-4.56.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:38b947de71748bab150259ee05a775e8a0635891568e9fdb3cdd7d0e0004e62f"}, + {file = "fonttools-4.56.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:86b2a1013ef7a64d2e94606632683f07712045ed86d937c11ef4dde97319c086"}, + {file = "fonttools-4.56.0-cp310-cp310-win32.whl", hash = "sha256:133bedb9a5c6376ad43e6518b7e2cd2f866a05b1998f14842631d5feb36b5786"}, + {file = "fonttools-4.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:17f39313b649037f6c800209984a11fc256a6137cbe5487091c6c7187cae4685"}, + {file = "fonttools-4.56.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ef04bc7827adb7532be3d14462390dd71287644516af3f1e67f1e6ff9c6d6df"}, + {file = "fonttools-4.56.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ffda9b8cd9cb8b301cae2602ec62375b59e2e2108a117746f12215145e3f786c"}, + {file = "fonttools-4.56.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e993e8db36306cc3f1734edc8ea67906c55f98683d6fd34c3fc5593fdbba4c"}, + {file = "fonttools-4.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:003548eadd674175510773f73fb2060bb46adb77c94854af3e0cc5bc70260049"}, + {file = "fonttools-4.56.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd9825822e7bb243f285013e653f6741954d8147427aaa0324a862cdbf4cbf62"}, + {file = "fonttools-4.56.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b23d30a2c0b992fb1c4f8ac9bfde44b5586d23457759b6cf9a787f1a35179ee0"}, + {file = "fonttools-4.56.0-cp311-cp311-win32.whl", hash = "sha256:47b5e4680002ae1756d3ae3b6114e20aaee6cc5c69d1e5911f5ffffd3ee46c6b"}, + {file = "fonttools-4.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:14a3e3e6b211660db54ca1ef7006401e4a694e53ffd4553ab9bc87ead01d0f05"}, + {file = "fonttools-4.56.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d6f195c14c01bd057bc9b4f70756b510e009c83c5ea67b25ced3e2c38e6ee6e9"}, + {file = "fonttools-4.56.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa760e5fe8b50cbc2d71884a1eff2ed2b95a005f02dda2fa431560db0ddd927f"}, + {file = "fonttools-4.56.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d54a45d30251f1d729e69e5b675f9a08b7da413391a1227781e2a297fa37f6d2"}, + {file = "fonttools-4.56.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:661a8995d11e6e4914a44ca7d52d1286e2d9b154f685a4d1f69add8418961563"}, + {file = "fonttools-4.56.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9d94449ad0a5f2a8bf5d2f8d71d65088aee48adbe45f3c5f8e00e3ad861ed81a"}, + {file = "fonttools-4.56.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f59746f7953f69cc3290ce2f971ab01056e55ddd0fb8b792c31a8acd7fee2d28"}, + {file = "fonttools-4.56.0-cp312-cp312-win32.whl", hash = "sha256:bce60f9a977c9d3d51de475af3f3581d9b36952e1f8fc19a1f2254f1dda7ce9c"}, + {file = "fonttools-4.56.0-cp312-cp312-win_amd64.whl", hash = "sha256:300c310bb725b2bdb4f5fc7e148e190bd69f01925c7ab437b9c0ca3e1c7cd9ba"}, + {file = "fonttools-4.56.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f20e2c0dfab82983a90f3d00703ac0960412036153e5023eed2b4641d7d5e692"}, + {file = "fonttools-4.56.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f36a0868f47b7566237640c026c65a86d09a3d9ca5df1cd039e30a1da73098a0"}, + {file = "fonttools-4.56.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62b4c6802fa28e14dba010e75190e0e6228513573f1eeae57b11aa1a39b7e5b1"}, + {file = "fonttools-4.56.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a05d1f07eb0a7d755fbe01fee1fd255c3a4d3730130cf1bfefb682d18fd2fcea"}, + {file = "fonttools-4.56.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0073b62c3438cf0058488c002ea90489e8801d3a7af5ce5f7c05c105bee815c3"}, + {file = "fonttools-4.56.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cad98c94833465bcf28f51c248aaf07ca022efc6a3eba750ad9c1e0256d278"}, + {file = "fonttools-4.56.0-cp313-cp313-win32.whl", hash = "sha256:d0cb73ccf7f6d7ca8d0bc7ea8ac0a5b84969a41c56ac3ac3422a24df2680546f"}, + {file = "fonttools-4.56.0-cp313-cp313-win_amd64.whl", hash = "sha256:62cc1253827d1e500fde9dbe981219fea4eb000fd63402283472d38e7d8aa1c6"}, + {file = "fonttools-4.56.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3fd3fccb7b9adaaecfa79ad51b759f2123e1aba97f857936ce044d4f029abd71"}, + {file = "fonttools-4.56.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:193b86e9f769320bc98ffdb42accafb5d0c8c49bd62884f1c0702bc598b3f0a2"}, + {file = "fonttools-4.56.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e81c1cc80c1d8bf071356cc3e0e25071fbba1c75afc48d41b26048980b3c771"}, + {file = "fonttools-4.56.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9270505a19361e81eecdbc2c251ad1e1a9a9c2ad75fa022ccdee533f55535dc"}, + {file = "fonttools-4.56.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:53f5e9767978a4daf46f28e09dbeb7d010319924ae622f7b56174b777258e5ba"}, + {file = "fonttools-4.56.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:9da650cb29bc098b8cfd15ef09009c914b35c7986c8fa9f08b51108b7bc393b4"}, + {file = "fonttools-4.56.0-cp38-cp38-win32.whl", hash = "sha256:965d0209e6dbdb9416100123b6709cb13f5232e2d52d17ed37f9df0cc31e2b35"}, + {file = "fonttools-4.56.0-cp38-cp38-win_amd64.whl", hash = "sha256:654ac4583e2d7c62aebc6fc6a4c6736f078f50300e18aa105d87ce8925cfac31"}, + {file = "fonttools-4.56.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca7962e8e5fc047cc4e59389959843aafbf7445b6c08c20d883e60ced46370a5"}, + {file = "fonttools-4.56.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1af375734018951c31c0737d04a9d5fd0a353a0253db5fbed2ccd44eac62d8c"}, + {file = "fonttools-4.56.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:442ad4122468d0e47d83bc59d0e91b474593a8c813839e1872e47c7a0cb53b10"}, + {file = "fonttools-4.56.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3cf4f8d2a30b454ac682e12c61831dcb174950c406011418e739de592bbf8f76"}, + {file = "fonttools-4.56.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:96a4271f63a615bcb902b9f56de00ea225d6896052c49f20d0c91e9f43529a29"}, + {file = "fonttools-4.56.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6c1d38642ca2dddc7ae992ef5d026e5061a84f10ff2b906be5680ab089f55bb8"}, + {file = "fonttools-4.56.0-cp39-cp39-win32.whl", hash = "sha256:2d351275f73ebdd81dd5b09a8b8dac7a30f29a279d41e1c1192aedf1b6dced40"}, + {file = "fonttools-4.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:d6ca96d1b61a707ba01a43318c9c40aaf11a5a568d1e61146fafa6ab20890793"}, + {file = "fonttools-4.56.0-py3-none-any.whl", hash = "sha256:1088182f68c303b50ca4dc0c82d42083d176cba37af1937e1a976a31149d4d14"}, + {file = "fonttools-4.56.0.tar.gz", hash = "sha256:a114d1567e1a1586b7e9e7fc2ff686ca542a82769a296cef131e4c4af51e58f4"}, +] + +[package.extras] +all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "pycairo", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0)", "xattr", "zopfli (>=0.1.4)"] +graphite = ["lz4 (>=1.7.4.2)"] +interpolatable = ["munkres", "pycairo", "scipy"] +lxml = ["lxml (>=4.0)"] +pathops = ["skia-pathops (>=0.5.0)"] +plot = ["matplotlib"] +repacker = ["uharfbuzz (>=0.23.0)"] +symfont = ["sympy"] +type1 = ["xattr"] +ufo = ["fs (>=2.2.0,<3)"] +unicode = ["unicodedata2 (>=15.1.0)"] +woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] + +[[package]] +name = "frozenlist" +version = "1.5.0" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"}, + {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"}, + {file = "frozenlist-1.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15538c0cbf0e4fa11d1e3a71f823524b0c46299aed6e10ebb4c2089abd8c3bec"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e79225373c317ff1e35f210dd5f1344ff31066ba8067c307ab60254cd3a78ad5"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9272fa73ca71266702c4c3e2d4a28553ea03418e591e377a03b8e3659d94fa76"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:498524025a5b8ba81695761d78c8dd7382ac0b052f34e66939c42df860b8ff17"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92b5278ed9d50fe610185ecd23c55d8b307d75ca18e94c0e7de328089ac5dcba"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f3c8c1dacd037df16e85227bac13cca58c30da836c6f936ba1df0c05d046d8d"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2ac49a9bedb996086057b75bf93538240538c6d9b38e57c82d51f75a73409d2"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e66cc454f97053b79c2ab09c17fbe3c825ea6b4de20baf1be28919460dd7877f"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3ba5f9a0dfed20337d3e966dc359784c9f96503674c2faf015f7fe8e96798c"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6321899477db90bdeb9299ac3627a6a53c7399c8cd58d25da094007402b039ab"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76e4753701248476e6286f2ef492af900ea67d9706a0155335a40ea21bf3b2f5"}, + {file = "frozenlist-1.5.0-cp310-cp310-win32.whl", hash = "sha256:977701c081c0241d0955c9586ffdd9ce44f7a7795df39b9151cd9a6fd0ce4cfb"}, + {file = "frozenlist-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:189f03b53e64144f90990d29a27ec4f7997d91ed3d01b51fa39d2dbe77540fd4"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fd74520371c3c4175142d02a976aee0b4cb4a7cc912a60586ffd8d5929979b30"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f3f7a0fbc219fb4455264cae4d9f01ad41ae6ee8524500f381de64ffaa077d5"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f47c9c9028f55a04ac254346e92977bf0f166c483c74b4232bee19a6697e4778"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0996c66760924da6e88922756d99b47512a71cfd45215f3570bf1e0b694c206a"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2fe128eb4edeabe11896cb6af88fca5346059f6c8d807e3b910069f39157869"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a8ea951bbb6cacd492e3948b8da8c502a3f814f5d20935aae74b5df2b19cf3d"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de537c11e4aa01d37db0d403b57bd6f0546e71a82347a97c6a9f0dcc532b3a45"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c2623347b933fcb9095841f1cc5d4ff0b278addd743e0e966cb3d460278840d"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cee6798eaf8b1416ef6909b06f7dc04b60755206bddc599f52232606e18179d3"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f5f9da7f5dbc00a604fe74aa02ae7c98bcede8a3b8b9666f9f86fc13993bc71a"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:90646abbc7a5d5c7c19461d2e3eeb76eb0b204919e6ece342feb6032c9325ae9"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bdac3c7d9b705d253b2ce370fde941836a5f8b3c5c2b8fd70940a3ea3af7f4f2"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03d33c2ddbc1816237a67f66336616416e2bbb6beb306e5f890f2eb22b959cdf"}, + {file = "frozenlist-1.5.0-cp311-cp311-win32.whl", hash = "sha256:237f6b23ee0f44066219dae14c70ae38a63f0440ce6750f868ee08775073f942"}, + {file = "frozenlist-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:0cc974cc93d32c42e7b0f6cf242a6bd941c57c61b618e78b6c0a96cb72788c1d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f"}, + {file = "frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8"}, + {file = "frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7a1a048f9215c90973402e26c01d1cff8a209e1f1b53f72b95c13db61b00f953"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dd47a5181ce5fcb463b5d9e17ecfdb02b678cca31280639255ce9d0e5aa67af0"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1431d60b36d15cda188ea222033eec8e0eab488f39a272461f2e6d9e1a8e63c2"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6482a5851f5d72767fbd0e507e80737f9c8646ae7fd303def99bfe813f76cf7f"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44c49271a937625619e862baacbd037a7ef86dd1ee215afc298a417ff3270608"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:12f78f98c2f1c2429d42e6a485f433722b0061d5c0b0139efa64f396efb5886b"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce3aa154c452d2467487765e3adc730a8c153af77ad84096bc19ce19a2400840"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b7dc0c4338e6b8b091e8faf0db3168a37101943e687f373dce00959583f7439"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45e0896250900b5aa25180f9aec243e84e92ac84bd4a74d9ad4138ef3f5c97de"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:561eb1c9579d495fddb6da8959fd2a1fca2c6d060d4113f5844b433fc02f2641"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:df6e2f325bfee1f49f81aaac97d2aa757c7646534a06f8f577ce184afe2f0a9e"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:140228863501b44b809fb39ec56b5d4071f4d0aa6d216c19cbb08b8c5a7eadb9"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7707a25d6a77f5d27ea7dc7d1fc608aa0a478193823f88511ef5e6b8a48f9d03"}, + {file = "frozenlist-1.5.0-cp313-cp313-win32.whl", hash = "sha256:31a9ac2b38ab9b5a8933b693db4939764ad3f299fcaa931a3e605bc3460e693c"}, + {file = "frozenlist-1.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:11aabdd62b8b9c4b84081a3c246506d1cddd2dd93ff0ad53ede5defec7886b28"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:dd94994fc91a6177bfaafd7d9fd951bc8689b0a98168aa26b5f543868548d3ca"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0da8bbec082bf6bf18345b180958775363588678f64998c2b7609e34719b10"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:73f2e31ea8dd7df61a359b731716018c2be196e5bb3b74ddba107f694fbd7604"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:828afae9f17e6de596825cf4228ff28fbdf6065974e5ac1410cecc22f699d2b3"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1577515d35ed5649d52ab4319db757bb881ce3b2b796d7283e6634d99ace307"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2150cc6305a2c2ab33299453e2968611dacb970d2283a14955923062c8d00b10"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a72b7a6e3cd2725eff67cd64c8f13335ee18fc3c7befc05aed043d24c7b9ccb9"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c16d2fa63e0800723139137d667e1056bee1a1cf7965153d2d104b62855e9b99"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:17dcc32fc7bda7ce5875435003220a457bcfa34ab7924a49a1c19f55b6ee185c"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:97160e245ea33d8609cd2b8fd997c850b56db147a304a262abc2b3be021a9171"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f1e6540b7fa044eee0bb5111ada694cf3dc15f2b0347ca125ee9ca984d5e9e6e"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:91d6c171862df0a6c61479d9724f22efb6109111017c87567cfeb7b5d1449fdf"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c1fac3e2ace2eb1052e9f7c7db480818371134410e1f5c55d65e8f3ac6d1407e"}, + {file = "frozenlist-1.5.0-cp38-cp38-win32.whl", hash = "sha256:b97f7b575ab4a8af9b7bc1d2ef7f29d3afee2226bd03ca3875c16451ad5a7723"}, + {file = "frozenlist-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:374ca2dabdccad8e2a76d40b1d037f5bd16824933bf7bcea3e59c891fd4a0923"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9bbcdfaf4af7ce002694a4e10a0159d5a8d20056a12b05b45cea944a4953f972"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1893f948bf6681733aaccf36c5232c231e3b5166d607c5fa77773611df6dc336"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2b5e23253bb709ef57a8e95e6ae48daa9ac5f265637529e4ce6b003a37b2621f"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f253985bb515ecd89629db13cb58d702035ecd8cfbca7d7a7e29a0e6d39af5f"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04a5c6babd5e8fb7d3c871dc8b321166b80e41b637c31a995ed844a6139942b6"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9fe0f1c29ba24ba6ff6abf688cb0b7cf1efab6b6aa6adc55441773c252f7411"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:226d72559fa19babe2ccd920273e767c96a49b9d3d38badd7c91a0fdeda8ea08"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b731db116ab3aedec558573c1a5eec78822b32292fe4f2f0345b7f697745c2"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:366d8f93e3edfe5a918c874702f78faac300209a4d5bf38352b2c1bdc07a766d"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1b96af8c582b94d381a1c1f51ffaedeb77c821c690ea5f01da3d70a487dd0a9b"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:c03eff4a41bd4e38415cbed054bbaff4a075b093e2394b6915dca34a40d1e38b"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:50cf5e7ee9b98f22bdecbabf3800ae78ddcc26e4a435515fc72d97903e8488e0"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1e76bfbc72353269c44e0bc2cfe171900fbf7f722ad74c9a7b638052afe6a00c"}, + {file = "frozenlist-1.5.0-cp39-cp39-win32.whl", hash = "sha256:666534d15ba8f0fda3f53969117383d5dc021266b3c1a42c9ec4855e4b58b9d3"}, + {file = "frozenlist-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:5c28f4b5dbef8a0d8aad0d4de24d1e9e981728628afaf4ea0792f5d0939372f0"}, + {file = "frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3"}, + {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, +] + +[[package]] +name = "fsspec" +version = "2024.12.0" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "fsspec-2024.12.0-py3-none-any.whl", hash = "sha256:b520aed47ad9804237ff878b504267a3b0b441e97508bd6d2d8774e3db85cee2"}, + {file = "fsspec-2024.12.0.tar.gz", hash = "sha256:670700c977ed2fb51e0d9f9253177ed20cbde4a3e5c0283cc5385b5870c8533f"}, +] + +[package.dependencies] +aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dev = ["pre-commit", "ruff"] +doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] +test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] +test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] +tqdm = ["tqdm"] + +[[package]] +name = "huggingface-hub" +version = "0.29.2" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +optional = false +python-versions = ">=3.8.0" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "huggingface_hub-0.29.2-py3-none-any.whl", hash = "sha256:c56f20fca09ef19da84dcde2b76379ecdaddf390b083f59f166715584953307d"}, + {file = "huggingface_hub-0.29.2.tar.gz", hash = "sha256:590b29c0dcbd0ee4b7b023714dc1ad8563fe4a68a91463438b74e980d28afaf3"}, +] + +[package.dependencies] +filelock = "*" +fsspec = ">=2023.5.0" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +hf-transfer = ["hf-transfer (>=0.1.4)"] +inference = ["aiohttp"] +quality = ["libcst (==1.4.0)", "mypy (==1.5.1)", "ruff (>=0.9.0)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +tensorflow-testing = ["keras (<3.0)", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +torch = ["safetensors[torch]", "torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] + +[[package]] +name = "idna" +version = "3.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "jinja2" +version = "3.1.6" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "joblib" +version = "1.4.2" +description = "Lightweight pipelining with Python functions" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, + {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, +] + +[[package]] +name = "kiwisolver" +version = "1.4.8" +description = "A fast implementation of the Cassowary constraint solver" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "kiwisolver-1.4.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88c6f252f6816a73b1f8c904f7bbe02fd67c09a69f7cb8a0eecdbf5ce78e63db"}, + {file = "kiwisolver-1.4.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c72941acb7b67138f35b879bbe85be0f6c6a70cab78fe3ef6db9c024d9223e5b"}, + {file = "kiwisolver-1.4.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce2cf1e5688edcb727fdf7cd1bbd0b6416758996826a8be1d958f91880d0809d"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c8bf637892dc6e6aad2bc6d4d69d08764166e5e3f69d469e55427b6ac001b19d"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:034d2c891f76bd3edbdb3ea11140d8510dca675443da7304205a2eaa45d8334c"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d47b28d1dfe0793d5e96bce90835e17edf9a499b53969b03c6c47ea5985844c3"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb158fe28ca0c29f2260cca8c43005329ad58452c36f0edf298204de32a9a3ed"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5536185fce131780ebd809f8e623bf4030ce1b161353166c49a3c74c287897f"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:369b75d40abedc1da2c1f4de13f3482cb99e3237b38726710f4a793432b1c5ff"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:641f2ddf9358c80faa22e22eb4c9f54bd3f0e442e038728f500e3b978d00aa7d"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d561d2d8883e0819445cfe58d7ddd673e4015c3c57261d7bdcd3710d0d14005c"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1732e065704b47c9afca7ffa272f845300a4eb959276bf6970dc07265e73b605"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bcb1ebc3547619c3b58a39e2448af089ea2ef44b37988caf432447374941574e"}, + {file = "kiwisolver-1.4.8-cp310-cp310-win_amd64.whl", hash = "sha256:89c107041f7b27844179ea9c85d6da275aa55ecf28413e87624d033cf1f6b751"}, + {file = "kiwisolver-1.4.8-cp310-cp310-win_arm64.whl", hash = "sha256:b5773efa2be9eb9fcf5415ea3ab70fc785d598729fd6057bea38d539ead28271"}, + {file = "kiwisolver-1.4.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a4d3601908c560bdf880f07d94f31d734afd1bb71e96585cace0e38ef44c6d84"}, + {file = "kiwisolver-1.4.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:856b269c4d28a5c0d5e6c1955ec36ebfd1651ac00e1ce0afa3e28da95293b561"}, + {file = "kiwisolver-1.4.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c2b9a96e0f326205af81a15718a9073328df1173a2619a68553decb7097fd5d7"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5020c83e8553f770cb3b5fc13faac40f17e0b205bd237aebd21d53d733adb03"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dace81d28c787956bfbfbbfd72fdcef014f37d9b48830829e488fdb32b49d954"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11e1022b524bd48ae56c9b4f9296bce77e15a2e42a502cceba602f804b32bb79"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b9b4d2892fefc886f30301cdd80debd8bb01ecdf165a449eb6e78f79f0fabd6"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a96c0e790ee875d65e340ab383700e2b4891677b7fcd30a699146f9384a2bb0"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23454ff084b07ac54ca8be535f4174170c1094a4cff78fbae4f73a4bcc0d4dab"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:87b287251ad6488e95b4f0b4a79a6d04d3ea35fde6340eb38fbd1ca9cd35bbbc"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b21dbe165081142b1232a240fc6383fd32cdd877ca6cc89eab93e5f5883e1c25"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:768cade2c2df13db52475bd28d3a3fac8c9eff04b0e9e2fda0f3760f20b3f7fc"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d47cfb2650f0e103d4bf68b0b5804c68da97272c84bb12850d877a95c056bd67"}, + {file = "kiwisolver-1.4.8-cp311-cp311-win_amd64.whl", hash = "sha256:ed33ca2002a779a2e20eeb06aea7721b6e47f2d4b8a8ece979d8ba9e2a167e34"}, + {file = "kiwisolver-1.4.8-cp311-cp311-win_arm64.whl", hash = "sha256:16523b40aab60426ffdebe33ac374457cf62863e330a90a0383639ce14bf44b2"}, + {file = "kiwisolver-1.4.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d6af5e8815fd02997cb6ad9bbed0ee1e60014438ee1a5c2444c96f87b8843502"}, + {file = "kiwisolver-1.4.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bade438f86e21d91e0cf5dd7c0ed00cda0f77c8c1616bd83f9fc157fa6760d31"}, + {file = "kiwisolver-1.4.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b83dc6769ddbc57613280118fb4ce3cd08899cc3369f7d0e0fab518a7cf37fdb"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111793b232842991be367ed828076b03d96202c19221b5ebab421ce8bcad016f"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:257af1622860e51b1a9d0ce387bf5c2c4f36a90594cb9514f55b074bcc787cfc"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b5637c3f316cab1ec1c9a12b8c5f4750a4c4b71af9157645bf32830e39c03a"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:782bb86f245ec18009890e7cb8d13a5ef54dcf2ebe18ed65f795e635a96a1c6a"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc978a80a0db3a66d25767b03688f1147a69e6237175c0f4ffffaaedf744055a"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:36dbbfd34838500a31f52c9786990d00150860e46cd5041386f217101350f0d3"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:eaa973f1e05131de5ff3569bbba7f5fd07ea0595d3870ed4a526d486fe57fa1b"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a66f60f8d0c87ab7f59b6fb80e642ebb29fec354a4dfad687ca4092ae69d04f4"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858416b7fb777a53f0c59ca08190ce24e9abbd3cffa18886a5781b8e3e26f65d"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:085940635c62697391baafaaeabdf3dd7a6c3643577dde337f4d66eba021b2b8"}, + {file = "kiwisolver-1.4.8-cp312-cp312-win_amd64.whl", hash = "sha256:01c3d31902c7db5fb6182832713d3b4122ad9317c2c5877d0539227d96bb2e50"}, + {file = "kiwisolver-1.4.8-cp312-cp312-win_arm64.whl", hash = "sha256:a3c44cb68861de93f0c4a8175fbaa691f0aa22550c331fefef02b618a9dcb476"}, + {file = "kiwisolver-1.4.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1c8ceb754339793c24aee1c9fb2485b5b1f5bb1c2c214ff13368431e51fc9a09"}, + {file = "kiwisolver-1.4.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a62808ac74b5e55a04a408cda6156f986cefbcf0ada13572696b507cc92fa1"}, + {file = "kiwisolver-1.4.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:68269e60ee4929893aad82666821aaacbd455284124817af45c11e50a4b42e3c"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34d142fba9c464bc3bbfeff15c96eab0e7310343d6aefb62a79d51421fcc5f1b"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc373e0eef45b59197de815b1b28ef89ae3955e7722cc9710fb91cd77b7f47"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:77e6f57a20b9bd4e1e2cedda4d0b986ebd0216236f0106e55c28aea3d3d69b16"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08e77738ed7538f036cd1170cbed942ef749137b1311fa2bbe2a7fda2f6bf3cc"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5ce1e481a74b44dd5e92ff03ea0cb371ae7a0268318e202be06c8f04f4f1246"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fc2ace710ba7c1dfd1a3b42530b62b9ceed115f19a1656adefce7b1782a37794"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:3452046c37c7692bd52b0e752b87954ef86ee2224e624ef7ce6cb21e8c41cc1b"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e9a60b50fe8b2ec6f448fe8d81b07e40141bfced7f896309df271a0b92f80f3"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:918139571133f366e8362fa4a297aeba86c7816b7ecf0bc79168080e2bd79957"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e063ef9f89885a1d68dd8b2e18f5ead48653176d10a0e324e3b0030e3a69adeb"}, + {file = "kiwisolver-1.4.8-cp313-cp313-win_amd64.whl", hash = "sha256:a17b7c4f5b2c51bb68ed379defd608a03954a1845dfed7cc0117f1cc8a9b7fd2"}, + {file = "kiwisolver-1.4.8-cp313-cp313-win_arm64.whl", hash = "sha256:3cd3bc628b25f74aedc6d374d5babf0166a92ff1317f46267f12d2ed54bc1d30"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:370fd2df41660ed4e26b8c9d6bbcad668fbe2560462cba151a721d49e5b6628c"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:84a2f830d42707de1d191b9490ac186bf7997a9495d4e9072210a1296345f7dc"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7a3ad337add5148cf51ce0b55642dc551c0b9d6248458a757f98796ca7348712"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7506488470f41169b86d8c9aeff587293f530a23a23a49d6bc64dab66bedc71e"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f0121b07b356a22fb0414cec4666bbe36fd6d0d759db3d37228f496ed67c880"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d6d6bd87df62c27d4185de7c511c6248040afae67028a8a22012b010bc7ad062"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:291331973c64bb9cce50bbe871fb2e675c4331dab4f31abe89f175ad7679a4d7"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:893f5525bb92d3d735878ec00f781b2de998333659507d29ea4466208df37bed"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b47a465040146981dc9db8647981b8cb96366fbc8d452b031e4f8fdffec3f26d"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:99cea8b9dd34ff80c521aef46a1dddb0dcc0283cf18bde6d756f1e6f31772165"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:151dffc4865e5fe6dafce5480fab84f950d14566c480c08a53c663a0020504b6"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:577facaa411c10421314598b50413aa1ebcf5126f704f1e5d72d7e4e9f020d90"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:be4816dc51c8a471749d664161b434912eee82f2ea66bd7628bd14583a833e85"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e7a019419b7b510f0f7c9dceff8c5eae2392037eae483a7f9162625233802b0a"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:286b18e86682fd2217a48fc6be6b0f20c1d0ed10958d8dc53453ad58d7be0bf8"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4191ee8dfd0be1c3666ccbac178c5a05d5f8d689bbe3fc92f3c4abec817f8fe0"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd2785b9391f2873ad46088ed7599a6a71e762e1ea33e87514b1a441ed1da1c"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c07b29089b7ba090b6f1a669f1411f27221c3662b3a1b7010e67b59bb5a6f10b"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:65ea09a5a3faadd59c2ce96dc7bf0f364986a315949dc6374f04396b0d60e09b"}, + {file = "kiwisolver-1.4.8.tar.gz", hash = "sha256:23d5f023bdc8c7e54eb65f03ca5d5bb25b601eac4d7f1a042888a1f45237987e"}, +] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "markupsafe" +version = "3.0.2" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, +] + +[[package]] +name = "matplotlib" +version = "3.10.1" +description = "Python plotting package" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "matplotlib-3.10.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ff2ae14910be903f4a24afdbb6d7d3a6c44da210fc7d42790b87aeac92238a16"}, + {file = "matplotlib-3.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0721a3fd3d5756ed593220a8b86808a36c5031fce489adb5b31ee6dbb47dd5b2"}, + {file = "matplotlib-3.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0673b4b8f131890eb3a1ad058d6e065fb3c6e71f160089b65f8515373394698"}, + {file = "matplotlib-3.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e875b95ac59a7908978fe307ecdbdd9a26af7fa0f33f474a27fcf8c99f64a19"}, + {file = "matplotlib-3.10.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2589659ea30726284c6c91037216f64a506a9822f8e50592d48ac16a2f29e044"}, + {file = "matplotlib-3.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:a97ff127f295817bc34517255c9db6e71de8eddaab7f837b7d341dee9f2f587f"}, + {file = "matplotlib-3.10.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:057206ff2d6ab82ff3e94ebd94463d084760ca682ed5f150817b859372ec4401"}, + {file = "matplotlib-3.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a144867dd6bf8ba8cb5fc81a158b645037e11b3e5cf8a50bd5f9917cb863adfe"}, + {file = "matplotlib-3.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56c5d9fcd9879aa8040f196a235e2dcbdf7dd03ab5b07c0696f80bc6cf04bedd"}, + {file = "matplotlib-3.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f69dc9713e4ad2fb21a1c30e37bd445d496524257dfda40ff4a8efb3604ab5c"}, + {file = "matplotlib-3.10.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c59af3e8aca75d7744b68e8e78a669e91ccbcf1ac35d0102a7b1b46883f1dd7"}, + {file = "matplotlib-3.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:11b65088c6f3dae784bc72e8d039a2580186285f87448babb9ddb2ad0082993a"}, + {file = "matplotlib-3.10.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:66e907a06e68cb6cfd652c193311d61a12b54f56809cafbed9736ce5ad92f107"}, + {file = "matplotlib-3.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b4bb156abb8fa5e5b2b460196f7db7264fc6d62678c03457979e7d5254b7be"}, + {file = "matplotlib-3.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1985ad3d97f51307a2cbfc801a930f120def19ba22864182dacef55277102ba6"}, + {file = "matplotlib-3.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c96f2c2f825d1257e437a1482c5a2cf4fee15db4261bd6fc0750f81ba2b4ba3d"}, + {file = "matplotlib-3.10.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:35e87384ee9e488d8dd5a2dd7baf471178d38b90618d8ea147aced4ab59c9bea"}, + {file = "matplotlib-3.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:cfd414bce89cc78a7e1d25202e979b3f1af799e416010a20ab2b5ebb3a02425c"}, + {file = "matplotlib-3.10.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42eee41e1b60fd83ee3292ed83a97a5f2a8239b10c26715d8a6172226988d7b"}, + {file = "matplotlib-3.10.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4f0647b17b667ae745c13721602b540f7aadb2a32c5b96e924cd4fea5dcb90f1"}, + {file = "matplotlib-3.10.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa3854b5f9473564ef40a41bc922be978fab217776e9ae1545c9b3a5cf2092a3"}, + {file = "matplotlib-3.10.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e496c01441be4c7d5f96d4e40f7fca06e20dcb40e44c8daa2e740e1757ad9e6"}, + {file = "matplotlib-3.10.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5d45d3f5245be5b469843450617dcad9af75ca50568acf59997bed9311131a0b"}, + {file = "matplotlib-3.10.1-cp313-cp313-win_amd64.whl", hash = "sha256:8e8e25b1209161d20dfe93037c8a7f7ca796ec9aa326e6e4588d8c4a5dd1e473"}, + {file = "matplotlib-3.10.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:19b06241ad89c3ae9469e07d77efa87041eac65d78df4fcf9cac318028009b01"}, + {file = "matplotlib-3.10.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01e63101ebb3014e6e9f80d9cf9ee361a8599ddca2c3e166c563628b39305dbb"}, + {file = "matplotlib-3.10.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f06bad951eea6422ac4e8bdebcf3a70c59ea0a03338c5d2b109f57b64eb3972"}, + {file = "matplotlib-3.10.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3dfb036f34873b46978f55e240cff7a239f6c4409eac62d8145bad3fc6ba5a3"}, + {file = "matplotlib-3.10.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dc6ab14a7ab3b4d813b88ba957fc05c79493a037f54e246162033591e770de6f"}, + {file = "matplotlib-3.10.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bc411ebd5889a78dabbc457b3fa153203e22248bfa6eedc6797be5df0164dbf9"}, + {file = "matplotlib-3.10.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:648406f1899f9a818cef8c0231b44dcfc4ff36f167101c3fd1c9151f24220fdc"}, + {file = "matplotlib-3.10.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:02582304e352f40520727984a5a18f37e8187861f954fea9be7ef06569cf85b4"}, + {file = "matplotlib-3.10.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3809916157ba871bcdd33d3493acd7fe3037db5daa917ca6e77975a94cef779"}, + {file = "matplotlib-3.10.1.tar.gz", hash = "sha256:e8d2d0e3881b129268585bf4765ad3ee73a4591d77b9a18c214ac7e3a79fb2ba"}, +] + +[package.dependencies] +contourpy = ">=1.0.1" +cycler = ">=0.10" +fonttools = ">=4.22.0" +kiwisolver = ">=1.3.1" +numpy = ">=1.23" +packaging = ">=20.0" +pillow = ">=8" +pyparsing = ">=2.3.1" +python-dateutil = ">=2.7" + +[package.extras] +dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setuptools (>=64)", "setuptools_scm (>=7)"] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4)"] +tests = ["pytest (>=4.6)"] + +[[package]] +name = "multidict" +version = "6.1.0" +description = "multidict implementation" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, + {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, + {file = "multidict-6.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a114d03b938376557927ab23f1e950827c3b893ccb94b62fd95d430fd0e5cf53"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1c416351ee6271b2f49b56ad7f308072f6f44b37118d69c2cad94f3fa8a40d5"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b5d83030255983181005e6cfbac1617ce9746b219bc2aad52201ad121226581"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e97b5e938051226dc025ec80980c285b053ffb1e25a3db2a3aa3bc046bf7f56"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d618649d4e70ac6efcbba75be98b26ef5078faad23592f9b51ca492953012429"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10524ebd769727ac77ef2278390fb0068d83f3acb7773792a5080f2b0abf7748"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ff3827aef427c89a25cc96ded1759271a93603aba9fb977a6d264648ebf989db"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06809f4f0f7ab7ea2cabf9caca7d79c22c0758b58a71f9d32943ae13c7ace056"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f179dee3b863ab1c59580ff60f9d99f632f34ccb38bf67a33ec6b3ecadd0fd76"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:aaed8b0562be4a0876ee3b6946f6869b7bcdb571a5d1496683505944e268b160"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c8b88a2ccf5493b6c8da9076fb151ba106960a2df90c2633f342f120751a9e7"}, + {file = "multidict-6.1.0-cp310-cp310-win32.whl", hash = "sha256:4a9cb68166a34117d6646c0023c7b759bf197bee5ad4272f420a0141d7eb03a0"}, + {file = "multidict-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:20b9b5fbe0b88d0bdef2012ef7dee867f874b72528cf1d08f1d59b0e3850129d"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753"}, + {file = "multidict-6.1.0-cp311-cp311-win32.whl", hash = "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80"}, + {file = "multidict-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3"}, + {file = "multidict-6.1.0-cp312-cp312-win32.whl", hash = "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133"}, + {file = "multidict-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d569388c381b24671589335a3be6e1d45546c2988c2ebe30fdcada8457a31008"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f90c822a402cb865e396a504f9fc8173ef34212a342d92e362ca498cad308e28"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b225d95519a5bf73860323e633a664b0d85ad3d5bede6d30d95b35d4dfe8805b"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:23bfd518810af7de1116313ebd9092cb9aa629beb12f6ed631ad53356ed6b86c"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c09fcfdccdd0b57867577b719c69e347a436b86cd83747f179dbf0cc0d4c1f3"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf6bea52ec97e95560af5ae576bdac3aa3aae0b6758c6efa115236d9e07dae44"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57feec87371dbb3520da6192213c7d6fc892d5589a93db548331954de8248fd2"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0c3f390dc53279cbc8ba976e5f8035eab997829066756d811616b652b00a23a3"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:59bfeae4b25ec05b34f1956eaa1cb38032282cd4dfabc5056d0a1ec4d696d3aa"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b2f59caeaf7632cc633b5cf6fc449372b83bbdf0da4ae04d5be36118e46cc0aa"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:37bb93b2178e02b7b618893990941900fd25b6b9ac0fa49931a40aecdf083fe4"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e9f48f58c2c523d5a06faea47866cd35b32655c46b443f163d08c6d0ddb17d6"}, + {file = "multidict-6.1.0-cp313-cp313-win32.whl", hash = "sha256:3a37ffb35399029b45c6cc33640a92bef403c9fd388acce75cdc88f58bd19a81"}, + {file = "multidict-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9aa71e15d9d9beaad2c6b9319edcdc0a49a43ef5c0a4c8265ca9ee7d6c67774"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:db7457bac39421addd0c8449933ac32d8042aae84a14911a757ae6ca3eef1392"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d094ddec350a2fb899fec68d8353c78233debde9b7d8b4beeafa70825f1c281a"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5845c1fd4866bb5dd3125d89b90e57ed3138241540897de748cdf19de8a2fca2"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9079dfc6a70abe341f521f78405b8949f96db48da98aeb43f9907f342f627cdc"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3914f5aaa0f36d5d60e8ece6a308ee1c9784cd75ec8151062614657a114c4478"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c08be4f460903e5a9d0f76818db3250f12e9c344e79314d1d570fc69d7f4eae4"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d093be959277cb7dee84b801eb1af388b6ad3ca6a6b6bf1ed7585895789d027d"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3702ea6872c5a2a4eeefa6ffd36b042e9773f05b1f37ae3ef7264b1163c2dcf6"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2090f6a85cafc5b2db085124d752757c9d251548cedabe9bd31afe6363e0aff2"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:f67f217af4b1ff66c68a87318012de788dd95fcfeb24cc889011f4e1c7454dfd"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:189f652a87e876098bbc67b4da1049afb5f5dfbaa310dd67c594b01c10388db6"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:6bb5992037f7a9eff7991ebe4273ea7f51f1c1c511e6a2ce511d0e7bdb754492"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f4c2b9e770c4e393876e35a7046879d195cd123b4f116d299d442b335bcd"}, + {file = "multidict-6.1.0-cp38-cp38-win32.whl", hash = "sha256:e27bbb6d14416713a8bd7aaa1313c0fc8d44ee48d74497a0ff4c3a1b6ccb5167"}, + {file = "multidict-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:22f3105d4fb15c8f57ff3959a58fcab6ce36814486500cd7485651230ad4d4ef"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4e18b656c5e844539d506a0a06432274d7bd52a7487e6828c63a63d69185626c"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a185f876e69897a6f3325c3f19f26a297fa058c5e456bfcff8015e9a27e83ae1"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab7c4ceb38d91570a650dba194e1ca87c2b543488fe9309b4212694174fd539c"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e617fb6b0b6953fffd762669610c1c4ffd05632c138d61ac7e14ad187870669c"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16e5f4bf4e603eb1fdd5d8180f1a25f30056f22e55ce51fb3d6ad4ab29f7d96f"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c035da3f544b1882bac24115f3e2e8760f10a0107614fc9839fd232200b875"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:957cf8e4b6e123a9eea554fa7ebc85674674b713551de587eb318a2df3e00255"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:483a6aea59cb89904e1ceabd2b47368b5600fb7de78a6e4a2c2987b2d256cf30"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:87701f25a2352e5bf7454caa64757642734da9f6b11384c1f9d1a8e699758057"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:682b987361e5fd7a139ed565e30d81fd81e9629acc7d925a205366877d8c8657"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce2186a7df133a9c895dea3331ddc5ddad42cdd0d1ea2f0a51e5d161e4762f28"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9f636b730f7e8cb19feb87094949ba54ee5357440b9658b2a32a5ce4bce53972"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:73eae06aa53af2ea5270cc066dcaf02cc60d2994bbb2c4ef5764949257d10f43"}, + {file = "multidict-6.1.0-cp39-cp39-win32.whl", hash = "sha256:1ca0083e80e791cffc6efce7660ad24af66c8d4079d2a750b29001b53ff59ada"}, + {file = "multidict-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:aa466da5b15ccea564bdab9c89175c762bc12825f4659c11227f515cee76fa4a"}, + {file = "multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506"}, + {file = "multidict-6.1.0.tar.gz", hash = "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a"}, +] + +[[package]] +name = "multiprocess" +version = "0.70.16" +description = "better multiprocessing and multithreading in Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, + {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, + {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, + {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, + {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, + {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, + {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, + {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, + {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, + {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, + {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, + {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, +] + +[package.dependencies] +dill = ">=0.3.8" + +[[package]] +name = "networkx" +version = "3.4.2" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, + {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, +] + +[package.extras] +default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] +example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] +extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] + +[[package]] +name = "numpy" +version = "2.2.3" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "numpy-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cbc6472e01952d3d1b2772b720428f8b90e2deea8344e854df22b0618e9cce71"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdfe0c22692a30cd830c0755746473ae66c4a8f2e7bd508b35fb3b6a0813d787"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:e37242f5324ffd9f7ba5acf96d774f9276aa62a966c0bad8dae692deebec7716"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:95172a21038c9b423e68be78fd0be6e1b97674cde269b76fe269a5dfa6fadf0b"}, + {file = "numpy-2.2.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5b47c440210c5d1d67e1cf434124e0b5c395eee1f5806fdd89b553ed1acd0a3"}, + {file = "numpy-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0391ea3622f5c51a2e29708877d56e3d276827ac5447d7f45e9bc4ade8923c52"}, + {file = "numpy-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f6b3dfc7661f8842babd8ea07e9897fe3d9b69a1d7e5fbb743e4160f9387833b"}, + {file = "numpy-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1ad78ce7f18ce4e7df1b2ea4019b5817a2f6a8a16e34ff2775f646adce0a5027"}, + {file = "numpy-2.2.3-cp310-cp310-win32.whl", hash = "sha256:5ebeb7ef54a7be11044c33a17b2624abe4307a75893c001a4800857956b41094"}, + {file = "numpy-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:596140185c7fa113563c67c2e894eabe0daea18cf8e33851738c19f70ce86aeb"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:16372619ee728ed67a2a606a614f56d3eabc5b86f8b615c79d01957062826ca8"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5521a06a3148686d9269c53b09f7d399a5725c47bbb5b35747e1cb76326b714b"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:7c8dde0ca2f77828815fd1aedfdf52e59071a5bae30dac3b4da2a335c672149a"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:77974aba6c1bc26e3c205c2214f0d5b4305bdc719268b93e768ddb17e3fdd636"}, + {file = "numpy-2.2.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d42f9c36d06440e34226e8bd65ff065ca0963aeecada587b937011efa02cdc9d"}, + {file = "numpy-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2712c5179f40af9ddc8f6727f2bd910ea0eb50206daea75f58ddd9fa3f715bb"}, + {file = "numpy-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c8b0451d2ec95010d1db8ca733afc41f659f425b7f608af569711097fd6014e2"}, + {file = "numpy-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9b4a8148c57ecac25a16b0e11798cbe88edf5237b0df99973687dd866f05e1b"}, + {file = "numpy-2.2.3-cp311-cp311-win32.whl", hash = "sha256:1f45315b2dc58d8a3e7754fe4e38b6fce132dab284a92851e41b2b344f6441c5"}, + {file = "numpy-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f48ba6f6c13e5e49f3d3efb1b51c8193215c42ac82610a04624906a9270be6f"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12c045f43b1d2915eca6b880a7f4a256f59d62df4f044788c8ba67709412128d"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87eed225fd415bbae787f93a457af7f5990b92a334e346f72070bf569b9c9c95"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:712a64103d97c404e87d4d7c47fb0c7ff9acccc625ca2002848e0d53288b90ea"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a5ae282abe60a2db0fd407072aff4599c279bcd6e9a2475500fc35b00a57c532"}, + {file = "numpy-2.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5266de33d4c3420973cf9ae3b98b54a2a6d53a559310e3236c4b2b06b9c07d4e"}, + {file = "numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe"}, + {file = "numpy-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34c1b7e83f94f3b564b35f480f5652a47007dd91f7c839f404d03279cc8dd021"}, + {file = "numpy-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4d8335b5f1b6e2bce120d55fb17064b0262ff29b459e8493d1785c18ae2553b8"}, + {file = "numpy-2.2.3-cp312-cp312-win32.whl", hash = "sha256:4d9828d25fb246bedd31e04c9e75714a4087211ac348cb39c8c5f99dbb6683fe"}, + {file = "numpy-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bfdb06b395385ea9b91bf55c1adf1b297c9fdb531552845ff1d3ea6e40d5aba"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:23c9f4edbf4c065fddb10a4f6e8b6a244342d95966a48820c614891e5059bb50"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:a0c03b6be48aaf92525cccf393265e02773be8fd9551a2f9adbe7db1fa2b60f1"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:2376e317111daa0a6739e50f7ee2a6353f768489102308b0d98fcf4a04f7f3b5"}, + {file = "numpy-2.2.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fb62fe3d206d72fe1cfe31c4a1106ad2b136fcc1606093aeab314f02930fdf2"}, + {file = "numpy-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52659ad2534427dffcc36aac76bebdd02b67e3b7a619ac67543bc9bfe6b7cdb1"}, + {file = "numpy-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b416af7d0ed3271cad0f0a0d0bee0911ed7eba23e66f8424d9f3dfcdcae1304"}, + {file = "numpy-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1402da8e0f435991983d0a9708b779f95a8c98c6b18a171b9f1be09005e64d9d"}, + {file = "numpy-2.2.3-cp313-cp313-win32.whl", hash = "sha256:136553f123ee2951bfcfbc264acd34a2fc2f29d7cdf610ce7daf672b6fbaa693"}, + {file = "numpy-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:5b732c8beef1d7bc2d9e476dbba20aaff6167bf205ad9aa8d30913859e82884b"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:435e7a933b9fda8126130b046975a968cc2d833b505475e588339e09f7672890"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7678556eeb0152cbd1522b684dcd215250885993dd00adb93679ec3c0e6e091c"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2e8da03bd561504d9b20e7a12340870dfc206c64ea59b4cfee9fceb95070ee94"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:c9aa4496fd0e17e3843399f533d62857cef5900facf93e735ef65aa4bbc90ef0"}, + {file = "numpy-2.2.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4ca91d61a4bf61b0f2228f24bbfa6a9facd5f8af03759fe2a655c50ae2c6610"}, + {file = "numpy-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deaa09cd492e24fd9b15296844c0ad1b3c976da7907e1c1ed3a0ad21dded6f76"}, + {file = "numpy-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:246535e2f7496b7ac85deffe932896a3577be7af8fb7eebe7146444680297e9a"}, + {file = "numpy-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:daf43a3d1ea699402c5a850e5313680ac355b4adc9770cd5cfc2940e7861f1bf"}, + {file = "numpy-2.2.3-cp313-cp313t-win32.whl", hash = "sha256:cf802eef1f0134afb81fef94020351be4fe1d6681aadf9c5e862af6602af64ef"}, + {file = "numpy-2.2.3-cp313-cp313t-win_amd64.whl", hash = "sha256:aee2512827ceb6d7f517c8b85aa5d3923afe8fc7a57d028cffcd522f1c6fd082"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3c2ec8a0f51d60f1e9c0c5ab116b7fc104b165ada3f6c58abf881cb2eb16044d"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:ed2cf9ed4e8ebc3b754d398cba12f24359f018b416c380f577bbae112ca52fc9"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39261798d208c3095ae4f7bc8eaeb3481ea8c6e03dc48028057d3cbdbdb8937e"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:783145835458e60fa97afac25d511d00a1eca94d4a8f3ace9fe2043003c678e4"}, + {file = "numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020"}, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.4.5.8" +description = "CUBLAS native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0f8aa1706812e00b9f19dfe0cdb3999b092ccb8ca168c0db5b8ea712456fd9b3"}, + {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b"}, + {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-win_amd64.whl", hash = "sha256:5a796786da89203a0657eda402bcdcec6180254a8ac22d72213abc42069522dc"}, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.4.127" +description = "CUDA profiling tools runtime libs." +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:79279b35cf6f91da114182a5ce1864997fd52294a87a16179ce275773799458a"}, + {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb"}, + {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:5688d203301ab051449a2b1cb6690fbe90d2b372f411521c86018b950f3d7922"}, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.4.127" +description = "NVRTC native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0eedf14185e04b76aa05b1fea04133e59f465b6f960c0cbf4e37c3cb6b0ea198"}, + {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338"}, + {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:a961b2f1d5f17b14867c619ceb99ef6fcec12e46612711bcec78eb05068a60ec"}, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.4.127" +description = "CUDA Runtime native Libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:961fe0e2e716a2a1d967aab7caee97512f71767f852f67432d572e36cb3a11f3"}, + {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5"}, + {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:09c2e35f48359752dfa822c09918211844a3d93c100a715d79b59591130c5e1e"}, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.1.0.70" +description = "cuDNN runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f"}, + {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-win_amd64.whl", hash = "sha256:6278562929433d68365a07a4a1546c237ba2849852c0d4b2262a486e805b977a"}, +] + +[package.dependencies] +nvidia-cublas-cu12 = "*" + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.2.1.3" +description = "CUFFT native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5dad8008fc7f92f5ddfa2101430917ce2ffacd86824914c82e28990ad7f00399"}, + {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9"}, + {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-win_amd64.whl", hash = "sha256:d802f4954291101186078ccbe22fc285a902136f974d369540fd4a5333d1440b"}, +] + +[package.dependencies] +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.5.147" +description = "CURAND native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1f173f09e3e3c76ab084aba0de819c49e56614feae5c12f69883f4ae9bb5fad9"}, + {file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b"}, + {file = "nvidia_curand_cu12-10.3.5.147-py3-none-win_amd64.whl", hash = "sha256:f307cc191f96efe9e8f05a87096abc20d08845a841889ef78cb06924437f6771"}, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.6.1.9" +description = "CUDA solver native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d338f155f174f90724bbde3758b7ac375a70ce8e706d70b018dd3375545fc84e"}, + {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260"}, + {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-win_amd64.whl", hash = "sha256:e77314c9d7b694fcebc84f58989f3aa4fb4cb442f12ca1a9bde50f5e8f6d1b9c"}, +] + +[package.dependencies] +nvidia-cublas-cu12 = "*" +nvidia-cusparse-cu12 = "*" +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.3.1.170" +description = "CUSPARSE native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9d32f62896231ebe0480efd8a7f702e143c98cfaa0e8a76df3386c1ba2b54df3"}, + {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1"}, + {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-win_amd64.whl", hash = "sha256:9bc90fb087bc7b4c15641521f31c0371e9a612fc2ba12c338d3ae032e6b6797f"}, +] + +[package.dependencies] +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.2" +description = "NVIDIA cuSPARSELt" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_cusparselt_cu12-0.6.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:067a7f6d03ea0d4841c85f0c6f1991c5dda98211f6302cb83a4ab234ee95bef8"}, + {file = "nvidia_cusparselt_cu12-0.6.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:df2c24502fd76ebafe7457dbc4716b2fec071aabaed4fb7691a201cde03704d9"}, + {file = "nvidia_cusparselt_cu12-0.6.2-py3-none-win_amd64.whl", hash = "sha256:0057c91d230703924c0422feabe4ce768841f9b4b44d28586b6f6d2eb86fbe70"}, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.21.5" +description = "NVIDIA Collective Communication Library (NCCL) Runtime" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:8579076d30a8c24988834445f8d633c697d42397e92ffc3f63fa26766d25e0a0"}, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.4.127" +description = "Nvidia JIT LTO Library" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83"}, + {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57"}, + {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:fd9020c501d27d135f983c6d3e244b197a7ccad769e34df53a42e276b0e25fa1"}, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.4.127" +description = "NVIDIA Tools Extension" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7959ad635db13edf4fc65c06a6e9f9e55fc2f92596db928d169c0bb031e88ef3"}, + {file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a"}, + {file = "nvidia_nvtx_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:641dccaaa1139f3ffb0d3164b4b84f9d253397e38246a4f2f36728b48566d485"}, +] + +[[package]] +name = "packaging" +version = "24.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, +] + +[[package]] +name = "pandas" +version = "2.2.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"}, + {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, + {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, + {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"}, + {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pillow" +version = "11.1.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pillow-11.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:e1abe69aca89514737465752b4bcaf8016de61b3be1397a8fc260ba33321b3a8"}, + {file = "pillow-11.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c640e5a06869c75994624551f45e5506e4256562ead981cce820d5ab39ae2192"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a07dba04c5e22824816b2615ad7a7484432d7f540e6fa86af60d2de57b0fcee2"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e267b0ed063341f3e60acd25c05200df4193e15a4a5807075cd71225a2386e26"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bd165131fd51697e22421d0e467997ad31621b74bfc0b75956608cb2906dda07"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:abc56501c3fd148d60659aae0af6ddc149660469082859fa7b066a298bde9482"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:54ce1c9a16a9561b6d6d8cb30089ab1e5eb66918cb47d457bd996ef34182922e"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73ddde795ee9b06257dac5ad42fcb07f3b9b813f8c1f7f870f402f4dc54b5269"}, + {file = "pillow-11.1.0-cp310-cp310-win32.whl", hash = "sha256:3a5fe20a7b66e8135d7fd617b13272626a28278d0e578c98720d9ba4b2439d49"}, + {file = "pillow-11.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:b6123aa4a59d75f06e9dd3dac5bf8bc9aa383121bb3dd9a7a612e05eabc9961a"}, + {file = "pillow-11.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:a76da0a31da6fcae4210aa94fd779c65c75786bc9af06289cd1c184451ef7a65"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e06695e0326d05b06833b40b7ef477e475d0b1ba3a6d27da1bb48c23209bf457"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96f82000e12f23e4f29346e42702b6ed9a2f2fea34a740dd5ffffcc8c539eb35"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3cd561ded2cf2bbae44d4605837221b987c216cff94f49dfeed63488bb228d2"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f189805c8be5ca5add39e6f899e6ce2ed824e65fb45f3c28cb2841911da19070"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dd0052e9db3474df30433f83a71b9b23bd9e4ef1de13d92df21a52c0303b8ab6"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:837060a8599b8f5d402e97197d4924f05a2e0d68756998345c829c33186217b1"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa8dd43daa836b9a8128dbe7d923423e5ad86f50a7a14dc688194b7be5c0dea2"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0a2f91f8a8b367e7a57c6e91cd25af510168091fb89ec5146003e424e1558a96"}, + {file = "pillow-11.1.0-cp311-cp311-win32.whl", hash = "sha256:c12fc111ef090845de2bb15009372175d76ac99969bdf31e2ce9b42e4b8cd88f"}, + {file = "pillow-11.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbd43429d0d7ed6533b25fc993861b8fd512c42d04514a0dd6337fb3ccf22761"}, + {file = "pillow-11.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:f7955ecf5609dee9442cbface754f2c6e541d9e6eda87fad7f7a989b0bdb9d71"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2062ffb1d36544d42fcaa277b069c88b01bb7298f4efa06731a7fd6cc290b81a"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a85b653980faad27e88b141348707ceeef8a1186f75ecc600c395dcac19f385b"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9409c080586d1f683df3f184f20e36fb647f2e0bc3988094d4fd8c9f4eb1b3b3"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fdadc077553621911f27ce206ffcbec7d3f8d7b50e0da39f10997e8e2bb7f6a"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:93a18841d09bcdd774dcdc308e4537e1f867b3dec059c131fde0327899734aa1"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9aa9aeddeed452b2f616ff5507459e7bab436916ccb10961c4a382cd3e03f47f"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cdcdb0b896e981678eee140d882b70092dac83ac1cdf6b3a60e2216a73f2b91"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36ba10b9cb413e7c7dfa3e189aba252deee0602c86c309799da5a74009ac7a1c"}, + {file = "pillow-11.1.0-cp312-cp312-win32.whl", hash = "sha256:cfd5cd998c2e36a862d0e27b2df63237e67273f2fc78f47445b14e73a810e7e6"}, + {file = "pillow-11.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a697cd8ba0383bba3d2d3ada02b34ed268cb548b369943cd349007730c92bddf"}, + {file = "pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae98e14432d458fc3de11a77ccb3ae65ddce70f730e7c76140653048c71bfcbc"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cc1331b6d5a6e144aeb5e626f4375f5b7ae9934ba620c0ac6b3e43d5e683a0f0"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:758e9d4ef15d3560214cddbc97b8ef3ef86ce04d62ddac17ad39ba87e89bd3b1"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b523466b1a31d0dcef7c5be1f20b942919b62fd6e9a9be199d035509cbefc0ec"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9044b5e4f7083f209c4e35aa5dd54b1dd5b112b108648f5c902ad586d4f945c5"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3764d53e09cdedd91bee65c2527815d315c6b90d7b8b79759cc48d7bf5d4f114"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:31eba6bbdd27dde97b0174ddf0297d7a9c3a507a8a1480e1e60ef914fe23d352"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5d658fbd9f0d6eea113aea286b21d3cd4d3fd978157cbf2447a6035916506d3"}, + {file = "pillow-11.1.0-cp313-cp313-win32.whl", hash = "sha256:f86d3a7a9af5d826744fabf4afd15b9dfef44fe69a98541f666f66fbb8d3fef9"}, + {file = "pillow-11.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:593c5fd6be85da83656b93ffcccc2312d2d149d251e98588b14fbc288fd8909c"}, + {file = "pillow-11.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:11633d58b6ee5733bde153a8dafd25e505ea3d32e261accd388827ee987baf65"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:70ca5ef3b3b1c4a0812b5c63c57c23b63e53bc38e758b37a951e5bc466449861"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8000376f139d4d38d6851eb149b321a52bb8893a88dae8ee7d95840431977081"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ee85f0696a17dd28fbcfceb59f9510aa71934b483d1f5601d1030c3c8304f3c"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:dd0e081319328928531df7a0e63621caf67652c8464303fd102141b785ef9547"}, + {file = "pillow-11.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e63e4e5081de46517099dc30abe418122f54531a6ae2ebc8680bcd7096860eab"}, + {file = "pillow-11.1.0-cp313-cp313t-win32.whl", hash = "sha256:dda60aa465b861324e65a78c9f5cf0f4bc713e4309f83bc387be158b077963d9"}, + {file = "pillow-11.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ad5db5781c774ab9a9b2c4302bbf0c1014960a0a7be63278d13ae6fdf88126fe"}, + {file = "pillow-11.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:67cd427c68926108778a9005f2a04adbd5e67c442ed21d95389fe1d595458756"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:bf902d7413c82a1bfa08b06a070876132a5ae6b2388e2712aab3a7cbc02205c6"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c1eec9d950b6fe688edee07138993e54ee4ae634c51443cfb7c1e7613322718e"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e275ee4cb11c262bd108ab2081f750db2a1c0b8c12c1897f27b160c8bd57bbc"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4db853948ce4e718f2fc775b75c37ba2efb6aaea41a1a5fc57f0af59eee774b2"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ab8a209b8485d3db694fa97a896d96dd6533d63c22829043fd9de627060beade"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:54251ef02a2309b5eec99d151ebf5c9904b77976c8abdcbce7891ed22df53884"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5bb94705aea800051a743aa4874bb1397d4695fb0583ba5e425ee0328757f196"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89dbdb3e6e9594d512780a5a1c42801879628b38e3efc7038094430844e271d8"}, + {file = "pillow-11.1.0-cp39-cp39-win32.whl", hash = "sha256:e5449ca63da169a2e6068dd0e2fcc8d91f9558aba89ff6d02121ca8ab11e79e5"}, + {file = "pillow-11.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:3362c6ca227e65c54bf71a5f88b3d4565ff1bcbc63ae72c34b07bbb1cc59a43f"}, + {file = "pillow-11.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:b20be51b37a75cc54c2c55def3fa2c65bb94ba859dde241cd0a4fd302de5ae0a"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8c730dc3a83e5ac137fbc92dfcfe1511ce3b2b5d7578315b63dbbb76f7f51d90"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7d33d2fae0e8b170b6a6c57400e077412240f6f5bb2a342cf1ee512a787942bb"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8d65b38173085f24bc07f8b6c505cbb7418009fa1a1fcb111b1f4961814a442"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:015c6e863faa4779251436db398ae75051469f7c903b043a48f078e437656f83"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d44ff19eea13ae4acdaaab0179fa68c0c6f2f45d66a4d8ec1eda7d6cecbcc15f"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3d8da4a631471dfaf94c10c85f5277b1f8e42ac42bade1ac67da4b4a7359b73"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4637b88343166249fe8aa94e7c4a62a180c4b3898283bb5d3d2fd5fe10d8e4e0"}, + {file = "pillow-11.1.0.tar.gz", hash = "sha256:368da70808b36d73b4b390a8ffac11069f8a5c85f29eff1f1b01bcf3ef5b2a20"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] + +[[package]] +name = "propcache" +version = "0.3.0" +description = "Accelerated property cache" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "propcache-0.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:efa44f64c37cc30c9f05932c740a8b40ce359f51882c70883cc95feac842da4d"}, + {file = "propcache-0.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2383a17385d9800b6eb5855c2f05ee550f803878f344f58b6e194de08b96352c"}, + {file = "propcache-0.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d3e7420211f5a65a54675fd860ea04173cde60a7cc20ccfbafcccd155225f8bc"}, + {file = "propcache-0.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3302c5287e504d23bb0e64d2a921d1eb4a03fb93a0a0aa3b53de059f5a5d737d"}, + {file = "propcache-0.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7e2e068a83552ddf7a39a99488bcba05ac13454fb205c847674da0352602082f"}, + {file = "propcache-0.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d913d36bdaf368637b4f88d554fb9cb9d53d6920b9c5563846555938d5450bf"}, + {file = "propcache-0.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ee1983728964d6070ab443399c476de93d5d741f71e8f6e7880a065f878e0b9"}, + {file = "propcache-0.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:36ca5e9a21822cc1746023e88f5c0af6fce3af3b85d4520efb1ce4221bed75cc"}, + {file = "propcache-0.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:9ecde3671e62eeb99e977f5221abcf40c208f69b5eb986b061ccec317c82ebd0"}, + {file = "propcache-0.3.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:d383bf5e045d7f9d239b38e6acadd7b7fdf6c0087259a84ae3475d18e9a2ae8b"}, + {file = "propcache-0.3.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:8cb625bcb5add899cb8ba7bf716ec1d3e8f7cdea9b0713fa99eadf73b6d4986f"}, + {file = "propcache-0.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:5fa159dcee5dba00c1def3231c249cf261185189205073bde13797e57dd7540a"}, + {file = "propcache-0.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a7080b0159ce05f179cfac592cda1a82898ca9cd097dacf8ea20ae33474fbb25"}, + {file = "propcache-0.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7161bccab7696a473fe7ddb619c1d75963732b37da4618ba12e60899fefe4f"}, + {file = "propcache-0.3.0-cp310-cp310-win32.whl", hash = "sha256:bf0d9a171908f32d54f651648c7290397b8792f4303821c42a74e7805bfb813c"}, + {file = "propcache-0.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:42924dc0c9d73e49908e35bbdec87adedd651ea24c53c29cac103ede0ea1d340"}, + {file = "propcache-0.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9ddd49258610499aab83b4f5b61b32e11fce873586282a0e972e5ab3bcadee51"}, + {file = "propcache-0.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2578541776769b500bada3f8a4eeaf944530516b6e90c089aa368266ed70c49e"}, + {file = "propcache-0.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8074c5dd61c8a3e915fa8fc04754fa55cfa5978200d2daa1e2d4294c1f136aa"}, + {file = "propcache-0.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b58229a844931bca61b3a20efd2be2a2acb4ad1622fc026504309a6883686fbf"}, + {file = "propcache-0.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e45377d5d6fefe1677da2a2c07b024a6dac782088e37c0b1efea4cfe2b1be19b"}, + {file = "propcache-0.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ec5060592d83454e8063e487696ac3783cc48c9a329498bafae0d972bc7816c9"}, + {file = "propcache-0.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15010f29fbed80e711db272909a074dc79858c6d28e2915704cfc487a8ac89c6"}, + {file = "propcache-0.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a254537b9b696ede293bfdbc0a65200e8e4507bc9f37831e2a0318a9b333c85c"}, + {file = "propcache-0.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2b975528998de037dfbc10144b8aed9b8dd5a99ec547f14d1cb7c5665a43f075"}, + {file = "propcache-0.3.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:19d36bb351ad5554ff20f2ae75f88ce205b0748c38b146c75628577020351e3c"}, + {file = "propcache-0.3.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6032231d4a5abd67c7f71168fd64a47b6b451fbcb91c8397c2f7610e67683810"}, + {file = "propcache-0.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6985a593417cdbc94c7f9c3403747335e450c1599da1647a5af76539672464d3"}, + {file = "propcache-0.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6a1948df1bb1d56b5e7b0553c0fa04fd0e320997ae99689488201f19fa90d2e7"}, + {file = "propcache-0.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8319293e85feadbbfe2150a5659dbc2ebc4afdeaf7d98936fb9a2f2ba0d4c35c"}, + {file = "propcache-0.3.0-cp311-cp311-win32.whl", hash = "sha256:63f26258a163c34542c24808f03d734b338da66ba91f410a703e505c8485791d"}, + {file = "propcache-0.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:cacea77ef7a2195f04f9279297684955e3d1ae4241092ff0cfcef532bb7a1c32"}, + {file = "propcache-0.3.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e53d19c2bf7d0d1e6998a7e693c7e87300dd971808e6618964621ccd0e01fe4e"}, + {file = "propcache-0.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a61a68d630e812b67b5bf097ab84e2cd79b48c792857dc10ba8a223f5b06a2af"}, + {file = "propcache-0.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5"}, + {file = "propcache-0.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67054e47c01b7b349b94ed0840ccae075449503cf1fdd0a1fdd98ab5ddc2667b"}, + {file = "propcache-0.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:997e7b8f173a391987df40f3b52c423e5850be6f6df0dcfb5376365440b56667"}, + {file = "propcache-0.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d663fd71491dde7dfdfc899d13a067a94198e90695b4321084c6e450743b8c7"}, + {file = "propcache-0.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8884ba1a0fe7210b775106b25850f5e5a9dc3c840d1ae9924ee6ea2eb3acbfe7"}, + {file = "propcache-0.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa806bbc13eac1ab6291ed21ecd2dd426063ca5417dd507e6be58de20e58dfcf"}, + {file = "propcache-0.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6f4d7a7c0aff92e8354cceca6fe223973ddf08401047920df0fcb24be2bd5138"}, + {file = "propcache-0.3.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:9be90eebc9842a93ef8335291f57b3b7488ac24f70df96a6034a13cb58e6ff86"}, + {file = "propcache-0.3.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:bf15fc0b45914d9d1b706f7c9c4f66f2b7b053e9517e40123e137e8ca8958b3d"}, + {file = "propcache-0.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5a16167118677d94bb48bfcd91e420088854eb0737b76ec374b91498fb77a70e"}, + {file = "propcache-0.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:41de3da5458edd5678b0f6ff66691507f9885f5fe6a0fb99a5d10d10c0fd2d64"}, + {file = "propcache-0.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:728af36011bb5d344c4fe4af79cfe186729efb649d2f8b395d1572fb088a996c"}, + {file = "propcache-0.3.0-cp312-cp312-win32.whl", hash = "sha256:6b5b7fd6ee7b54e01759f2044f936dcf7dea6e7585f35490f7ca0420fe723c0d"}, + {file = "propcache-0.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:2d15bc27163cd4df433e75f546b9ac31c1ba7b0b128bfb1b90df19082466ff57"}, + {file = "propcache-0.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a2b9bf8c79b660d0ca1ad95e587818c30ccdb11f787657458d6f26a1ea18c568"}, + {file = "propcache-0.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b0c1a133d42c6fc1f5fbcf5c91331657a1ff822e87989bf4a6e2e39b818d0ee9"}, + {file = "propcache-0.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bb2f144c6d98bb5cbc94adeb0447cfd4c0f991341baa68eee3f3b0c9c0e83767"}, + {file = "propcache-0.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1323cd04d6e92150bcc79d0174ce347ed4b349d748b9358fd2e497b121e03c8"}, + {file = "propcache-0.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b812b3cb6caacd072276ac0492d249f210006c57726b6484a1e1805b3cfeea0"}, + {file = "propcache-0.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:742840d1d0438eb7ea4280f3347598f507a199a35a08294afdcc560c3739989d"}, + {file = "propcache-0.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c6e7e4f9167fddc438cd653d826f2222222564daed4116a02a184b464d3ef05"}, + {file = "propcache-0.3.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a94ffc66738da99232ddffcf7910e0f69e2bbe3a0802e54426dbf0714e1c2ffe"}, + {file = "propcache-0.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c6ec957025bf32b15cbc6b67afe233c65b30005e4c55fe5768e4bb518d712f1"}, + {file = "propcache-0.3.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:549722908de62aa0b47a78b90531c022fa6e139f9166be634f667ff45632cc92"}, + {file = "propcache-0.3.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:5d62c4f6706bff5d8a52fd51fec6069bef69e7202ed481486c0bc3874912c787"}, + {file = "propcache-0.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:24c04f8fbf60094c531667b8207acbae54146661657a1b1be6d3ca7773b7a545"}, + {file = "propcache-0.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:7c5f5290799a3f6539cc5e6f474c3e5c5fbeba74a5e1e5be75587746a940d51e"}, + {file = "propcache-0.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4fa0e7c9c3cf7c276d4f6ab9af8adddc127d04e0fcabede315904d2ff76db626"}, + {file = "propcache-0.3.0-cp313-cp313-win32.whl", hash = "sha256:ee0bd3a7b2e184e88d25c9baa6a9dc609ba25b76daae942edfb14499ac7ec374"}, + {file = "propcache-0.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1c8f7d896a16da9455f882870a507567d4f58c53504dc2d4b1e1d386dfe4588a"}, + {file = "propcache-0.3.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e560fd75aaf3e5693b91bcaddd8b314f4d57e99aef8a6c6dc692f935cc1e6bbf"}, + {file = "propcache-0.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65a37714b8ad9aba5780325228598a5b16c47ba0f8aeb3dc0514701e4413d7c0"}, + {file = "propcache-0.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:07700939b2cbd67bfb3b76a12e1412405d71019df00ca5697ce75e5ef789d829"}, + {file = "propcache-0.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c0fdbdf6983526e269e5a8d53b7ae3622dd6998468821d660d0daf72779aefa"}, + {file = "propcache-0.3.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:794c3dd744fad478b6232289c866c25406ecdfc47e294618bdf1697e69bd64a6"}, + {file = "propcache-0.3.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4544699674faf66fb6b4473a1518ae4999c1b614f0b8297b1cef96bac25381db"}, + {file = "propcache-0.3.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54"}, + {file = "propcache-0.3.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f857034dc68d5ceb30fb60afb6ff2103087aea10a01b613985610e007053a121"}, + {file = "propcache-0.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e"}, + {file = "propcache-0.3.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f47d52fd9b2ac418c4890aad2f6d21a6b96183c98021f0a48497a904199f006e"}, + {file = "propcache-0.3.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:9ff4e9ecb6e4b363430edf2c6e50173a63e0820e549918adef70515f87ced19a"}, + {file = "propcache-0.3.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ecc2920630283e0783c22e2ac94427f8cca29a04cfdf331467d4f661f4072dac"}, + {file = "propcache-0.3.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:c441c841e82c5ba7a85ad25986014be8d7849c3cfbdb6004541873505929a74e"}, + {file = "propcache-0.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6c929916cbdb540d3407c66f19f73387f43e7c12fa318a66f64ac99da601bcdf"}, + {file = "propcache-0.3.0-cp313-cp313t-win32.whl", hash = "sha256:0c3e893c4464ebd751b44ae76c12c5f5c1e4f6cbd6fbf67e3783cd93ad221863"}, + {file = "propcache-0.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:75e872573220d1ee2305b35c9813626e620768248425f58798413e9c39741f46"}, + {file = "propcache-0.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:03c091bb752349402f23ee43bb2bff6bd80ccab7c9df6b88ad4322258d6960fc"}, + {file = "propcache-0.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:46ed02532cb66612d42ae5c3929b5e98ae330ea0f3900bc66ec5f4862069519b"}, + {file = "propcache-0.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:11ae6a8a01b8a4dc79093b5d3ca2c8a4436f5ee251a9840d7790dccbd96cb649"}, + {file = "propcache-0.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df03cd88f95b1b99052b52b1bb92173229d7a674df0ab06d2b25765ee8404bce"}, + {file = "propcache-0.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe"}, + {file = "propcache-0.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd54895e4ae7d32f1e3dd91261df46ee7483a735017dc6f987904f194aa5fd14"}, + {file = "propcache-0.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a67e5c04e3119594d8cfae517f4b9330c395df07ea65eab16f3d559b7068fe"}, + {file = "propcache-0.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee25f1ac091def37c4b59d192bbe3a206298feeb89132a470325bf76ad122a1e"}, + {file = "propcache-0.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:58e6d2a5a7cb3e5f166fd58e71e9a4ff504be9dc61b88167e75f835da5764d07"}, + {file = "propcache-0.3.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:be90c94570840939fecedf99fa72839aed70b0ced449b415c85e01ae67422c90"}, + {file = "propcache-0.3.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:49ea05212a529c2caffe411e25a59308b07d6e10bf2505d77da72891f9a05641"}, + {file = "propcache-0.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:119e244ab40f70a98c91906d4c1f4c5f2e68bd0b14e7ab0a06922038fae8a20f"}, + {file = "propcache-0.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:507c5357a8d8b4593b97fb669c50598f4e6cccbbf77e22fa9598aba78292b4d7"}, + {file = "propcache-0.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:8526b0941ec5a40220fc4dfde76aed58808e2b309c03e9fa8e2260083ef7157f"}, + {file = "propcache-0.3.0-cp39-cp39-win32.whl", hash = "sha256:7cedd25e5f678f7738da38037435b340694ab34d424938041aa630d8bac42663"}, + {file = "propcache-0.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:bf4298f366ca7e1ad1d21bbb58300a6985015909964077afd37559084590c929"}, + {file = "propcache-0.3.0-py3-none-any.whl", hash = "sha256:67dda3c7325691c2081510e92c561f465ba61b975f481735aefdfc845d2cd043"}, + {file = "propcache-0.3.0.tar.gz", hash = "sha256:a8fd93de4e1d278046345f49e2238cdb298589325849b2645d4a94c53faeffc5"}, +] + +[[package]] +name = "psutil" +version = "7.0.0" +description = "Cross-platform lib for process and system monitoring in Python. NOTE: the syntax of this script MUST be kept compatible with Python 2.7." +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25"}, + {file = "psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993"}, + {file = "psutil-7.0.0-cp36-cp36m-win32.whl", hash = "sha256:84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17"}, + {file = "psutil-7.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e"}, + {file = "psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99"}, + {file = "psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553"}, + {file = "psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456"}, +] + +[package.extras] +dev = ["abi3audit", "black (==24.10.0)", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest", "pytest-cov", "pytest-xdist", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] +test = ["pytest", "pytest-xdist", "setuptools"] + +[[package]] +name = "pyarrow" +version = "19.0.1" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pyarrow-19.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:fc28912a2dc924dddc2087679cc8b7263accc71b9ff025a1362b004711661a69"}, + {file = "pyarrow-19.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec"}, + {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad76aef7f5f7e4a757fddcdcf010a8290958f09e3470ea458c80d26f4316ae89"}, + {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d03c9d6f2a3dffbd62671ca070f13fc527bb1867b4ec2b98c7eeed381d4f389a"}, + {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:65cf9feebab489b19cdfcfe4aa82f62147218558d8d3f0fc1e9dea0ab8e7905a"}, + {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:41f9706fbe505e0abc10e84bf3a906a1338905cbbcf1177b71486b03e6ea6608"}, + {file = "pyarrow-19.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:c6cb2335a411b713fdf1e82a752162f72d4a7b5dbc588e32aa18383318b05866"}, + {file = "pyarrow-19.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:cc55d71898ea30dc95900297d191377caba257612f384207fe9f8293b5850f90"}, + {file = "pyarrow-19.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:7a544ec12de66769612b2d6988c36adc96fb9767ecc8ee0a4d270b10b1c51e00"}, + {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0148bb4fc158bfbc3d6dfe5001d93ebeed253793fff4435167f6ce1dc4bddeae"}, + {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f24faab6ed18f216a37870d8c5623f9c044566d75ec586ef884e13a02a9d62c5"}, + {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4982f8e2b7afd6dae8608d70ba5bd91699077323f812a0448d8b7abdff6cb5d3"}, + {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:49a3aecb62c1be1d822f8bf629226d4a96418228a42f5b40835c1f10d42e4db6"}, + {file = "pyarrow-19.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466"}, + {file = "pyarrow-19.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b"}, + {file = "pyarrow-19.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294"}, + {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14"}, + {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34"}, + {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6"}, + {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832"}, + {file = "pyarrow-19.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960"}, + {file = "pyarrow-19.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e45274b20e524ae5c39d7fc1ca2aa923aab494776d2d4b316b49ec7572ca324c"}, + {file = "pyarrow-19.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d9dedeaf19097a143ed6da37f04f4051aba353c95ef507764d344229b2b740ae"}, + {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ebfb5171bb5f4a52319344ebbbecc731af3f021e49318c74f33d520d31ae0c4"}, + {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a21d39fbdb948857f67eacb5bbaaf36802de044ec36fbef7a1c8f0dd3a4ab2"}, + {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:99bc1bec6d234359743b01e70d4310d0ab240c3d6b0da7e2a93663b0158616f6"}, + {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1b93ef2c93e77c442c979b0d596af45e4665d8b96da598db145b0fec014b9136"}, + {file = "pyarrow-19.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:d9d46e06846a41ba906ab25302cf0fd522f81aa2a85a71021826f34639ad31ef"}, + {file = "pyarrow-19.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:c0fe3dbbf054a00d1f162fda94ce236a899ca01123a798c561ba307ca38af5f0"}, + {file = "pyarrow-19.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:96606c3ba57944d128e8a8399da4812f56c7f61de8c647e3470b417f795d0ef9"}, + {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f04d49a6b64cf24719c080b3c2029a3a5b16417fd5fd7c4041f94233af732f3"}, + {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a9137cf7e1640dce4c190551ee69d478f7121b5c6f323553b319cac936395f6"}, + {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7c1bca1897c28013db5e4c83944a2ab53231f541b9e0c3f4791206d0c0de389a"}, + {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:58d9397b2e273ef76264b45531e9d552d8ec8a6688b7390b5be44c02a37aade8"}, + {file = "pyarrow-19.0.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:b9766a47a9cb56fefe95cb27f535038b5a195707a08bf61b180e642324963b46"}, + {file = "pyarrow-19.0.1-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:6c5941c1aac89a6c2f2b16cd64fe76bcdb94b2b1e99ca6459de4e6f07638d755"}, + {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8"}, + {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:335d170e050bcc7da867a1ed8ffb8b44c57aaa6e0843b156a501298657b1e972"}, + {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:1c7556165bd38cf0cd992df2636f8bcdd2d4b26916c6b7e646101aff3c16f76f"}, + {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:699799f9c80bebcf1da0983ba86d7f289c5a2a5c04b945e2f2bcf7e874a91911"}, + {file = "pyarrow-19.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8464c9fbe6d94a7fe1599e7e8965f350fd233532868232ab2596a71586c5a429"}, + {file = "pyarrow-19.0.1.tar.gz", hash = "sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e"}, +] + +[package.extras] +test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"] + +[[package]] +name = "pydot" +version = "3.0.4" +description = "Python interface to Graphviz's Dot" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pydot-3.0.4-py3-none-any.whl", hash = "sha256:bfa9c3fc0c44ba1d132adce131802d7df00429d1a79cc0346b0a5cd374dbe9c6"}, + {file = "pydot-3.0.4.tar.gz", hash = "sha256:3ce88b2558f3808b0376f22bfa6c263909e1c3981e2a7b629b65b451eee4a25d"}, +] + +[package.dependencies] +pyparsing = ">=3.0.9" + +[package.extras] +dev = ["chardet", "parameterized", "ruff"] +release = ["zest.releaser[recommended]"] +tests = ["chardet", "parameterized", "pytest", "pytest-cov", "pytest-xdist[psutil]", "ruff", "tox"] + +[[package]] +name = "pygments" +version = "2.19.1" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, + {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pyparsing" +version = "3.2.1" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pyparsing-3.2.1-py3-none-any.whl", hash = "sha256:506ff4f4386c4cec0590ec19e6302d3aedb992fdc02c761e90416f158dacf8e1"}, + {file = "pyparsing-3.2.1.tar.gz", hash = "sha256:61980854fd66de3a90028d679a954d5f2623e83144b5afe5ee86f43d762e5f0a"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2025.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57"}, + {file = "pytz-2025.1.tar.gz", hash = "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + +[[package]] +name = "regex" +version = "2024.11.6" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91"}, + {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0"}, + {file = "regex-2024.11.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164d8b7b3b4bcb2068b97428060b2a53be050085ef94eca7f240e7947f1b080e"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3660c82f209655a06b587d55e723f0b813d3a7db2e32e5e7dc64ac2a9e86fde"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d22326fcdef5e08c154280b71163ced384b428343ae16a5ab2b3354aed12436e"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1ac758ef6aebfc8943560194e9fd0fa18bcb34d89fd8bd2af18183afd8da3a2"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:997d6a487ff00807ba810e0f8332c18b4eb8d29463cfb7c820dc4b6e7562d0cf"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02a02d2bb04fec86ad61f3ea7f49c015a0681bf76abb9857f945d26159d2968c"}, + {file = "regex-2024.11.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f02f93b92358ee3f78660e43b4b0091229260c5d5c408d17d60bf26b6c900e86"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:06eb1be98df10e81ebaded73fcd51989dcf534e3c753466e4b60c4697a003b67"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:040df6fe1a5504eb0f04f048e6d09cd7c7110fef851d7c567a6b6e09942feb7d"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabbfc59f2c6edba2a6622c647b716e34e8e3867e0ab975412c5c2f79b82da2"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8447d2d39b5abe381419319f942de20b7ecd60ce86f16a23b0698f22e1b70008"}, + {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:da8f5fc57d1933de22a9e23eec290a0d8a5927a5370d24bda9a6abe50683fe62"}, + {file = "regex-2024.11.6-cp310-cp310-win32.whl", hash = "sha256:b489578720afb782f6ccf2840920f3a32e31ba28a4b162e13900c3e6bd3f930e"}, + {file = "regex-2024.11.6-cp310-cp310-win_amd64.whl", hash = "sha256:5071b2093e793357c9d8b2929dfc13ac5f0a6c650559503bb81189d0a3814519"}, + {file = "regex-2024.11.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5478c6962ad548b54a591778e93cd7c456a7a29f8eca9c49e4f9a806dcc5d638"}, + {file = "regex-2024.11.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c89a8cc122b25ce6945f0423dc1352cb9593c68abd19223eebbd4e56612c5b7"}, + {file = "regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94d87b689cdd831934fa3ce16cc15cd65748e6d689f5d2b8f4f4df2065c9fa20"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1062b39a0a2b75a9c694f7a08e7183a80c63c0d62b301418ffd9c35f55aaa114"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:167ed4852351d8a750da48712c3930b031f6efdaa0f22fa1933716bfcd6bf4a3"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d548dafee61f06ebdb584080621f3e0c23fff312f0de1afc776e2a2ba99a74f"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a19f302cd1ce5dd01a9099aaa19cae6173306d1302a43b627f62e21cf18ac0"}, + {file = "regex-2024.11.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bec9931dfb61ddd8ef2ebc05646293812cb6b16b60cf7c9511a832b6f1854b55"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9714398225f299aa85267fd222f7142fcb5c769e73d7733344efc46f2ef5cf89"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:202eb32e89f60fc147a41e55cb086db2a3f8cb82f9a9a88440dcfc5d37faae8d"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4181b814e56078e9b00427ca358ec44333765f5ca1b45597ec7446d3a1ef6e34"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:068376da5a7e4da51968ce4c122a7cd31afaaec4fccc7856c92f63876e57b51d"}, + {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f2c4184420d881a3475fb2c6f4d95d53a8d50209a2500723d831036f7c45"}, + {file = "regex-2024.11.6-cp311-cp311-win32.whl", hash = "sha256:c36f9b6f5f8649bb251a5f3f66564438977b7ef8386a52460ae77e6070d309d9"}, + {file = "regex-2024.11.6-cp311-cp311-win_amd64.whl", hash = "sha256:02e28184be537f0e75c1f9b2f8847dc51e08e6e171c6bde130b2687e0c33cf60"}, + {file = "regex-2024.11.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:52fb28f528778f184f870b7cf8f225f5eef0a8f6e3778529bdd40c7b3920796a"}, + {file = "regex-2024.11.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9"}, + {file = "regex-2024.11.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805e6b60c54bf766b251e94526ebad60b7de0c70f70a4e6210ee2891acb70bf2"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b85c2530be953a890eaffde05485238f07029600e8f098cdf1848d414a8b45e4"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb26437975da7dc36b7efad18aa9dd4ea569d2357ae6b783bf1118dabd9ea577"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abfa5080c374a76a251ba60683242bc17eeb2c9818d0d30117b4486be10c59d3"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b7fa6606c2881c1db9479b0eaa11ed5dfa11c8d60a474ff0e095099f39d98e"}, + {file = "regex-2024.11.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c32f75920cf99fe6b6c539c399a4a128452eaf1af27f39bce8909c9a3fd8cbe"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:982e6d21414e78e1f51cf595d7f321dcd14de1f2881c5dc6a6e23bbbbd68435e"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a7c2155f790e2fb448faed6dd241386719802296ec588a8b9051c1f5c481bc29"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149f5008d286636e48cd0b1dd65018548944e495b0265b45e1bffecce1ef7f39"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e5364a4502efca094731680e80009632ad6624084aff9a23ce8c8c6820de3e51"}, + {file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0a86e7eeca091c09e021db8eb72d54751e527fa47b8d5787caf96d9831bd02ad"}, + {file = "regex-2024.11.6-cp312-cp312-win32.whl", hash = "sha256:32f9a4c643baad4efa81d549c2aadefaeba12249b2adc5af541759237eee1c54"}, + {file = "regex-2024.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:a93c194e2df18f7d264092dc8539b8ffb86b45b899ab976aa15d48214138e81b"}, + {file = "regex-2024.11.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a6ba92c0bcdf96cbf43a12c717eae4bc98325ca3730f6b130ffa2e3c3c723d84"}, + {file = "regex-2024.11.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:525eab0b789891ac3be914d36893bdf972d483fe66551f79d3e27146191a37d4"}, + {file = "regex-2024.11.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:086a27a0b4ca227941700e0b31425e7a28ef1ae8e5e05a33826e17e47fbfdba0"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bde01f35767c4a7899b7eb6e823b125a64de314a8ee9791367c9a34d56af18d0"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b583904576650166b3d920d2bcce13971f6f9e9a396c673187f49811b2769dc7"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c4de13f06a0d54fa0d5ab1b7138bfa0d883220965a29616e3ea61b35d5f5fc7"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3cde6e9f2580eb1665965ce9bf17ff4952f34f5b126beb509fee8f4e994f143c"}, + {file = "regex-2024.11.6-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d7f453dca13f40a02b79636a339c5b62b670141e63efd511d3f8f73fba162b3"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59dfe1ed21aea057a65c6b586afd2a945de04fc7db3de0a6e3ed5397ad491b07"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b97c1e0bd37c5cd7902e65f410779d39eeda155800b65fc4d04cc432efa9bc6e"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f9d1e379028e0fc2ae3654bac3cbbef81bf3fd571272a42d56c24007979bafb6"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:13291b39131e2d002a7940fb176e120bec5145f3aeb7621be6534e46251912c4"}, + {file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f51f88c126370dcec4908576c5a627220da6c09d0bff31cfa89f2523843316d"}, + {file = "regex-2024.11.6-cp313-cp313-win32.whl", hash = "sha256:63b13cfd72e9601125027202cad74995ab26921d8cd935c25f09c630436348ff"}, + {file = "regex-2024.11.6-cp313-cp313-win_amd64.whl", hash = "sha256:2b3361af3198667e99927da8b84c1b010752fa4b1115ee30beaa332cabc3ef1a"}, + {file = "regex-2024.11.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3a51ccc315653ba012774efca4f23d1d2a8a8f278a6072e29c7147eee7da446b"}, + {file = "regex-2024.11.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ad182d02e40de7459b73155deb8996bbd8e96852267879396fb274e8700190e3"}, + {file = "regex-2024.11.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ba9b72e5643641b7d41fa1f6d5abda2c9a263ae835b917348fc3c928182ad467"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40291b1b89ca6ad8d3f2b82782cc33807f1406cf68c8d440861da6304d8ffbbd"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cdf58d0e516ee426a48f7b2c03a332a4114420716d55769ff7108c37a09951bf"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a36fdf2af13c2b14738f6e973aba563623cb77d753bbbd8d414d18bfaa3105dd"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1cee317bfc014c2419a76bcc87f071405e3966da434e03e13beb45f8aced1a6"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50153825ee016b91549962f970d6a4442fa106832e14c918acd1c8e479916c4f"}, + {file = "regex-2024.11.6-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea1bfda2f7162605f6e8178223576856b3d791109f15ea99a9f95c16a7636fb5"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:df951c5f4a1b1910f1a99ff42c473ff60f8225baa1cdd3539fe2819d9543e9df"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:072623554418a9911446278f16ecb398fb3b540147a7828c06e2011fa531e773"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f654882311409afb1d780b940234208a252322c24a93b442ca714d119e68086c"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89d75e7293d2b3e674db7d4d9b1bee7f8f3d1609428e293771d1a962617150cc"}, + {file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:f65557897fc977a44ab205ea871b690adaef6b9da6afda4790a2484b04293a5f"}, + {file = "regex-2024.11.6-cp38-cp38-win32.whl", hash = "sha256:6f44ec28b1f858c98d3036ad5d7d0bfc568bdd7a74f9c24e25f41ef1ebfd81a4"}, + {file = "regex-2024.11.6-cp38-cp38-win_amd64.whl", hash = "sha256:bb8f74f2f10dbf13a0be8de623ba4f9491faf58c24064f32b65679b021ed0001"}, + {file = "regex-2024.11.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5704e174f8ccab2026bd2f1ab6c510345ae8eac818b613d7d73e785f1310f839"}, + {file = "regex-2024.11.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:220902c3c5cc6af55d4fe19ead504de80eb91f786dc102fbd74894b1551f095e"}, + {file = "regex-2024.11.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7e351589da0850c125f1600a4c4ba3c722efefe16b297de54300f08d734fbf"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5056b185ca113c88e18223183aa1a50e66507769c9640a6ff75859619d73957b"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e34b51b650b23ed3354b5a07aab37034d9f923db2a40519139af34f485f77d0"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5670bce7b200273eee1840ef307bfa07cda90b38ae56e9a6ebcc9f50da9c469b"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08986dce1339bc932923e7d1232ce9881499a0e02925f7402fb7c982515419ef"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93c0b12d3d3bc25af4ebbf38f9ee780a487e8bf6954c115b9f015822d3bb8e48"}, + {file = "regex-2024.11.6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:764e71f22ab3b305e7f4c21f1a97e1526a25ebdd22513e251cf376760213da13"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f056bf21105c2515c32372bbc057f43eb02aae2fda61052e2f7622c801f0b4e2"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:69ab78f848845569401469da20df3e081e6b5a11cb086de3eed1d48f5ed57c95"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:86fddba590aad9208e2fa8b43b4c098bb0ec74f15718bb6a704e3c63e2cef3e9"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:684d7a212682996d21ca12ef3c17353c021fe9de6049e19ac8481ec35574a70f"}, + {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a03e02f48cd1abbd9f3b7e3586d97c8f7a9721c436f51a5245b3b9483044480b"}, + {file = "regex-2024.11.6-cp39-cp39-win32.whl", hash = "sha256:41758407fc32d5c3c5de163888068cfee69cb4c2be844e7ac517a52770f9af57"}, + {file = "regex-2024.11.6-cp39-cp39-win_amd64.whl", hash = "sha256:b2837718570f95dd41675328e111345f9b7095d821bac435aac173ac80b19983"}, + {file = "regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "rich" +version = "13.9.4" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.8.0" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, + {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "safetensors" +version = "0.5.3" +description = "" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "safetensors-0.5.3-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd20eb133db8ed15b40110b7c00c6df51655a2998132193de2f75f72d99c7073"}, + {file = "safetensors-0.5.3-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:21d01c14ff6c415c485616b8b0bf961c46b3b343ca59110d38d744e577f9cce7"}, + {file = "safetensors-0.5.3-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11bce6164887cd491ca75c2326a113ba934be596e22b28b1742ce27b1d076467"}, + {file = "safetensors-0.5.3-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4a243be3590bc3301c821da7a18d87224ef35cbd3e5f5727e4e0728b8172411e"}, + {file = "safetensors-0.5.3-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8bd84b12b1670a6f8e50f01e28156422a2bc07fb16fc4e98bded13039d688a0d"}, + {file = "safetensors-0.5.3-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:391ac8cab7c829452175f871fcaf414aa1e292b5448bd02620f675a7f3e7abb9"}, + {file = "safetensors-0.5.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cead1fa41fc54b1e61089fa57452e8834f798cb1dc7a09ba3524f1eb08e0317a"}, + {file = "safetensors-0.5.3-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1077f3e94182d72618357b04b5ced540ceb71c8a813d3319f1aba448e68a770d"}, + {file = "safetensors-0.5.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:799021e78287bac619c7b3f3606730a22da4cda27759ddf55d37c8db7511c74b"}, + {file = "safetensors-0.5.3-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:df26da01aaac504334644e1b7642fa000bfec820e7cef83aeac4e355e03195ff"}, + {file = "safetensors-0.5.3-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:32c3ef2d7af8b9f52ff685ed0bc43913cdcde135089ae322ee576de93eae5135"}, + {file = "safetensors-0.5.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:37f1521be045e56fc2b54c606d4455573e717b2d887c579ee1dbba5f868ece04"}, + {file = "safetensors-0.5.3-cp38-abi3-win32.whl", hash = "sha256:cfc0ec0846dcf6763b0ed3d1846ff36008c6e7290683b61616c4b040f6a54ace"}, + {file = "safetensors-0.5.3-cp38-abi3-win_amd64.whl", hash = "sha256:836cbbc320b47e80acd40e44c8682db0e8ad7123209f69b093def21ec7cafd11"}, + {file = "safetensors-0.5.3.tar.gz", hash = "sha256:b6b0d6ecacec39a4fdd99cc19f4576f5219ce858e6fd8dbe7609df0b8dc56965"}, +] + +[package.extras] +all = ["safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[pinned-tf]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] +dev = ["safetensors[all]"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] +mlx = ["mlx (>=0.0.9)"] +numpy = ["numpy (>=1.21.6)"] +paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] +pinned-tf = ["safetensors[numpy]", "tensorflow (==2.18.0)"] +quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] +tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] +torch = ["safetensors[numpy]", "torch (>=1.10)"] + +[[package]] +name = "scikit-learn" +version = "1.6.1" +description = "A set of python modules for machine learning and data mining" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "scikit_learn-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d056391530ccd1e501056160e3c9673b4da4805eb67eb2bdf4e983e1f9c9204e"}, + {file = "scikit_learn-1.6.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0c8d036eb937dbb568c6242fa598d551d88fb4399c0344d95c001980ec1c7d36"}, + {file = "scikit_learn-1.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8634c4bd21a2a813e0a7e3900464e6d593162a29dd35d25bdf0103b3fce60ed5"}, + {file = "scikit_learn-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:775da975a471c4f6f467725dff0ced5c7ac7bda5e9316b260225b48475279a1b"}, + {file = "scikit_learn-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:8a600c31592bd7dab31e1c61b9bbd6dea1b3433e67d264d17ce1017dbdce8002"}, + {file = "scikit_learn-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:72abc587c75234935e97d09aa4913a82f7b03ee0b74111dcc2881cba3c5a7b33"}, + {file = "scikit_learn-1.6.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b3b00cdc8f1317b5f33191df1386c0befd16625f49d979fe77a8d44cae82410d"}, + {file = "scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc4765af3386811c3ca21638f63b9cf5ecf66261cc4815c1db3f1e7dc7b79db2"}, + {file = "scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25fc636bdaf1cc2f4a124a116312d837148b5e10872147bdaf4887926b8c03d8"}, + {file = "scikit_learn-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:fa909b1a36e000a03c382aade0bd2063fd5680ff8b8e501660c0f59f021a6415"}, + {file = "scikit_learn-1.6.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:926f207c804104677af4857b2c609940b743d04c4c35ce0ddc8ff4f053cddc1b"}, + {file = "scikit_learn-1.6.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2c2cae262064e6a9b77eee1c8e768fc46aa0b8338c6a8297b9b6759720ec0ff2"}, + {file = "scikit_learn-1.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1061b7c028a8663fb9a1a1baf9317b64a257fcb036dae5c8752b2abef31d136f"}, + {file = "scikit_learn-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e69fab4ebfc9c9b580a7a80111b43d214ab06250f8a7ef590a4edf72464dd86"}, + {file = "scikit_learn-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:70b1d7e85b1c96383f872a519b3375f92f14731e279a7b4c6cfd650cf5dffc52"}, + {file = "scikit_learn-1.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2ffa1e9e25b3d93990e74a4be2c2fc61ee5af85811562f1288d5d055880c4322"}, + {file = "scikit_learn-1.6.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:dc5cf3d68c5a20ad6d571584c0750ec641cc46aeef1c1507be51300e6003a7e1"}, + {file = "scikit_learn-1.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c06beb2e839ecc641366000ca84f3cf6fa9faa1777e29cf0c04be6e4d096a348"}, + {file = "scikit_learn-1.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8ca8cb270fee8f1f76fa9bfd5c3507d60c6438bbee5687f81042e2bb98e5a97"}, + {file = "scikit_learn-1.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:7a1c43c8ec9fde528d664d947dc4c0789be4077a3647f232869f41d9bf50e0fb"}, + {file = "scikit_learn-1.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a17c1dea1d56dcda2fac315712f3651a1fea86565b64b48fa1bc090249cbf236"}, + {file = "scikit_learn-1.6.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a7aa5f9908f0f28f4edaa6963c0a6183f1911e63a69aa03782f0d924c830a35"}, + {file = "scikit_learn-1.6.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0650e730afb87402baa88afbf31c07b84c98272622aaba002559b614600ca691"}, + {file = "scikit_learn-1.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:3f59fe08dc03ea158605170eb52b22a105f238a5d512c4470ddeca71feae8e5f"}, + {file = "scikit_learn-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6849dd3234e87f55dce1db34c89a810b489ead832aaf4d4550b7ea85628be6c1"}, + {file = "scikit_learn-1.6.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:e7be3fa5d2eb9be7d77c3734ff1d599151bb523674be9b834e8da6abe132f44e"}, + {file = "scikit_learn-1.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44a17798172df1d3c1065e8fcf9019183f06c87609b49a124ebdf57ae6cb0107"}, + {file = "scikit_learn-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b7a3b86e411e4bce21186e1c180d792f3d99223dcfa3b4f597ecc92fa1a422"}, + {file = "scikit_learn-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:7a73d457070e3318e32bdb3aa79a8d990474f19035464dfd8bede2883ab5dc3b"}, + {file = "scikit_learn-1.6.1.tar.gz", hash = "sha256:b4fc2525eca2c69a59260f583c56a7557c6ccdf8deafdba6e060f94c1c59738e"}, +] + +[package.dependencies] +joblib = ">=1.2.0" +numpy = ">=1.19.5" +scipy = ">=1.6.0" +threadpoolctl = ">=3.1.0" + +[package.extras] +benchmark = ["matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "pandas (>=1.1.5)"] +build = ["cython (>=3.0.10)", "meson-python (>=0.16.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.17.1)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)", "towncrier (>=24.8.0)"] +examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] +install = ["joblib (>=1.2.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)", "threadpoolctl (>=3.1.0)"] +maintenance = ["conda-lock (==2.5.6)"] +tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.5.1)", "scikit-image (>=0.17.2)"] + +[[package]] +name = "scipy" +version = "1.15.2" +description = "Fundamental algorithms for scientific computing in Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "scipy-1.15.2-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a2ec871edaa863e8213ea5df811cd600734f6400b4af272e1c011e69401218e9"}, + {file = "scipy-1.15.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:6f223753c6ea76983af380787611ae1291e3ceb23917393079dcc746ba60cfb5"}, + {file = "scipy-1.15.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:ecf797d2d798cf7c838c6d98321061eb3e72a74710e6c40540f0e8087e3b499e"}, + {file = "scipy-1.15.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:9b18aa747da280664642997e65aab1dd19d0c3d17068a04b3fe34e2559196cb9"}, + {file = "scipy-1.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87994da02e73549dfecaed9e09a4f9d58a045a053865679aeb8d6d43747d4df3"}, + {file = "scipy-1.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69ea6e56d00977f355c0f84eba69877b6df084516c602d93a33812aa04d90a3d"}, + {file = "scipy-1.15.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:888307125ea0c4466287191e5606a2c910963405ce9671448ff9c81c53f85f58"}, + {file = "scipy-1.15.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9412f5e408b397ff5641080ed1e798623dbe1ec0d78e72c9eca8992976fa65aa"}, + {file = "scipy-1.15.2-cp310-cp310-win_amd64.whl", hash = "sha256:b5e025e903b4f166ea03b109bb241355b9c42c279ea694d8864d033727205e65"}, + {file = "scipy-1.15.2-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:92233b2df6938147be6fa8824b8136f29a18f016ecde986666be5f4d686a91a4"}, + {file = "scipy-1.15.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:62ca1ff3eb513e09ed17a5736929429189adf16d2d740f44e53270cc800ecff1"}, + {file = "scipy-1.15.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4c6676490ad76d1c2894d77f976144b41bd1a4052107902238047fb6a473e971"}, + {file = "scipy-1.15.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a8bf5cb4a25046ac61d38f8d3c3426ec11ebc350246a4642f2f315fe95bda655"}, + {file = "scipy-1.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a8e34cf4c188b6dd004654f88586d78f95639e48a25dfae9c5e34a6dc34547e"}, + {file = "scipy-1.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28a0d2c2075946346e4408b211240764759e0fabaeb08d871639b5f3b1aca8a0"}, + {file = "scipy-1.15.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:42dabaaa798e987c425ed76062794e93a243be8f0f20fff6e7a89f4d61cb3d40"}, + {file = "scipy-1.15.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6f5e296ec63c5da6ba6fa0343ea73fd51b8b3e1a300b0a8cae3ed4b1122c7462"}, + {file = "scipy-1.15.2-cp311-cp311-win_amd64.whl", hash = "sha256:597a0c7008b21c035831c39927406c6181bcf8f60a73f36219b69d010aa04737"}, + {file = "scipy-1.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c4697a10da8f8765bb7c83e24a470da5797e37041edfd77fd95ba3811a47c4fd"}, + {file = "scipy-1.15.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:869269b767d5ee7ea6991ed7e22b3ca1f22de73ab9a49c44bad338b725603301"}, + {file = "scipy-1.15.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:bad78d580270a4d32470563ea86c6590b465cb98f83d760ff5b0990cb5518a93"}, + {file = "scipy-1.15.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b09ae80010f52efddb15551025f9016c910296cf70adbf03ce2a8704f3a5ad20"}, + {file = "scipy-1.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a6fd6eac1ce74a9f77a7fc724080d507c5812d61e72bd5e4c489b042455865e"}, + {file = "scipy-1.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b871df1fe1a3ba85d90e22742b93584f8d2b8e6124f8372ab15c71b73e428b8"}, + {file = "scipy-1.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:03205d57a28e18dfd39f0377d5002725bf1f19a46f444108c29bdb246b6c8a11"}, + {file = "scipy-1.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:601881dfb761311045b03114c5fe718a12634e5608c3b403737ae463c9885d53"}, + {file = "scipy-1.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:e7c68b6a43259ba0aab737237876e5c2c549a031ddb7abc28c7b47f22e202ded"}, + {file = "scipy-1.15.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:01edfac9f0798ad6b46d9c4c9ca0e0ad23dbf0b1eb70e96adb9fa7f525eff0bf"}, + {file = "scipy-1.15.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:08b57a9336b8e79b305a143c3655cc5bdbe6d5ece3378578888d2afbb51c4e37"}, + {file = "scipy-1.15.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:54c462098484e7466362a9f1672d20888f724911a74c22ae35b61f9c5919183d"}, + {file = "scipy-1.15.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:cf72ff559a53a6a6d77bd8eefd12a17995ffa44ad86c77a5df96f533d4e6c6bb"}, + {file = "scipy-1.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9de9d1416b3d9e7df9923ab23cd2fe714244af10b763975bea9e4f2e81cebd27"}, + {file = "scipy-1.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb530e4794fc8ea76a4a21ccb67dea33e5e0e60f07fc38a49e821e1eae3b71a0"}, + {file = "scipy-1.15.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5ea7ed46d437fc52350b028b1d44e002646e28f3e8ddc714011aaf87330f2f32"}, + {file = "scipy-1.15.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:11e7ad32cf184b74380f43d3c0a706f49358b904fa7d5345f16ddf993609184d"}, + {file = "scipy-1.15.2-cp313-cp313-win_amd64.whl", hash = "sha256:a5080a79dfb9b78b768cebf3c9dcbc7b665c5875793569f48bf0e2b1d7f68f6f"}, + {file = "scipy-1.15.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:447ce30cee6a9d5d1379087c9e474628dab3db4a67484be1b7dc3196bfb2fac9"}, + {file = "scipy-1.15.2-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:c90ebe8aaa4397eaefa8455a8182b164a6cc1d59ad53f79943f266d99f68687f"}, + {file = "scipy-1.15.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:def751dd08243934c884a3221156d63e15234a3155cf25978b0a668409d45eb6"}, + {file = "scipy-1.15.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:302093e7dfb120e55515936cb55618ee0b895f8bcaf18ff81eca086c17bd80af"}, + {file = "scipy-1.15.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd5b77413e1855351cdde594eca99c1f4a588c2d63711388b6a1f1c01f62274"}, + {file = "scipy-1.15.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d0194c37037707b2afa7a2f2a924cf7bac3dc292d51b6a925e5fcb89bc5c776"}, + {file = "scipy-1.15.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:bae43364d600fdc3ac327db99659dcb79e6e7ecd279a75fe1266669d9a652828"}, + {file = "scipy-1.15.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f031846580d9acccd0044efd1a90e6f4df3a6e12b4b6bd694a7bc03a89892b28"}, + {file = "scipy-1.15.2-cp313-cp313t-win_amd64.whl", hash = "sha256:fe8a9eb875d430d81755472c5ba75e84acc980e4a8f6204d402849234d3017db"}, + {file = "scipy-1.15.2.tar.gz", hash = "sha256:cd58a314d92838f7e6f755c8a2167ead4f27e1fd5c1251fd54289569ef3495ec"}, +] + +[package.dependencies] +numpy = ">=1.23.5,<2.5" + +[package.extras] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] +doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.16.5)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + +[[package]] +name = "seqeval" +version = "1.2.2" +description = "Testing framework for sequence labeling" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "seqeval-1.2.2.tar.gz", hash = "sha256:f28e97c3ab96d6fcd32b648f6438ff2e09cfba87f05939da9b3970713ec56e6f"}, +] + +[package.dependencies] +numpy = ">=1.14.0" +scikit-learn = ">=0.21.3" + +[[package]] +name = "setuptools" +version = "75.8.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.12\"" +files = [ + {file = "setuptools-75.8.2-py3-none-any.whl", hash = "sha256:558e47c15f1811c1fa7adbd0096669bf76c1d3f433f58324df69f3f5ecac4e8f"}, + {file = "setuptools-75.8.2.tar.gz", hash = "sha256:4880473a969e5f23f2a2be3646b2dfd84af9028716d398e46192f84bc36900d2"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] +core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] + +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + +[[package]] +name = "sympy" +version = "1.13.1" +description = "Computer algebra system (CAS) in Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sympy-1.13.1-py3-none-any.whl", hash = "sha256:db36cdc64bf61b9b24578b6f7bab1ecdd2452cf008f34faa33776680c26d66f8"}, + {file = "sympy-1.13.1.tar.gz", hash = "sha256:9cebf7e04ff162015ce31c9c6c9144daa34a93bd082f54fd8f12deca4f47515f"}, +] + +[package.dependencies] +mpmath = ">=1.1.0,<1.4" + +[package.extras] +dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] + +[[package]] +name = "threadpoolctl" +version = "3.5.0" +description = "threadpoolctl" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"}, + {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, +] + +[[package]] +name = "tokenizers" +version = "0.20.3" +description = "" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "tokenizers-0.20.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:31ccab28dbb1a9fe539787210b0026e22debeab1662970f61c2d921f7557f7e4"}, + {file = "tokenizers-0.20.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6361191f762bda98c773da418cf511cbaa0cb8d0a1196f16f8c0119bde68ff8"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f128d5da1202b78fa0a10d8d938610472487da01b57098d48f7e944384362514"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79c4121a2e9433ad7ef0769b9ca1f7dd7fa4c0cd501763d0a030afcbc6384481"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7850fde24197fe5cd6556e2fdba53a6d3bae67c531ea33a3d7c420b90904141"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b357970c095dc134978a68c67d845a1e3803ab7c4fbb39195bde914e7e13cf8b"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a333d878c4970b72d6c07848b90c05f6b045cf9273fc2bc04a27211721ad6118"}, + {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd9fee817f655a8f50049f685e224828abfadd436b8ff67979fc1d054b435f1"}, + {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9e7816808b402129393a435ea2a509679b41246175d6e5e9f25b8692bfaa272b"}, + {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba96367db9d8a730d3a1d5996b4b7babb846c3994b8ef14008cd8660f55db59d"}, + {file = "tokenizers-0.20.3-cp310-none-win32.whl", hash = "sha256:ee31ba9d7df6a98619426283e80c6359f167e2e9882d9ce1b0254937dbd32f3f"}, + {file = "tokenizers-0.20.3-cp310-none-win_amd64.whl", hash = "sha256:a845c08fdad554fe0871d1255df85772f91236e5fd6b9287ef8b64f5807dbd0c"}, + {file = "tokenizers-0.20.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:585b51e06ca1f4839ce7759941e66766d7b060dccfdc57c4ca1e5b9a33013a90"}, + {file = "tokenizers-0.20.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61cbf11954f3b481d08723ebd048ba4b11e582986f9be74d2c3bdd9293a4538d"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef820880d5e4e8484e2fa54ff8d297bb32519eaa7815694dc835ace9130a3eea"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:67ef4dcb8841a4988cd00dd288fb95dfc8e22ed021f01f37348fd51c2b055ba9"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff1ef8bd47a02b0dc191688ccb4da53600df5d4c9a05a4b68e1e3de4823e78eb"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:444d188186eab3148baf0615b522461b41b1f0cd58cd57b862ec94b6ac9780f1"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37c04c032c1442740b2c2d925f1857885c07619224a533123ac7ea71ca5713da"}, + {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:453c7769d22231960ee0e883d1005c93c68015025a5e4ae56275406d94a3c907"}, + {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4bb31f7b2847e439766aaa9cc7bccf7ac7088052deccdb2275c952d96f691c6a"}, + {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:843729bf0f991b29655a069a2ff58a4c24375a553c70955e15e37a90dd4e045c"}, + {file = "tokenizers-0.20.3-cp311-none-win32.whl", hash = "sha256:efcce3a927b1e20ca694ba13f7a68c59b0bd859ef71e441db68ee42cf20c2442"}, + {file = "tokenizers-0.20.3-cp311-none-win_amd64.whl", hash = "sha256:88301aa0801f225725b6df5dea3d77c80365ff2362ca7e252583f2b4809c4cc0"}, + {file = "tokenizers-0.20.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:49d12a32e190fad0e79e5bdb788d05da2f20d8e006b13a70859ac47fecf6ab2f"}, + {file = "tokenizers-0.20.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:282848cacfb9c06d5e51489f38ec5aa0b3cd1e247a023061945f71f41d949d73"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abe4e08c7d0cd6154c795deb5bf81d2122f36daf075e0c12a8b050d824ef0a64"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca94fc1b73b3883c98f0c88c77700b13d55b49f1071dfd57df2b06f3ff7afd64"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef279c7e239f95c8bdd6ff319d9870f30f0d24915b04895f55b1adcf96d6c60d"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16384073973f6ccbde9852157a4fdfe632bb65208139c9d0c0bd0176a71fd67f"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:312d522caeb8a1a42ebdec87118d99b22667782b67898a76c963c058a7e41d4f"}, + {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2b7cb962564785a83dafbba0144ecb7f579f1d57d8c406cdaa7f32fe32f18ad"}, + {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:124c5882ebb88dadae1fc788a582299fcd3a8bd84fc3e260b9918cf28b8751f5"}, + {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2b6e54e71f84c4202111a489879005cb14b92616a87417f6c102c833af961ea2"}, + {file = "tokenizers-0.20.3-cp312-none-win32.whl", hash = "sha256:83d9bfbe9af86f2d9df4833c22e94d94750f1d0cd9bfb22a7bb90a86f61cdb1c"}, + {file = "tokenizers-0.20.3-cp312-none-win_amd64.whl", hash = "sha256:44def74cee574d609a36e17c8914311d1b5dbcfe37c55fd29369d42591b91cf2"}, + {file = "tokenizers-0.20.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e0b630e0b536ef0e3c8b42c685c1bc93bd19e98c0f1543db52911f8ede42cf84"}, + {file = "tokenizers-0.20.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a02d160d2b19bcbfdf28bd9a4bf11be4cb97d0499c000d95d4c4b1a4312740b6"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e3d80d89b068bc30034034b5319218c7c0a91b00af19679833f55f3becb6945"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:174a54910bed1b089226512b4458ea60d6d6fd93060254734d3bc3540953c51c"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:098b8a632b8656aa5802c46689462c5c48f02510f24029d71c208ec2c822e771"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:78c8c143e3ae41e718588281eb3e212c2b31623c9d6d40410ec464d7d6221fb5"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b26b0aadb18cd8701077362ba359a06683662d5cafe3e8e8aba10eb05c037f1"}, + {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07d7851a72717321022f3774e84aa9d595a041d643fafa2e87fbc9b18711dac0"}, + {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:bd44e48a430ada902c6266a8245f5036c4fe744fcb51f699999fbe82aa438797"}, + {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a4c186bb006ccbe1f5cc4e0380d1ce7806f5955c244074fd96abc55e27b77f01"}, + {file = "tokenizers-0.20.3-cp313-none-win32.whl", hash = "sha256:6e19e0f1d854d6ab7ea0c743d06e764d1d9a546932be0a67f33087645f00fe13"}, + {file = "tokenizers-0.20.3-cp313-none-win_amd64.whl", hash = "sha256:d50ede425c7e60966a9680d41b58b3a0950afa1bb570488e2972fa61662c4273"}, + {file = "tokenizers-0.20.3-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:9adda1ff5fb9dcdf899ceca672a4e2ce9e797adb512a6467305ca3d8bfcfbdd0"}, + {file = "tokenizers-0.20.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:6dde2cae6004ba7a3badff4a11911cae03ebf23e97eebfc0e71fef2530e5074f"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4a7fd678b35614fca708579eb95b7587a5e8a6d328171bd2488fd9f27d82be4"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b80e3c7283a01a356bd2210f53d1a4a5d32b269c2024389ed0173137708d50e"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8cc0e8176b762973758a77f0d9c4467d310e33165fb74173418ca3734944da4"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5634b2e2f5f3d2b4439d2d74066e22eb4b1f04f3fea05cb2a3c12d89b5a3bcd"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b4ba635165bc1ea46f2da8e5d80b5f70f6ec42161e38d96dbef33bb39df73964"}, + {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18e4c7c64172e7789bd8b07aa3087ea87c4c4de7e90937a2aa036b5d92332536"}, + {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1f74909ef7675c26d4095a817ec3393d67f3158ca4836c233212e5613ef640c4"}, + {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0e9b81321a1e05b16487d312b4264984513f8b4a7556229cafac6e88c2036b09"}, + {file = "tokenizers-0.20.3-cp37-none-win32.whl", hash = "sha256:ab48184cd58b4a03022a2ec75b54c9f600ffea9a733612c02325ed636f353729"}, + {file = "tokenizers-0.20.3-cp37-none-win_amd64.whl", hash = "sha256:60ac483cebee1c12c71878523e768df02fa17e4c54412966cb3ac862c91b36c1"}, + {file = "tokenizers-0.20.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3229ef103c89583d10b9378afa5d601b91e6337530a0988e17ca8d635329a996"}, + {file = "tokenizers-0.20.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6ac52cc24bad3de865c7e65b1c4e7b70d00938a8ae09a92a453b8f676e714ad5"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04627b7b502fa6a2a005e1bd446fa4247d89abcb1afaa1b81eb90e21aba9a60f"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c27ceb887f0e81a3c377eb4605dca7a95a81262761c0fba308d627b2abb98f2b"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65ab780194da4e1fcf5670523a2f377c4838ebf5249efe41fa1eddd2a84fb49d"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98d343134f47159e81f7f242264b0eb222e6b802f37173c8d7d7b64d5c9d1388"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2475bb004ab2009d29aff13b5047bfdb3d4b474f0aa9d4faa13a7f34dbbbb43"}, + {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b6583a65c01db1197c1eb36857ceba8ec329d53afadd268b42a6b04f4965724"}, + {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:62d00ba208358c037eeab7bfc00a905adc67b2d31b68ab40ed09d75881e114ea"}, + {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0fc7a39e5bedc817bda395a798dfe2d9c5f7c71153c90d381b5135a0328d9520"}, + {file = "tokenizers-0.20.3-cp38-none-win32.whl", hash = "sha256:84d40ee0f8550d64d3ea92dd7d24a8557a9172165bdb986c9fb2503b4fe4e3b6"}, + {file = "tokenizers-0.20.3-cp38-none-win_amd64.whl", hash = "sha256:205a45246ed7f1718cf3785cff88450ba603352412aaf220ace026384aa3f1c0"}, + {file = "tokenizers-0.20.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:93e37f0269a11dc3b1a953f1fca9707f0929ebf8b4063c591c71a0664219988e"}, + {file = "tokenizers-0.20.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f4cb0c614b0135e781de96c2af87e73da0389ac1458e2a97562ed26e29490d8d"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7eb2fb1c432f5746b22f8a7f09fc18c4156cb0031c77f53cb19379d82d43297a"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfa8d029bb156181b006643309d6b673615a24e4ed24cf03aa191d599b996f51"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f90549622de3bf476ad9f1dd6f3f952ec3ed6ab8615ae88ef060d0c5bfad55d"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1d469c74eebf5c43fd61cd9b030e271d17198edd7bd45392e03a3c091d7d6d4"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bee8f53b2594749f4460d53253bae55d718f04e9b633efa0f5df8938bd98e4f0"}, + {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:938441babf3e5720e4459e306ef2809fb267680df9d1ff2873458b22aef60248"}, + {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7310ab23d7b0caebecc0e8be11a1146f320f5f07284000f6ea54793e83de1b75"}, + {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:16121eb030a2b13094cfec936b0c12e8b4063c5f839591ea7d0212336d8f9921"}, + {file = "tokenizers-0.20.3-cp39-none-win32.whl", hash = "sha256:401cc21ef642ee235985d747f65e18f639464d377c70836c9003df208d582064"}, + {file = "tokenizers-0.20.3-cp39-none-win_amd64.whl", hash = "sha256:7498f3ea7746133335a6adb67a77cf77227a8b82c8483f644a2e5f86fea42b8d"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e919f2e3e68bb51dc31de4fcbbeff3bdf9c1cad489044c75e2b982a91059bd3c"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b8e9608f2773996cc272156e305bd79066163a66b0390fe21750aff62df1ac07"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39270a7050deaf50f7caff4c532c01b3c48f6608d42b3eacdebdc6795478c8df"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e005466632b1c5d2d2120f6de8aa768cc9d36cd1ab7d51d0c27a114c91a1e6ee"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a07962340b36189b6c8feda552ea1bfeee6cf067ff922a1d7760662c2ee229e5"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:55046ad3dd5f2b3c67501fcc8c9cbe3e901d8355f08a3b745e9b57894855f85b"}, + {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:efcf0eb939988b627558aaf2b9dc3e56d759cad2e0cfa04fcab378e4b48fc4fd"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f3558a7ae6a6d38a77dfce12172a1e2e1bf3e8871e744a1861cd7591ea9ebe24"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d53029fe44bc70c3ff14ef512460a0cf583495a0f8e2f4b70e26eb9438e38a9"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a2a56397b2bec5a629b516b23f0f8a3e4f978c7488d4a299980f8375954b85"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e5bfaae740ef9ece000f8a07e78ac0e2b085c5ce9648f8593ddf0243c9f76d"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fbaf3ea28fedfb2283da60e710aff25492e795a7397cad8a50f1e079b65a5a70"}, + {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c47c037116310dc976eb96b008e41b9cfaba002ed8005848d4d632ee0b7ba9ae"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c31751f0721f58f5e19bb27c1acc259aeff860d8629c4e1a900b26a1979ada8e"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:c697cbd3be7a79ea250ea5f380d6f12e534c543cfb137d5c734966b3ee4f34cc"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48971b88ef9130bf35b41b35fd857c3c4dae4a9cd7990ebc7fc03e59cc92438"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e615de179bbe060ab33773f0d98a8a8572b5883dd7dac66c1de8c056c7e748c"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da1ec842035ed9999c62e45fbe0ff14b7e8a7e02bb97688cc6313cf65e5cd755"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6ee4954c1dd23aadc27958dad759006e71659d497dcb0ef0c7c87ea992c16ebd"}, + {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3eda46ca402751ec82553a321bf35a617b76bbed7586e768c02ccacbdda94d6d"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:de082392a85eb0055cc055c535bff2f0cc15d7a000bdc36fbf601a0f3cf8507a"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c3db46cc0647bfd88263afdb739b92017a02a87ee30945cb3e86c7e25c7c9917"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a292392f24ab9abac5cfa8197e5a6208f2e43723420217e1ceba0b4ec77816ac"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dcd91f4e60f62b20d83a87a84fe062035a1e3ff49a8c2bbdeb2d441c8e311f4"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900991a2b8ee35961b1095db7e265342e0e42a84c1a594823d5ee9f8fb791958"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5a8d8261ca2133d4f98aa9627c748189502b3787537ba3d7e2beb4f7cfc5d627"}, + {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c4fd4d71e6deb6ddf99d8d0eab87d1d16f635898906e631914a9bae8ae9f2cfb"}, + {file = "tokenizers-0.20.3.tar.gz", hash = "sha256:2278b34c5d0dd78e087e1ca7f9b1dcbf129d80211afa645f214bd6e051037539"}, +] + +[package.dependencies] +huggingface-hub = ">=0.16.4,<1.0" + +[package.extras] +dev = ["tokenizers[testing]"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"] + +[[package]] +name = "torch" +version = "2.6.0" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +optional = false +python-versions = ">=3.9.0" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "torch-2.6.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:6860df13d9911ac158f4c44031609700e1eba07916fff62e21e6ffa0a9e01961"}, + {file = "torch-2.6.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c4f103a49830ce4c7561ef4434cc7926e5a5fe4e5eb100c19ab36ea1e2b634ab"}, + {file = "torch-2.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:56eeaf2ecac90da5d9e35f7f35eb286da82673ec3c582e310a8d1631a1c02341"}, + {file = "torch-2.6.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:09e06f9949e1a0518c5b09fe95295bc9661f219d9ecb6f9893e5123e10696628"}, + {file = "torch-2.6.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:7979834102cd5b7a43cc64e87f2f3b14bd0e1458f06e9f88ffa386d07c7446e1"}, + {file = "torch-2.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:ccbd0320411fe1a3b3fec7b4d3185aa7d0c52adac94480ab024b5c8f74a0bf1d"}, + {file = "torch-2.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:46763dcb051180ce1ed23d1891d9b1598e07d051ce4c9d14307029809c4d64f7"}, + {file = "torch-2.6.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:94fc63b3b4bedd327af588696559f68c264440e2503cc9e6954019473d74ae21"}, + {file = "torch-2.6.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:2bb8987f3bb1ef2675897034402373ddfc8f5ef0e156e2d8cfc47cacafdda4a9"}, + {file = "torch-2.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b789069020c5588c70d5c2158ac0aa23fd24a028f34a8b4fcb8fcb4d7efcf5fb"}, + {file = "torch-2.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:7e1448426d0ba3620408218b50aa6ada88aeae34f7a239ba5431f6c8774b1239"}, + {file = "torch-2.6.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:9a610afe216a85a8b9bc9f8365ed561535c93e804c2a317ef7fabcc5deda0989"}, + {file = "torch-2.6.0-cp313-cp313-manylinux1_x86_64.whl", hash = "sha256:4874a73507a300a5d089ceaff616a569e7bb7c613c56f37f63ec3ffac65259cf"}, + {file = "torch-2.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a0d5e1b9874c1a6c25556840ab8920569a7a4137afa8a63a32cee0bc7d89bd4b"}, + {file = "torch-2.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:510c73251bee9ba02ae1cb6c9d4ee0907b3ce6020e62784e2d7598e0cfa4d6cc"}, + {file = "torch-2.6.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:ff96f4038f8af9f7ec4231710ed4549da1bdebad95923953a25045dcf6fd87e2"}, + {file = "torch-2.6.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9ea955317cfcd3852b1402b62af258ce735c2edeee42ca9419b6bc889e5ae053"}, + {file = "torch-2.6.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:bb2c6c3e65049f081940f5ab15c9136c7de40d3f01192541c920a07c7c585b7e"}, + {file = "torch-2.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:683410f97984103148e31b38a8631acf31c3034c020c0f4d26171e7626d8317a"}, + {file = "torch-2.6.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:265f70de5fd45b864d924b64be1797f86e76c8e48a02c2a3a6fc7ec247d2226c"}, +] + +[package.dependencies] +filelock = "*" +fsspec = "*" +jinja2 = "*" +networkx = "*" +nvidia-cublas-cu12 = {version = "12.4.5.8", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-cupti-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-nvrtc-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-runtime-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cudnn-cu12 = {version = "9.1.0.70", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufft-cu12 = {version = "11.2.1.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-curand-cu12 = {version = "10.3.5.147", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusolver-cu12 = {version = "11.6.1.9", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparse-cu12 = {version = "12.3.1.170", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparselt-cu12 = {version = "0.6.2", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nccl-cu12 = {version = "2.21.5", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvjitlink-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvtx-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +setuptools = {version = "*", markers = "python_version >= \"3.12\""} +sympy = {version = "1.13.1", markers = "python_version >= \"3.9\""} +triton = {version = "3.2.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +typing-extensions = ">=4.10.0" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] +optree = ["optree (>=0.13.0)"] + +[[package]] +name = "tqdm" +version = "4.67.1" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] +discord = ["requests"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "transformers" +version = "4.46.1" +description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +optional = false +python-versions = ">=3.8.0" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "transformers-4.46.1-py3-none-any.whl", hash = "sha256:f77b251a648fd32e3d14b5e7e27c913b7c29154940f519e4c8c3aa6061df0f05"}, + {file = "transformers-4.46.1.tar.gz", hash = "sha256:16d79927d772edaf218820a96f9254e2211f9cd7fb3c308562d2d636c964a68c"}, +] + +[package.dependencies] +accelerate = {version = ">=0.26.0", optional = true, markers = "extra == \"torch\""} +filelock = "*" +huggingface-hub = ">=0.23.2,<1.0" +numpy = ">=1.17" +packaging = ">=20.0" +pyyaml = ">=5.1" +regex = "!=2019.12.17" +requests = "*" +safetensors = ">=0.4.1" +tokenizers = ">=0.20,<0.21" +torch = {version = "*", optional = true, markers = "extra == \"torch\""} +tqdm = ">=4.27" + +[package.extras] +accelerate = ["accelerate (>=0.26.0)"] +agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision"] +audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +benchmark = ["optimum-benchmark (>=0.3.0)"] +codecarbon = ["codecarbon (==1.2.0)"] +deepspeed = ["accelerate (>=0.26.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.26.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk (<=3.8.1)", "optuna", "parameterized", "protobuf", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "libcst", "librosa", "nltk (<=3.8.1)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "libcst", "librosa", "nltk (<=3.8.1)", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.20,<0.21)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "libcst", "librosa", "nltk (<=3.8.1)", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy (<1.13.0)"] +flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +ftfy = ["ftfy"] +integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] +modelcreation = ["cookiecutter (==1.7.3)"] +natten = ["natten (>=0.14.6,<0.15.0)"] +onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] +onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +optuna = ["optuna"] +quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "isort (>=5.5.4)", "libcst", "rich", "ruff (==0.5.1)", "urllib3 (<2.0.0)"] +ray = ["ray[tune] (>=2.7.0)"] +retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] +ruff = ["ruff (==0.5.1)"] +sagemaker = ["sagemaker (>=2.31.0)"] +sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["fastapi", "pydantic", "starlette", "uvicorn"] +sigopt = ["sigopt"] +sklearn = ["scikit-learn"] +speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk (<=3.8.1)", "parameterized", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] +tf-cpu = ["keras (>2.9,<2.16)", "keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow-cpu (>2.9,<2.16)", "tensorflow-probability (<0.24)", "tensorflow-text (<2.16)", "tf2onnx"] +tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +tiktoken = ["blobfile", "tiktoken"] +timm = ["timm (<=0.9.16)"] +tokenizers = ["tokenizers (>=0.20,<0.21)"] +torch = ["accelerate (>=0.26.0)", "torch"] +torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.23.2,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.20,<0.21)", "torch", "tqdm (>=4.27)"] +video = ["av (==9.2.0)"] +vision = ["Pillow (>=10.0.1,<=15.0)"] + +[[package]] +name = "triton" +version = "3.2.0" +description = "A language and compiler for custom Deep Learning operations" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version == \"3.11\" and platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\" and platform_system == \"Linux\" and platform_machine == \"x86_64\"" +files = [ + {file = "triton-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3e54983cd51875855da7c68ec05c05cf8bb08df361b1d5b69e05e40b0c9bd62"}, + {file = "triton-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8009a1fb093ee8546495e96731336a33fb8856a38e45bb4ab6affd6dbc3ba220"}, + {file = "triton-3.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d9b215efc1c26fa7eefb9a157915c92d52e000d2bf83e5f69704047e63f125c"}, + {file = "triton-3.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5dfa23ba84541d7c0a531dfce76d8bcd19159d50a4a8b14ad01e91734a5c1b0"}, + {file = "triton-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30ceed0eff2c4a73b14eb63e052992f44bbdf175f3fad21e1ac8097a772de7ee"}, +] + +[package.extras] +build = ["cmake (>=3.20)", "lit"] +tests = ["autopep8", "flake8", "isort", "llnl-hatchet", "numpy", "pytest", "scipy (>=1.7.1)"] +tutorials = ["matplotlib", "pandas", "tabulate"] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "tzdata" +version = "2025.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "tzdata-2025.1-py2.py3-none-any.whl", hash = "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639"}, + {file = "tzdata-2025.1.tar.gz", hash = "sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694"}, +] + +[[package]] +name = "urllib3" +version = "2.3.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, + {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "xdis" +version = "6.1.1" +description = "Python cross-version byte-code disassembler and marshal routines" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "xdis-6.1.1-py2-none-any.whl", hash = "sha256:0f7bf67cd9c3a2fa3b14857f05ee159308e4aec97168bbf3cc8db902a14a0e24"}, + {file = "xdis-6.1.1-py310-none-any.whl", hash = "sha256:326bbeb04566eae9c9b679d4fca47262d920edd841fff93ed2efccaf9fec6237"}, + {file = "xdis-6.1.1-py311-none-any.whl", hash = "sha256:37ad24cd9fb72410f1973ea892ca46375b4a9b497bd3b3ed93f440182dfd0e15"}, + {file = "xdis-6.1.1-py312-none-any.whl", hash = "sha256:73365e0645429a3823ee612ae79a2a5a9ed83a0fbd51e1711bbe102884c485d9"}, + {file = "xdis-6.1.1.tar.gz", hash = "sha256:1cbeaf56b996ba1b1e744a3f66753692be5ef9855131b7332601aa0a124e5d13"}, +] + +[package.dependencies] +click = "*" +six = ">=1.10.0" + +[package.extras] +dev = ["pre-commit", "pytest"] + +[[package]] +name = "xxhash" +version = "3.5.0" +description = "Python binding for xxHash" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "xxhash-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212"}, + {file = "xxhash-3.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3171f693dbc2cef6477054a665dc255d996646b4023fe56cb4db80e26f4cc520"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c5d3e570ef46adaf93fc81b44aca6002b5a4d8ca11bd0580c07eac537f36680"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7cb29a034301e2982df8b1fe6328a84f4b676106a13e9135a0d7e0c3e9f806da"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0d307d27099bb0cbeea7260eb39ed4fdb99c5542e21e94bb6fd29e49c57a23"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0342aafd421795d740e514bc9858ebddfc705a75a8c5046ac56d85fe97bf196"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3dbbd9892c5ebffeca1ed620cf0ade13eb55a0d8c84e0751a6653adc6ac40d0c"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4cc2d67fdb4d057730c75a64c5923abfa17775ae234a71b0200346bfb0a7f482"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ec28adb204b759306a3d64358a5e5c07d7b1dd0ccbce04aa76cb9377b7b70296"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1328f6d8cca2b86acb14104e381225a3d7b42c92c4b86ceae814e5c400dbb415"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8d47ebd9f5d9607fd039c1fbf4994e3b071ea23eff42f4ecef246ab2b7334198"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b96d559e0fcddd3343c510a0fe2b127fbff16bf346dd76280b82292567523442"}, + {file = "xxhash-3.5.0-cp310-cp310-win32.whl", hash = "sha256:61c722ed8d49ac9bc26c7071eeaa1f6ff24053d553146d5df031802deffd03da"}, + {file = "xxhash-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:9bed5144c6923cc902cd14bb8963f2d5e034def4486ab0bbe1f58f03f042f9a9"}, + {file = "xxhash-3.5.0-cp310-cp310-win_arm64.whl", hash = "sha256:893074d651cf25c1cc14e3bea4fceefd67f2921b1bb8e40fcfeba56820de80c6"}, + {file = "xxhash-3.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02c2e816896dc6f85922ced60097bcf6f008dedfc5073dcba32f9c8dd786f3c1"}, + {file = "xxhash-3.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6027dcd885e21581e46d3c7f682cfb2b870942feeed58a21c29583512c3f09f8"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1308fa542bbdbf2fa85e9e66b1077eea3a88bef38ee8a06270b4298a7a62a166"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c28b2fdcee797e1c1961cd3bcd3d545cab22ad202c846235197935e1df2f8ef7"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:924361811732ddad75ff23e90efd9ccfda4f664132feecb90895bade6a1b4623"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89997aa1c4b6a5b1e5b588979d1da048a3c6f15e55c11d117a56b75c84531f5a"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:685c4f4e8c59837de103344eb1c8a3851f670309eb5c361f746805c5471b8c88"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbd2ecfbfee70bc1a4acb7461fa6af7748ec2ab08ac0fa298f281c51518f982c"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25b5a51dc3dfb20a10833c8eee25903fd2e14059e9afcd329c9da20609a307b2"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a8fb786fb754ef6ff8c120cb96629fb518f8eb5a61a16aac3a979a9dbd40a084"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a905ad00ad1e1c34fe4e9d7c1d949ab09c6fa90c919860c1534ff479f40fd12d"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:963be41bcd49f53af6d795f65c0da9b4cc518c0dd9c47145c98f61cb464f4839"}, + {file = "xxhash-3.5.0-cp311-cp311-win32.whl", hash = "sha256:109b436096d0a2dd039c355fa3414160ec4d843dfecc64a14077332a00aeb7da"}, + {file = "xxhash-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:b702f806693201ad6c0a05ddbbe4c8f359626d0b3305f766077d51388a6bac58"}, + {file = "xxhash-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:c4dcb4120d0cc3cc448624147dba64e9021b278c63e34a38789b688fd0da9bf3"}, + {file = "xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00"}, + {file = "xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e"}, + {file = "xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8"}, + {file = "xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e"}, + {file = "xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2"}, + {file = "xxhash-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:37889a0d13b0b7d739cfc128b1c902f04e32de17b33d74b637ad42f1c55101f6"}, + {file = "xxhash-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97a662338797c660178e682f3bc180277b9569a59abfb5925e8620fba00b9fc5"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f85e0108d51092bdda90672476c7d909c04ada6923c14ff9d913c4f7dc8a3bc"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2fd827b0ba763ac919440042302315c564fdb797294d86e8cdd4578e3bc7f3"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82085c2abec437abebf457c1d12fccb30cc8b3774a0814872511f0f0562c768c"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07fda5de378626e502b42b311b049848c2ef38784d0d67b6f30bb5008642f8eb"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c279f0d2b34ef15f922b77966640ade58b4ccdfef1c4d94b20f2a364617a493f"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:89e66ceed67b213dec5a773e2f7a9e8c58f64daeb38c7859d8815d2c89f39ad7"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bcd51708a633410737111e998ceb3b45d3dbc98c0931f743d9bb0a209033a326"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ff2c0a34eae7df88c868be53a8dd56fbdf592109e21d4bfa092a27b0bf4a7bf"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4e28503dccc7d32e0b9817aa0cbfc1f45f563b2c995b7a66c4c8a0d232e840c7"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a6c50017518329ed65a9e4829154626f008916d36295b6a3ba336e2458824c8c"}, + {file = "xxhash-3.5.0-cp313-cp313-win32.whl", hash = "sha256:53a068fe70301ec30d868ece566ac90d873e3bb059cf83c32e76012c889b8637"}, + {file = "xxhash-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:80babcc30e7a1a484eab952d76a4f4673ff601f54d5142c26826502740e70b43"}, + {file = "xxhash-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:4811336f1ce11cac89dcbd18f3a25c527c16311709a89313c3acaf771def2d4b"}, + {file = "xxhash-3.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6e5f70f6dca1d3b09bccb7daf4e087075ff776e3da9ac870f86ca316736bb4aa"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e76e83efc7b443052dd1e585a76201e40b3411fe3da7af4fe434ec51b2f163b"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33eac61d0796ca0591f94548dcfe37bb193671e0c9bcf065789b5792f2eda644"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ec70a89be933ea49222fafc3999987d7899fc676f688dd12252509434636622"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86b8e7f703ec6ff4f351cfdb9f428955859537125904aa8c963604f2e9d3e7"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0adfbd36003d9f86c8c97110039f7539b379f28656a04097e7434d3eaf9aa131"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:63107013578c8a730419adc05608756c3fa640bdc6abe806c3123a49fb829f43"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:683b94dbd1ca67557850b86423318a2e323511648f9f3f7b1840408a02b9a48c"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5d2a01dcce81789cf4b12d478b5464632204f4c834dc2d064902ee27d2d1f0ee"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:a9d360a792cbcce2fe7b66b8d51274ec297c53cbc423401480e53b26161a290d"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:f0b48edbebea1b7421a9c687c304f7b44d0677c46498a046079d445454504737"}, + {file = "xxhash-3.5.0-cp37-cp37m-win32.whl", hash = "sha256:7ccb800c9418e438b44b060a32adeb8393764da7441eb52aa2aa195448935306"}, + {file = "xxhash-3.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c3bc7bf8cb8806f8d1c9bf149c18708cb1c406520097d6b0a73977460ea03602"}, + {file = "xxhash-3.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:74752ecaa544657d88b1d1c94ae68031e364a4d47005a90288f3bab3da3c970f"}, + {file = "xxhash-3.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dee1316133c9b463aa81aca676bc506d3f80d8f65aeb0bba2b78d0b30c51d7bd"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:602d339548d35a8579c6b013339fb34aee2df9b4e105f985443d2860e4d7ffaa"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:695735deeddfb35da1677dbc16a083445360e37ff46d8ac5c6fcd64917ff9ade"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1030a39ba01b0c519b1a82f80e8802630d16ab95dc3f2b2386a0b5c8ed5cbb10"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5bc08f33c4966f4eb6590d6ff3ceae76151ad744576b5fc6c4ba8edd459fdec"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:160e0c19ee500482ddfb5d5570a0415f565d8ae2b3fd69c5dcfce8a58107b1c3"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f1abffa122452481a61c3551ab3c89d72238e279e517705b8b03847b1d93d738"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:d5e9db7ef3ecbfc0b4733579cea45713a76852b002cf605420b12ef3ef1ec148"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:23241ff6423378a731d84864bf923a41649dc67b144debd1077f02e6249a0d54"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:82b833d5563fefd6fceafb1aed2f3f3ebe19f84760fdd289f8b926731c2e6e91"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0a80ad0ffd78bef9509eee27b4a29e56f5414b87fb01a888353e3d5bda7038bd"}, + {file = "xxhash-3.5.0-cp38-cp38-win32.whl", hash = "sha256:50ac2184ffb1b999e11e27c7e3e70cc1139047e7ebc1aa95ed12f4269abe98d4"}, + {file = "xxhash-3.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:392f52ebbb932db566973693de48f15ce787cabd15cf6334e855ed22ea0be5b3"}, + {file = "xxhash-3.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfc8cdd7f33d57f0468b0614ae634cc38ab9202c6957a60e31d285a71ebe0301"}, + {file = "xxhash-3.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e0c48b6300cd0b0106bf49169c3e0536408dfbeb1ccb53180068a18b03c662ab"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe1a92cfbaa0a1253e339ccec42dbe6db262615e52df591b68726ab10338003f"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33513d6cc3ed3b559134fb307aae9bdd94d7e7c02907b37896a6c45ff9ce51bd"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eefc37f6138f522e771ac6db71a6d4838ec7933939676f3753eafd7d3f4c40bc"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a606c8070ada8aa2a88e181773fa1ef17ba65ce5dd168b9d08038e2a61b33754"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42eca420c8fa072cc1dd62597635d140e78e384a79bb4944f825fbef8bfeeef6"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:604253b2143e13218ff1ef0b59ce67f18b8bd1c4205d2ffda22b09b426386898"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6e93a5ad22f434d7876665444a97e713a8f60b5b1a3521e8df11b98309bff833"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:7a46e1d6d2817ba8024de44c4fd79913a90e5f7265434cef97026215b7d30df6"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:30eb2efe6503c379b7ab99c81ba4a779748e3830241f032ab46bd182bf5873af"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c8aa771ff2c13dd9cda8166d685d7333d389fae30a4d2bb39d63ab5775de8606"}, + {file = "xxhash-3.5.0-cp39-cp39-win32.whl", hash = "sha256:5ed9ebc46f24cf91034544b26b131241b699edbfc99ec5e7f8f3d02d6eb7fba4"}, + {file = "xxhash-3.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:220f3f896c6b8d0316f63f16c077d52c412619e475f9372333474ee15133a558"}, + {file = "xxhash-3.5.0-cp39-cp39-win_arm64.whl", hash = "sha256:a7b1d8315d9b5e9f89eb2933b73afae6ec9597a258d52190944437158b49d38e"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:2014c5b3ff15e64feecb6b713af12093f75b7926049e26a580e94dcad3c73d8c"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fab81ef75003eda96239a23eda4e4543cedc22e34c373edcaf744e721a163986"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e2febf914ace002132aa09169cc572e0d8959d0f305f93d5828c4836f9bc5a6"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d3a10609c51da2a1c0ea0293fc3968ca0a18bd73838455b5bca3069d7f8e32b"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2b4154c00eb22e4d543f472cfca430e7962a0f1d0f3778334f2e08a7ba59363c"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d30bbc1644f726b825b3278764240f449d75f1a8bdda892e641d4a688b1494ae"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa0b72f2423e2aa53077e54a61c28e181d23effeaafd73fcb9c494e60930c8e"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13de2b76c1835399b2e419a296d5b38dc4855385d9e96916299170085ef72f57"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:0691bfcc4f9c656bcb96cc5db94b4d75980b9d5589f2e59de790091028580837"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:297595fe6138d4da2c8ce9e72a04d73e58725bb60f3a19048bc96ab2ff31c692"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1276d369452040cbb943300dc8abeedab14245ea44056a2943183822513a18"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2061188a1ba352fc699c82bff722f4baacb4b4b8b2f0c745d2001e56d0dfb514"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38c384c434021e4f62b8d9ba0bc9467e14d394893077e2c66d826243025e1f81"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e6a4dd644d72ab316b580a1c120b375890e4c52ec392d4aef3c63361ec4d77d1"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:531af8845aaadcadf951b7e0c1345c6b9c68a990eeb74ff9acd8501a0ad6a1c9"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ce379bcaa9fcc00f19affa7773084dd09f5b59947b3fb47a1ceb0179f91aaa1"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd1b2281d01723f076df3c8188f43f2472248a6b63118b036e641243656b1b0f"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c770750cc80e8694492244bca7251385188bc5597b6a39d98a9f30e8da984e0"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b150b8467852e1bd844387459aa6fbe11d7f38b56e901f9f3b3e6aba0d660240"}, + {file = "xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f"}, +] + +[[package]] +name = "yarl" +version = "1.18.3" +description = "Yet another URL library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34"}, + {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7"}, + {file = "yarl-1.18.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690"}, + {file = "yarl-1.18.3-cp310-cp310-win32.whl", hash = "sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6"}, + {file = "yarl-1.18.3-cp310-cp310-win_amd64.whl", hash = "sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a"}, + {file = "yarl-1.18.3-cp311-cp311-win32.whl", hash = "sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1"}, + {file = "yarl-1.18.3-cp311-cp311-win_amd64.whl", hash = "sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285"}, + {file = "yarl-1.18.3-cp312-cp312-win32.whl", hash = "sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2"}, + {file = "yarl-1.18.3-cp312-cp312-win_amd64.whl", hash = "sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8"}, + {file = "yarl-1.18.3-cp313-cp313-win32.whl", hash = "sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d"}, + {file = "yarl-1.18.3-cp313-cp313-win_amd64.whl", hash = "sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1"}, + {file = "yarl-1.18.3-cp39-cp39-win32.whl", hash = "sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5"}, + {file = "yarl-1.18.3-cp39-cp39-win_amd64.whl", hash = "sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9"}, + {file = "yarl-1.18.3-py3-none-any.whl", hash = "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b"}, + {file = "yarl-1.18.3.tar.gz", hash = "sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" +propcache = ">=0.2.0" + +[metadata] +lock-version = "2.1" +python-versions = ">=3.11" +content-hash = "3a5b923fbe837083092ffde6b0fda8d18c82d265318c6da5e89c379513380b31" diff --git a/pylingual/__init__.py b/pylingual/__init__.py new file mode 100644 index 0000000..f3e3eb5 --- /dev/null +++ b/pylingual/__init__.py @@ -0,0 +1,3 @@ +from .decompiler import decompile + +__all__ = ["decompile"] diff --git a/pylingual/control_flow_reconstruction/cfg_utils.py b/pylingual/control_flow_reconstruction/cfg_utils.py new file mode 100644 index 0000000..a6ed9a7 --- /dev/null +++ b/pylingual/control_flow_reconstruction/cfg_utils.py @@ -0,0 +1,58 @@ +import networkx as nx + +from typing import Callable, Any + +from pylingual.editable_bytecode.control_flow_graph import ControlFlowEdgeType + + +def get_out_edge_dict(cfg: nx.DiGraph, node) -> dict: + edge_dict = {"natural": (None, None), "conditional": (None, None), "exception": (None, None)} + if node is None: + return edge_dict + + out_edges = cfg.out_edges(nbunch=node, data=True) + for source, target, edge_props in out_edges: + if edge_props["type"] in [ControlFlowEdgeType.NATURAL.value, ControlFlowEdgeType.JUMP.value]: + edge_dict["natural"] = (target, edge_props) + elif edge_props["type"] in [ControlFlowEdgeType.TRUE_JUMP.value, ControlFlowEdgeType.FALSE_JUMP.value]: + edge_dict["conditional"] = (target, edge_props) + elif edge_props["type"] == ControlFlowEdgeType.EXCEPTION.value: + edge_dict["exception"] = (target, edge_props) + elif edge_props["type"] == ControlFlowEdgeType.META.value: + pass # ignore meta edges in graph traversal + else: + raise ValueError(f"Unknown edge type {edge_props['type']}") + return edge_dict + + +def _to_iter(item): + """Converts something to an iterable version""" + if not hasattr(item, "__iter__") or isinstance(item, str): + return (item,) + return item + + +def create_dominator_tree(graph, start_node=None): + """Creates a dominator tree for the given graph""" + + # default start node is the minimum offset node + if start_node is None: + get_start_offset = lambda node: min(_to_iter(graph.nodes.data()[node].get("offset", ())), default=float("inf")) + start_node = min(graph.nodes, key=get_start_offset) + + dominator_tree = nx.create_empty_copy(graph) + dominator_tree.add_edges_from(nx.immediate_dominators(graph, start_node).items()) + dominator_tree.remove_edge(start_node, start_node) + return dominator_tree.reverse() + + +def get_dominator_function(cfg: nx.DiGraph) -> Callable[[Any, Any], bool]: + # preprocessing to identify loop headers; dominator tree cached in cfg so we don't recompute unless the graph changed + if not hasattr(cfg, "dominator_tree"): + cfg.dominator_tree = create_dominator_tree(cfg, start_node="START") + cfg.domination_relation = nx.transitive_closure_dag(cfg.dominator_tree) + + def dominates(a, b): + return cfg.domination_relation.has_edge(a, b) or a == b + + return dominates diff --git a/pylingual/control_flow_reconstruction/cflow.py b/pylingual/control_flow_reconstruction/cflow.py new file mode 100644 index 0000000..769bfa4 --- /dev/null +++ b/pylingual/control_flow_reconstruction/cflow.py @@ -0,0 +1,57 @@ +import os + +from pylingual.editable_bytecode import EditableBytecode +from pylingual.editable_bytecode.control_flow_graph import bytecode_to_control_flow_graph +from pylingual.utils.use_escape_sequences import use_escape_sequences + +from .structure_control_flow import structure_control_flow + + +def pyc_to_indented_sources(pyc: EditableBytecode, source_lines: list[str]) -> dict[object, str]: + sources = {} + for bytecode in pyc.iter_bytecodes(): + sources[bytecode.codeobj] = bytecode_to_indented_source(bytecode, source_lines) + return sources + + +def split_newlines(li): + return "\n".join(li).split("\n") + + +def bytecode_to_indented_source(bytecode: EditableBytecode, source_lines: list[str]) -> list[str]: + cfg = bytecode_to_control_flow_graph(bytecode) + + # breakpoint to debug control flow templates if DEBUG_CFLOW is set + if os.environ.get("DEBUG_CFLOW", None) == "1": + breakpoint() + + structured = structure_control_flow(cfg, bytecode) + indented_source = structured.to_indented_source(source_lines).split("\n") + + bytecode.ordered_instructions = structured.get_instructions() + # force generator if necessary + if bytecode.codeobj.co_flags & (0x20 | 0x200): + if not any(x.strip().startswith("yield ") or x.strip() == "yield" for x in split_newlines(indented_source)): + indented_source.insert(0, "if False: yield # inserted") + + # insert globals + for global_var in bytecode.globals: + indented_source.insert(0, f"global {global_var} # inserted") + + # insert nonlocals + parent_nonlocal = set() + parent = bytecode.parent + while parent: + parent_nonlocal |= parent.nonlocals + parent = parent.parent + for nonlocal_var in bytecode.nonlocals: + if nonlocal_var in parent_nonlocal: + indented_source.insert(0, f"nonlocal {nonlocal_var} # inserted") + + # add function docstring + if bytecode.codeobj.co_flags & 0x2: + if bytecode.codeobj.co_consts and isinstance(bytecode.codeobj.co_consts[0], str): + doc = use_escape_sequences(bytecode.codeobj.co_consts[0]) + indented_source.insert(0, f'"""{doc}""" # inserted') + + return [line for line in indented_source if line] # filter out empty strings diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/Subgraph.py b/pylingual/control_flow_reconstruction/control_flow_templates/Subgraph.py new file mode 100644 index 0000000..8ff255c --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/Subgraph.py @@ -0,0 +1,171 @@ +import networkx as nx + +from ..cfg_utils import get_out_edge_dict + +from typing import Any, Callable +from .abstract.AbstractTemplate import ControlFlowTemplate +# INVARIANT: each node will have at most one of each "natural", "conditional", and "exception" edge + + +class TemplateEdge: + def __init__(self, source: Any, dest: Any, edge_verification_func: Callable[[Any, Any, dict], bool] = None, commit_none_to_mapping: bool = True) -> None: + self.source = source + self.dest = dest + self.edge_verification_func = edge_verification_func + # for optional edges, toggle if the absence of a node will be committed to the mapping + # set to False for edges that may not exist, even when their template destination may be reachable from other nodes + self.commit_none_to_mapping = commit_none_to_mapping + + def check_edge(self, graph_source: Any, graph_dest: Any, graph_edge_properties: dict) -> bool: + # if no verification function is provided, just check that the edge exists + if self.edge_verification_func is None: + return graph_dest is not None + + return self.edge_verification_func(graph_source, graph_dest, graph_edge_properties) + + +class TemplateNode: + def __init__( + self, node_verification_func: Callable[[nx.DiGraph, Any], bool] = None, natural_edge: TemplateEdge = None, conditional_edge: TemplateEdge = None, exception_edge: TemplateEdge = None, subtemplate: ControlFlowTemplate = None + ) -> None: + self.node_verification_func = node_verification_func + self.natural_edge = natural_edge + self.conditional_edge = conditional_edge + self.exception_edge = exception_edge + self.subtemplate = subtemplate + + def check_node(self, cfg: nx.DiGraph, node: Any) -> bool: + # I am not a valid candidate, so this is not a valid mapping + # it is the job of the node verification func to check in_degree + if self.node_verification_func is None: + if node is None: + return False + elif not self.node_verification_func(cfg, node): + return False + + # check the outgoing edges for this node + node_out_edge_dict = get_out_edge_dict(cfg, node) + natural_target, natural_properties = node_out_edge_dict["natural"] if node_out_edge_dict["natural"] else (None, None) + # if the edge is in the template, it must be valid + if self.natural_edge and not self.natural_edge.check_edge(node, natural_target, natural_properties): + return False + # if the edge is not in the template, reject + if natural_target and not self.natural_edge: + return False + + conditional_target, conditional_properties = node_out_edge_dict["conditional"] if node_out_edge_dict["conditional"] else (None, None) + # if the edge is in the template, it must be valid + if self.conditional_edge and not self.conditional_edge.check_edge(node, conditional_target, conditional_properties): + return False + # if the edge is not in the template, reject + if conditional_target and not self.conditional_edge: + return False + + exception_target, exception_properties = node_out_edge_dict["exception"] if node_out_edge_dict["exception"] else (None, None) + # if the edge is in the template, it must be valid + if self.exception_edge and not self.exception_edge.check_edge(node, exception_target, exception_properties): + return False + # if the edge is not in the template, reject + if exception_target and not self.exception_edge: + return False + + # node is good and all outgoing edges are good + return True + + +class GraphTemplateMatcher: + def __init__(self, template_node_dict: dict[Any, TemplateNode], root_key: Any, mapping_verification_func: Callable[[nx.DiGraph, dict], bool]) -> None: + self.template_node_dict = template_node_dict + self.root_key = root_key + self.mapping_verification_func = mapping_verification_func + + def match_at_graph_node(self, cfg: nx.DiGraph, root_node: Any) -> dict: + mapping = dict() + mapped_nodes = set() + + dfs_stack = [(self.root_key, root_node)] + + original_cfg = cfg # save this reference for later + + while dfs_stack: + current_template_key, current_graph_node = dfs_stack.pop() + current_template_node = self.template_node_dict[current_template_key] + # if the template node has already been mapped, we don't process it again + if current_template_key in mapping: + # if the current template node has been mapped inconsistently, then the mapping failed + if mapping[current_template_key] != current_graph_node: + return None + else: + continue + if current_graph_node in mapped_nodes: + return None + + # try to match the node subtemplate if one was provided + # if there is a match, then update the cfg under consideration, ensuring that nodes don't get double-mapped + if current_template_node.subtemplate: + updated_cfg = current_template_node.subtemplate.try_to_match_node(cfg, current_graph_node) + # if we didn't match the subtemplate, then this node matching failed + if not updated_cfg: + return None + + # check that previously mapped nodes did not get removed + for mapped_node in mapping.values(): + if mapped_node is not None and mapped_node not in updated_cfg.nodes: + return None + + # update the current graph node + added_nodes = set(updated_cfg.nodes) - set(cfg.nodes) + # enforce invariant that templates add no more than one node + assert len(added_nodes) <= 1 + if added_nodes: + current_graph_node = added_nodes.pop() + + # update the cfg + cfg = updated_cfg + + # if the node is not a valid match, then the mapping failed + # check_node also checks all the outgoing edges + if not current_template_node.check_node(cfg, current_graph_node): + return None + + mapping[current_template_key] = current_graph_node + mapped_nodes.add(current_graph_node) + + graph_node_out_edge_dict = get_out_edge_dict(cfg, current_graph_node) + + # extend along the natural edge + if current_template_node.natural_edge: + next_template_key = current_template_node.natural_edge.dest + if next_template_key is not None: + next_graph_node, _ = graph_node_out_edge_dict["natural"] if graph_node_out_edge_dict["natural"] else (None, None) + if next_graph_node is not None or current_template_node.natural_edge.commit_none_to_mapping: + dfs_stack.append((next_template_key, next_graph_node)) + + # extend along the conditional edge + if current_template_node.conditional_edge: + next_template_key = current_template_node.conditional_edge.dest + if next_template_key is not None: + next_graph_node, _ = graph_node_out_edge_dict["conditional"] if graph_node_out_edge_dict["conditional"] else (None, None) + if next_graph_node is not None or current_template_node.conditional_edge.commit_none_to_mapping: + dfs_stack.append((next_template_key, next_graph_node)) + + # extend along the exception edge + if current_template_node.exception_edge: + next_template_key = current_template_node.exception_edge.dest + if next_template_key is not None: + next_graph_node, _ = graph_node_out_edge_dict["exception"] if graph_node_out_edge_dict["exception"] else (None, None) + if next_graph_node is not None or current_template_node.exception_edge.commit_none_to_mapping: + dfs_stack.append((next_template_key, next_graph_node)) + + # we have a final mapping, check any top-level verification stuff + if self.mapping_verification_func and not self.mapping_verification_func(cfg, mapping): + return None + + # mapping was successful + if cfg == original_cfg: + return mapping + + # commit changes to the original cfg by modifying the reference + original_cfg.clear() + original_cfg.update(cfg) + return mapping diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractExceptionBlockTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractExceptionBlockTemplate.py new file mode 100644 index 0000000..30f4bd5 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractExceptionBlockTemplate.py @@ -0,0 +1,5 @@ +# to make a base template for exception blocks + + +class AbstractExceptionBlockTemplate: + pass diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractNonSequentiableTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractNonSequentiableTemplate.py new file mode 100644 index 0000000..71550b3 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractNonSequentiableTemplate.py @@ -0,0 +1,5 @@ +# to make a base template to deal with end finallys so we can whitelist templates + + +class AbstractNonSequentiable: + pass diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractTemplate.py new file mode 100644 index 0000000..32599aa --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/abstract/AbstractTemplate.py @@ -0,0 +1,42 @@ +from abc import ABC, abstractmethod + +import networkx as nx + +from pylingual.editable_bytecode import Inst + + +class ControlFlowTemplate(ABC): + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + ... + + @abstractmethod + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + ... + + @staticmethod + def _indent_multiline_string(multiline_string: str, indentation_level: int = 1) -> str: + return "\n".join("\t" * indentation_level + line.rstrip() for line in multiline_string.split("\n") if line) + + def __repr__(self) -> str: + name = f"{type(self).__name__}" + components = ControlFlowTemplate._indent_multiline_string(",\n".join(f"{key}={repr(value)}" for key, value in vars(self).items())) + return f"{name}[\n{components}]" + + def get_instructions(self) -> list[Inst]: + insts: list[Inst] = [] + for key, value in vars(self).items(): + if hasattr(value, "get_instructions"): + insts.extend(value.get_instructions()) + elif isinstance(value, Inst): + insts.append(value) + return insts + return sorted(insts, key=lambda i: i.offset) diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ChainedComparisonTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ChainedComparisonTemplate.py new file mode 100644 index 0000000..0fc263c --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ChainedComparisonTemplate.py @@ -0,0 +1,321 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_node_has_no_backwards_edges, node_match_all, assert_no_linestarts + + + +class ChainedComparisonTemplate(ControlFlowTemplate): + """ + A chained comparison such as a == b == c. + (0) + / \\j + (1) (2) (0123) + / \\j/j / \\j + (3) (5) --> (4) (5) + |j + (4) + + not (a == b == c) + + (0) + j/ \\ + (2) (1) --> (0123) + / / \\j / \\j + | (3) (5) (4) (5) + | /j + (4) + + optionally, all nodes in the pattern can have a shared exception handler. + This condenses the chained comparison down to be matched against an if-like template later + """ + + _subgraph = { + "first_condition": TemplateNode( + node_verification_func=assert_node_has_no_backwards_edges, + natural_edge=TemplateEdge( + source="first_condition", + dest="second_condition", + ), + conditional_edge=TemplateEdge( + source="first_condition", + dest="cleanup", + ), + exception_edge=TemplateEdge( + source="first_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "second_condition": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + assert_no_linestarts, + ), + natural_edge=TemplateEdge( + source="second_condition", + dest="j2if_body", + ), + conditional_edge=TemplateEdge( + source="second_condition", + dest="tail", + ), + exception_edge=TemplateEdge( + source="second_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "cleanup": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + assert_no_linestarts, + ), + natural_edge=TemplateEdge( + source="cleanup", + dest="tail", + ), + exception_edge=TemplateEdge( + source="cleanup", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "j2if_body": TemplateNode( + natural_edge=TemplateEdge( + source="j2if_body", + dest="if_body", + ), + exception_edge=TemplateEdge( + source="j2if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + natural_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + _subgraph2 = { + "first_condition": TemplateNode( + node_verification_func=assert_node_has_no_backwards_edges, + natural_edge=TemplateEdge( + source="first_condition", + dest="second_condition", + ), + conditional_edge=TemplateEdge( + source="first_condition", + dest="cleanup", + ), + exception_edge=TemplateEdge( + source="first_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "second_condition": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + assert_no_linestarts, + ), + natural_edge=TemplateEdge( + source="second_condition", + dest="j2if_body", + ), + conditional_edge=TemplateEdge( + source="second_condition", + dest="tail", + ), + exception_edge=TemplateEdge( + source="second_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "cleanup": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + assert_no_linestarts, + ), + natural_edge=TemplateEdge( + source="cleanup", + dest="if_body", + ), + exception_edge=TemplateEdge( + source="j2if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "j2if_body": TemplateNode( + natural_edge=TemplateEdge( + source="j2if_body", + dest="if_body", + ), + exception_edge=TemplateEdge( + source="j2if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + natural_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, first_condition: ControlFlowTemplate, second_condition: ControlFlowTemplate, cleanup: ControlFlowTemplate, j2if_body: ControlFlowTemplate): + self.first_condition = first_condition + self.second_condition = second_condition + self.cleanup = cleanup + self.j2if_body = j2if_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ChainedComparisonTemplate._subgraph, root_key="first_condition", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + matcher = GraphTemplateMatcher(template_node_dict=ChainedComparisonTemplate._subgraph2, root_key="first_condition", mapping_verification_func=None) + mapping = matcher.match_at_graph_node(cfg, node) + if not mapping: + return None + + chained_comparison_template = ChainedComparisonTemplate(first_condition=mapping["first_condition"], second_condition=mapping["second_condition"], cleanup=mapping["cleanup"], j2if_body=mapping["j2if_body"]) + + in_edges = ((src, chained_comparison_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(chained_comparison_template, mapping["if_body"], {"type": ControlFlowEdgeType.NATURAL.value}), (chained_comparison_template, mapping["tail"], {"type": ControlFlowEdgeType.TRUE_JUMP.value})] + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([chained_comparison_template.first_condition, chained_comparison_template.second_condition, chained_comparison_template.cleanup, chained_comparison_template.j2if_body]) + reduced_cfg.add_node(chained_comparison_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + first_condition = self.first_condition.to_indented_source(source_lines) + second_condition = self.second_condition.to_indented_source(source_lines) + return "\n".join([first_condition, second_condition]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitAndTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitAndTemplate.py new file mode 100644 index 0000000..a9778d0 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitAndTemplate.py @@ -0,0 +1,274 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..loop.LoopExitTemplate import LoopExitTemplate + + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_node_has_no_backwards_edges, node_match_all, assert_no_linestarts, assert_node_type, node_match_none + +from ..loop.PreRefinedLoopTemplate import PreRefinedLoopTemplate + + +class ShortCircuitAndTemplate(ControlFlowTemplate): + """ + A short-circuit evaluated boolean AND. Typically these are all part of one line. + (0) + / \\ (01) + (1) |j --> / \\j + |\\j| (2) (3) + (2) (3) + + optionally, all nodes in the pattern can have a shared exception handler. + This condenses the short-circuit down to be matched against an if-like template later + """ + + _subgraph = { + "first_condition": TemplateNode( + node_verification_func=assert_node_has_no_backwards_edges, + natural_edge=TemplateEdge( + source="first_condition", + dest="second_condition", + ), + conditional_edge=TemplateEdge( + source="first_condition", + dest="tail", + ), + exception_edge=TemplateEdge( + source="first_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "second_condition": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + assert_no_linestarts, + node_match_none(assert_node_type(PreRefinedLoopTemplate)), + ), + natural_edge=TemplateEdge( + source="second_condition", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="second_condition", + dest="tail", + ), + exception_edge=TemplateEdge( + source="second_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + natural_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + _subgraph_for_loop_exits = { + "first_condition": TemplateNode( + node_verification_func=assert_node_has_no_backwards_edges, + natural_edge=TemplateEdge( + source="first_condition", + dest="second_condition", + ), + conditional_edge=TemplateEdge( + source="first_condition", + dest="first_loop_exit_tail", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "second_condition": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + assert_no_linestarts, + ), + natural_edge=TemplateEdge( + source="second_condition", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="second_condition", + dest="second_loop_exit_tail", + ), + exception_edge=TemplateEdge( + source="second_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + natural_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "first_loop_exit_tail": TemplateNode( + node_verification_func=assert_node_type(LoopExitTemplate), + exception_edge=TemplateEdge( + source="first_loop_exit_tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "second_loop_exit_tail": TemplateNode( + node_verification_func=assert_node_type(LoopExitTemplate), + exception_edge=TemplateEdge( + source="second_loop_exit_tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def _verify_loop_exit_match(cfg: nx.DiGraph, mapping: dict) -> bool: + first_tail = mapping["first_loop_exit_tail"] + second_tail = mapping["second_loop_exit_tail"] + if not isinstance(first_tail, LoopExitTemplate) or not isinstance(second_tail, LoopExitTemplate): + return False + + # the loop exits should have no code associated with them + # this part of the pattern is just to deal with implicit continues that got split into separate nodes + if first_tail.tail or second_tail.tail: + return False + + return first_tail.exit_statement == second_tail.exit_statement + + def __init__(self, first_condition: ControlFlowTemplate, second_condition: ControlFlowTemplate): + self.first_condition = first_condition + self.second_condition = second_condition + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ShortCircuitAndTemplate._subgraph, root_key="first_condition", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + matcher = GraphTemplateMatcher(template_node_dict=ShortCircuitAndTemplate._subgraph_for_loop_exits, root_key="first_condition", mapping_verification_func=ShortCircuitAndTemplate._verify_loop_exit_match) + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + short_circuit_template = ShortCircuitAndTemplate( + first_condition=mapping["first_condition"], + second_condition=mapping["second_condition"], + ) + + in_edges = ((src, short_circuit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(short_circuit_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(short_circuit_template.second_condition, data=True)] + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([short_circuit_template.first_condition, short_circuit_template.second_condition]) + if first_loop_exit_tail := mapping.get("first_loop_exit_tail", None): + reduced_cfg.remove_node(first_loop_exit_tail) + reduced_cfg.add_node(short_circuit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + first_condition = self.first_condition.to_indented_source(source_lines) + second_condition = self.second_condition.to_indented_source(source_lines) + return "\n".join([first_condition, second_condition]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitOrContinueTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitOrContinueTemplate.py new file mode 100644 index 0000000..fa5af56 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitOrContinueTemplate.py @@ -0,0 +1,188 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import get_dominator_function + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_unconditional_jump + + + +class ShortCircuitOrContinueTemplate(ControlFlowTemplate): + """ + A short-circuit evaluated boolean OR. Typically these are all part of one line. + This variant occurs only when the content of the if statement is just a continue. + + (-1) + | \\ + (0) \\ + / \\j | (-10) + (1) (2) |j --> / \\j + \\j / (1) (2) + (3) -/ \\j + (3) + + optionally, all nodes in the pattern can have a shared exception handler. + This condenses the short-circuit down to be matched against an if-like template later + """ + + _subgraph = { + "first_condition": TemplateNode( + natural_edge=TemplateEdge( + source="first_condition", + dest="second_condition", + ), + conditional_edge=TemplateEdge( + source="first_condition", + dest="loop_header", + ), + exception_edge=TemplateEdge( + source="first_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "second_condition": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="second_condition", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="second_condition", + dest="tail", + ), + exception_edge=TemplateEdge( + source="second_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + node_verification_func=assert_unconditional_jump, # this is the continue statement + natural_edge=TemplateEdge( + source="if_body", + dest="loop_header", + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "loop_header": TemplateNode( + natural_edge=TemplateEdge( + source="loop_header", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="loop_header", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="loop_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, first_condition: ControlFlowTemplate, second_condition: ControlFlowTemplate): + self.first_condition = first_condition + self.second_condition = second_condition + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + if cfg.in_degree(node) != 1: + return None + + # to avoid being treated as an if-else, we actually need to greedily search up one layer + pred = next(cfg.predecessors(node)) + + def verify_loop_header(cfg: nx.DiGraph, mapping: dict[str, ControlFlowTemplate]) -> bool: + # check to make sure that all non-stack/control instructions match between the two finally blocks + # this list was made for 3.9, so it may need to be expanded for other versions + dominates = get_dominator_function(cfg) + return dominates(mapping["loop_header"], mapping["first_condition"]) + + matcher = GraphTemplateMatcher(template_node_dict=ShortCircuitOrContinueTemplate._subgraph, root_key="first_condition", mapping_verification_func=verify_loop_header) + + mapping = matcher.match_at_graph_node(cfg, pred) + + if not mapping: + return None + + short_circuit_template = ShortCircuitOrContinueTemplate( + first_condition=mapping["first_condition"], + second_condition=mapping["second_condition"], + ) + + in_edges = ((src, short_circuit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=short_circuit_template.first_condition, data=True)) + out_edges = [(short_circuit_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(short_circuit_template.second_condition, data=True)] + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([short_circuit_template.first_condition, short_circuit_template.second_condition]) + reduced_cfg.add_node(short_circuit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + first_condition = self.first_condition.to_indented_source(source_lines) + second_condition = self.second_condition.to_indented_source(source_lines) + return "\n".join([first_condition, second_condition]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitOrTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitOrTemplate.py new file mode 100644 index 0000000..b8f153d --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/booleans/ShortCircuitOrTemplate.py @@ -0,0 +1,157 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + + +class ShortCircuitOrTemplate(ControlFlowTemplate): + """ + A short-circuit evaluated boolean OR. Typically these are all part of one line. + (0) + / \\ (01) + (1) |j --> / \\j + |j\\| (2) (3) + (2) (3) + + optionally, all nodes in the pattern can have a shared exception handler. + This condenses the short-circuit down to be matched against an if-like template later + """ + + _subgraph = { + "first_condition": TemplateNode( + natural_edge=TemplateEdge( + source="first_condition", + dest="second_condition", + ), + conditional_edge=TemplateEdge( + source="first_condition", + dest="if_body", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "second_condition": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="second_condition", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="second_condition", + dest="tail", + ), + exception_edge=TemplateEdge( + source="second_condition", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + natural_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="if_body", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, first_condition: ControlFlowTemplate, second_condition: ControlFlowTemplate): + self.first_condition = first_condition + self.second_condition = second_condition + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ShortCircuitOrTemplate._subgraph, root_key="first_condition", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + short_circuit_template = ShortCircuitOrTemplate( + first_condition=mapping["first_condition"], + second_condition=mapping["second_condition"], + ) + + in_edges = ((src, short_circuit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(short_circuit_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(short_circuit_template.second_condition, data=True)] + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([short_circuit_template.first_condition, short_circuit_template.second_condition]) + reduced_cfg.add_node(short_circuit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + first_condition = self.first_condition.to_indented_source(source_lines) + second_condition = self.second_condition.to_indented_source(source_lines) + return "\n".join([first_condition, second_condition]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/AsyncWithCleanup312.py b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/AsyncWithCleanup312.py new file mode 100644 index 0000000..b4792ff --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/AsyncWithCleanup312.py @@ -0,0 +1,130 @@ +import networkx as nx + +import itertools + + +from ..abstract.AbstractTemplate import ControlFlowTemplate + + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_edge, assert_in_degree, node_match_all, is_exactly_opname, contains_opname_sequence + + +class AsyncWithCleanup312(ControlFlowTemplate): + _subgraph = { + "start": TemplateNode( + node_verification_func=is_exactly_opname("PUSH_EXC_INFO", "WITH_EXCEPT_START", "GET_AWAITABLE", "LOAD_CONST"), + natural_edge=TemplateEdge( + source="start", + dest="send", + ), + exception_edge=TemplateEdge(source="start", dest="exc"), + ), + "send": TemplateNode( + node_verification_func=node_match_all(is_exactly_opname("SEND"), assert_in_degree(2)), + natural_edge=TemplateEdge( + source="send", + dest="yield", + ), + conditional_edge=TemplateEdge( + source="send", + dest="ifthen", + ), + exception_edge=TemplateEdge( + source="send", + dest="exc", + ), + ), + "yield": TemplateNode( + node_verification_func=node_match_all(is_exactly_opname("YIELD_VALUE"), assert_in_degree(1)), natural_edge=TemplateEdge(source="yield", dest="jump_back"), exception_edge=TemplateEdge(source="yield", dest="ifthen") + ), + "jump_back": TemplateNode( + node_verification_func=node_match_all(is_exactly_opname("JUMP_BACKWARD_NO_INTERRUPT"), assert_in_degree(1)), natural_edge=TemplateEdge(source="jump_back", dest="send"), exception_edge=TemplateEdge(source="jump_back", dest="exc") + ), + "ifthen": TemplateNode( + node_verification_func=node_match_all(is_exactly_opname("CLEANUP_THROW", "END_SEND", "POP_JUMP_IF_TRUE", "RERAISE", "POP_TOP"), assert_in_degree(2)), + natural_edge=TemplateEdge(source="ifthen", dest="tail"), + exception_edge=TemplateEdge( + source="ifthen", + dest="exc", + ), + ), + "exc": TemplateNode( + node_verification_func=node_match_all(is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), assert_in_degree(4)), + natural_edge=TemplateEdge( + source="exc", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exc", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exc", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + node_verification_func=node_match_all(contains_opname_sequence("POP_EXCEPT", "POP_TOP", "POP_TOP"), assert_in_degree(1)), + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self): + pass + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher( + template_node_dict=AsyncWithCleanup312._subgraph, + root_key="start", + mapping_verification_func=None, + ) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + with_template = AsyncWithCleanup312() + + in_edges = ((src, with_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + # out_edges = ((with_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(nbunch=mapping['exc'], data=True)) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from(mapping.values()) + reduced_cfg.add_node(with_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + return "" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/Await312Template.py b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/Await312Template.py new file mode 100644 index 0000000..7dfba4c --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/Await312Template.py @@ -0,0 +1,136 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_edge, assert_in_degree, assert_node_has_no_backwards_edges, node_match_all, is_exactly_opname, node_match_any +from ...cfg_utils import ControlFlowEdgeType + + +class Await312Template(ControlFlowTemplate): + _subgraph = { + "awaited": TemplateNode( + natural_edge=TemplateEdge( + source="awaited", + dest="send", + ), + exception_edge=TemplateEdge( + source="awaited", + dest="exception_handler", + ), + ), + "send": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(2), assert_node_has_no_backwards_edges, is_exactly_opname("SEND")), + natural_edge=TemplateEdge( + source="send", + dest="yield", + ), + conditional_edge=TemplateEdge( + source="send", + dest="jump_back", + ), + exception_edge=TemplateEdge( + source="send", + dest="exception_handler", + ), + ), + "yield": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), assert_node_has_no_backwards_edges, is_exactly_opname("YIELD_VALUE")), + natural_edge=TemplateEdge( + source="yield", + dest="jump_back", + ), + exception_edge=TemplateEdge( + source="yield", + dest="cleanup_throw", + ), + ), + "jump_back": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(2), is_exactly_opname("JUMP_BACKWARD_NO_INTERRUPT")), + natural_edge=TemplateEdge( + source="jump_back", + dest="send", + ), + exception_edge=TemplateEdge( + source="jump_back", + dest="exception_handler", + ), + ), + "cleanup_throw": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), is_exactly_opname("CLEANUP_THROW")), + natural_edge=TemplateEdge( + source="cleanup_throw", + dest="jump_back2", + ), + exception_edge=TemplateEdge( + source="cleanup_throw", + dest="exception_handler", + ), + ), + "jump_back2": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), node_match_any(is_exactly_opname("JUMP_BACKWARD"), is_exactly_opname("JUMP_BACKWARD_NO_INTERRUPT"))), + natural_edge=TemplateEdge( + source="jump_back2", + dest=None, + ), + ), + "exception_handler": TemplateNode( + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, awaited): + self.awaited = awaited + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=Await312Template._subgraph, root_key="awaited", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + template = Await312Template( + awaited=mapping["awaited"], + ) + + in_edges = ((src, template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(template, next(cfg.successors(mapping["jump_back2"])), {"type": ControlFlowEdgeType.NATURAL.value}), (template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})] + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([template.awaited, mapping["send"], mapping["yield"], mapping["jump_back"], mapping["cleanup_throw"], mapping["jump_back2"]]) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + return self.awaited.to_indented_source(source_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithCleanup312.py b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithCleanup312.py new file mode 100644 index 0000000..0574ff8 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithCleanup312.py @@ -0,0 +1,128 @@ +import networkx as nx + + + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_edge, assert_in_degree, node_match_all, is_exactly_opname, contains_opname_sequence, node_match_any + + +class WithCleanup312(ControlFlowTemplate, AbstractNonSequentiable): + _subgraph = { + "start": TemplateNode( + node_verification_func=node_match_any( + is_exactly_opname("PUSH_EXC_INFO", "WITH_EXCEPT_START", "POP_JUMP_IF_TRUE"), + is_exactly_opname("PUSH_EXC_INFO", "WITH_EXCEPT_START", "TO_BOOL", "POP_JUMP_IF_TRUE"), + ), + natural_edge=TemplateEdge( + source="start", + dest="reraise", + ), + conditional_edge=TemplateEdge( + source="start", + dest="poptop", + ), + exception_edge=TemplateEdge(source="start", dest="exc"), + ), + "reraise": TemplateNode( + node_verification_func=node_match_all(is_exactly_opname("RERAISE"), assert_in_degree(1)), + exception_edge=TemplateEdge( + source="reraise", + dest="exc", + ), + ), + "poptop": TemplateNode( + node_verification_func=node_match_all(is_exactly_opname("POP_TOP"), assert_in_degree(1)), + natural_edge=TemplateEdge( + source="poptop", + dest="tail", + ), + exception_edge=TemplateEdge( + source="poptop", + dest="exc", + ), + ), + "exc": TemplateNode( + node_verification_func=node_match_all(is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), assert_in_degree(3)), + natural_edge=TemplateEdge( + source="exc", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exc", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exc", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + node_verification_func=node_match_all(contains_opname_sequence("POP_EXCEPT", "POP_TOP", "POP_TOP"), assert_in_degree(1)), + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self): + pass + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + # to avoid being treated as an try-except, we actually need to greedily search up one layer + node = next(cfg.predecessors(node)) + + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher( + template_node_dict=WithCleanup312._subgraph, + root_key="start", + mapping_verification_func=None, + ) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + with_template = WithCleanup312() + + in_edges = ((src, with_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from(mapping.values()) + reduced_cfg.add_node(with_template) + reduced_cfg.add_edges_from(in_edges) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + return "" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate.py new file mode 100644 index 0000000..f286f5f --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate.py @@ -0,0 +1,177 @@ +import networkx as nx + +import itertools + + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_with, node_match_all +from ..subtemplates.OptionalExitSubtemplate import ExitSubTemplate + + +class WithTemplate(ControlFlowTemplate, AbstractNonSequentiable): + r""" + + A basic with template as a catch for normal withs + + (0) node 2 may point to an outer exception handler + | + (1) + e/ | + / (2) + \ | + (3) + """ + + _subgraph = { + "setup_with": TemplateNode( + node_verification_func=assert_with, + natural_edge=TemplateEdge( + source="setup_with", + dest="body", + ), + exception_edge=TemplateEdge( + source="setup_with", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="body", + dest="begin_finally", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, # since it is possible to not have a begin finally block we need to commit it to mapping + ), + exception_edge=TemplateEdge(source="body", dest="with_cleanup"), + ), + "begin_finally": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=node_match_all( + optional_node, + assert_in_degree(1), + ), + natural_edge=TemplateEdge( + source="begin_finally", + dest="with_cleanup", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, # if the destination node is None, don't commit to the mapping + ), + exception_edge=TemplateEdge(source="begin_finally", dest="exception_handler", edge_verification_func=optional_edge), + ), + "with_cleanup": TemplateNode( + natural_edge=TemplateEdge( + source="with_cleanup", + dest="tail", + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="with_cleanup", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, setup_with: ControlFlowTemplate, body: ControlFlowTemplate, begin_finally: ControlFlowTemplate, with_cleanup: ControlFlowTemplate): + self.setup_with = setup_with + self.body = body + self.begin_finally = begin_finally + self.with_cleanup = with_cleanup + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + # to avoid being treated as an try-except, we actually need to greedily search up one layer + node = next(cfg.predecessors(node)) + + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher( + template_node_dict=WithTemplate._subgraph, + root_key="setup_with", + mapping_verification_func=None, + ) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + with_template = WithTemplate( + setup_with=mapping["setup_with"], + body=mapping["body"], + begin_finally=mapping.get("begin_finally", None), + with_cleanup=mapping["with_cleanup"], + ) + + in_edges = ((src, with_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [] + if mapping["tail"]: + out_edges.append((with_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})) + else: + out_edges.extend([(with_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(nbunch=mapping["with_cleanup"], data=True)]) + if mapping["exception_handler"]: + out_edges.append((with_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([with_template.setup_with, with_template.body, with_template.begin_finally, with_template.with_cleanup]) + reduced_cfg.add_node(with_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + header = self.setup_with.to_indented_source(source_lines) + body = self.body._indent_multiline_string(self.body.to_indented_source(source_lines)) + # cleanup = self.with_cleanup.to_indented_source(source_lines) + return f"{header}\n{body}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate312.py b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate312.py new file mode 100644 index 0000000..17c819b --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate312.py @@ -0,0 +1,124 @@ +import networkx as nx + +import itertools + + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher +from .WithCleanup312 import WithCleanup312 +from .AsyncWithCleanup312 import AsyncWithCleanup312 + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_with, node_match_all, assert_node_type + + +class WithTemplate312(ControlFlowTemplate, AbstractNonSequentiable): + _subgraph = { + "setup_with": TemplateNode( + node_verification_func=assert_with, + natural_edge=TemplateEdge( + source="setup_with", + dest="body", + ), + exception_edge=TemplateEdge( + source="setup_with", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="body", dest="with_cleanup2", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="body", + dest="with_cleanup", + ), + ), + "with_cleanup": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), assert_node_type(WithCleanup312, AsyncWithCleanup312)), + ), + "with_cleanup2": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="with_cleanup2", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="with_cleanup2", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="with_cleanup2", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, setup_with: ControlFlowTemplate, body: ControlFlowTemplate, with_cleanup: ControlFlowTemplate, with_cleanup2: ControlFlowTemplate): + self.setup_with = setup_with + self.body = body + self.with_cleanup = with_cleanup + self.with_cleanup2 = with_cleanup2 + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + node = next(cfg.predecessors(node)) + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher( + template_node_dict=WithTemplate312._subgraph, + root_key="setup_with", + mapping_verification_func=None, + ) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + with_template = WithTemplate312( + setup_with=mapping["setup_with"], + body=mapping["body"], + with_cleanup=mapping["with_cleanup"], + with_cleanup2=mapping.get("with_cleanup2"), + ) + + in_edges = ((src, with_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + if "with_cleanup2" in mapping: + out_edges = ( + (with_template, dst, {"type": ControlFlowEdgeType.NATURAL.value} if edge_properties["type"] == ControlFlowEdgeType.JUMP.value else edge_properties) + for src, dst, edge_properties in cfg.out_edges(nbunch=mapping["with_cleanup2"], data=True) + ) + else: + out_edges = () + out_edges2 = ((with_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(nbunch=mapping["setup_with"], data=True) if edge_properties["type"] == ControlFlowEdgeType.EXCEPTION.value) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([with_template.setup_with, with_template.body, with_template.with_cleanup, with_template.with_cleanup2]) + reduced_cfg.add_node(with_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges, out_edges2)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + header = self.setup_with.to_indented_source(source_lines) + body = self._indent_multiline_string(self.body.to_indented_source(source_lines)) + if self.with_cleanup2 is not None: + clean = self.with_cleanup2.to_indented_source(source_lines) + else: + clean = "" + return f"{header}\n{body}\n{clean}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate39.py b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate39.py new file mode 100644 index 0000000..56e6303 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/context_managers/WithTemplate39.py @@ -0,0 +1,115 @@ +import networkx as nx + +import itertools + + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..try_except.TryExceptTemplate import TryExceptTemplate + + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_with, node_match_all, assert_node_type + + +class WithTemplate39(ControlFlowTemplate, AbstractNonSequentiable): + _subgraph = { + "setup_with": TemplateNode( + node_verification_func=assert_with, + natural_edge=TemplateEdge( + source="setup_with", + dest="body", + ), + exception_edge=TemplateEdge( + source="setup_with", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "body": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), assert_node_type(TryExceptTemplate)), + natural_edge=TemplateEdge( + source="body", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="body", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="body", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, setup_with: ControlFlowTemplate, body: ControlFlowTemplate): + self.setup_with = setup_with + self.body = body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + node = next(cfg.predecessors(node)) + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher( + template_node_dict=WithTemplate39._subgraph, + root_key="setup_with", + mapping_verification_func=None, + ) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + with_template = WithTemplate39( + setup_with=mapping["setup_with"], + body=mapping["body"], + ) + + in_edges = ((src, with_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = ((with_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(nbunch=mapping["body"], data=True)) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([with_template.setup_with, with_template.body]) + reduced_cfg.add_node(with_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + __import__("pdb").set_trace() + header = self.setup_with.to_indented_source(source_lines) + body = self._indent_multiline_string(self.body.try_body.to_indented_source(source_lines)) + return f"{header}\n{body}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ElseExitExceptTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ElseExitExceptTemplate.py new file mode 100644 index 0000000..595009f --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ElseExitExceptTemplate.py @@ -0,0 +1,180 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + +class ElseExitExceptTemplate(ControlFlowTemplate, AbstractNonSequentiable): + """ + An if-else block where only the else has no further control flow (structured breaks/continues and returns). + When the exit leaves an exception block, the final exit statement does not have the same exception handler. + (0) + j/ \\ --> (0123) + (1) (2) | + | |j (...) + (3) (...) + + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "if_header": TemplateNode( + natural_edge=TemplateEdge( + source="if_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="else_body", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="if_body", + dest="tail", + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "else_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="else_body", + dest="exit_node", + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="else_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exit_node": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="exit_node", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, if_header: ControlFlowTemplate, if_body: ControlFlowTemplate, else_body: ControlFlowTemplate, exit_node: ControlFlowTemplate): + self.if_header = if_header + self.if_body = if_body + self.else_body = else_body + self.exit_node = exit_node + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ElseExitExceptTemplate._subgraph, root_key="if_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + if_else_template = ElseExitExceptTemplate(if_header=mapping["if_header"], if_body=mapping["if_body"], else_body=mapping["else_body"], exit_node=mapping["exit_node"]) + + in_edges = ((src, if_else_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(if_else_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})] + if mapping["exception_handler"]: + out_edges.append((if_else_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([if_else_template.if_header, if_else_template.if_body, if_else_template.else_body, if_else_template.exit_node]) + reduced_cfg.add_node(if_else_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.if_header.to_indented_source(source_lines) + if_body = ControlFlowTemplate._indent_multiline_string(self.if_body.to_indented_source(source_lines)) + else_body = ControlFlowTemplate._indent_multiline_string(self.else_body.to_indented_source(source_lines)) + exit_node = ControlFlowTemplate._indent_multiline_string(self.exit_node.to_indented_source(source_lines)) + return "\n".join([header, if_body, "else: # inserted", else_body, exit_node]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ElseExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ElseExitTemplate.py new file mode 100644 index 0000000..4680ed2 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ElseExitTemplate.py @@ -0,0 +1,144 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + +class ElseExitTemplate(ControlFlowTemplate): + """ + An if-else block where only the else has no further control flow (structured breaks/continues and returns). + (0) + j/ \\ --> (012) + (1) (2) | + |j (3) + (3) + + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "if_header": TemplateNode( + natural_edge=TemplateEdge( + source="if_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="else_body", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="if_body", + dest="tail", + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "else_body": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="else_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, if_header: ControlFlowTemplate, if_body: ControlFlowTemplate, else_body: ControlFlowTemplate): + self.if_header = if_header + self.if_body = if_body + self.else_body = else_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ElseExitTemplate._subgraph, root_key="if_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + if_else_template = ElseExitTemplate(if_header=mapping["if_header"], if_body=mapping["if_body"], else_body=mapping["else_body"]) + + in_edges = ((src, if_else_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(if_else_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})] + if mapping["exception_handler"]: + out_edges.append((if_else_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([if_else_template.if_header, if_else_template.if_body, if_else_template.else_body]) + reduced_cfg.add_node(if_else_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.if_header.to_indented_source(source_lines) + if_body = ControlFlowTemplate._indent_multiline_string(self.if_body.to_indented_source(source_lines)) + else_body = ControlFlowTemplate._indent_multiline_string(self.else_body.to_indented_source(source_lines)) + return "\n".join([header, if_body, "else: # inserted", else_body]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ExceptExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ExceptExitTemplate.py new file mode 100644 index 0000000..c6f2462 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/ExceptExitTemplate.py @@ -0,0 +1,148 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import assert_edge_type, optional_node, optional_edge, assert_in_degree + +from ..try_except.ExceptAsTemplate import ExceptAsTemplate +from ..try_except.ExceptAsExceptTemplate import ExceptAsExceptTemplate + + +class ExceptExitTemplate(ControlFlowTemplate, AbstractNonSequentiable): + """ + A `try-except` block where the except has no further control flow. + (0) + / \\e --> (012) + (1) (2) | + | (3) + (3) + """ + + _subgraph = { + "try_body": TemplateNode( + natural_edge=TemplateEdge( + source="try_body", + dest="try_footer", + ), + exception_edge=TemplateEdge( + source="try_body", + dest="except_body", + ), + ), + "try_footer": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="try_footer", dest="after_try_except", edge_verification_func=assert_edge_type(ControlFlowEdgeType.JUMP)), + exception_edge=TemplateEdge( + source="try_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_try_except": TemplateNode( + natural_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_try_except", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, try_body: ControlFlowTemplate, try_footer: ControlFlowTemplate, except_body: ControlFlowTemplate): + self.try_body = try_body + self.try_footer = try_footer + self.except_body = except_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptExitTemplate._subgraph, root_key="try_body", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + try_except_template = ExceptExitTemplate(try_body=mapping["try_body"], try_footer=mapping["try_footer"], except_body=mapping["except_body"]) + + in_edges = ((src, try_except_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + # insert a continuation edge to after the try except + out_edges = [(try_except_template, mapping["after_try_except"], {"type": ControlFlowEdgeType.NATURAL.value})] + if mapping["outer_exception_handler"]: + out_edges.append((try_except_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([try_except_template.try_body, try_except_template.try_footer, try_except_template.except_body]) + reduced_cfg.add_node(try_except_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + try_body = ControlFlowTemplate._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + + try_except_lines = ["try:", try_body] + # if we matched against an "Except ... as" chain, then omit the inserted except: block + if isinstance(self.except_body, ExceptAsTemplate) or isinstance(self.except_body, ExceptAsExceptTemplate): + except_body = self.except_body.to_indented_source(source_lines) + else: + except_body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + try_except_lines.append("except:") + try_except_lines.append(except_body) + + return "\n".join(try_except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfElseExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfElseExitTemplate.py new file mode 100644 index 0000000..9bb2cbe --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfElseExitTemplate.py @@ -0,0 +1,185 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import assert_edge_type, optional_node, optional_edge, assert_in_degree, node_is_none_or_matches, edge_is_none_or_matches + + +class IfElseExitTemplate(ControlFlowTemplate): + """ + An if-else block where both options have no further control flow (structured breaks/continues and returns). + (0) + j/ \\ --> (012) + (1) (2) + + optionally, all nodes in the pattern can have a shared exception handler. + nodes 1 and 2 can optionally have a "tail" that is an exit statement that breaks out of the current exception handler. + """ + + _subgraph = { + "if_header": TemplateNode( + natural_edge=TemplateEdge( + source="if_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="else_body", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="if_body", + dest="if_tail", + edge_verification_func=edge_is_none_or_matches(assert_edge_type(ControlFlowEdgeType.NATURAL)), + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_tail": TemplateNode( + node_verification_func=node_is_none_or_matches(assert_in_degree(1)), + exception_edge=TemplateEdge( + source="if_tail", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "else_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="else_body", + dest="else_tail", + edge_verification_func=edge_is_none_or_matches(assert_edge_type(ControlFlowEdgeType.NATURAL)), + ), + exception_edge=TemplateEdge( + source="else_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "else_tail": TemplateNode( + node_verification_func=node_is_none_or_matches(assert_in_degree(1)), + exception_edge=TemplateEdge( + source="else_tail", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__( + self, + if_header: ControlFlowTemplate, + if_body: ControlFlowTemplate, + if_tail: ControlFlowTemplate, + else_body: ControlFlowTemplate, + else_tail: ControlFlowTemplate, + ): + self.if_header = if_header + self.if_body = if_body + self.if_tail = if_tail # may be none + self.else_body = else_body + self.else_tail = else_tail # may be none + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=IfElseExitTemplate._subgraph, root_key="if_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + if_else_template = IfElseExitTemplate( + if_header=mapping["if_header"], + if_body=mapping["if_body"], + if_tail=mapping["if_tail"], + else_body=mapping["else_body"], + else_tail=mapping["else_tail"], + ) + + in_edges = ((src, if_else_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + # only preserve meta edges + out_edges = [(if_else_template, "END", data) for _, _, data in cfg.out_edges([if_else_template.if_body, if_else_template.else_body], data=True) if data["type"] == ControlFlowEdgeType.META.value] + if mapping["exception_handler"]: + out_edges.append((if_else_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([if_else_template.if_header, if_else_template.if_body, if_else_template.if_tail, if_else_template.else_body, if_else_template.else_tail]) + reduced_cfg.add_node(if_else_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.if_header.to_indented_source(source_lines) + if_body = self.if_body.to_indented_source(source_lines) + if header.strip(): + if_body = ControlFlowTemplate._indent_multiline_string(if_body) + else_body = self.else_body.to_indented_source(source_lines) + + return "\n".join([header, if_body, else_body]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfExitExceptTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfExitExceptTemplate.py new file mode 100644 index 0000000..6d3313d --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfExitExceptTemplate.py @@ -0,0 +1,162 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import get_out_edge_dict, ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + +class IfExitExceptTemplate(ControlFlowTemplate): + """ + An if block where the if has no further control flow (structured breaks/continues and returns). + When the exit leaves an exception block, the final exit statement does not have the same exception handler. + + (0) + j/ \\ --> (023) + (1) (2) | + | | (1) + ... (3) + + In this configuration, (0,1,2) share an exception handler, but 3 does not + """ + + _subgraph = { + "if_header": TemplateNode( + natural_edge=TemplateEdge( + source="if_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="tail", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + ), + ), + "if_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="if_body", + dest="exit_node", + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + ), + ), + "exit_node": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="exit_node", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + ), + ), + "exception_handler": TemplateNode( + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, if_header: ControlFlowTemplate, if_body: ControlFlowTemplate, exit_node: ControlFlowTemplate): + self.if_header = if_header + self.if_body = if_body + self.exit_node = exit_node + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + # try to match happy non-exception version + matcher = GraphTemplateMatcher(template_node_dict=IfExitExceptTemplate._subgraph, root_key="if_header", mapping_verification_func=None) + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + if_exit_template = IfExitExceptTemplate(if_header=mapping["if_header"], if_body=mapping["if_body"], exit_node=mapping["exit_node"]) + + in_edges = ((src, if_exit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + node_edge_dict = get_out_edge_dict(cfg, node) + out_edges = [(if_exit_template, node_edge_dict["conditional"][0], {"type": ControlFlowEdgeType.NATURAL.value})] + if node_edge_dict["exception"]: + out_edges.append((if_exit_template, *(node_edge_dict["exception"]))) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([if_exit_template.if_header, if_exit_template.if_body, if_exit_template.exit_node]) + reduced_cfg.add_node(if_exit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.if_header.to_indented_source(source_lines) + if_body = ControlFlowTemplate._indent_multiline_string(self.if_body.to_indented_source(source_lines)) + exit_node = ControlFlowTemplate._indent_multiline_string(self.exit_node.to_indented_source(source_lines)) + return "\n".join([header, if_body, exit_node]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfExitTemplate.py new file mode 100644 index 0000000..d61c3cb --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/IfExitTemplate.py @@ -0,0 +1,133 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import get_out_edge_dict, ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + +class IfExitTemplate(ControlFlowTemplate): + """ + An if block where the if has no further control flow (structured breaks/continues and returns). + (0) + j/ \\ --> (02) + (1) (2) | + | (1) + ... + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "if_header": TemplateNode( + natural_edge=TemplateEdge( + source="if_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="tail", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, if_header: ControlFlowTemplate, if_body: ControlFlowTemplate): + self.if_header = if_header + self.if_body = if_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=IfExitTemplate._subgraph, root_key="if_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + if_exit_template = IfExitTemplate(if_header=mapping["if_header"], if_body=mapping["if_body"]) + + in_edges = ((src, if_exit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + node_edge_dict = get_out_edge_dict(cfg, node) + out_edges = [(if_exit_template, node_edge_dict["conditional"][0], {"type": ControlFlowEdgeType.NATURAL.value})] + if node_edge_dict["exception"]: + out_edges.append((if_exit_template, *(node_edge_dict["exception"]))) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([if_exit_template.if_header, if_exit_template.if_body]) + reduced_cfg.add_node(if_exit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.if_header.to_indented_source(source_lines) + if_body = self.if_body.to_indented_source(source_lines) + # sometimes there is no header in the case of short-circuit boolean AND + if header: + if_body = ControlFlowTemplate._indent_multiline_string(if_body) + return "\n".join([header, if_body]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/TryExitExceptExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/TryExitExceptExitTemplate.py new file mode 100644 index 0000000..0a86317 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/TryExitExceptExitTemplate.py @@ -0,0 +1,139 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..try_except.ExceptAsExceptTemplate import ExceptAsExceptTemplate +from ..try_except.ExceptAsTemplate import ExceptAsTemplate +from ..try_except.ExceptAsExitTemplate import ExceptAsExitTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_instruction_opname, assert_except_as + + +class TryExitExceptExitTemplate(ControlFlowTemplate, AbstractNonSequentiable): + """ + An try block where neither the try body nor the except body has no further control flow (structured breaks/continues and returns). + (0) + | --> (012) + (1) + |e + (2) + """ + + _subgraph = { + "setup_finally": TemplateNode( + node_verification_func=assert_instruction_opname("SETUP_FINALLY"), + natural_edge=TemplateEdge( + source="setup_finally", + dest="try_body", + ), + exception_edge=TemplateEdge( + source="setup_finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "try_body": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="try_body", + dest="except_body", + ), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, setup_finally: ControlFlowTemplate, try_body: ControlFlowTemplate, except_body: ControlFlowTemplate): + self.setup_finally = setup_finally + self.try_body = try_body + self.except_body = except_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + # an except as exit looks exactly like this, so we need to check that we are not part of the larger pattern + def assert_not_in_except_as(cfg: nx.DiGraph, mapping: dict) -> bool: + setup_finally = mapping["setup_finally"] + if cfg.in_degree(setup_finally) != 1: + return True + + pred = next(cfg.predecessors(setup_finally)) + return not assert_except_as(cfg, pred) + + matcher = GraphTemplateMatcher(template_node_dict=TryExitExceptExitTemplate._subgraph, root_key="setup_finally", mapping_verification_func=assert_not_in_except_as) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + try_exit_template = TryExitExceptExitTemplate(setup_finally=mapping["setup_finally"], try_body=mapping["try_body"], except_body=mapping["except_body"]) + + in_edges = ((src, try_exit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((try_exit_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([try_exit_template.setup_finally, try_exit_template.try_body, try_exit_template.except_body]) + reduced_cfg.add_node(try_exit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + setup_finally = self.setup_finally.to_indented_source(source_lines) + try_body = ControlFlowTemplate._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + + try_except_lines = [setup_finally, "try:", try_body] + # if we matched against an "Except ... as" chain, then omit the inserted except: block + if isinstance(self.except_body, ExceptAsTemplate) or isinstance(self.except_body, ExceptAsExceptTemplate) or isinstance(self.except_body, ExceptAsExitTemplate): + except_body = self.except_body.to_indented_source(source_lines) + else: + except_body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + try_except_lines.append("except:") + try_except_lines.append(except_body) + + return "\n".join(try_except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/TryExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/TryExitTemplate.py new file mode 100644 index 0000000..f73ea67 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/deprecated/TryExitTemplate.py @@ -0,0 +1,153 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractExceptionTemplate +from ..try_except.ExceptAsExceptTemplate import ExceptAsExceptTemplate +from ..try_except.ExceptAsTemplate import ExceptAsTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + +class TryExitTemplate(ControlFlowTemplate, AbstractExceptionTemplate): + """ + An try block where the try body has no further control flow (structured breaks/continues and returns). + (0) + e/ \\ --> (012) + (1) (2) | + | (3) + (3) + """ + + _subgraph = { + "try_body": TemplateNode( + natural_edge=TemplateEdge( + source="try_body", + dest="try_exit", + ), + exception_edge=TemplateEdge( + source="try_body", + dest="except_body", + ), + ), + "try_exit": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="try_exit", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="except_body", + dest="after_try_except", + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_try_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_try_except", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, try_body: ControlFlowTemplate, try_exit: ControlFlowTemplate, except_body: ControlFlowTemplate): + self.try_body = try_body + self.try_exit = try_exit + self.except_body = except_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=TryExitTemplate._subgraph, root_key="try_body", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + try_exit_template = TryExitTemplate(try_body=mapping["try_body"], try_exit=mapping["try_exit"], except_body=mapping["except_body"]) + + in_edges = ((src, try_exit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((try_exit_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping["after_try_except"]: + out_edges.append((try_exit_template, mapping["after_try_except"], {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([try_exit_template.try_body, try_exit_template.try_exit, try_exit_template.except_body]) + reduced_cfg.add_node(try_exit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + try_body = ControlFlowTemplate._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + try_exit = ControlFlowTemplate._indent_multiline_string(self.try_exit.to_indented_source(source_lines)) + + try_except_lines = ["try:", try_body, try_exit] + # if we matched against an "Except ... as" chain, then omit the inserted except: block + if isinstance(self.except_body, ExceptAsTemplate) or isinstance(self.except_body, ExceptAsExceptTemplate): + except_body = self.except_body.to_indented_source(source_lines) + else: + except_body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + try_except_lines.append("except:") + try_except_lines.append(except_body) + + return "\n".join(try_except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/if_then/ConditionalExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/if_then/ConditionalExitTemplate.py new file mode 100644 index 0000000..8077877 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/if_then/ConditionalExitTemplate.py @@ -0,0 +1,115 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge + + +class ConditionalExitTemplate(ControlFlowTemplate): + """ + A conditional exit within a line. Typically due to an assert statement. + (0) + j| --> (01) + (1) + + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "exit_header": TemplateNode( + conditional_edge=TemplateEdge( + source="exit_header", + dest="tail", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, exit_header: ControlFlowTemplate, tail: ControlFlowTemplate): + self.exit_header = exit_header + self.tail = tail + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ConditionalExitTemplate._subgraph, root_key="exit_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + conditional_exit_template = ConditionalExitTemplate( + exit_header=mapping["exit_header"], + tail=mapping["tail"], + ) + + in_edges = ((src, conditional_exit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = ((conditional_exit_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(nbunch=mapping["tail"], data=True)) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([conditional_exit_template.exit_header, conditional_exit_template.tail]) + reduced_cfg.add_node(conditional_exit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.exit_header.to_indented_source(source_lines) + tail = self.tail.to_indented_source(source_lines) + return "\n".join([header, tail]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfElseTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfElseTemplate.py new file mode 100644 index 0000000..aad08e3 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfElseTemplate.py @@ -0,0 +1,194 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..natural.InstructionTemplate import InstructionTemplate + +from ..subtemplates.OptionalExitSubtemplate import ExitSubTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, node_match_all, assert_node_has_no_backwards_edges, node_match_none, assert_except_as, is_exactly_opname + +from ..natural.LinearSequenceTemplate import LinearSequenceTemplate + + +class IfElseTemplate(ControlFlowTemplate): + """ + A standard if-else-block with no extra control flow. + (0) + j/ \\ (012) + (1) (2) --> | + \\ /j (3) + (3) + + optionally, all nodes in the pattern can have a shared exception handler. + + Interestingly, this template also covers loops with guaranteed breaks and an else block. + """ + + _subgraph = { + "if_header": TemplateNode( + node_verification_func=node_match_none(assert_except_as, is_exactly_opname("CLEANUP_THROW", "END_SEND", "POP_JUMP_IF_TRUE")), + natural_edge=TemplateEdge( + source="if_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="else_body", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ), + natural_edge=TemplateEdge( + source="if_body", + dest="tail", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "else_body": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ), + natural_edge=TemplateEdge( + source="else_body", + dest="tail", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="else_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + commit_none_to_mapping=False, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + commit_none_to_mapping=False, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + commit_none_to_mapping=False, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + commit_none_to_mapping=False, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + commit_none_to_mapping=False, + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + commit_none_to_mapping=False, + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, if_header: ControlFlowTemplate, if_body: ControlFlowTemplate, else_body: ControlFlowTemplate): + self.if_header = if_header + self.if_body = if_body + self.else_body = else_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=IfElseTemplate._subgraph, root_key="if_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + if_else_template = IfElseTemplate(if_header=mapping["if_header"], if_body=mapping["if_body"], else_body=mapping["else_body"]) + + in_edges = ((src, if_else_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + if "tail" in mapping: + out_edges = [(if_else_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})] + else: + out_edges = [] + if mapping["exception_handler"]: + out_edges.append((if_else_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([if_else_template.if_header, if_else_template.if_body, if_else_template.else_body]) + reduced_cfg.add_node(if_else_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + if_lines = [] + header = self.if_header.to_indented_source(source_lines).rstrip() + if header and header.split("\n")[-1].strip().startswith("assert "): + return "\n".join([header, self.if_body.to_indented_source(source_lines), self.else_body.to_indented_source(source_lines)]) + if header: + if_lines.append(header) + if_body = ControlFlowTemplate._indent_multiline_string(self.if_body.to_indented_source(source_lines)) + if if_body: + if_lines.append(if_body) + else_body = ControlFlowTemplate._indent_multiline_string(self.else_body.to_indented_source(source_lines)) + if else_body: + if_lines.extend(["else: # inserted", else_body]) + + # edge case hack to deal with for loops that have guaranteed breaks (they look exactly like if statements) + # while loops should be translated as if statements in this case, so we don't have to worry there + if isinstance(self.if_header, LinearSequenceTemplate): + last_member = self.if_header.members[-1] + if isinstance(last_member, InstructionTemplate) and last_member.instruction.opname == "FOR_ITER": + if_lines.insert(2, "\tbreak # inserted") + + return "\n".join(if_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfThenJumpTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfThenJumpTemplate.py new file mode 100644 index 0000000..cefd54d --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfThenJumpTemplate.py @@ -0,0 +1,162 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..natural.InstructionTemplate import InstructionTemplate +from ..natural.LinearSequenceTemplate import LinearSequenceTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_unconditional_jump + + +class IfThenJumpTemplate(ControlFlowTemplate): + """ + A standard if-block with no extra control flow. + This variant has an absolute jump from the end of the if body to the outside. + This occurs when there are nested if-else blocks and the inner if statements jump out directly to the top level. + (0) + | \\ (01) + j| (1) --> | + | | (2) + | (2) |j + | /j (3) + (3) + + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "if_header": TemplateNode( + natural_edge=TemplateEdge( + source="if_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="tail", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="if_body", + dest="jump", + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "jump": TemplateNode( + node_verification_func=assert_unconditional_jump, + natural_edge=TemplateEdge( + source="jump", + dest="tail", + ), + exception_edge=TemplateEdge( + source="jump", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, if_header: ControlFlowTemplate, if_body: ControlFlowTemplate): + self.if_header = if_header + self.if_body = if_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=IfThenJumpTemplate._subgraph, root_key="if_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + if_then_template = IfThenJumpTemplate(if_header=mapping["if_header"], if_body=mapping["if_body"]) + + in_edges = ((src, if_then_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(if_then_template, mapping["jump"], {"type": ControlFlowEdgeType.NATURAL.value})] + if mapping["exception_handler"]: + out_edges.append((if_then_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([if_then_template.if_header, if_then_template.if_body]) + reduced_cfg.add_node(if_then_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.if_header.to_indented_source(source_lines).strip() + body = ControlFlowTemplate._indent_multiline_string(self.if_body.to_indented_source(source_lines)) + + if_lines = [header, body] + + # edge case hack to deal with for loops that have guaranteed breaks (they look exactly like if statements) + # while loops should be translated as if statements in this case, so we don't have to worry there + if isinstance(self.if_header, LinearSequenceTemplate): + last_member = self.if_header.members[-1] + if isinstance(last_member, InstructionTemplate) and last_member.instruction.opname == "FOR_ITER": + if_lines.insert(2, "\tbreak # inserted") + + return "\n".join(if_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfThenTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfThenTemplate.py new file mode 100644 index 0000000..41c7d0e --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/if_then/IfThenTemplate.py @@ -0,0 +1,173 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..natural.InstructionTemplate import InstructionTemplate +from ..natural.LinearSequenceTemplate import LinearSequenceTemplate + +from ..subtemplates.OptionalExitSubtemplate import ExitSubTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_node_has_no_backwards_edges, node_match_all, assert_except_as, node_match_none + + +class IfThenTemplate(ControlFlowTemplate): + """ + A standard if-block with no extra control flow. + (0) + | \\ (01) + j| (1) --> | + | / (2) + (2) + + optionally, all nodes in the pattern can have a shared exception handler. + + Interestingly, this template also covers loops with guaranteed breaks. + """ + + _subgraph = { + "if_header": TemplateNode( + node_verification_func=node_match_none(assert_except_as), + natural_edge=TemplateEdge( + source="if_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="tail", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ), + natural_edge=TemplateEdge( + source="if_body", + dest="tail", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest=None, + commit_none_to_mapping=False, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, if_header: ControlFlowTemplate, if_body: ControlFlowTemplate): + self.if_header = if_header + self.if_body = if_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=IfThenTemplate._subgraph, root_key="if_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + if_then_template = IfThenTemplate(if_header=mapping["if_header"], if_body=mapping["if_body"]) + + in_edges = ((src, if_then_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(if_then_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})] + if mapping["exception_handler"]: + out_edges.append((if_then_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([if_then_template.if_header, if_then_template.if_body]) + reduced_cfg.add_node(if_then_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.if_header.to_indented_source(source_lines).strip() + """ + if header.startswith('while ') and isinstance(self.if_body, RefinedLoopTemplate) and isinstance(self.if_body.loop_header, WhileTruePlaceholderTemplate): + if isinstance(self.if_body.loop_body, LinearSequenceTemplate): + last = self.if_body.loop_body.members[-1] + else: + last = self.if_body.loop_body + assert isinstance(last, IfElseTemplate) + last.to_indented_source = last.if_header.to_indented_source + self.if_body.loop_header.to_indented_source = lambda x: '' + if isinstance(self.if_body, LoopExitTemplate) and not header.startswith('if '): + body = '' + else: + body = ControlFlowTemplate._indent_multiline_string(self.if_body.to_indented_source(source_lines)) + """ + body = ControlFlowTemplate._indent_multiline_string(self.if_body.to_indented_source(source_lines)) + + if_lines = [header, body] + + # edge case hack to deal with for loops that have guaranteed breaks (they look exactly like if statements) + # while loops should be translated as if statements in this case, so we don't have to worry there + if isinstance(self.if_header, LinearSequenceTemplate): + last_member = self.if_header.members[-1] + if isinstance(last_member, InstructionTemplate) and last_member.instruction.opname == "FOR_ITER": + if_lines.insert(2, "\tbreak # inserted") + + return "\n".join(if_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/AsyncForTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/AsyncForTemplate.py new file mode 100644 index 0000000..1f809e5 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/AsyncForTemplate.py @@ -0,0 +1,152 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_instruction_opname, node_match_all, assert_first_instruction_opname + + +class AsyncForTemplate(ControlFlowTemplate): + """ + An async for loop. + (-1) + | ^ + (0) |j + | \\| (-101) + e| (1) --> | + | (2) + (2) + + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "loop_header": TemplateNode( + node_verification_func=assert_instruction_opname("SETUP_FINALLY"), + natural_edge=TemplateEdge( + source="loop_header", + dest="loop_iter", + ), + exception_edge=TemplateEdge( + source="loop_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "loop_iter": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), assert_first_instruction_opname("GET_ANEXT")), + natural_edge=TemplateEdge( + source="loop_iter", + dest="loop_body", + ), + exception_edge=TemplateEdge( + source="loop_iter", + dest="tail", + ), + ), + "loop_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="loop_body", + dest="loop_header", + ), + exception_edge=TemplateEdge( + source="loop_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, loop_header: ControlFlowTemplate, loop_iter: ControlFlowTemplate, loop_body: ControlFlowTemplate): + self.loop_header = loop_header + self.loop_iter = loop_iter + self.loop_body = loop_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + if cfg.in_degree(node) != 1: + return None + + # to avoid being treated as a try-except, we actually need to greedily search up one layer + pred = next(cfg.predecessors(node)) + + matcher = GraphTemplateMatcher(template_node_dict=AsyncForTemplate._subgraph, root_key="loop_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, pred) + + if not mapping: + return None + + loop_template = AsyncForTemplate(loop_header=mapping["loop_header"], loop_iter=mapping["loop_iter"], loop_body=mapping["loop_body"]) + + in_edges = ((src, loop_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=pred, data=True) if src != mapping["loop_body"]) + out_edges = [(loop_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})] + if mapping["exception_handler"]: + out_edges.append((loop_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([loop_template.loop_header, loop_template.loop_iter, loop_template.loop_body]) + reduced_cfg.add_node(loop_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.loop_header.to_indented_source(source_lines) + loop_iter = self.loop_iter.to_indented_source(source_lines) + body = ControlFlowTemplate._indent_multiline_string(self.loop_body.to_indented_source(source_lines)) + return "\n".join([header, loop_iter, body]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/ForIf312Template.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/ForIf312Template.py new file mode 100644 index 0000000..5fcf0c8 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/ForIf312Template.py @@ -0,0 +1,120 @@ +import networkx as nx + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher +from ..natural.InstructionTemplate import InstructionTemplate +from .LoopExitTemplate import LoopExitTemplate + +from ..match_utils import optional_node, optional_edge, assert_in_degree, node_match_all +from ...cfg_utils import ControlFlowEdgeType + + +def is_j(cfg: nx.DiGraph, node) -> bool: + return isinstance(node, LoopExitTemplate) and isinstance(node.tail, InstructionTemplate) and node.tail.instruction.opname == "JUMP_BACKWARD" and node.exit_statement == "continue" and node.tail.instruction.target.opname == "FOR_ITER" + + +class ForIf312Template(ControlFlowTemplate): + _subgraph = { + "if_header": TemplateNode( + natural_edge=TemplateEdge( + source="if_header", + dest="jump_back", + ), + conditional_edge=TemplateEdge( + source="if_header", + dest="real_body", + ), + exception_edge=TemplateEdge( + source="if_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "jump_back": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), is_j), + exception_edge=TemplateEdge( + source="jump_back", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "real_body": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1)), + exception_edge=TemplateEdge( + source="real_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, if_header: ControlFlowTemplate, body: ControlFlowTemplate, jb: ControlFlowTemplate): + self.if_header = if_header + self.body = body + self.jb = jb + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ForIf312Template._subgraph, root_key="if_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + template = ForIf312Template(if_header=mapping["if_header"], body=mapping["real_body"], jb=mapping["jump_back"]) + + in_edges = ((src, template, edge) for src, dst, edge in cfg.in_edges(node, data=True)) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([mapping["if_header"], mapping["real_body"], mapping["jump_back"]]) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(in_edges) + if mapping["exception_handler"]: + reduced_cfg.add_edge(template, mapping["exception_handler"], type=ControlFlowEdgeType.EXCEPTION.value) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + header = self.if_header.to_indented_source(source_lines) + body = self._indent_multiline_string(self.body.to_indented_source(source_lines)) + """ + n = header.strip().split('\n')[-1].strip().startswith('if not ') + fj = self.if_header.get_instructions()[-1].opname == 'POP_JUMP_IF_FALSE' + breakpoint() + if fj != n: + header += '\n\tpass\nelse: # inserted' + """ + last = max((i.starts_line for i in self.if_header.get_instructions() if i.starts_line is not None), default=None) + if last is not None and last < len(source_lines) and body.split("\n")[0].strip() != source_lines[last].strip(): + header += "\n\tpass\nelse: # inserted" + return header + "\n" + body + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/InlinedComprehension.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/InlinedComprehension.py new file mode 100644 index 0000000..a7e2b79 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/InlinedComprehension.py @@ -0,0 +1,97 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, node_match_all + + +def is_cleanup(cfg: nx.DiGraph, node: ControlFlowTemplate) -> bool: + insts = node.get_instructions() + if not insts or insts[-1].opname != "RERAISE": + return False + if [i.opname for i in insts[:3]] != ["SWAP", "POP_TOP", "SWAP"]: + return False + return all(i.opname == "STORE_FAST" for i in insts[3:-1]) + + +class InlinedComprehensionTemplate(ControlFlowTemplate): + _subgraph = { + "comp": TemplateNode( + natural_edge=TemplateEdge( + source="comp", + dest="tail", + ), + exception_edge=TemplateEdge( + source="comp", + dest="cleanup", + ), + ), + "cleanup": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), is_cleanup), + exception_edge=TemplateEdge( + source="cleanup", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge(source="tail", dest=None, edge_verification_func=optional_edge), + conditional_edge=TemplateEdge(source="tail", dest=None, edge_verification_func=optional_edge), + exception_edge=TemplateEdge(source="tail", dest="exception_handler", edge_verification_func=optional_edge), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge(source="exception_handler", dest=None, edge_verification_func=optional_edge), + conditional_edge=TemplateEdge(source="exception_handler", dest=None, edge_verification_func=optional_edge), + exception_edge=TemplateEdge(source="exception_handler", dest=None, edge_verification_func=optional_edge), + ), + } + + def __init__(self, comp: ControlFlowTemplate, cleanup: ControlFlowTemplate): + self.comp = comp + self.cleanup = cleanup + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=InlinedComprehensionTemplate._subgraph, root_key="comp", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + template = InlinedComprehensionTemplate(comp=mapping["comp"], cleanup=mapping["cleanup"]) + + in_edges = ((src, template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [(template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})] + if mapping["exception_handler"]: + out_edges.append((template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([template.comp, template.cleanup]) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + return "" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/LoopExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/LoopExitTemplate.py new file mode 100644 index 0000000..cb684c4 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/LoopExitTemplate.py @@ -0,0 +1,54 @@ +import networkx as nx + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..try_except.ExceptAsTemplate import ExceptAsTemplate + +from ...cfg_utils import get_out_edge_dict, ControlFlowEdgeType + + +class LoopExitTemplate(ControlFlowTemplate): + """ + A wrapper for identified break and continue statements. + """ + + def __init__(self, exit_statement: str, tail: ControlFlowTemplate = None): + self.tail = tail + self.exit_statement = exit_statement + assert self.exit_statement in ["break", "continue"] + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + raise NotImplementedError("Loop Exits do not have localized matching logic. These are assigned in refine_loops.") + + @staticmethod + def structure_edge_inplace(cfg: nx.DiGraph, edge: tuple, exit_statment: str) -> None: + src, dst = edge + edge_properties = cfg.get_edge_data(src, dst) + + cfg.remove_edge(src, dst) + # for an unconditional jump, integrate the tail into the exit template + if edge_properties.get("type", None) == ControlFlowEdgeType.JUMP.value: + template = LoopExitTemplate(exit_statement=exit_statment, tail=src) + nx.relabel_nodes(cfg, {src: template}, copy=False) + else: + template = LoopExitTemplate(exit_statement=exit_statment) + cfg.add_edge(src, template, **edge_properties) + src_exception_handler = get_out_edge_dict(cfg, src).get("exception") + if src_exception_handler != (None, None): + cfg.add_edge(template, src_exception_handler[0], type=ControlFlowEdgeType.EXCEPTION.value) + return template + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + tail_source = self.tail.to_indented_source(source_lines) + "\n" if self.tail else "" + + exit_statement = self.exit_statement + if isinstance(self.tail, ExceptAsTemplate): + exit_statement = ControlFlowTemplate._indent_multiline_string(self.exit_statement) + + return tail_source + exit_statement + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/LoopTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/LoopTemplate.py new file mode 100644 index 0000000..446cd8f --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/LoopTemplate.py @@ -0,0 +1,143 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import ControlFlowEdgeType, get_dominator_function + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + +class LoopTemplate(ControlFlowTemplate): + """ + A natural non-infinite loop with no extra control flow. + (0) + | \\ (01) + j| (1) --> | + | (2) + (2) + + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "loop_header": TemplateNode( + natural_edge=TemplateEdge( + source="loop_header", + dest="loop_body", + ), + conditional_edge=TemplateEdge( + source="loop_header", + dest="tail", + ), + exception_edge=TemplateEdge( + source="loop_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "loop_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="loop_body", + dest="loop_header", + ), + exception_edge=TemplateEdge( + source="loop_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, loop_header: ControlFlowTemplate, loop_body: ControlFlowTemplate): + self.loop_header = loop_header + self.loop_body = loop_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + def verify_tail_not_in_loop(cfg: nx.DiGraph, mapping: dict) -> bool: + dominates = get_dominator_function(cfg) + # subgraph containing all nodes dominated by the loop header + dominated_subgraph: nx.DiGraph = cfg.subgraph(n for n in cfg.nodes if dominates(mapping["loop_header"], n)) + reverse_reachability_map = nx.single_source_shortest_path_length(dominated_subgraph.reverse(), source=mapping["loop_header"]) + # a node is in the loop if there is a backwards path to the header that doesn't leave the loop + loop_nodes = [loop_node for loop_node, distance in reverse_reachability_map.items() if distance >= 0] + return mapping["tail"] not in loop_nodes + + matcher = GraphTemplateMatcher(template_node_dict=LoopTemplate._subgraph, root_key="loop_header", mapping_verification_func=verify_tail_not_in_loop) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + loop_template = LoopTemplate(loop_header=mapping["loop_header"], loop_body=mapping["loop_body"]) + + in_edges = ((src, loop_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True) if src != mapping["loop_body"]) + out_edges = [(loop_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})] + if mapping["exception_handler"]: + out_edges.append((loop_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([loop_template.loop_header, loop_template.loop_body]) + reduced_cfg.add_node(loop_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.loop_header.to_indented_source(source_lines) + body = ControlFlowTemplate._indent_multiline_string(self.loop_body.to_indented_source(source_lines)) + return "\n".join([header, body]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/PreRefinedLoopTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/PreRefinedLoopTemplate.py new file mode 100644 index 0000000..073d81c --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/PreRefinedLoopTemplate.py @@ -0,0 +1,57 @@ +import networkx as nx + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import get_out_edge_dict + +from ..placeholders.WhileTruePlaceholderTemplate import WhileTruePlaceholderTemplate + + +class PreRefinedLoopTemplate(ControlFlowTemplate): + """ + Matches a loop header for an unrefined loop containing breaks and continues. + Results in a RefinedLoopTemplate header and replaces all breaks and continues with LoopExitTemplates + """ + + def __init__(self, loop_header: ControlFlowTemplate, loop_else: ControlFlowTemplate): + self.loop_header = loop_header + self.loop_else = loop_else + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + raise NotImplementedError("PreRefinedLoopTemplate does not have local matching logic. These are created in refine_loop") + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + return self.loop_header.to_indented_source(source_lines) + + @staticmethod + def structure_nodes_inplace(cfg: nx.DiGraph, loop_header, canonical_loop_exit, loop_successor): + if not canonical_loop_exit: + # while true; use a placeholder that makes the while true "look like" a normal loop + loop_header = WhileTruePlaceholderTemplate.structure_node_inplace(cfg, loop_header, loop_successor) + loop_template = PreRefinedLoopTemplate(loop_header=loop_header, loop_else=None) + if canonical_loop_exit != loop_successor: + loop_template = PreRefinedLoopTemplate(loop_header=loop_header, loop_else=canonical_loop_exit) + else: + loop_template = PreRefinedLoopTemplate(loop_header=loop_header, loop_else=None) + + in_edges = ((src, loop_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=loop_header, data=True)) + out_edges = [(loop_template, loop_successor if dst == canonical_loop_exit else dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(nbunch=loop_header, data=True)] + + loop_header_out_dict = get_out_edge_dict(cfg, loop_header) + exception_target, edge_type = loop_header_out_dict["exception"] + if exception_target: + out_edges.append((loop_template, exception_target, edge_type)) + + cfg.remove_node(loop_template.loop_header) + if loop_template.loop_else: + cfg.remove_node(loop_template.loop_else) + cfg.add_node(loop_template) + cfg.add_edges_from(in_edges) + cfg.add_edges_from(out_edges) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/RefinedLoopTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/RefinedLoopTemplate.py new file mode 100644 index 0000000..cd8bb77 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/RefinedLoopTemplate.py @@ -0,0 +1,147 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from .PreRefinedLoopTemplate import PreRefinedLoopTemplate +from ..placeholders.WhileTruePlaceholderTemplate import WhileTruePlaceholderTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + +class RefinedLoopTemplate(ControlFlowTemplate): + """ + The second stage of matching loops with breaks an continues; matches fully-structured PreRefinedLoopTemplates. + (0) = PreRefinedLoopTemplate + // \\j --> (01) + (1) (2) | + (2) + + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "pre_refined_loop": TemplateNode( + natural_edge=TemplateEdge( + source="pre_refined_loop", + dest="loop_body", + ), + conditional_edge=TemplateEdge(source="pre_refined_loop", dest="loop_successor", edge_verification_func=optional_edge), + exception_edge=TemplateEdge( + source="pre_refined_loop", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "loop_body": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="loop_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "loop_successor": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="loop_successor", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="loop_successor", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="loop_successor", + dest="exception_handler", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, loop_header: ControlFlowTemplate, loop_body: ControlFlowTemplate, loop_else: ControlFlowTemplate, has_successor: bool = True): + self.loop_header = loop_header + self.loop_body = loop_body + self.loop_else = loop_else + self.has_successor = has_successor + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + # this pattern is only for matching on PreRefinedLoops + if not isinstance(node, PreRefinedLoopTemplate): + return None + + matcher = GraphTemplateMatcher(template_node_dict=RefinedLoopTemplate._subgraph, root_key="pre_refined_loop", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + loop_template = RefinedLoopTemplate(loop_header=mapping["pre_refined_loop"].loop_header, loop_body=mapping["loop_body"], loop_else=mapping["pre_refined_loop"].loop_else, has_successor=bool(mapping["loop_successor"])) + + in_edges = ((src, loop_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [] + if mapping["loop_successor"]: + out_edges.append((loop_template, mapping["loop_successor"], {"type": ControlFlowEdgeType.NATURAL.value})) + if mapping["exception_handler"]: + out_edges.append((loop_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([mapping["pre_refined_loop"], loop_template.loop_body]) + reduced_cfg.add_node(loop_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + loop_lines = [] + header = self.loop_header.to_indented_source(source_lines) + if not self.has_successor and not isinstance(self.loop_header, WhileTruePlaceholderTemplate): + header = ControlFlowTemplate._indent_multiline_string(header) + loop_lines.append("while True: # inserted") + loop_body = ControlFlowTemplate._indent_multiline_string(self.loop_body.to_indented_source(source_lines)) + loop_lines.extend([header, loop_body]) + if self.loop_else: + loop_else = ControlFlowTemplate._indent_multiline_string(self.loop_else.to_indented_source(source_lines)) + loop_lines.extend(["else: # inserted", loop_else]) + + return "\n".join(loop_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/SelfLoopTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/SelfLoopTemplate.py new file mode 100644 index 0000000..18ef05b --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/SelfLoopTemplate.py @@ -0,0 +1,85 @@ +import networkx as nx + + +from ..abstract.AbstractTemplate import ControlFlowTemplate + + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge + + +class SelfLoopTemplate(ControlFlowTemplate): + """ + An infinite loop with no extra control flow. + (0)-< --> (0) + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "loop_body": TemplateNode( + natural_edge=TemplateEdge( + source="loop_body", + dest="loop_body", + ), + exception_edge=TemplateEdge( + source="loop_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, loop_body: ControlFlowTemplate): + self.loop_body = loop_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=SelfLoopTemplate._subgraph, root_key="loop_body", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + loop_template = SelfLoopTemplate(loop_body=mapping["loop_body"]) + + reduced_cfg: nx.DiGraph = nx.relabel_nodes(cfg, {mapping["loop_body"]: loop_template}) + reduced_cfg.remove_edge(loop_template, loop_template) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + body = ControlFlowTemplate._indent_multiline_string(self.loop_body.to_indented_source(source_lines)) + return f"while True: # inserted\n{body}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/loop/WhileTrueIfElseTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/loop/WhileTrueIfElseTemplate.py new file mode 100644 index 0000000..cc4bcb8 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/loop/WhileTrueIfElseTemplate.py @@ -0,0 +1,135 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..if_then.IfElseTemplate import IfElseTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree + + +class WhileTrueIfElseTemplate(ControlFlowTemplate): + """ + A while true that contains in if-else statement at the top level. + (0) + j| \\ + (2) (1) --> (012) + + nodes 1 and 2 have a backwards unconditional jump to 0 + optionally, all nodes in the pattern can have a shared exception handler. + """ + + _subgraph = { + "loop_header": TemplateNode( + natural_edge=TemplateEdge( + source="loop_header", + dest="if_body", + ), + conditional_edge=TemplateEdge( + source="loop_header", + dest="else_body", + ), + exception_edge=TemplateEdge( + source="loop_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "if_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="if_body", + dest="loop_header", + ), + exception_edge=TemplateEdge( + source="if_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "else_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="else_body", + dest="loop_header", + ), + exception_edge=TemplateEdge( + source="else_body", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, loop_header: ControlFlowTemplate, if_body: ControlFlowTemplate, else_body: ControlFlowTemplate): + self.loop_header = loop_header + self.if_body = if_body + self.else_body = else_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=WhileTrueIfElseTemplate._subgraph, root_key="loop_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + loop_template = WhileTrueIfElseTemplate( + loop_header=mapping["loop_header"], + if_body=mapping["if_body"], + else_body=mapping["else_body"], + ) + + in_edges = ((src, loop_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True) if src != mapping["if_body"] and src != mapping["else_body"]) + out_edges = [] + if mapping["exception_handler"]: + out_edges.append((loop_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([loop_template.loop_header, loop_template.if_body, loop_template.else_body]) + reduced_cfg.add_node(loop_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + + if_else_template = IfElseTemplate(if_header=self.loop_header, if_body=self.if_body, else_body=self.else_body) + body = ControlFlowTemplate._indent_multiline_string(if_else_template.to_indented_source(source_lines)) + return "\n".join(["while True: # inserted", body]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/match_utils.py b/pylingual/control_flow_reconstruction/control_flow_templates/match_utils.py new file mode 100644 index 0000000..bc06f7f --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/match_utils.py @@ -0,0 +1,232 @@ +import networkx as nx + +import itertools +import collections + +from ..cfg_utils import ControlFlowEdgeType, get_dominator_function +from .natural.InstructionTemplate import InstructionTemplate + +from .abstract.AbstractTemplate import ControlFlowTemplate +from .natural.LinearSequenceTemplate import LinearSequenceTemplate + +from typing import Callable, Any + +# common node/edge/mapping verification functions and factories + + +def assert_edge_type(*edge_types: ControlFlowEdgeType) -> Callable[[Any, Any, dict], bool]: + def initialized_assert_edge_type(graph_source, graph_dest, graph_edge_properties: dict) -> bool: + if graph_edge_properties is None: + return False + return graph_edge_properties.get("type", None) in [edge_type.value for edge_type in edge_types] + + return initialized_assert_edge_type + + +def assert_node_type(*node_types: type) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_assert_node_type(cfg: nx.DiGraph, node) -> bool: + return any(isinstance(node, node_type) for node_type in node_types) + + return initialized_assert_node_type + + +def assert_in_degree(in_degree: int) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_assert_in_degree(cfg: nx.DiGraph, node) -> bool: + if node is None: + return False + return cfg.in_degree(node) == in_degree + + return initialized_assert_in_degree + + +def assert_instruction_opname(*opnames: str) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_assert_instruction_opname(cfg: nx.DiGraph, node) -> bool: + if node is None: + return False + + if isinstance(node, LinearSequenceTemplate): + candidate = node.members[-1] + else: + candidate = node + + if not isinstance(candidate, InstructionTemplate): + return False + return candidate.instruction.opname in opnames + + return initialized_assert_instruction_opname + + +def assert_first_instruction_opname(*opnames: str) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_assert_first_instruction_opname(cfg: nx.DiGraph, node) -> bool: + """ + if node is None: + return False + + if isinstance(node, LinearSequenceTemplate): + candidate = node.members[0] + else: + candidate = node + + if not isinstance(candidate, InstructionTemplate): + return False + return candidate.instruction.opname in opnames + """ + i = node.get_instructions() + return i and i[0].opname in opnames + + return initialized_assert_first_instruction_opname + + +def assert_unconditional_jump(cfg: nx.DiGraph, node) -> bool: + if not isinstance(node, InstructionTemplate): + return False + return node.instruction.is_uncond_jump + + +def optional_node(cfg, node) -> bool: + # returns true even when node is None! + # overrides default behavior of checking if the node exists + return True + + +def optional_edge(graph_source, graph_dest, graph_edge_properties: dict) -> bool: + # returns true even when the edge is None! + # overrides default behavior of checking if the edge exists + return True + + +def edge_is_none_or_matches(verification_func: Callable[[nx.DiGraph, Any], bool]) -> Callable[[Any, Any, dict], bool]: + def initialized_edge_is_none_or_matches(graph_source, graph_dest, graph_edge_properties: dict) -> bool: + return graph_dest is None or verification_func(graph_source, graph_dest, graph_edge_properties) + + return initialized_edge_is_none_or_matches + + +def node_is_none_or_matches(verification_func: Callable[[nx.DiGraph, Any], bool]) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_node_is_none_or_matches(cfg: nx.DiGraph, node) -> bool: + return node is None or verification_func(cfg, node) + + return initialized_node_is_none_or_matches + + +def node_match_all(*verification_funcs: Callable[[nx.DiGraph, Any], bool]) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_match_all(cfg: nx.DiGraph, node) -> bool: + return all(f(cfg, node) for f in verification_funcs) + + return initialized_match_all + + +def node_match_none(*verification_funcs: Callable[[nx.DiGraph, Any], bool]) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_match_all(cfg: nx.DiGraph, node) -> bool: + return not any(f(cfg, node) for f in verification_funcs) + + return initialized_match_all + + +def node_match_any(*verification_funcs: Callable[[nx.DiGraph, Any], bool]) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_match_any(cfg: nx.DiGraph, node) -> bool: + return any(f(cfg, node) for f in verification_funcs) + + return initialized_match_any + + +def assert_no_linestarts(cfg: nx.DiGraph, node: ControlFlowTemplate) -> bool: + return not any(inst.starts_line for inst in node.get_instructions()) + + +def contains_opname_sequence(*opnames: str) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_contains_opname_sequence(cfg: nx.DiGraph, node: ControlFlowTemplate) -> bool: + for window in sliding_window(node.get_instructions(), n=len(opnames)): + if tuple(inst.opname for inst in window) == opnames: + return True + return False + + return initialized_contains_opname_sequence + + +def starts_with_opname_sequence(*opnames: str) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_starts_with_opname_sequence(cfg: nx.DiGraph, node: ControlFlowTemplate) -> bool: + i = node.get_instructions() + return len(i) >= len(opnames) and tuple(x.opname for x in i[: len(opnames)]) == opnames + + return initialized_starts_with_opname_sequence + + +def ends_with_opname_sequence(*opnames: str) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_ends_with_opname_sequence(cfg: nx.DiGraph, node: ControlFlowTemplate) -> bool: + i = node.get_instructions() + return len(i) >= len(opnames) and tuple(x.opname for x in i[-len(opnames) :]) == opnames + + return initialized_ends_with_opname_sequence + + +def is_exactly_opname(*opnames: str) -> Callable[[nx.DiGraph, Any], bool]: + def initialized_is_exactly_opname(cfg: nx.DiGraph, node: ControlFlowTemplate) -> bool: + return isinstance(node, ControlFlowTemplate) and tuple(x.opname for x in node.get_instructions()) == opnames + + return initialized_is_exactly_opname + + +def assert_node_has_no_backwards_edges(cfg, node) -> bool: + dominates = get_dominator_function(cfg) + return not any(dominates(successor, node) for successor in cfg.successors(node)) + + +def assert_except_as(cfg: nx.DiGraph, node: ControlFlowTemplate) -> bool: + # specialized node verification function for the header + # the header must be a LinearSequence where the last instruction is JUMP_IF_NOT_EXC_MATCH + # this instruction is used *exclusively* for except-as constructions in pre-3.11 bytecode + # this rule only applies to versions 3.9 and 3.10 + if not isinstance(node, LinearSequenceTemplate): + return False + + # version 3.9-3.10 + exc_match_member = node.members[-1] + if not isinstance(exc_match_member, InstructionTemplate): + return False + if exc_match_member.instruction.opname == "JUMP_IF_NOT_EXC_MATCH": + return True + + # so we dont throw errors + + if len(node.members) < 2: + return False + + # pre-3.9 + exc_match_member = node.members[-2] + if not isinstance(exc_match_member, InstructionTemplate): + return False + if exc_match_member.instruction.opname == "COMPARE_OP" and exc_match_member.instruction.argval == "exception-match": + return True + + # 3.11 + if exc_match_member.instruction.opname == "CHECK_EXC_MATCH": + return True + return False + + +def assert_with(cfg: nx.DiGraph, node: ControlFlowTemplate) -> bool: + # these statements begin in a linear sequence template + # so if the node is not in a linear sequence template then this is not + # a with statement + + if not isinstance(node, LinearSequenceTemplate): + return False + + # designed for version 3.8 might be different for other versions + with_match_member = node.members[-1] # get the last element that should be a SETUP_WITH + if not isinstance(with_match_member, InstructionTemplate): + return False + if with_match_member.instruction.opname in ("SETUP_WITH", "SETUP_ASYNC_WITH", "BEFORE_WITH", "END_SEND"): + return True + + +# iteration helper +def sliding_window(iterable, n): + "Collect data into overlapping fixed-length chunks or blocks." + # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG + it = iter(iterable) + window = collections.deque(itertools.islice(it, n - 1), maxlen=n) + for x in it: + window.append(x) + yield tuple(window) diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/natural/InstructionTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/natural/InstructionTemplate.py new file mode 100644 index 0000000..d5d636b --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/natural/InstructionTemplate.py @@ -0,0 +1,69 @@ +import networkx as nx + +from pylingual.editable_bytecode import Inst + +from ..abstract.AbstractTemplate import ControlFlowTemplate + + +class InstructionTemplate(ControlFlowTemplate): + """ + A thin wrapper around the Inst class to support formatting source code + """ + + def __init__(self, instruction: Inst): + self.instruction = instruction + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if not isinstance(node, Inst): + return None + + if node not in cfg.nodes: + return None + + inst_template = InstructionTemplate(node) + return nx.relabel_nodes(cfg, mapping={node: inst_template}, copy=True) + + @staticmethod + def match_graph(cfg: nx.DiGraph) -> nx.DiGraph: + """ + Attempts to match this template on the whole graph + Returns an updated cfg with the appropriate nodes condensed into an instance of this template. + """ + node_mapping = dict() + for node in cfg.nodes: + if not isinstance(node, Inst): + continue + + inst_template = InstructionTemplate(node) + node_mapping[node] = inst_template + + return nx.relabel_nodes(cfg, mapping=node_mapping, copy=True) + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + if not self.instruction.starts_line: + return "" + + line = source_lines[self.instruction.starts_line - 1].strip() + if line.startswith("elif "): + line = line[2:] + elif line in ("break", "continue", "except:", "try:"): + line = "" + + return line + + def get_instructions(self) -> list[Inst]: + return [self.instruction] + + def __repr__(self) -> str: + if self.instruction.starts_line: + return f"({self.instruction.starts_line}) <{self.instruction.get_dis_view()}>" + return f"<{self.instruction.get_dis_view()}>" diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/natural/LineTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/natural/LineTemplate.py new file mode 100644 index 0000000..c1000a7 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/natural/LineTemplate.py @@ -0,0 +1,37 @@ +import networkx as nx + +from pylingual.editable_bytecode import Inst + +from ..abstract.AbstractTemplate import ControlFlowTemplate + + +class LineTemplate(ControlFlowTemplate): + """ + A natural progression of control flow templates with the same exception handler. + No conditional jumps are allowed. + """ + + def __init__(self, *members: ControlFlowTemplate): + self.members = members + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + raise NotImplementedError("LineTemplates do not have local matching logic.") + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + return "\n".join(member.to_indented_source(source_lines) for member in self.members) + + def get_instructions(self) -> list[Inst]: + insts: list[Inst] = [] + for member in self.members: + insts.extend(member.get_instructions()) + return insts + return sorted(insts, key=lambda i: i.offset) + + def __repr__(self) -> str: + name = f"{type(self).__name__}" + components = ControlFlowTemplate._indent_multiline_string("\n".join(repr(member) for member in self.members)) + return f"{name}[\n{components}]" diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/natural/LinearSequenceTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/natural/LinearSequenceTemplate.py new file mode 100644 index 0000000..7075cb9 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/natural/LinearSequenceTemplate.py @@ -0,0 +1,127 @@ +import networkx as nx + +import itertools + +from pylingual.editable_bytecode import Inst + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +# imports for our exception whitelist so we do not have to absorb any tails and affect +# control flow in the future (hopefully fingers crossed) +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..loop.PreRefinedLoopTemplate import PreRefinedLoopTemplate + +from ...cfg_utils import get_out_edge_dict, ControlFlowEdgeType, get_dominator_function + + +class LinearSequenceTemplate(ControlFlowTemplate): + """ + A natural progression of control flow templates with the same exception handler. + No conditional jumps are allowed. + """ + + def __init__(self, *members: ControlFlowTemplate): + self.members = members + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes or not isinstance(node, ControlFlowTemplate): + return None + + dominates = get_dominator_function(cfg) + + def is_back_edge(src, dst): + return dominates(dst, src) + + base_edge_dict = get_out_edge_dict(cfg, node) + base_exception_handler = base_edge_dict["exception"] + + # validate that the current node is able to start a linear sequence + # jumps cannot start a linear sequence + if base_edge_dict["conditional"][0] or (base_edge_dict["natural"][1] and base_edge_dict["natural"][1]["type"] != ControlFlowEdgeType.NATURAL.value): + return None + # back edges cannot start a linear sequence + if any(is_back_edge(*edge) for edge in cfg.out_edges(node)): + return None + + matched_sequence = [node] + current_edge_dict = base_edge_dict + + # while there is a natural progression, try to extend the linear sequence + while (next_node_and_edge_properties := current_edge_dict["natural"])[0]: + next_node, _ = next_node_and_edge_properties + next_edge_dict = get_out_edge_dict(cfg, next_node) + + # all elements of a linear sequence must have the same exception handler + if next_edge_dict["exception"] != base_exception_handler: + break + + # only the natural incoming edge from the previous node is allowed in linear sequences + if cfg.in_degree(nbunch=next_node) > 1: + break + + # do not extend after an END_FINALLY + if isinstance(matched_sequence[-1], ControlFlowTemplate) and not isinstance(matched_sequence[-1], AbstractNonSequentiable): + insts = matched_sequence[-1].get_instructions() + if insts and insts[-1].opname == "END_FINALLY": + break + + # do not merge in prerefined loop templates; they still need to be refined + if isinstance(next_node, PreRefinedLoopTemplate): + break + + # conditional jumps are only allowed in the last element of a linear sequence + if current_edge_dict["conditional"][0] and current_edge_dict["natural"][0]: + break + + # absolute jumps are only allowed in the last element of a linear sequence + if current_edge_dict["natural"][1] and current_edge_dict["natural"][1]["type"] != ControlFlowEdgeType.NATURAL.value: + break + + matched_sequence.append(next_node) + current_edge_dict = next_edge_dict + + # if we didn't reduce the graph size, match failed + if len(matched_sequence) < 2: + return None + + # unpack nested LinearSequenceTemplates for improved readability of the parse tree + unpacked_matched_sequence = [] + for match_item in matched_sequence: + if isinstance(match_item, LinearSequenceTemplate): + unpacked_matched_sequence.extend(match_item.members) + else: + unpacked_matched_sequence.append(match_item) + # preserve the incoming edges from the first node and the outgoing edges from the last node + linear_sequence_template = LinearSequenceTemplate(*unpacked_matched_sequence) + in_edges = ((linear_sequence_template if src in matched_sequence else src, linear_sequence_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = ((linear_sequence_template, linear_sequence_template if dst in matched_sequence else dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(nbunch=matched_sequence[-1], data=True)) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from(matched_sequence) + reduced_cfg.add_node(linear_sequence_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + return "\n".join(member.to_indented_source(source_lines) for member in self.members) + + def get_instructions(self) -> list[Inst]: + insts: list[Inst] = [] + for member in self.members: + insts.extend(member.get_instructions()) + return insts + return sorted(insts, key=lambda i: i.offset) + + def __repr__(self) -> str: + name = f"{type(self).__name__}" + components = ControlFlowTemplate._indent_multiline_string("\n".join(repr(member) for member in self.members)) + return f"{name}[\n{components}]" diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/ExceptPlaceholderTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/ExceptPlaceholderTemplate.py new file mode 100644 index 0000000..e4f3b47 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/ExceptPlaceholderTemplate.py @@ -0,0 +1,30 @@ +import networkx as nx + + +from ..abstract.AbstractTemplate import ControlFlowTemplate + + + + + +class ExceptPlaceholderTemplate(ControlFlowTemplate): + """ + Placeholder for except; used in ExceptAs.py + """ + + def __init__(self, body: ControlFlowTemplate): + self.body = body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + raise NotImplementedError("ExceptPlaceholderTemplate does not have local matching logic. These are created in ExceptAs") + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + body = ControlFlowTemplate._indent_multiline_string(self.body.to_indented_source(source_lines)) + return f"except:\n{body}" + + def __repr__(self) -> str: + return super().__repr__() if self.body else type(self).__name__ diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/IrreduciblePlaceholderTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/IrreduciblePlaceholderTemplate.py new file mode 100644 index 0000000..0edc9c2 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/IrreduciblePlaceholderTemplate.py @@ -0,0 +1,9 @@ +from ..abstract.AbstractTemplate import ControlFlowTemplate + + +class IrreduciblePlaceholderTemplate(ControlFlowTemplate): + def __init__(self, msg): + self.msg = msg + + def to_indented_source(self, source_lines: list[str]) -> str: + return f"pass # cflow: {self.msg}" diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/WhileTruePlaceholderTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/WhileTruePlaceholderTemplate.py new file mode 100644 index 0000000..0e85f0c --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/placeholders/WhileTruePlaceholderTemplate.py @@ -0,0 +1,42 @@ +import networkx as nx + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import ControlFlowEdgeType + + +class WhileTruePlaceholderTemplate(ControlFlowTemplate): + """ + Placeholder for While True; used in PreRefinedLoopTemplate + """ + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + raise NotImplementedError("WhileTruePlaceholderTemplate does not have local matching logic. These are created in PreRefinedLoopTemplate") + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + return "while True: # inserted" + + @staticmethod + def structure_node_inplace(cfg: nx.DiGraph, loop_header, loop_successor): + # insert a WhileTruePlaceholderTemplate before the loop_header, and add a conditional edge to the loop successor + # this "looks like" a normal while loop, which allows structuring to continue + placeholder = WhileTruePlaceholderTemplate() + + # replace the incoming edges + in_edges = [(src, placeholder, data) for src, _, data in cfg.in_edges(loop_header, data=True)] + cfg.add_edges_from(in_edges) + cfg.remove_edges_from(list(cfg.in_edges(loop_header))) + + # add outgoing edges to the placeholder + cfg.add_edge(placeholder, loop_header, type=ControlFlowEdgeType.NATURAL.value) + if loop_successor: + cfg.add_edge(placeholder, loop_successor, type=ControlFlowEdgeType.FALSE_JUMP.value) + + return placeholder + + def __repr__(self) -> str: + return type(self).__name__ diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/subtemplates/OptionalExitSubtemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/subtemplates/OptionalExitSubtemplate.py new file mode 100644 index 0000000..22c3fae --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/subtemplates/OptionalExitSubtemplate.py @@ -0,0 +1,130 @@ +import networkx as nx + +import itertools + + +from ..abstract.AbstractTemplate import ControlFlowTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import assert_edge_type, optional_node, optional_edge, assert_in_degree, node_is_none_or_matches, edge_is_none_or_matches + + +class ExitSubTemplate(ControlFlowTemplate): + """ + + A basic with template as a catch all for exits + first case + + (1) or simply (1) + e/ \ + (2) + + + """ + + _subgraph = { + "exit_header": TemplateNode( + natural_edge=TemplateEdge(source="exit_header", dest="exit_flow", edge_verification_func=edge_is_none_or_matches(assert_edge_type(ControlFlowEdgeType.NATURAL))), + exception_edge=TemplateEdge( + source="exit_header", + dest="exception_handler", + edge_verification_func=optional_edge, + ), + ), + "exit_flow": TemplateNode(node_verification_func=node_is_none_or_matches(assert_in_degree(1)), exception_edge=TemplateEdge(source="exit_flow", dest="outer_exception_handler", edge_verification_func=optional_edge)), + "exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, exit_header: ControlFlowTemplate, exit_flow: ControlFlowTemplate): + self.exit_header = exit_header + self.exit_flow = exit_flow + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template or we are happy and return the base cfg. + Otherwise, returns None. + """ + + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher( + template_node_dict=ExitSubTemplate._subgraph, + root_key="exit_header", + mapping_verification_func=None, + ) + + mapping = matcher.match_at_graph_node(cfg, node) + + # not standard; if we didn't match the exit, then continue matching the rest of the parent template + if not mapping: + return cfg + + # this is an appropriate match, but there is nothing to do + if not mapping["exit_flow"]: + return cfg + + exit_template = ExitSubTemplate( + exit_header=mapping["exit_header"], + exit_flow=mapping["exit_flow"], + ) + + in_edges = ((src, exit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = [] + + if mapping["exception_handler"]: + out_edges.append((exit_template, mapping["exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([exit_template.exit_flow, exit_template.exit_header]) + reduced_cfg.add_node(exit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + header = self.exit_header.to_indented_source(source_lines) + exit_flow = self.exit_flow.to_indented_source(source_lines) + return "\n".join([header, exit_flow]) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsCleanup.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsCleanup.py new file mode 100644 index 0000000..a39affc --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsCleanup.py @@ -0,0 +1,139 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import assert_edge_type, optional_node, optional_edge, assert_in_degree, node_match_all, node_match_any, contains_opname_sequence, edge_is_none_or_matches + + +class ExceptAsCleanupTemplate(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + The boilerplate cleanup at the end of an `except as` block. + The "happy cleanup" (1) is when there is no exception, and it jumps out to the next code segment. + The "angry cleanup" (2) is when there is an exception, and it reraises. + (0) + / \\e --> (012) + (1) (2) |j + |j (3) + (3) + """ + + _subgraph = { + "except_body": TemplateNode( + natural_edge=TemplateEdge( + source="except_body", + dest="happy_cleanup", + ), + exception_edge=TemplateEdge( + source="except_body", + dest="angry_cleanup", + ), + ), + "happy_cleanup": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + node_match_any( + contains_opname_sequence( + "LOAD_CONST", + "STORE_NAME", + "DELETE_NAME", + ), + contains_opname_sequence( + "LOAD_CONST", + "STORE_FAST", + "DELETE_FAST", + ), + ), + ), + natural_edge=TemplateEdge(source="happy_cleanup", dest=None, edge_verification_func=edge_is_none_or_matches(assert_edge_type(ControlFlowEdgeType.JUMP))), + exception_edge=TemplateEdge( + source="happy_cleanup", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "angry_cleanup": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + node_match_any( + contains_opname_sequence("LOAD_CONST", "STORE_NAME", "DELETE_NAME", "RERAISE"), + contains_opname_sequence("LOAD_CONST", "STORE_FAST", "DELETE_FAST", "RERAISE"), + ), + ), + exception_edge=TemplateEdge( + source="angry_cleanup", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_body: ControlFlowTemplate, happy_cleanup: ControlFlowTemplate, angry_cleanup: ControlFlowTemplate): + self.except_body = except_body + self.happy_cleanup = happy_cleanup + self.angry_cleanup = angry_cleanup + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsCleanupTemplate._subgraph, root_key="except_body", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + except_as_cleanup_template = ExceptAsCleanupTemplate(except_body=mapping["except_body"], happy_cleanup=mapping.get("happy_cleanup", None), angry_cleanup=mapping.get("angry_cleanup", None)) + + in_edges = ((src, except_as_cleanup_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + out_edges = [(except_as_cleanup_template, dst, data) for _, dst, data in cfg.out_edges([except_as_cleanup_template.happy_cleanup, except_as_cleanup_template.angry_cleanup], data=True)] + if mapping["outer_exception_handler"]: + out_edges.append((except_as_cleanup_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([except_as_cleanup_template.except_body, except_as_cleanup_template.happy_cleanup, except_as_cleanup_template.angry_cleanup]) + reduced_cfg.add_node(except_as_cleanup_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + # cleanup code is implicit! only report the body code + return self.except_body.to_indented_source(source_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsExceptTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsExceptTemplate.py new file mode 100644 index 0000000..1f8e0af --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsExceptTemplate.py @@ -0,0 +1,154 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_except_as + +from ..placeholders.ExceptPlaceholderTemplate import ExceptPlaceholderTemplate +from .ExceptAsTemplate import ExceptAsTemplate + + +class ExceptAsExceptTemplate(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + An `except as` block, after its cleanup has been structured. + If there are multiple, this will match the last block in the series and set up the next one to be matched + (0) + / \\j --> (012) + (1) (2) |j + \\j //j (3) + (3) + """ + + _subgraph = { + "except_as_header": TemplateNode( + node_verification_func=assert_except_as, + natural_edge=TemplateEdge( + source="except_as_header", + dest="except_body", + ), + conditional_edge=TemplateEdge(source="except_as_header", dest="non_match_path"), + exception_edge=TemplateEdge(source="except_as_header", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="except_body", + dest="after_except", + ), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "non_match_path": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="non_match_path", + dest="after_except", + ), + exception_edge=TemplateEdge( + source="non_match_path", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_as_header: ControlFlowTemplate, except_body: ControlFlowTemplate, non_match_path: ControlFlowTemplate): + self.except_as_header = except_as_header + self.except_body = except_body + self.non_match_path = non_match_path + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsExceptTemplate._subgraph, root_key="except_as_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + non_match_path = mapping["non_match_path"] + if not isinstance(non_match_path, ExceptAsExceptTemplate) and not isinstance(non_match_path, ExceptAsTemplate): + non_match_path = ExceptPlaceholderTemplate(body=non_match_path) + + except_as_cleanup_template = ExceptAsExceptTemplate(except_as_header=mapping["except_as_header"], except_body=mapping["except_body"], non_match_path=non_match_path) + + in_edges = ((src, except_as_cleanup_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((except_as_cleanup_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping["after_except"]: + out_edges.append((except_as_cleanup_template, mapping["after_except"], {"type": ControlFlowEdgeType.JUMP.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([except_as_cleanup_template.except_as_header, except_as_cleanup_template.except_body, mapping["non_match_path"]]) + reduced_cfg.add_node(except_as_cleanup_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.except_as_header.to_indented_source(source_lines).rstrip() + body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)).rstrip() + non_match = self.non_match_path.to_indented_source(source_lines).rstrip() + return f"{header}\n{body}\n{non_match}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsExitTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsExitTemplate.py new file mode 100644 index 0000000..449a9e5 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsExitTemplate.py @@ -0,0 +1,159 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_except_as, node_match_all, node_match_any, contains_opname_sequence + + + +class ExceptAsExitTemplate(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + An `except as` block, but with an exit statement. + If there are multiple, this will match the last block in the series and set up the next one to be matched + (0) + / \\j --> (01234) + (1) (2) + | + (3) + |e + (4) + """ + + _subgraph = { + "except_as_header": TemplateNode( + node_verification_func=assert_except_as, + natural_edge=TemplateEdge( + source="except_as_header", + dest="except_body_setup", + ), + conditional_edge=TemplateEdge(source="except_as_header", dest="non_match_path"), + exception_edge=TemplateEdge(source="except_as_header", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "except_body_setup": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="except_body_setup", dest="except_body", edge_verification_func=optional_edge), + exception_edge=TemplateEdge( + source="except_body_setup", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="except_body", + dest="except_as_cleanup", + edge_verification_func=optional_edge, + ), + ), + "except_as_cleanup": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + node_match_any( + contains_opname_sequence("LOAD_CONST", "STORE_NAME", "DELETE_NAME", "RERAISE"), + contains_opname_sequence("LOAD_CONST", "STORE_FAST", "DELETE_FAST", "RERAISE"), + contains_opname_sequence("LOAD_CONST", "STORE_NAME", "DELETE_NAME", "END_FINALLY"), + contains_opname_sequence("LOAD_CONST", "STORE_FAST", "DELETE_FAST", "END_FINALLY"), + ), + ), + exception_edge=TemplateEdge( + source="except_as_cleanup", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "non_match_path": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="non_match_path", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="non_match_path", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="non_match_path", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_as_header: ControlFlowTemplate, except_body_setup: ControlFlowTemplate, except_body: ControlFlowTemplate, except_as_cleanup: ControlFlowTemplate, non_match_path: ControlFlowTemplate): + self.except_as_header = except_as_header + self.except_body_setup = except_body_setup + self.except_body = except_body + self.except_as_cleanup = except_as_cleanup + self.non_match_path = non_match_path + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsExitTemplate._subgraph, root_key="except_as_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + except_as_exit_template = ExceptAsExitTemplate( + except_as_header=mapping["except_as_header"], except_body_setup=mapping["except_body_setup"], except_body=mapping["except_body"], except_as_cleanup=mapping["except_as_cleanup"], non_match_path=mapping["non_match_path"] + ) + + in_edges = ((src, except_as_exit_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + out_edges = ((except_as_exit_template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(except_as_exit_template.non_match_path, data=True)) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from( + [except_as_exit_template.except_as_header, except_as_exit_template.except_body_setup, except_as_exit_template.except_body, except_as_exit_template.except_as_cleanup, except_as_exit_template.non_match_path] + ) + reduced_cfg.add_node(except_as_exit_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.except_as_header.to_indented_source(source_lines) + self.except_body_setup.to_indented_source(source_lines) + body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + non_match = self.non_match_path.to_indented_source(source_lines) + return f"{header}\n{body}\n{non_match}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsTemplate.py new file mode 100644 index 0000000..87d8442 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptAsTemplate.py @@ -0,0 +1,141 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_except_as + + + +class ExceptAsTemplate(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + An `except as` block, after its cleanup has been structured. + If there are multiple, this will match the last block in the series and set up the next one to be matched + (0) + / \\j --> (012) + (1) (2) |j + |j (3) + (3) + """ + + _subgraph = { + "except_as_header": TemplateNode( + node_verification_func=assert_except_as, + natural_edge=TemplateEdge( + source="except_as_header", + dest="except_body", + ), + conditional_edge=TemplateEdge(source="except_as_header", dest="non_match_path"), + exception_edge=TemplateEdge(source="except_as_header", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="except_body", dest="after_except", edge_verification_func=optional_edge), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "non_match_path": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="non_match_path", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_as_header: ControlFlowTemplate, except_body: ControlFlowTemplate, non_match_path: ControlFlowTemplate): + self.except_as_header = except_as_header + self.except_body = except_body + self.non_match_path = non_match_path + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsTemplate._subgraph, root_key="except_as_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + except_as_cleanup_template = ExceptAsTemplate(except_as_header=mapping["except_as_header"], except_body=mapping["except_body"], non_match_path=mapping["non_match_path"]) + + in_edges = ((src, except_as_cleanup_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((except_as_cleanup_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping["after_except"]: + out_edges.append((except_as_cleanup_template, mapping["after_except"], {"type": ControlFlowEdgeType.JUMP.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([except_as_cleanup_template.except_as_header, except_as_cleanup_template.except_body, except_as_cleanup_template.non_match_path]) + reduced_cfg.add_node(except_as_cleanup_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.except_as_header.to_indented_source(source_lines) + body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + non_match = self.non_match_path.to_indented_source(source_lines) + return f"{header}\n{body}\n{non_match}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptException.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptException.py new file mode 100644 index 0000000..036da5a --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/ExceptException.py @@ -0,0 +1,120 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_except_as + + + +class ExceptException(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + An `except exception block' currently confirmed to work from 3.6 to 3.8 + If there are multiple, this will match the last block in the series and set up the next one to be matched + (0) + / \\j --> (012) + (1) (2) |j + + """ + + _subgraph = { + "except_as_header": TemplateNode( + node_verification_func=assert_except_as, + natural_edge=TemplateEdge( + source="except_as_header", + dest="except_body", + ), + conditional_edge=TemplateEdge(source="except_as_header", dest="non_match_path"), + exception_edge=TemplateEdge(source="except_as_header", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + # natural_edge=TemplateEdge( + # source='except_body', + # dest=None, #might need a tail I am not too sure about any potential natural edges as of now + # edge_verification_func=optional_edge + # ), + exception_edge=TemplateEdge(source="except_body", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "non_match_path": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="non_match_path", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_as_header: ControlFlowTemplate, except_body: ControlFlowTemplate, non_match_path: ControlFlowTemplate): + self.except_as_header = except_as_header + self.except_body = except_body + self.non_match_path = non_match_path + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptException._subgraph, root_key="except_as_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + except_exception_template = ExceptException(except_as_header=mapping["except_as_header"], except_body=mapping["except_body"], non_match_path=mapping["non_match_path"]) + + in_edges = ((src, except_exception_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((except_exception_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([except_exception_template.except_as_header, except_exception_template.except_body, except_exception_template.non_match_path]) + reduced_cfg.add_node(except_exception_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.except_as_header.to_indented_source(source_lines) + body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + non_match = self.non_match_path.to_indented_source(source_lines) + return f"{header}\n{body}\n{non_match}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/FinallyTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/FinallyTemplate.py new file mode 100644 index 0000000..57f773a --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/FinallyTemplate.py @@ -0,0 +1,167 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, assert_instruction_opname + + + +class FinallyTemplate(ControlFlowTemplate, AbstractNonSequentiable): + """ + A basic `finally` block after the related try-excepts have been structured. + (0) + / \\ --> (01234) + |e (2) + | /e| + (3) (4) + does not cover additional finally blocks that will be inserted in the bytecode as a result of returns / breaking out of loops + """ + + _subgraph = { + "setup_finally": TemplateNode( + node_verification_func=assert_instruction_opname("SETUP_FINALLY"), + natural_edge=TemplateEdge( + source="setup_finally", + dest="try_except", + ), + exception_edge=TemplateEdge( + source="setup_finally", + dest="angry_finally", + ), + ), + "try_except": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="try_except", + dest="happy_finally", + ), + exception_edge=TemplateEdge( + source="try_except", + dest="angry_finally", + edge_verification_func=optional_edge, + ), + ), + "happy_finally": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="happy_finally", dest="tail", edge_verification_func=optional_edge), + exception_edge=TemplateEdge( + source="happy_finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "angry_finally": TemplateNode( + node_verification_func=assert_in_degree(2), + exception_edge=TemplateEdge( + source="angry_finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, setup_finally: ControlFlowTemplate, try_except: ControlFlowTemplate, happy_finally: ControlFlowTemplate, angry_finally: ControlFlowTemplate): + self.setup_finally = setup_finally + self.try_except = try_except + self.happy_finally = happy_finally + self.angry_finally = angry_finally + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + def verify_finally_match(cfg: nx.DiGraph, mapping: dict[str, ControlFlowTemplate]) -> bool: + # check to make sure that all non-stack/control instructions match between the two finally blocks + # this list was made for 3.9, so it may need to be expanded for other versions + stack_and_control_insts = {"POP_TOP", "POP_EXCEPT", "ROT_TWO", "ROT_THREE", "ROT_FOUR", "JUMP_FORWARD", "JUMP_BACKWARD", "JUMP_ABSOLUTE", "RERAISE"} + happy_insts = [(inst.opname, inst.arg) for inst in mapping["happy_finally"].get_instructions() if inst.opname not in stack_and_control_insts] + angry_insts = [(inst.opname, inst.arg) for inst in mapping["angry_finally"].get_instructions() if inst.opname not in stack_and_control_insts] + return happy_insts == angry_insts + + matcher = GraphTemplateMatcher(template_node_dict=FinallyTemplate._subgraph, root_key="setup_finally", mapping_verification_func=verify_finally_match) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + finally_template = FinallyTemplate(setup_finally=mapping["setup_finally"], try_except=mapping["try_except"], happy_finally=mapping["happy_finally"], angry_finally=mapping["angry_finally"]) + + in_edges = [(src, finally_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)] + # only preserve exception handling edges + # insert a continuation edge to after the finally + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((finally_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping["tail"]: + out_edges.append((finally_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([finally_template.setup_finally, finally_template.try_except, finally_template.happy_finally, finally_template.angry_finally]) + reduced_cfg.add_node(finally_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + try_block = self.try_except.to_indented_source(source_lines) + # pick one of the finally bodies to get the source code from + finally_body = ControlFlowTemplate._indent_multiline_string(self.happy_finally.to_indented_source(source_lines)) + if not finally_body: + finally_body = ControlFlowTemplate._indent_multiline_string(self.angry_finally.to_indented_source(source_lines)) + finally_lines = [try_block, "finally: # inserted", finally_body] + return "\n".join(finally_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/GeneratorCleanupTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/GeneratorCleanupTemplate.py new file mode 100644 index 0000000..2ea4505 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/GeneratorCleanupTemplate.py @@ -0,0 +1,70 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate + + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import assert_in_degree, node_match_all + + +def is_cleanup(cfg: nx.DiGraph, node) -> bool: + i = node.get_instructions() + return len(i) == 2 and i[0].opname == "CALL_INTRINSIC_1" and i[1].opname == "RERAISE" + + +class GeneratorCleanupTemplate(ControlFlowTemplate): + _subgraph = { + "generator": TemplateNode( + exception_edge=TemplateEdge( + source="generator", + dest="cleanup", + ) + ), + "cleanup": TemplateNode( + node_verification_func=node_match_all(assert_in_degree(1), is_cleanup), + ), + } + + def __init__(self, generator: ControlFlowTemplate, cleanup: ControlFlowTemplate): + self.generator = generator + self.cleanup = cleanup + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=GeneratorCleanupTemplate._subgraph, root_key="generator", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + template = GeneratorCleanupTemplate(generator=mapping["generator"], cleanup=mapping["cleanup"]) + + in_edges = ((src, template, edge_properties) for src, dst, edge_properties in cfg.in_edges(nbunch=node, data=True)) + out_edges = ((template, dst, edge_properties) for src, dst, edge_properties in cfg.out_edges(nbunch=node, data=True) if dst != mapping["cleanup"]) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([template.cleanup, template.generator]) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + return self.generator.to_indented_source(source_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryExceptElseTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryExceptElseTemplate.py new file mode 100644 index 0000000..410b29a --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryExceptElseTemplate.py @@ -0,0 +1,167 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from .TryExceptTemplate import TryExceptTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import assert_edge_type, optional_node, optional_edge, assert_in_degree, edge_is_none_or_matches + + + +class TryExceptElseTemplate(ControlFlowTemplate, AbstractNonSequentiable): + """ + A `try-except` with an else and a structured except. + (0) + / \\e --> (0123) + (1) (2) | + |j |j (4) + (3) | + \\ / + (4) + """ + + _subgraph = { + "try_body": TemplateNode( + natural_edge=TemplateEdge( + source="try_body", + dest="try_footer", + ), + exception_edge=TemplateEdge( + source="try_body", + dest="except_body", + ), + ), + "try_footer": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="try_footer", dest="else_body", edge_verification_func=assert_edge_type(ControlFlowEdgeType.JUMP)), + exception_edge=TemplateEdge( + source="try_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "else_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="else_body", + dest="after_try_except", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="else_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="except_body", + dest="after_try_except", + edge_verification_func=edge_is_none_or_matches(assert_edge_type(ControlFlowEdgeType.JUMP)), + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_try_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, try_body: ControlFlowTemplate, try_footer: ControlFlowTemplate, else_body: ControlFlowTemplate, except_body: ControlFlowTemplate): + self.try_body = try_body + self.try_footer = try_footer + self.else_body = else_body + self.except_body = except_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=TryExceptElseTemplate._subgraph, root_key="try_body", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + try_except_template = TryExceptElseTemplate(try_body=mapping["try_body"], try_footer=mapping["try_footer"], else_body=mapping["else_body"], except_body=mapping["except_body"]) + + in_edges = ((src, try_except_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + # insert a continuation edge to after the try except + out_edges = [] + if mapping.get("after_try_except", None): + out_edges.append((try_except_template, mapping["after_try_except"], {"type": ControlFlowEdgeType.NATURAL.value})) + if mapping["outer_exception_handler"]: + out_edges.append((try_except_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([try_except_template.try_body, try_except_template.try_footer, try_except_template.else_body, try_except_template.except_body]) + reduced_cfg.add_node(try_except_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + try_except_template = TryExceptTemplate(try_body=self.try_body, try_footer=self.try_footer, except_body=self.except_body) + try_except_lines = [try_except_template.to_indented_source(source_lines)] + else_body = ControlFlowTemplate._indent_multiline_string(self.else_body.to_indented_source(source_lines)) + try_except_lines.extend(["else: # inserted", else_body]) + + return "\n".join(try_except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryExceptTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryExceptTemplate.py new file mode 100644 index 0000000..26f72af --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryExceptTemplate.py @@ -0,0 +1,188 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ..abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate +from ..loop.LoopExitTemplate import LoopExitTemplate + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import optional_node, optional_edge, assert_in_degree, node_match_all, assert_node_has_no_backwards_edges, node_is_none_or_matches + +from .ExceptAsTemplate import ExceptAsTemplate +from .ExceptAsExceptTemplate import ExceptAsExceptTemplate +from ..subtemplates.OptionalExitSubtemplate import ExitSubTemplate + + +class TryExceptTemplate(ControlFlowTemplate, AbstractNonSequentiable): + """ + A `try-except` block with just a naked except. + (0) + / \\e --> (012) + (1) (2) | + \\j /j (3) + (3) + One or more of the try/except may have no further control flow. + However, if both have successors, they must go to the same place. + """ + + _subgraph = { + "try_body": TemplateNode( + natural_edge=TemplateEdge( + source="try_body", + dest="try_footer", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="try_body", + dest="except_body", + ), + ), + "try_footer": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=node_is_none_or_matches( + node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ) + ), + natural_edge=TemplateEdge(source="try_footer", dest="after_try_except", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="try_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "except_body": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ), + natural_edge=TemplateEdge(source="except_body", dest="after_try_except", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_try_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_try_except", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, try_body: ControlFlowTemplate, try_footer: ControlFlowTemplate, except_body: ControlFlowTemplate): + self.try_body = try_body + self.try_footer = try_footer + self.except_body = except_body + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=TryExceptTemplate._subgraph, root_key="try_body", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + # have to make sure there is a try_footer before trying to map it as it is an optional node (this is mostly here for 3.7 + # since there are cases where there is not a try footer at all) + + try_except_template = TryExceptTemplate(try_body=mapping["try_body"], try_footer=mapping.get("try_footer", None), except_body=mapping["except_body"]) + + in_edges = ((src, try_except_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + # insert a continuation edge to after the try except + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((try_except_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + if "after_try_except" in mapping.keys(): + after_try_except = mapping["after_try_except"] + out_edges.append((try_except_template, after_try_except, {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([try_except_template.try_body, try_except_template.try_footer, try_except_template.except_body]) + reduced_cfg.add_node(try_except_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + try_body = ControlFlowTemplate._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + + # check if there is a try footer as in 3.7 there may not be a try footer at all + if self.try_footer: + try_footer = ControlFlowTemplate._indent_multiline_string(self.try_footer.to_indented_source(source_lines)) + else: + try_footer = "" + + try_except_lines = ["try:", try_body, try_footer] + + # if we matched against an "Except ... as" chain, then omit the inserted except: block + omit_except = False + if isinstance(self.except_body, AbstractExceptionBlockTemplate): + omit_except = True + elif isinstance(self.except_body, LoopExitTemplate): + if isinstance(self.except_body.tail, ExceptAsTemplate) or isinstance(self.except_body.tail, ExceptAsExceptTemplate): + omit_except = True + + except_body = self.except_body.to_indented_source(source_lines) + if not omit_except: + try_except_lines.append("except:") + except_body = ControlFlowTemplate._indent_multiline_string(except_body) + + try_except_lines.append(except_body) + + return "\n".join(try_except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryFinallyTemplate.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryFinallyTemplate.py new file mode 100644 index 0000000..057848a --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/TryFinallyTemplate.py @@ -0,0 +1,173 @@ +import networkx as nx + +import itertools + +from ..abstract.AbstractTemplate import ControlFlowTemplate +from ..abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + +from ...cfg_utils import ControlFlowEdgeType + +from ..Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ..match_utils import assert_edge_type, optional_node, optional_edge, assert_in_degree, assert_instruction_opname, edge_is_none_or_matches + + + +class TryFinallyTemplate(ControlFlowTemplate, AbstractNonSequentiable): + """ + A `try` block with only a `finally` following it. + (0) + | + (1) + /e\\ --> (0123) + (3) (2) | + |j (4) + (4) + does not cover additional finally blocks that will be inserted in the bytecode as a result of returns / breaking out of loops + """ + + _subgraph = { + "setup_finally": TemplateNode( + node_verification_func=assert_instruction_opname("SETUP_FINALLY"), + natural_edge=TemplateEdge( + source="setup_finally", + dest="try_body", + ), + exception_edge=TemplateEdge(source="setup_finally", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "try_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="try_body", + dest="happy_finally", + ), + exception_edge=TemplateEdge( + source="try_body", + dest="angry_finally", + ), + ), + "happy_finally": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="happy_finally", dest="tail", edge_verification_func=edge_is_none_or_matches(assert_edge_type(ControlFlowEdgeType.JUMP))), + exception_edge=TemplateEdge( + source="happy_finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "angry_finally": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="angry_finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, setup_finally: ControlFlowTemplate, try_body: ControlFlowTemplate, happy_finally: ControlFlowTemplate, angry_finally: ControlFlowTemplate): + self.setup_finally = setup_finally + self.try_body = try_body + self.happy_finally = happy_finally + self.angry_finally = angry_finally + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + if cfg.in_degree(node) != 1: + return None + + # to avoid being treated as a try-except, we actually need to greedily search up one layer + pred = next(cfg.predecessors(node)) + + def verify_finally_match(cfg: nx.DiGraph, mapping: dict[str, ControlFlowTemplate]) -> bool: + # check to make sure that all non-stack/control instructions match between the two finally blocks + # this list was made for 3.9, so it may need to be expanded for other versions + stack_and_control_insts = {"POP_TOP", "POP_EXCEPT", "ROT_TWO", "ROT_THREE", "ROT_FOUR", "JUMP_FORWARD", "JUMP_BACKWARD", "JUMP_ABSOLUTE", "RERAISE"} + happy_insts = [(inst.opname, inst.arg) for inst in mapping["happy_finally"].get_instructions() if inst.opname not in stack_and_control_insts] + angry_insts = [(inst.opname, inst.arg) for inst in mapping["angry_finally"].get_instructions() if inst.opname not in stack_and_control_insts] + return happy_insts == angry_insts + + matcher = GraphTemplateMatcher(template_node_dict=TryFinallyTemplate._subgraph, root_key="setup_finally", mapping_verification_func=verify_finally_match) + + mapping = matcher.match_at_graph_node(cfg, pred) + + if not mapping: + return None + + finally_template = TryFinallyTemplate(setup_finally=mapping["setup_finally"], try_body=mapping["try_body"], happy_finally=mapping["happy_finally"], angry_finally=mapping["angry_finally"]) + + in_edges = [(src, finally_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(finally_template.setup_finally, data=True)] + # only preserve exception handling edges + # insert a continuation edge to after the finally + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((finally_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping["tail"]: + out_edges.append((finally_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([finally_template.setup_finally, finally_template.try_body, finally_template.happy_finally, finally_template.angry_finally]) + reduced_cfg.add_node(finally_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + # sometimes the setup finally is included in a linear sequence, so we need to include that source + setup_finally = self.setup_finally.to_indented_source(source_lines) + try_block = ControlFlowTemplate._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + # pick one of the finally bodies to get the source code from + finally_body = ControlFlowTemplate._indent_multiline_string(self.happy_finally.to_indented_source(source_lines)) + if not finally_body: + finally_body = ControlFlowTemplate._indent_multiline_string(self.angry_finally.to_indented_source(source_lines)) + finally_lines = [setup_finally, "try:", try_block, "finally: # inserted", finally_body] + return "\n".join(finally_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsCleanupSubTemplate311.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsCleanupSubTemplate311.py new file mode 100644 index 0000000..902efe7 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsCleanupSubTemplate311.py @@ -0,0 +1,159 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ...abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import optional_node, optional_edge, assert_in_degree, node_match_all, node_match_any, contains_opname_sequence + + +class ExceptAsCleanupSubTemplate311(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + The boilerplate cleanup at the end of an `except as` block after 3.11. + The "happy cleanup" (3) is when there is no exception, and it jumps out to the next code segment (except footer in 3.11). + The "angry cleanup" (2) is when there is an exception, and it reraises. + (0) + | \\e + (1) | + / |e | --> (012) + (3)(2) | | \\e + |e / (3) (4) + (4) + """ + + _subgraph = { + "except_header": TemplateNode( + natural_edge=TemplateEdge( + source="except_header", + dest="except_body", + ), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + ), + ), + "except_body": TemplateNode( + natural_edge=TemplateEdge( + source="except_body", + dest="happy_cleanup", + ), + exception_edge=TemplateEdge( + source="except_body", + dest="angry_cleanup", + ), + ), + "happy_cleanup": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + node_match_any( + contains_opname_sequence( + "LOAD_CONST", + "STORE_NAME", + "DELETE_NAME", + ), + contains_opname_sequence( + "LOAD_CONST", + "STORE_FAST", + "DELETE_FAST", + ), + ), + ), + natural_edge=TemplateEdge(source="happy_cleanup", dest=None, edge_verification_func=optional_edge), + exception_edge=TemplateEdge( + source="happy_cleanup", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "angry_cleanup": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + node_match_any( + contains_opname_sequence("LOAD_CONST", "STORE_NAME", "DELETE_NAME", "RERAISE"), + contains_opname_sequence("LOAD_CONST", "STORE_FAST", "DELETE_FAST", "RERAISE"), + ), + ), + exception_edge=TemplateEdge( + source="angry_cleanup", + dest="panic_except", + ), + ), + "panic_except": TemplateNode( + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ) + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_header: ControlFlowTemplate, except_body: ControlFlowTemplate, angry_cleanup: ControlFlowTemplate): + self.except_header = except_header + self.except_body = except_body + self.angry_cleanup = angry_cleanup + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsCleanupSubTemplate311._subgraph, root_key="except_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return cfg # if we didn't match the subtemplate, keep trying with the main template + + except_as_cleanup_template = ExceptAsCleanupSubTemplate311(except_header=mapping["except_header"], except_body=mapping["except_body"], angry_cleanup=mapping["angry_cleanup"]) + + in_edges = ((src, except_as_cleanup_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + out_edges = [] + if mapping["happy_cleanup"]: + out_edges.append((except_as_cleanup_template, mapping["happy_cleanup"], {"type": ControlFlowEdgeType.NATURAL.value})) + if mapping["panic_except"]: + out_edges.append((except_as_cleanup_template, mapping["panic_except"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([except_as_cleanup_template.except_header, except_as_cleanup_template.except_body, except_as_cleanup_template.angry_cleanup]) + reduced_cfg.add_node(except_as_cleanup_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + # cleanup code is implicit! only report the body code + return self.except_body.to_indented_source(source_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsNonMatchSubtemplate311.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsNonMatchSubtemplate311.py new file mode 100644 index 0000000..0847860 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsNonMatchSubtemplate311.py @@ -0,0 +1,228 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ...abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + +from ...subtemplates.OptionalExitSubtemplate import ExitSubTemplate + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import optional_node, optional_edge, assert_in_degree, node_is_none_or_matches, assert_instruction_opname, assert_node_type + + + +class ExceptAsNonMatchSubTemplate311(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + The non-match path of an except-as, which can be: + 1. a standalone reraise (end of an except as chain) + 2. an except block, which may exit + 3. a structured except-as + """ + + _reraise_subgraph = { + "reraise": TemplateNode(node_verification_func=assert_instruction_opname("RERAISE"), exception_edge=TemplateEdge(source="reraise", dest="panic_except")), + "panic_except": TemplateNode( + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ) + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + _except_subgraph = { + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="except_body", + dest="except_footer", + ), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + ), + ), + "except_footer": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=node_is_none_or_matches(assert_in_degree(1)), + natural_edge=TemplateEdge( + source="except_footer", + dest="after_except", + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="except_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "panic_except": TemplateNode( + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ) + ), + "after_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + _structered_except_as_subgraph = { + "except_as": TemplateNode(node_verification_func=assert_node_type(AbstractNonSequentiable), natural_edge=TemplateEdge(source="except_as", dest=None), exception_edge=TemplateEdge(source="except_as", dest="panic_except")), + "panic_except": TemplateNode( + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ) + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_body: ControlFlowTemplate, except_footer: ControlFlowTemplate): + self.except_body = except_body + self.except_footer = except_footer + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + # start by trying to match reraise + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsNonMatchSubTemplate311._reraise_subgraph, root_key="reraise", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if mapping: + # single-node subgraph does not need to by updated + return cfg + + # didn't match reraise; try to match structured except as + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsNonMatchSubTemplate311._structered_except_as_subgraph, root_key="except_as", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if mapping: + # single-node subgraph does not need to by updated + return cfg + + # didn't match structured except as; try to match except block + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsNonMatchSubTemplate311._except_subgraph, root_key="except_body", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + except_template = ExceptAsNonMatchSubTemplate311(except_body=mapping["except_body"], except_footer=mapping["except_footer"]) + + in_edges = ((src, except_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + out_edges = [] + if mapping["panic_except"]: + out_edges.append((except_template, mapping["panic_except"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping["after_except"]: + out_edges.append((except_template, mapping["after_except"], {"type": ControlFlowEdgeType.JUMP.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_node(except_template.except_body) + if except_template.except_footer: + reduced_cfg.remove_node(except_template.except_footer) + reduced_cfg.add_node(except_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + + except_lines = ["except:"] + body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + except_lines.append(body) + footer = ControlFlowTemplate._indent_multiline_string(self.except_footer.to_indented_source(source_lines)) + except_lines.append(footer) + + return "\n".join(except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsTemplate311.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsTemplate311.py new file mode 100644 index 0000000..a8a95a2 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptAsTemplate311.py @@ -0,0 +1,196 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ...abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + + +from ...subtemplates.OptionalExitSubtemplate import ExitSubTemplate +from .ExceptAsNonMatchSubtemplate311 import ExceptAsNonMatchSubTemplate311 +from .ExceptAsCleanupSubTemplate311 import ExceptAsCleanupSubTemplate311 + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import optional_node, optional_edge, assert_in_degree, assert_except_as + + + +class ExceptAsTemplate311(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + An `except as` block, after its cleanup has been structured. + If there are multiple, this will match the last block in the series and set up the next one to be matched + (0) + / \\j --> (0123) + (1) (2) |j + | (4) + (3) + |j + (4) + + 0,1,2 all have an exception edge to the panic cleanup from the current try block + """ + + _subgraph = { + "except_as_header": TemplateNode( + node_verification_func=assert_except_as, + natural_edge=TemplateEdge( + source="except_as_header", + dest="except_body", + ), + conditional_edge=TemplateEdge(source="except_as_header", dest="non_match_path"), + exception_edge=TemplateEdge( + source="except_as_header", + dest="panic_except", + ), + ), + "except_body": TemplateNode( + subtemplate=ExceptAsCleanupSubTemplate311, + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="except_body", + dest="except_footer", + ), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + ), + ), + "non_match_path": TemplateNode( + subtemplate=ExceptAsNonMatchSubTemplate311, + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="non_match_path", + dest="panic_except", + ), + natural_edge=TemplateEdge( + edge_verification_func=optional_edge, + source="non_match_path", + dest="after_except", + commit_none_to_mapping=False, + ), + ), + "except_footer": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="except_footer", + dest="after_except", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="except_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "panic_except": TemplateNode( + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ) + ), + "after_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_as_header: ControlFlowTemplate, except_body: ControlFlowTemplate, except_footer: ControlFlowTemplate, non_match_path: ControlFlowTemplate): + self.except_as_header = except_as_header + self.except_body = except_body + self.except_footer = except_footer + self.non_match_path = non_match_path + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptAsTemplate311._subgraph, root_key="except_as_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + except_as_template = ExceptAsTemplate311(except_as_header=mapping["except_as_header"], except_body=mapping["except_body"], except_footer=mapping["except_footer"], non_match_path=mapping["non_match_path"]) + + in_edges = ((src, except_as_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + out_edges = [] + if mapping["panic_except"]: + out_edges.append((except_as_template, mapping["panic_except"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping.get("after_except", None): + out_edges.append((except_as_template, mapping["after_except"], {"type": ControlFlowEdgeType.JUMP.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([except_as_template.except_as_header, except_as_template.except_body, except_as_template.except_footer, except_as_template.non_match_path]) + reduced_cfg.add_node(except_as_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + except_lines = [] + + header = self.except_as_header.to_indented_source(source_lines) + except_lines.append(header) + + body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + except_lines.append(body) + + footer = ControlFlowTemplate._indent_multiline_string(self.except_footer.to_indented_source(source_lines)) + except_lines.append(footer) + + non_match = self.non_match_path.to_indented_source(source_lines) + except_lines.append(non_match) + + return "\n".join(except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptTemplate311.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptTemplate311.py new file mode 100644 index 0000000..2262384 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/ExceptTemplate311.py @@ -0,0 +1,430 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ...abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate + +from ...natural.InstructionTemplate import InstructionTemplate + + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import ( + optional_node, + optional_edge, + assert_in_degree, + node_match_all, + assert_first_instruction_opname, + ends_with_opname_sequence, + is_exactly_opname, + node_match_any, +) + + + +class ExceptTemplate311(ControlFlowTemplate): + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + if isinstance(node, InstructionTemplate) and node.instruction.opname == "RERAISE": + return cfg + if isinstance(node, ExceptETemplate311): + return cfg + new_cfg = ExceptETemplate311.try_to_match_node(cfg, node) + if new_cfg is not None: + return new_cfg + new_cfg = BareExcept311.try_to_match_node(cfg, node) + if new_cfg is not None: + return new_cfg + + +class Footer(ControlFlowTemplate): + _subgraph = { + "swap": TemplateNode(node_verification_func=node_match_all(is_exactly_opname("SWAP"), assert_in_degree(1)), natural_edge=TemplateEdge(source="swap", dest="footer"), exception_edge=TemplateEdge(source="swap", dest="panic")), + "footer": TemplateNode( + natural_edge=TemplateEdge(source="footer", dest=None, edge_verification_func=optional_edge), + conditional_edge=TemplateEdge(source="footer", dest=None, edge_verification_func=optional_edge), + exception_edge=TemplateEdge(source="footer", dest=None, edge_verification_func=optional_edge), + ), + "panic": TemplateNode(node_verification_func=is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), exception_edge=TemplateEdge(source="panic", dest=None, edge_verification_func=optional_edge)), + } + + def __init__(self, footer): + self.footer = footer + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=Footer._subgraph, root_key="swap", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return cfg + + template = Footer(mapping["footer"]) + + edges = [(next(cfg.predecessors(node)), template, {"type": ControlFlowEdgeType.NATURAL.value})] + edges.extend((template, dst, prop) for src, dst, prop in cfg.out_edges(mapping["footer"], data=True)) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from((node, mapping["footer"])) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(edges) + return reduced_cfg + + def to_indented_source(self, source_lines): + return self.footer.to_indented_source(source_lines) + + +class ExceptBody(ControlFlowTemplate): + _subgraph = { + "store": TemplateNode( + node_verification_func=node_match_any(is_exactly_opname("STORE_FAST"), is_exactly_opname("STORE_NAME")), + natural_edge=TemplateEdge( + source="store", + dest="body", + ), + exception_edge=TemplateEdge( + source="store", + dest="panic", + ), + ), + "body": TemplateNode( + natural_edge=TemplateEdge( + source="body", + dest="footer", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="body", + dest="cleanup", + ), + ), + "cleanup": TemplateNode( + node_verification_func=node_match_any( + is_exactly_opname("LOAD_CONST", "STORE_FAST", "DELETE_FAST", "RERAISE"), + is_exactly_opname("LOAD_CONST", "STORE_NAME", "DELETE_NAME", "RERAISE"), + ), + exception_edge=TemplateEdge(source="cleanup", dest="panic"), + ), + "panic": TemplateNode(exception_edge=TemplateEdge(source="panic", dest=None, edge_verification_func=optional_edge)), + "footer": TemplateNode( + subtemplate=Footer, + node_verification_func=optional_node, + natural_edge=TemplateEdge(source="footer", dest=None, edge_verification_func=optional_edge), + conditional_edge=TemplateEdge(source="footer", dest=None, edge_verification_func=optional_edge), + exception_edge=TemplateEdge(source="footer", dest=None, edge_verification_func=optional_edge), + ), + } + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptBody._subgraph, root_key="store", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return cfg + + header = next(cfg.predecessors(node)) + footer = mapping.get("footer") + + template = ExceptBody(mapping["body"]) + edges = [(header, template, {"type": ControlFlowEdgeType.NATURAL.value}), (template, mapping["panic"], {"type": ControlFlowEdgeType.EXCEPTION.value})] + if footer: + edges.append((template, footer, {"type": ControlFlowEdgeType.NATURAL.value})) + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([mapping["store"], template.body, mapping["cleanup"]]) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(edges) + return reduced_cfg + + def __init__(self, body): + self.body = body + + def to_indented_source(self, source_lines): + return self.body.to_indented_source(source_lines) + + +class BareExcept311(ControlFlowTemplate): + _subgraph = { + "body": TemplateNode( + natural_edge=TemplateEdge(source="body", dest="footer"), + exception_edge=TemplateEdge( + source="body", + dest="panic", + ), + ), + "footer": TemplateNode( + node_verification_func=node_match_all(assert_first_instruction_opname("POP_EXCEPT"), assert_in_degree(1)), + natural_edge=TemplateEdge( + source="footer", + dest="after_except", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="except_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + ), + "panic": TemplateNode(node_verification_func=is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), exception_edge=TemplateEdge(source="panic", dest="outer_exception_handler", edge_verification_func=optional_edge)), + "after_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + conditional_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + conditional_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + ), + } + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=BareExcept311._subgraph, root_key="body", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + template = BareExcept311( + body=mapping["body"], + footer=mapping["footer"], + ) + + in_edges = ((src, template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + out_edges = [] + if mapping["panic"]: + out_edges.append((template, mapping["panic"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping.get("after_except", None): + out_edges.append((template, mapping["after_except"], {"type": ControlFlowEdgeType.JUMP.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([template.body, template.footer]) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def __init__(self, body, footer): + self.body = body + self.footer = footer + + def to_indented_source(self, source_lines): + return "\n".join(["except:", self._indent_multiline_string(self.body.to_indented_source(source_lines)), self._indent_multiline_string(self.footer.to_indented_source(source_lines))]) + + +class ExceptETemplate311(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + _subgraph = { + "except_header": TemplateNode( + node_verification_func=node_match_any( + ends_with_opname_sequence("CHECK_EXC_MATCH", "POP_JUMP_FORWARD_IF_FALSE"), + ends_with_opname_sequence("CHECK_EXC_MATCH", "POP_JUMP_IF_FALSE"), + ), + natural_edge=TemplateEdge( + source="except_header", + dest="except_body", + ), + conditional_edge=TemplateEdge(source="except_header", dest="non_match_path"), + exception_edge=TemplateEdge( + source="except_header", + dest="panic_except", + ), + ), + "except_body": TemplateNode( + subtemplate=ExceptBody, + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="except_body", dest="except_footer", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + ), + ), + "non_match_path": TemplateNode( + subtemplate=ExceptTemplate311, + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="non_match_path", + dest="panic_except", + ), + natural_edge=TemplateEdge( + edge_verification_func=optional_edge, + source="non_match_path", + dest="after_except", + commit_none_to_mapping=False, + ), + ), + "except_footer": TemplateNode( + node_verification_func=node_match_all(assert_first_instruction_opname("POP_EXCEPT"), assert_in_degree(1)), + natural_edge=TemplateEdge( + source="except_footer", + dest="after_except", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="except_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "panic_except": TemplateNode( + node_verification_func=is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), + exception_edge=TemplateEdge( + source="panic_except", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_except", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_header: ControlFlowTemplate, except_body: ControlFlowTemplate, except_footer: ControlFlowTemplate, non_match_path: ControlFlowTemplate): + self.except_header = except_header + self.except_body = except_body + self.except_footer = except_footer + self.non_match_path = non_match_path + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=ExceptETemplate311._subgraph, root_key="except_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + template = ExceptETemplate311(except_header=mapping["except_header"], except_body=mapping["except_body"], except_footer=mapping.get("except_footer"), non_match_path=mapping["non_match_path"]) + + in_edges = ((src, template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + out_edges = [] + if mapping["panic_except"]: + out_edges.append((template, mapping["panic_except"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + if mapping.get("after_except", None): + out_edges.append((template, mapping["after_except"], {"type": ControlFlowEdgeType.JUMP.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([template.except_header, template.except_body, template.non_match_path]) + if template.except_footer: + reduced_cfg.remove_node(template.except_footer) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + except_lines = [] + + header = self.except_header.to_indented_source(source_lines) + except_lines.append(header) + + body = ControlFlowTemplate._indent_multiline_string(self.except_body.to_indented_source(source_lines)) + except_lines.append(body) + + if self.except_footer: + footer = ControlFlowTemplate._indent_multiline_string(self.except_footer.to_indented_source(source_lines)) + except_lines.append(footer) + + non_match = self.non_match_path.to_indented_source(source_lines) + except_lines.append(non_match) + + return "\n".join(except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/FinallyTemplate312.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/FinallyTemplate312.py new file mode 100644 index 0000000..6d5acc8 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/FinallyTemplate312.py @@ -0,0 +1,243 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...natural.InstructionTemplate import InstructionTemplate +from ...natural.LinearSequenceTemplate import LinearSequenceTemplate +from ...if_then.IfThenTemplate import IfThenTemplate +from ...if_then.IfElseTemplate import IfElseTemplate +from .TryTemplate311 import TryTemplate311 +from .TryTemplate312 import TryTemplate312 + + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import ( + assert_edge_type, + optional_node, + optional_edge, + assert_in_degree, + node_match_all, + assert_node_has_no_backwards_edges, + assert_instruction_opname, + is_exactly_opname, + assert_node_type, +) + + + +class FinallyTemplate312(ControlFlowTemplate): + _subgraph = { + "try_header": TemplateNode( + node_verification_func=assert_instruction_opname("NOP"), + natural_edge=TemplateEdge(source="try_header", dest="try_body", edge_verification_func=assert_edge_type(ControlFlowEdgeType.NATURAL)), + ), + "try_body": TemplateNode( + natural_edge=TemplateEdge(source="try_body", dest="finally_body", edge_verification_func=assert_edge_type(ControlFlowEdgeType.NATURAL)), + exception_edge=TemplateEdge( + source="try_body", + dest="fail", + ), + ), + "finally_body": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ), + natural_edge=TemplateEdge(source="finally_body", dest=None, edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="finally_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "fail": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + edge_verification_func=optional_edge, + ), + ), + "panic_except": TemplateNode( + node_verification_func=is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + _subgraph2 = { + "try_except": TemplateNode( + node_verification_func=assert_node_type(TryTemplate311, TryTemplate312), + natural_edge=TemplateEdge(source="try_except", dest="finally_body", edge_verification_func=assert_edge_type(ControlFlowEdgeType.NATURAL)), + exception_edge=TemplateEdge( + source="try_except", + dest="fail", + ), + ), + "finally_body": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ), + natural_edge=TemplateEdge(source="finally_body", dest=None, edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="finally_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "fail": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + edge_verification_func=optional_edge, + ), + ), + "panic_except": TemplateNode( + node_verification_func=is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, try_header: ControlFlowTemplate, try_body: ControlFlowTemplate, finally_body: ControlFlowTemplate, fail: ControlFlowTemplate, panic_except: ControlFlowTemplate, cutoff): + self.try_header = try_header + self.try_body = try_body + self.finally_body = finally_body + self.fail = fail + self.panic_except = panic_except + self.cutoff = cutoff + + @staticmethod + def mapping_verification_func(cfg, mapping): + finally_body = mapping["finally_body"] + fail = mapping["fail"] + if any(x.starts_line is not None for x in fail.get_instructions()): + return False + if not isinstance(finally_body, LinearSequenceTemplate): + finally_body = LinearSequenceTemplate(finally_body) + if not isinstance(fail, LinearSequenceTemplate): + fail = LinearSequenceTemplate(fail) + if isinstance(fail.members[0], InstructionTemplate) and fail.members[0].instruction.opname == "PUSH_EXC_INFO": + fail.members = fail.members[1:] + if isinstance(fail.members[-1], InstructionTemplate) and fail.members[-1].instruction.opname == "RERAISE": + fail.members = fail.members[:-1] + for x, y in zip(finally_body.members, fail.members): + if type(x) is not type(y) and not all(type(a) in [IfThenTemplate, IfElseTemplate] for a in (x, y)): + return False + mapping["cutoff"] = x + return True + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=FinallyTemplate312._subgraph, root_key="try_header", mapping_verification_func=FinallyTemplate312.mapping_verification_func) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + matcher = GraphTemplateMatcher(template_node_dict=FinallyTemplate312._subgraph2, root_key="try_except", mapping_verification_func=FinallyTemplate312.mapping_verification_func) + + mapping = matcher.match_at_graph_node(cfg, node) + if not mapping: + return None + mapping["try_header"] = None + mapping["try_body"] = mapping["try_except"] + + reduced_cfg = cfg.copy() + + # "bite off" the NOP from a linear sequence template + if isinstance(mapping["try_header"], LinearSequenceTemplate): + # grab the nop and update the linear sequence + nop_inst_template = mapping["try_header"].members[-1] + mapping["try_header"].members = mapping["try_header"].members[:-1] + if len(mapping["try_header"].members) == 1: + nx.relabel_nodes(reduced_cfg, {mapping["try_header"]: mapping["try_header"].members[0]}, copy=False) + mapping["try_header"] = mapping["try_header"].members[0] + + # transfer outgoing edges to the bitten off chunk + header_out_edges = list(reduced_cfg.out_edges(mapping["try_header"], data=True)) + reduced_cfg.add_node(nop_inst_template) + reduced_cfg.remove_edges_from(header_out_edges) + reduced_cfg.add_edges_from((nop_inst_template, dst, data) for src, dst, data in header_out_edges) + reduced_cfg.add_edge(mapping["try_header"], nop_inst_template, type=ControlFlowEdgeType.NATURAL.value) + mapping["try_header"] = nop_inst_template + + template = FinallyTemplate312(try_header=mapping["try_header"], try_body=mapping["try_body"], finally_body=mapping["finally_body"], fail=mapping["fail"], panic_except=mapping["panic_except"], cutoff=mapping["cutoff"]) + + in_edges = ((src, template, edge_properties) for src, dst, edge_properties in reduced_cfg.in_edges(template.try_header or template.try_body, data=True)) + out_edges = [(template, dst, edge_properties) for src, dst, edge_properties in reduced_cfg.out_edges(template.finally_body, data=True)] + if mapping["outer_exception_handler"]: + out_edges.append((template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg.remove_nodes_from([template.try_header, template.try_body, template.finally_body, template.fail, template.panic_except]) + reduced_cfg.add_node(template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + try_header = self.try_header.to_indented_source(source_lines) if self.try_header else "" + try_body = self._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + + if isinstance(self.finally_body, LinearSequenceTemplate): + i = self.finally_body.members.index(self.cutoff) + 1 + in_finally = self._indent_multiline_string(LinearSequenceTemplate(*self.finally_body.members[:i]).to_indented_source(source_lines)) + after = LinearSequenceTemplate(*self.finally_body.members[i:]).to_indented_source(source_lines) + else: + in_finally = self._indent_multiline_string(self.finally_body.to_indented_source(source_lines)) + after = "" + + lines = [try_header, "try:", try_body, "finally: # inserted", in_finally, after] + + return "\n".join(lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryExceptTemplate311.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryExceptTemplate311.py new file mode 100644 index 0000000..9c5f5b7 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryExceptTemplate311.py @@ -0,0 +1,213 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate +from ...natural.LinearSequenceTemplate import LinearSequenceTemplate +from ...loop.LoopExitTemplate import LoopExitTemplate + +from .ExceptAsNonMatchSubtemplate311 import ExceptAsNonMatchSubTemplate311 + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import assert_edge_type, optional_node, optional_edge, assert_in_degree, node_match_all, assert_node_has_no_backwards_edges, assert_instruction_opname +from ...subtemplates.OptionalExitSubtemplate import ExitSubTemplate + + + +class TryExceptTemplate311(ControlFlowTemplate): + """ + A `try-except` block with just a naked except in Python 3.11+. + (-1) + | (-1) + (0) | + / \\e --> (01235) + (1) (2) | + | | \\e (4) + | (3) (5) + \\j /j + (4) + One or more of the try/except may have no further control flow. + However, if both have successors, they must go to the same place. + """ + + _subgraph = { + "try_header": TemplateNode( + node_verification_func=assert_instruction_opname("NOP"), + natural_edge=TemplateEdge(source="try_header", dest="try_body", edge_verification_func=assert_edge_type(ControlFlowEdgeType.NATURAL)), + ), + "try_body": TemplateNode( + natural_edge=TemplateEdge(source="try_body", dest="try_footer", edge_verification_func=assert_edge_type(ControlFlowEdgeType.NATURAL)), + exception_edge=TemplateEdge( + source="try_body", + dest="except_body", + ), + ), + "try_footer": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ), + natural_edge=TemplateEdge(source="try_footer", dest="after_try_except", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="try_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "except_body": TemplateNode( + subtemplate=ExceptAsNonMatchSubTemplate311, + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="except_body", dest="after_try_except", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + edge_verification_func=optional_edge, + ), + ), + "panic_except": TemplateNode( + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ) + ), + "after_try_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_try_except", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, try_header: ControlFlowTemplate, try_body: ControlFlowTemplate, try_footer: ControlFlowTemplate, except_body: ControlFlowTemplate, panic_except: ControlFlowTemplate): + self.try_header = try_header + self.try_body = try_body + self.try_footer = try_footer + self.except_body = except_body + self.panic_except = panic_except + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=TryExceptTemplate311._subgraph, root_key="try_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + reduced_cfg: nx.DiGraph = cfg.copy() + # "bite off" the NOP from a linear sequence template + if isinstance(mapping["try_header"], LinearSequenceTemplate): + # grab the nop and update the linear sequence + nop_inst_template = mapping["try_header"].members[-1] + mapping["try_header"].members = mapping["try_header"].members[:-1] + if len(mapping["try_header"].members) == 1: + nx.relabel_nodes(reduced_cfg, {mapping["try_header"]: mapping["try_header"].members[0]}, copy=False) + mapping["try_header"] = mapping["try_header"].members[0] + + # transfer outgoing edges to the bitten off chunk + header_out_edges = list(reduced_cfg.out_edges(mapping["try_header"], data=True)) + reduced_cfg.add_node(nop_inst_template) + reduced_cfg.remove_edges_from(header_out_edges) + reduced_cfg.add_edges_from((nop_inst_template, dst, data) for src, dst, data in header_out_edges) + reduced_cfg.add_edge(mapping["try_header"], nop_inst_template, type=ControlFlowEdgeType.NATURAL.value) + mapping["try_header"] = nop_inst_template + + try_except_template = TryExceptTemplate311(try_header=mapping["try_header"], try_body=mapping["try_body"], try_footer=mapping.get("try_footer", None), except_body=mapping["except_body"], panic_except=mapping["panic_except"]) + + in_edges = ((src, try_except_template, edge_properties) for src, dst, edge_properties in reduced_cfg.in_edges(try_except_template.try_header, data=True)) + # only preserve exception handling edges + # insert a continuation edge to after the try except + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((try_except_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + if "after_try_except" in mapping.keys(): + after_try_except = mapping["after_try_except"] + out_edges.append((try_except_template, after_try_except, {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg.remove_nodes_from([try_except_template.try_header, try_except_template.try_body, try_except_template.try_footer, try_except_template.except_body, try_except_template.panic_except]) + reduced_cfg.add_node(try_except_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + try_header = self.try_header.to_indented_source(source_lines) + try_body = ControlFlowTemplate._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + + # check if there is a try footer as in 3.7 there may not be a try footer at all + if self.try_footer: + try_footer = ControlFlowTemplate._indent_multiline_string(self.try_footer.to_indented_source(source_lines)) + else: + try_footer = "" + + try_except_lines = [try_header, "try:", try_body, try_footer] + + # if we matched against an "Except ... as" chain, then omit the inserted except: block + omit_except = False + if isinstance(self.except_body, AbstractExceptionBlockTemplate): + omit_except = True + elif isinstance(self.except_body, LoopExitTemplate): + if isinstance(self.except_body.tail, AbstractExceptionBlockTemplate): + omit_except = True + + except_body = self.except_body.to_indented_source(source_lines) + if not omit_except: + try_except_lines.append("except:") + except_body = ControlFlowTemplate._indent_multiline_string(except_body) + + try_except_lines.append(except_body) + + # the panic except should never have a line + + return "\n".join(try_except_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryTemplate311.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryTemplate311.py new file mode 100644 index 0000000..e7900ff --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryTemplate311.py @@ -0,0 +1,178 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...natural.LinearSequenceTemplate import LinearSequenceTemplate + +from .ExceptTemplate311 import ExceptTemplate311 + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import ( + assert_edge_type, + optional_node, + optional_edge, + assert_in_degree, + node_match_all, + assert_node_has_no_backwards_edges, + assert_instruction_opname, + is_exactly_opname, +) + + + +class TryTemplate311(ControlFlowTemplate): + _subgraph = { + "try_header": TemplateNode( + node_verification_func=assert_instruction_opname("NOP"), + natural_edge=TemplateEdge(source="try_header", dest="try_body", edge_verification_func=assert_edge_type(ControlFlowEdgeType.NATURAL)), + ), + "try_body": TemplateNode( + natural_edge=TemplateEdge(source="try_body", dest="try_footer", edge_verification_func=assert_edge_type(ControlFlowEdgeType.NATURAL)), + exception_edge=TemplateEdge( + source="try_body", + dest="except_body", + ), + ), + "try_footer": TemplateNode( + node_verification_func=node_match_all( + assert_in_degree(1), + assert_node_has_no_backwards_edges, + ), + natural_edge=TemplateEdge(source="try_footer", dest="after_try_except", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="try_footer", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "except_body": TemplateNode( + subtemplate=ExceptTemplate311, + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="except_body", dest="after_try_except", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + edge_verification_func=optional_edge, + ), + ), + "panic_except": TemplateNode( + node_verification_func=is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_try_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_try_except", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, try_header: ControlFlowTemplate, try_body: ControlFlowTemplate, try_footer: ControlFlowTemplate, except_body: ControlFlowTemplate, panic_except: ControlFlowTemplate): + self.try_header = try_header + self.try_body = try_body + self.try_footer = try_footer + self.except_body = except_body + self.panic_except = panic_except + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=TryTemplate311._subgraph, root_key="try_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + reduced_cfg = cfg.copy() + + # "bite off" the NOP from a linear sequence template + if isinstance(mapping["try_header"], LinearSequenceTemplate): + # grab the nop and update the linear sequence + nop_inst_template = mapping["try_header"].members[-1] + mapping["try_header"].members = mapping["try_header"].members[:-1] + if len(mapping["try_header"].members) == 1: + nx.relabel_nodes(reduced_cfg, {mapping["try_header"]: mapping["try_header"].members[0]}, copy=False) + mapping["try_header"] = mapping["try_header"].members[0] + + # transfer outgoing edges to the bitten off chunk + header_out_edges = list(reduced_cfg.out_edges(mapping["try_header"], data=True)) + reduced_cfg.add_node(nop_inst_template) + reduced_cfg.remove_edges_from(header_out_edges) + reduced_cfg.add_edges_from((nop_inst_template, dst, data) for src, dst, data in header_out_edges) + reduced_cfg.add_edge(mapping["try_header"], nop_inst_template, type=ControlFlowEdgeType.NATURAL.value) + mapping["try_header"] = nop_inst_template + + try_except_template = TryTemplate311(try_header=mapping["try_header"], try_body=mapping["try_body"], try_footer=mapping["try_footer"], except_body=mapping["except_body"], panic_except=mapping["panic_except"]) + + in_edges = ((src, try_except_template, edge_properties) for src, dst, edge_properties in reduced_cfg.in_edges(try_except_template.try_header, data=True)) + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((try_except_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + if "after_try_except" in mapping.keys(): + after_try_except = mapping["after_try_except"] + out_edges.append((try_except_template, after_try_except, {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg.remove_nodes_from([try_except_template.try_header, try_except_template.try_body, try_except_template.try_footer, try_except_template.except_body, try_except_template.panic_except]) + reduced_cfg.add_node(try_except_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + try_header = self.try_header.to_indented_source(source_lines) + try_body = self._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + + except_body = self.except_body.to_indented_source(source_lines) + + lines = [try_header, "try:", try_body, except_body] + + try_footer = self.try_footer.to_indented_source(source_lines) + if try_footer.strip(): + lines.extend(["else: # inserted", self._indent_multiline_string(try_footer)]) + + return "\n".join(lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryTemplate312.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryTemplate312.py new file mode 100644 index 0000000..9dc6cfc --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/post_311/TryTemplate312.py @@ -0,0 +1,169 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...natural.LinearSequenceTemplate import LinearSequenceTemplate + +from .ExceptTemplate311 import ExceptTemplate311 + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import ( + assert_edge_type, + optional_node, + optional_edge, + assert_in_degree, + assert_instruction_opname, + is_exactly_opname, +) + + + +class TryTemplate312(ControlFlowTemplate): + _subgraph = { + "try_header": TemplateNode( + node_verification_func=assert_instruction_opname("NOP"), + natural_edge=TemplateEdge(source="try_header", dest="try_body", edge_verification_func=assert_edge_type(ControlFlowEdgeType.NATURAL)), + ), + "try_body": TemplateNode( + natural_edge=TemplateEdge( + source="try_body", + dest="after_try_except", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="try_body", + dest="except_body", + ), + ), + "except_body": TemplateNode( + subtemplate=ExceptTemplate311, + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="except_body", dest="after_try_except", edge_verification_func=optional_edge, commit_none_to_mapping=False), + exception_edge=TemplateEdge( + source="except_body", + dest="panic_except", + edge_verification_func=optional_edge, + ), + ), + "panic_except": TemplateNode( + node_verification_func=is_exactly_opname("COPY", "POP_EXCEPT", "RERAISE"), + exception_edge=TemplateEdge( + source="except_body", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "after_try_except": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="after_try_except", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="after_try_except", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, try_header: ControlFlowTemplate, try_body: ControlFlowTemplate, except_body: ControlFlowTemplate, panic_except: ControlFlowTemplate): + self.try_header = try_header + self.try_body = try_body + self.except_body = except_body + self.panic_except = panic_except + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=TryTemplate312._subgraph, root_key="try_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + reduced_cfg = cfg.copy() + + # "bite off" the NOP from a linear sequence template + if isinstance(mapping["try_header"], LinearSequenceTemplate): + # grab the nop and update the linear sequence + nop_inst_template = mapping["try_header"].members[-1] + mapping["try_header"].members = mapping["try_header"].members[:-1] + if len(mapping["try_header"].members) == 1: + nx.relabel_nodes(reduced_cfg, {mapping["try_header"]: mapping["try_header"].members[0]}, copy=False) + mapping["try_header"] = mapping["try_header"].members[0] + + # transfer outgoing edges to the bitten off chunk + header_out_edges = list(reduced_cfg.out_edges(mapping["try_header"], data=True)) + reduced_cfg.add_node(nop_inst_template) + reduced_cfg.remove_edges_from(header_out_edges) + reduced_cfg.add_edges_from((nop_inst_template, dst, data) for src, dst, data in header_out_edges) + reduced_cfg.add_edge(mapping["try_header"], nop_inst_template, type=ControlFlowEdgeType.NATURAL.value) + mapping["try_header"] = nop_inst_template + + try_except_template = TryTemplate312( + try_header=mapping["try_header"], + try_body=mapping["try_body"], + except_body=mapping["except_body"], + panic_except=mapping["panic_except"], + ) + + in_edges = ((src, try_except_template, edge_properties) for src, dst, edge_properties in reduced_cfg.in_edges(try_except_template.try_header, data=True)) + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((try_except_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + if "after_try_except" in mapping.keys(): + after_try_except = mapping["after_try_except"] + out_edges.append((try_except_template, after_try_except, {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg.remove_nodes_from([try_except_template.try_header, try_except_template.try_body, try_except_template.except_body, try_except_template.panic_except]) + reduced_cfg.add_node(try_except_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + try_header = self.try_header.to_indented_source(source_lines) + try_body = self._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + + except_body = self.except_body.to_indented_source(source_lines) + + lines = [try_header, "try:", try_body, except_body] + + return "\n".join(lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/ExceptAsPre39.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/ExceptAsPre39.py new file mode 100644 index 0000000..2988614 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/ExceptAsPre39.py @@ -0,0 +1,165 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable +from ...abstract.AbstractExceptionBlockTemplate import AbstractExceptionBlockTemplate +from ...natural.LinearSequenceTemplate import LinearSequenceTemplate +from ...try_except.pre_39.TryFinallyPre39 import Pre39TryFinallyTemplate + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import optional_node, optional_edge, assert_in_degree, assert_except_as + + + +class Pre39ExceptAsTemplate(ControlFlowTemplate, AbstractNonSequentiable, AbstractExceptionBlockTemplate): + """ + An `except as` block, after its cleanup has been structured. + If there are multiple, this will match the last block in the series and set up the next one to be matched + (0) + / \\j --> (012) + (1) (2) |j + |j (3) + (3) + """ + + _subgraph = { + "except_as_header": TemplateNode( + node_verification_func=assert_except_as, + natural_edge=TemplateEdge( + source="except_as_header", + dest="except_setup", + ), + conditional_edge=TemplateEdge(source="except_as_header", dest="non_match_path"), + exception_edge=TemplateEdge(source="except_as_header", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "except_setup": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="except_setup", + dest="except_body", + ), + exception_edge=TemplateEdge(source="except_setup", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "except_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge(source="except_body", dest="begin_finally", edge_verification_func=optional_edge), + exception_edge=TemplateEdge( + source="except_body", + dest="cleanup", + ), + ), + "begin_finally": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="begin_finally", + dest="cleanup", + ), + exception_edge=TemplateEdge(source="begin_finally", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "cleanup": TemplateNode( + exception_edge=TemplateEdge( + source="cleanup", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "non_match_path": TemplateNode( + node_verification_func=assert_in_degree(1), + exception_edge=TemplateEdge( + source="non_match_path", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, except_as_header: ControlFlowTemplate, except_setup: ControlFlowTemplate, except_body: ControlFlowTemplate, begin_finally: ControlFlowTemplate, cleanup: ControlFlowTemplate, non_match_path: ControlFlowTemplate): + self.except_as_header = except_as_header + self.except_setup = except_setup + self.except_body = except_body + self.begin_finally = begin_finally + self.cleanup = cleanup + self.non_match_path = non_match_path + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + matcher = GraphTemplateMatcher(template_node_dict=Pre39ExceptAsTemplate._subgraph, root_key="except_as_header", mapping_verification_func=None) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + except_as_cleanup_template = Pre39ExceptAsTemplate( + except_as_header=mapping["except_as_header"], except_setup=mapping["except_setup"], except_body=mapping["except_body"], begin_finally=mapping["begin_finally"], cleanup=mapping["cleanup"], non_match_path=mapping["non_match_path"] + ) + + in_edges = ((src, except_as_cleanup_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(node, data=True)) + # only preserve exception handling edges + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((except_as_cleanup_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from( + [ + except_as_cleanup_template.except_as_header, + except_as_cleanup_template.except_setup, + except_as_cleanup_template.except_body, + except_as_cleanup_template.begin_finally, + except_as_cleanup_template.cleanup, + except_as_cleanup_template.non_match_path, + ] + ) + reduced_cfg.add_node(except_as_cleanup_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + header = self.except_as_header.to_indented_source(source_lines) + if isinstance(self.except_body, LinearSequenceTemplate): + assert isinstance(self.except_body[0], Pre39TryFinallyTemplate) + _body = self.except_body[0].try_body.to_indented_source(source_lines) + else: + assert isinstance(self.except_body, Pre39TryFinallyTemplate) + _body = self.except_body.try_body.to_indented_source(source_lines) + body = ControlFlowTemplate._indent_multiline_string(_body) + non_match = self.non_match_path.to_indented_source(source_lines) + return f"{header}\n{body}\n{non_match}" + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/TryFinallyExitPre39.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/TryFinallyExitPre39.py new file mode 100644 index 0000000..bc2da20 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/TryFinallyExitPre39.py @@ -0,0 +1,211 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import optional_node, optional_edge, assert_in_degree, assert_instruction_opname, node_match_none, node_match_all, contains_opname_sequence + +from ...subtemplates.OptionalExitSubtemplate import ExitSubTemplate + + +class Pre39TryFinallyExitTemplate(ControlFlowTemplate, AbstractNonSequentiable): + r""" + A `try` block with only a `finally` following it. But 3.8 and below. This has a similar structure to the with template. + (0) only here because could not figure out a way to condense an exit without killing off the tail + | + (1) + / e\ --> (0123) + (2) \ | + \ / (4) + (3) + does not cover additional finally blocks that will be inserted in the bytecode as a result of returns / breaking out of loops + """ + + _subgraph = { + "setup_finally": TemplateNode( + node_verification_func=node_match_all( + assert_instruction_opname("SETUP_FINALLY"), + node_match_none( + contains_opname_sequence( + "POP_TOP", + "STORE_FAST", + "POP_TOP", + ), + ), + ), + natural_edge=TemplateEdge( + source="setup_finally", + dest="try_body", + ), + exception_edge=TemplateEdge(source="setup_finally", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "try_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="try_body", + dest="begin_finally", + ), + exception_edge=TemplateEdge( + source="try_body", + dest="finally", + ), + ), + "begin_finally": TemplateNode( + subtemplate=ExitSubTemplate, + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="begin_finally", + dest="finally", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="begin_finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "finally": TemplateNode( + subtemplate=ExitSubTemplate, + natural_edge=TemplateEdge( + source="finally", + dest="tail", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__( + self, + setup_finally: ControlFlowTemplate, + try_body: ControlFlowTemplate, + begin_finally: ControlFlowTemplate, + _finally: ControlFlowTemplate, + ): + self.setup_finally = setup_finally + self.try_body = try_body + self.begin_finally = begin_finally + self._finally = _finally + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + if cfg.in_degree(node) != 1: + return None + + # to avoid being treated as a try-except, we actually need to greedily search up one layer + node = next(cfg.predecessors(node)) + + matcher = GraphTemplateMatcher( + template_node_dict=Pre39TryFinallyExitTemplate._subgraph, + root_key="setup_finally", + mapping_verification_func=None, + ) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + finally_template = Pre39TryFinallyExitTemplate( + setup_finally=mapping["setup_finally"], + try_body=mapping["try_body"], + begin_finally=mapping["begin_finally"], + _finally=mapping["finally"], + ) + + in_edges = [(src, finally_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(finally_template.setup_finally, data=True)] + # only preserve exception handling edges + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((finally_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + # if there is a tail add a natural out edge + if mapping.get("tail", None): + out_edges.append((finally_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from( + [ + finally_template.setup_finally, + finally_template.try_body, + finally_template.begin_finally, + finally_template._finally, + ] + ) + reduced_cfg.add_node(finally_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + # sometimes the setup finally is included in a linear sequence, so we need to include that source + setup_finally = self.setup_finally.to_indented_source(source_lines) + try_block = ControlFlowTemplate._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + # pick one of the finally bodies to get the source code from + finally_body = ControlFlowTemplate._indent_multiline_string(self._finally.to_indented_source(source_lines)) + + if not finally_body: + finally_body = ControlFlowTemplate._indent_multiline_string(self._finally.to_indented_source(source_lines)) + finally_lines = [setup_finally, "try:", try_block, "finally:", finally_body] + return "\n".join(finally_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/TryFinallyPre39.py b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/TryFinallyPre39.py new file mode 100644 index 0000000..17d0107 --- /dev/null +++ b/pylingual/control_flow_reconstruction/control_flow_templates/try_except/pre_39/TryFinallyPre39.py @@ -0,0 +1,188 @@ +import networkx as nx + +import itertools + +from ...abstract.AbstractTemplate import ControlFlowTemplate +from ...abstract.AbstractNonSequentiableTemplate import AbstractNonSequentiable + +from ....cfg_utils import ControlFlowEdgeType + +from ...Subgraph import TemplateEdge, TemplateNode, GraphTemplateMatcher + +from ...match_utils import optional_node, optional_edge, assert_in_degree, assert_instruction_opname + + + +class Pre39TryFinallyTemplate(ControlFlowTemplate, AbstractNonSequentiable): + r""" + A `try` block with only a `finally` following it. But 3.8 and below. This has a similar structure to the with template + (0) + | + (1) + / e\ --> (0123) + (2) \ | + \ / (4) + (3) + does not cover additional finally blocks that will be inserted in the bytecode as a result of returns / breaking out of loops + """ + + _subgraph = { + "setup_finally": TemplateNode( + node_verification_func=assert_instruction_opname("SETUP_FINALLY"), + natural_edge=TemplateEdge( + source="setup_finally", + dest="try_body", + ), + exception_edge=TemplateEdge(source="setup_finally", dest="outer_exception_handler", edge_verification_func=optional_edge), + ), + "try_body": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="try_body", + dest="begin_finally", + ), + exception_edge=TemplateEdge( + source="try_body", + dest="finally", + ), + ), + "begin_finally": TemplateNode( + node_verification_func=assert_in_degree(1), + natural_edge=TemplateEdge( + source="begin_finally", + dest="finally", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="begin_finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "finally": TemplateNode( + natural_edge=TemplateEdge( + source="finally", + dest="tail", + edge_verification_func=optional_edge, + commit_none_to_mapping=False, + ), + exception_edge=TemplateEdge( + source="finally", + dest="outer_exception_handler", + edge_verification_func=optional_edge, + ), + ), + "tail": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="tail", + dest=None, + edge_verification_func=optional_edge, + ), + ), + "outer_exception_handler": TemplateNode( + node_verification_func=optional_node, + natural_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + exception_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + conditional_edge=TemplateEdge( + source="outer_exception_handler", + dest=None, + edge_verification_func=optional_edge, + ), + ), + } + + def __init__(self, setup_finally: ControlFlowTemplate, try_body: ControlFlowTemplate, begin_finally: ControlFlowTemplate, _finally: ControlFlowTemplate): + self.setup_finally = setup_finally + self.try_body = try_body + self.begin_finally = begin_finally + self._finally = _finally + + @staticmethod + def try_to_match_node(cfg: nx.DiGraph, node) -> nx.DiGraph: + """ + Attempts to match this template on the graph at the given node. + If successful, returns an updated cfg with the appropriate nodes condensed into an instance of this template. + Otherwise, returns None. + """ + if node not in cfg.nodes: + return None + + if cfg.in_degree(node) != 1: + return None + + # to avoid being treated as a try-except, we actually need to greedily search up one layer + node = next(cfg.predecessors(node)) + + matcher = GraphTemplateMatcher( + template_node_dict=Pre39TryFinallyTemplate._subgraph, + root_key="setup_finally", + mapping_verification_func=None, + ) + + mapping = matcher.match_at_graph_node(cfg, node) + + if not mapping: + return None + + finally_template = Pre39TryFinallyTemplate( + setup_finally=mapping["setup_finally"], + try_body=mapping["try_body"], + begin_finally=mapping["begin_finally"], + _finally=mapping["finally"], + ) + + in_edges = [(src, finally_template, edge_properties) for src, dst, edge_properties in cfg.in_edges(finally_template.setup_finally, data=True)] + + # only preserve exception handling edges + + out_edges = [] + if mapping["outer_exception_handler"]: + out_edges.append((finally_template, mapping["outer_exception_handler"], {"type": ControlFlowEdgeType.EXCEPTION.value})) + + # if there is a tail add it as an out edge + if mapping.get("tail", None): + out_edges.append((finally_template, mapping["tail"], {"type": ControlFlowEdgeType.NATURAL.value})) + + reduced_cfg = cfg.copy() + reduced_cfg.remove_nodes_from([finally_template.setup_finally, finally_template.try_body, finally_template.begin_finally, finally_template._finally]) + reduced_cfg.add_node(finally_template) + reduced_cfg.add_edges_from(itertools.chain(in_edges, out_edges)) + return reduced_cfg + + def to_indented_source(self, source_lines: list[str]) -> str: + """ + Returns the source code for this template, recursively calling into its children to create the full source code. + """ + # sometimes the setup finally is included in a linear sequence, so we need to include that source + setup_finally = self.setup_finally.to_indented_source(source_lines) + try_block = ControlFlowTemplate._indent_multiline_string(self.try_body.to_indented_source(source_lines)) + # pick one of the finally bodies to get the source code from + finally_body = ControlFlowTemplate._indent_multiline_string(self._finally.to_indented_source(source_lines)) + + if not finally_body: + finally_body = ControlFlowTemplate._indent_multiline_string(self._finally.to_indented_source(source_lines)) + finally_lines = [setup_finally, "try:", try_block, "finally:", finally_body] + return "\n".join(finally_lines) + + def __repr__(self) -> str: + return super().__repr__() diff --git a/pylingual/control_flow_reconstruction/reconstruct_control_indentation.py b/pylingual/control_flow_reconstruction/reconstruct_control_indentation.py new file mode 100644 index 0000000..6a48e06 --- /dev/null +++ b/pylingual/control_flow_reconstruction/reconstruct_control_indentation.py @@ -0,0 +1,114 @@ +from typing import Hashable + + +def postprocess(source_lines: list[str]): + i = 0 + tab = " " * 4 + decs = [] + while i < len(source_lines): + line = source_lines[i] + if line.startswith("global ") or line.startswith("nonlocal "): + decs.append(i) + elif line.startswith("__doc__ = "): + source_lines[i] = line[10:] + # should check for 'from __future__ import ', but lines are still masked now + # checking only for 'from ' doesn't make a difference though + elif not line.startswith('"""') and not line.startswith("from "): + break + i += 1 + for dec in reversed(decs): + source_lines.insert(i - 1, source_lines.pop(dec)) + block_level = None + can_have_return = [False] + + while i < len(source_lines): + line = source_lines[i] + tabs = len(line) - len(line.lstrip("\t")) + inserted = line.endswith("# inserted") + if inserted: + line = line[:-10] + line = line.strip() + while len(can_have_return) - 1 > tabs: + can_have_return.pop() + if line.startswith("def ") or line.startswith("class ") or line.startswith("async def "): + while len(can_have_return) - 1 < tabs + 1: + can_have_return.append(can_have_return[-1]) + can_have_return[tabs + 1] = not line.startswith("class") + # add newline between function and class defs + if line.startswith("@") or line.startswith("def ") or line.startswith("class ") or line.startswith("async def "): + if i and not source_lines[i - 1].strip().startswith("@") and block_level is None: + source_lines.insert(i, "") + i += 1 + if (line.startswith("return ") or line == "return") and not can_have_return[min(tabs, len(can_have_return) - 1)]: + source_lines.pop(i) + continue + # insert pass in empty blocks + if block_level is not None: + if tabs <= block_level or not line.strip(): + if source_lines[i - 1].strip().startswith("while True:"): + prev_line = source_lines[i - 1] + source_lines[i - 1] = " " * (len(prev_line) - len(prev_line.lstrip(" "))) + "pass" + else: + source_lines.insert(i, tab * (block_level + 1) + "pass # postinserted") + i += 1 + block_level = None + if line.endswith(":"): + block_level = tabs + + # convert tabs to spaces + source_lines[i] = tab * tabs + line + (" # inserted" if inserted else "") + i += 1 + if block_level is not None: + source_lines.insert(i, tab * (block_level + 1) + "pass # postinserted") + i += 1 + return "\n".join(source_lines) + + +def reconstruct_source(pyc, sources): + merged_source, blame = merge_indented_sources(pyc, sources) + return postprocess(merged_source), blame + + +def split_newlines(li): + return "\n".join(li).split("\n") + + +def indent_newlines(li, n=1): + li = [line for line in split_newlines(li)] + return ["\t" * n + line for line in li] + + +def merge_indented_sources(pyc, sources): + blame_dict = {} + for bytecode in pyc.child_bytecodes: + sources[bytecode.codeobj], blame_dict[bytecode.codeobj] = merge_indented_sources(bytecode, sources) + line = 0 + indented_source = split_newlines(sources[pyc.codeobj]) + blame = [pyc.codeobj] * len(indented_source) + lines_set = set() + for i, instruction in enumerate(pyc.ordered_instructions): + if instruction.starts_line and instruction.starts_line not in lines_set: + lines_set.add(instruction.starts_line) + # count implicit else/finally/while True + while line < len(indented_source) and indented_source[line].endswith("# inserted"): + line += 1 + line += 1 + if instruction.opname == "LOAD_CONST" and isinstance(instruction.argval, Hashable): + if instruction.argval in sources: + if instruction.argval.co_name not in ("", "", "", "", ""): + new_tabs = 1 + + # add indentation of previous line + prev_line = "" + if line > 0: + if line < len(indented_source): + prev_line = indented_source[line - 1] + else: + prev_line = indented_source[-1] + new_tabs += len(prev_line) - len(prev_line.lstrip("\t")) + + code_to_insert = indent_newlines(sources[instruction.argval], new_tabs) + indented_source[line:line] = code_to_insert + blame[line:line] = blame_dict[instruction.argval] + line += len(code_to_insert) + return indented_source, blame diff --git a/pylingual/control_flow_reconstruction/structure_control_flow.py b/pylingual/control_flow_reconstruction/structure_control_flow.py new file mode 100644 index 0000000..e39f704 --- /dev/null +++ b/pylingual/control_flow_reconstruction/structure_control_flow.py @@ -0,0 +1,452 @@ +import networkx as nx + +import os + +from pylingual.utils.lazy import lazy_import + +from .cfg_utils import ControlFlowEdgeType, get_out_edge_dict, get_dominator_function +from pylingual.editable_bytecode import Inst +from pylingual.editable_bytecode import EditableBytecode + +# abstract type +from .control_flow_templates.abstract.AbstractTemplate import ControlFlowTemplate + +from .control_flow_templates.placeholders.IrreduciblePlaceholderTemplate import IrreduciblePlaceholderTemplate + +# default flow +from .control_flow_templates.natural.InstructionTemplate import InstructionTemplate +from .control_flow_templates.natural.LinearSequenceTemplate import LinearSequenceTemplate +from .control_flow_templates.natural.LineTemplate import LineTemplate + +# if/else +from .control_flow_templates.if_then.IfThenTemplate import IfThenTemplate +from .control_flow_templates.if_then.IfElseTemplate import IfElseTemplate +from .control_flow_templates.if_then.IfThenJumpTemplate import IfThenJumpTemplate +from .control_flow_templates.if_then.ConditionalExitTemplate import ConditionalExitTemplate +from .control_flow_templates.booleans.ShortCircuitOrTemplate import ShortCircuitOrTemplate +from .control_flow_templates.booleans.ShortCircuitOrContinueTemplate import ShortCircuitOrContinueTemplate +from .control_flow_templates.booleans.ShortCircuitAndTemplate import ShortCircuitAndTemplate +from .control_flow_templates.booleans.ChainedComparisonTemplate import ChainedComparisonTemplate +from .control_flow_templates.context_managers.WithTemplate import WithTemplate +from .control_flow_templates.context_managers.WithTemplate39 import WithTemplate39 +from .control_flow_templates.context_managers.WithCleanup312 import WithCleanup312 +from .control_flow_templates.context_managers.AsyncWithCleanup312 import AsyncWithCleanup312 +from .control_flow_templates.context_managers.WithTemplate312 import WithTemplate312 +from .control_flow_templates.context_managers.Await312Template import Await312Template + +# loops +from .control_flow_templates.loop.LoopTemplate import LoopTemplate +from .control_flow_templates.loop.SelfLoopTemplate import SelfLoopTemplate +from .control_flow_templates.loop.LoopExitTemplate import LoopExitTemplate +from .control_flow_templates.loop.PreRefinedLoopTemplate import PreRefinedLoopTemplate +from .control_flow_templates.loop.RefinedLoopTemplate import RefinedLoopTemplate +from .control_flow_templates.loop.WhileTrueIfElseTemplate import WhileTrueIfElseTemplate +from .control_flow_templates.loop.AsyncForTemplate import AsyncForTemplate +from .control_flow_templates.loop.InlinedComprehension import InlinedComprehensionTemplate +from .control_flow_templates.loop.ForIf312Template import ForIf312Template + +# exceptions +from .control_flow_templates.try_except.TryExceptTemplate import TryExceptTemplate +from .control_flow_templates.try_except.TryExceptElseTemplate import TryExceptElseTemplate +from .control_flow_templates.try_except.ExceptAsExceptTemplate import ExceptAsExceptTemplate +from .control_flow_templates.try_except.ExceptAsCleanup import ExceptAsCleanupTemplate +from .control_flow_templates.try_except.ExceptAsExitTemplate import ExceptAsExitTemplate +from .control_flow_templates.try_except.FinallyTemplate import FinallyTemplate +from .control_flow_templates.try_except.TryFinallyTemplate import TryFinallyTemplate +from .control_flow_templates.try_except.pre_39.TryFinallyPre39 import Pre39TryFinallyTemplate +from .control_flow_templates.try_except.pre_39.TryFinallyExitPre39 import Pre39TryFinallyExitTemplate +from .control_flow_templates.try_except.pre_39.ExceptAsPre39 import Pre39ExceptAsTemplate +from .control_flow_templates.try_except.ExceptException import ExceptException +from .control_flow_templates.try_except.GeneratorCleanupTemplate import GeneratorCleanupTemplate + +# 3.11/3.12-specific exceptions +from .control_flow_templates.try_except.post_311.TryTemplate311 import TryTemplate311 +from .control_flow_templates.try_except.post_311.TryTemplate312 import TryTemplate312 +from .control_flow_templates.try_except.post_311.FinallyTemplate312 import FinallyTemplate312 + +import pathlib + +lazy_import("pydot") + +from typing import Generator, Any + + +def viz(graph, name, node_label="label"): + namepath = pathlib.Path(name) + dot = pydot.Dot(namepath.name) + nodes = {} + + for node, data in graph.nodes.data(): + n = pydot.Node(hash(node), label=data[node_label]) + dot.add_node(n) + nodes[hash(node)] = n + + for node1, node2, data in graph.edges.data(): + edge = pydot.Edge(nodes[hash(node1)], nodes[hash(node2)], **data) + dot.add_edge(edge) + + try: + dot.write_png(name) + except FileNotFoundError: + dot.write_raw(name.replace(".png", ".dot")) + + +# order matters! +# More specific templates should appear before more general templates for correctness +# More common templates should appear before more rare templates for efficiency +cyclic_templates: list[type[ControlFlowTemplate]] = [ + WhileTrueIfElseTemplate, + LoopTemplate, + SelfLoopTemplate, + ShortCircuitOrTemplate, # the short circuit templates aren't cyclic, but are needed to match certain while loops + ShortCircuitAndTemplate, +] + +# priority dict structure +# Template type : (pass number, priority number) # lower is earlier +acyclic_templates_priority_dict: dict[ControlFlowTemplate, tuple[int, int]] = { + RefinedLoopTemplate: (0, 0), + AsyncForTemplate: (0, 1), # technically a cyclic template, but it searches up one node to complete the loop + FinallyTemplate: (0, 10), + WithTemplate: (0, 11), + LinearSequenceTemplate: (0, 20), + ExceptAsExitTemplate: (0, 25), + ExceptAsExceptTemplate: (0, 27), + ShortCircuitOrContinueTemplate: (0, 30), + IfElseTemplate: (1, 43), + IfThenTemplate: (1, 44), + IfThenJumpTemplate: (0, 45), + ConditionalExitTemplate: (0, 46), + ExceptAsCleanupTemplate: (0, 50), + TryFinallyTemplate: (0, 60), + TryExceptElseTemplate: (0, 62), + ShortCircuitOrTemplate: (0, 70), + ShortCircuitAndTemplate: (0, 71), + ChainedComparisonTemplate: (0, 72), +} + +# dictionary structure +# version: {template: (pass, priority)} +version_specific_acyclic_templates_dict: dict[tuple[int, int], dict[ControlFlowTemplate, tuple[int, int]]] = { + (3, 13): { + TryTemplate312: (-1, 60), + TryTemplate311: (-1, 61), + WithCleanup312: (-1, 0), + AsyncWithCleanup312: (-1, 0), + WithTemplate312: (0, 10), + InlinedComprehensionTemplate: (-1, 0), + GeneratorCleanupTemplate: (0, 1), + Await312Template: (0, 2), + ForIf312Template: (0, 0), + FinallyTemplate312: (-1, 199), + }, + (3, 12): { + TryTemplate312: (-1, 60), + TryTemplate311: (-1, 61), + WithCleanup312: (-1, 0), + AsyncWithCleanup312: (-1, 0), + WithTemplate312: (0, 10), + InlinedComprehensionTemplate: (-1, 0), + GeneratorCleanupTemplate: (0, 1), + Await312Template: (0, 2), + ForIf312Template: (0, 0), + FinallyTemplate312: (-1, 199), + }, + (3, 11): { + TryTemplate311: (-1, 55), + }, + (3, 9): { + WithTemplate39: (0, 12), + TryExceptTemplate: (1, 61), + }, + (3, 8): { + Pre39ExceptAsTemplate: (0, 40), + Pre39TryFinallyTemplate: (0, 60), + Pre39TryFinallyExitTemplate: (0, 75), + ExceptException: (0, 38), + TryExceptTemplate: (1, 61), + }, + (3, 7): { + Pre39ExceptAsTemplate: (0, 40), + Pre39TryFinallyTemplate: (0, 60), + Pre39TryFinallyExitTemplate: (0, 75), + ExceptException: (0, 38), + TryExceptTemplate: (1, 61), + }, + (3, 6): { + Pre39ExceptAsTemplate: (0, 40), + Pre39TryFinallyTemplate: (0, 60), + Pre39TryFinallyExitTemplate: (0, 75), + ExceptException: (0, 38), + TryExceptTemplate: (1, 61), + }, +} + + +def get_acyclic_template_passes(version: tuple[int, int]) -> Generator[list[ControlFlowTemplate], None, None]: + pass_dict = dict() + # accumulate the passes, merging in version-specific templates + for template, (pass_number, priority) in (acyclic_templates_priority_dict | version_specific_acyclic_templates_dict.get(version, dict())).items(): + pass_list = pass_dict.get(pass_number, list()) + pass_list.append((template, priority)) + pass_dict[pass_number] = pass_list + # sort each pass by priority + for pass_number, pass_list in pass_dict.items(): + pass_dict[pass_number] = [template for template, priority in sorted(pass_list, key=lambda item: item[1])] + # yield the templates for each pass + for pass_number in sorted(pass_dict.keys()): + yield pass_dict[pass_number] + + +def visualize(graph: nx.DiGraph, name, suffix): + # visualization is slow + if os.environ.get("DEBUG_CFLOW", None) != "1": + return + for n in graph.nodes: + graph.nodes[n]["label"] = repr(n) + v = next(x for x in graph.nodes if not isinstance(x, str)).get_instructions()[0].bytecode.version + viz(graph, f"/tmp/graph/{name}_{v[1]}_{suffix}.png", edge_label="type") + + +def structure_loop(cfg: nx.DiGraph, node) -> nx.DiGraph: + dominates = get_dominator_function(cfg) + # a node is a loop header if there are back-edges to it + # a latching node is a node with a back-edge to the loop header + # a back-edge is an edge from any node that is dominated by this node + latching_nodes = [pred for pred in cfg.predecessors(node) if dominates(node, pred)] + if not latching_nodes: + return None + + # attempt to match a loop template + for template in cyclic_templates: + candidate_cfg = template.try_to_match_node(cfg, node) + if candidate_cfg is not None: + return candidate_cfg + + if len(node.get_instructions()) == 1 and node.get_instructions()[0].opname == "SEND": + return None + + # identify the canonical loop exit and outer exception handler by looking at the loop header + loop_header_edge_dict = get_out_edge_dict(cfg, node) + canonical_loop_exit, _ = loop_header_edge_dict["conditional"] + outer_exception_handler, _ = loop_header_edge_dict["exception"] + + # subgraph containing all nodes dominated by the loop header + dominated_subgraph: nx.DiGraph = cfg.subgraph(n for n in cfg.nodes if dominates(node, n)) + reverse_reachability_map = nx.single_source_shortest_path_length(dominated_subgraph.reverse(), source=node) + # a node is in the loop if there is a backwards path to the header that doesn't leave the loop + loop_nodes = [loop_node for loop_node, distance in reverse_reachability_map.items() if distance >= 0] + # extend loop nodes with their natural edges; you can't leave the loop without a jump of some kind + # also extend loop nodes with exception edges that do not leave the loop + natural_edges = [(u, v) for u, v, data in dominated_subgraph.edges(data=True) if data["type"] == ControlFlowEdgeType.NATURAL.value] + # also extend loop nodes with their conditional edges, excluding the loop header + conditional_edges = [(u, v) for u, v, data in dominated_subgraph.edges(data=True) if data["type"] in [ControlFlowEdgeType.TRUE_JUMP.value, ControlFlowEdgeType.FALSE_JUMP.value] and u != node] + internal_exception_edges = [(u, v) for u, v, data in dominated_subgraph.edges(data=True) if data["type"] == ControlFlowEdgeType.EXCEPTION.value and v is not outer_exception_handler] + natural_dominated_subgraph = dominated_subgraph.edge_subgraph(natural_edges + internal_exception_edges + conditional_edges) + loop_nodes = set(loop_nodes + [v for _, v in nx.edge_dfs(natural_dominated_subgraph, source=loop_nodes)]) + + # canonical loop exit can be misidentified in while trues that start with if statements + if canonical_loop_exit and any(exit_successor in loop_nodes for exit_successor in cfg.successors(canonical_loop_exit)): + canonical_loop_exit = None + + # There are 4 kinds of exits: + # 1. canonical exit (the conditional branch from the loop header) + # 2. break statement + # 3. return statement + # 4. raised exception caught outside loop + loop_exit_edges = [(src, dst) for src, dst in cfg.edges if src in loop_nodes and dst not in loop_nodes and cfg.get_edge_data(src, dst)["type"] != ControlFlowEdgeType.META.value] + + loop_successor = None + break_edges = [] + for loop_node, exit_node in loop_exit_edges: + # skip the canonical exit + if loop_node is node and exit_node is canonical_loop_exit: + continue + + # skip exception edges to the outer handler + if cfg.get_edge_data(loop_node, exit_node)["type"] == ControlFlowEdgeType.EXCEPTION.value and exit_node is outer_exception_handler: + continue + + # all other cases are exhausted, so we are now only considering break statements + if loop_successor is None: + loop_successor = exit_node + elif loop_successor != exit_node: + if os.environ.get("DEBUG_CFLOW", None) == "1": + breakpoint() + raise RuntimeError("Found multiple break targets in the same loop!") + + break_edges.append((loop_node, exit_node)) + + # if there are no break statements, then the successor is the canonical exit + # the canonical exit may be different in the case of a loop-else, but that only matters if there are breaks + if loop_successor is None: + loop_successor = canonical_loop_exit + + # continue edges are all the latching nodes; may be explicit or implicit + continue_edges = [(src, node) for src in latching_nodes] + + # if we found nothing to refine, then exit + if not continue_edges and not break_edges: + return None + + # reduce the break/continue edges + reduced_cfg = cfg.copy() + for continue_edge in set(continue_edges): + LoopExitTemplate.structure_edge_inplace(reduced_cfg, continue_edge, exit_statment="continue") + + for break_edge in set(break_edges): + LoopExitTemplate.structure_edge_inplace(reduced_cfg, break_edge, exit_statment="break") + + # partially structure the loop while we have the information available + # if the canonical exit is not the successor, then the canonical exit is a loop else + if canonical_loop_exit is not None and loop_successor is not None and canonical_loop_exit != loop_successor: + loop_else_out_edges = get_out_edge_dict(reduced_cfg, canonical_loop_exit) + if loop_else_out_edges["natural"] is not None and loop_else_out_edges["natural"][0] != loop_successor: + # todo: fix triple nested loop w else break + e = (canonical_loop_exit, loop_else_out_edges["natural"][0]) + if dominates(e[1], e[0]): + # backwards edge + canonical_loop_exit = LoopExitTemplate.structure_edge_inplace(reduced_cfg, e, exit_statment="continue") + else: + canonical_loop_exit = LoopExitTemplate.structure_edge_inplace(reduced_cfg, e, exit_statment="break") + PreRefinedLoopTemplate.structure_nodes_inplace(reduced_cfg, loop_header=node, canonical_loop_exit=canonical_loop_exit, loop_successor=loop_successor) + + return reduced_cfg + + +def get_line_out_edge_dict(cfg: nx.DiGraph, insts: list[Inst]) -> dict[str, tuple[Any, ControlFlowEdgeType]]: + # check that all outgoing edges of a given category have the same target + line_out_edge_dict = dict() + for inst in insts: + for edge_category, (edge_target, edge_data) in get_out_edge_dict(cfg, inst).items(): + # skip considering internal control flow + if edge_target is None or edge_target in insts: + continue + # add edge to line-level mapping if this is the first time we've seen it + if edge_category not in line_out_edge_dict: + line_out_edge_dict[edge_category] = (edge_target, edge_data["type"]) + # reject inconsistent mappings; this line cannot be condensed + elif edge_target != line_out_edge_dict[edge_category]: + return None + return line_out_edge_dict + + +def condense_lines(cfg: nx.DiGraph, bytecode: EditableBytecode) -> nx.DiGraph: + lno_insts = bytecode.get_lno_insts() + for line_number, insts in lno_insts.items(): + insts = [inst for inst in insts if inst in cfg.nodes] # discard unreachable instructions + if not insts: + continue + line_in_edges = cfg.in_edges(nbunch=insts, data=True) + # check that no edges come from the outside to the middle of the line (sanity check) + incoming_edges = [(src, dst, data) for src, dst, data in line_in_edges if src not in insts] + if any(dst != insts[0] for src, dst, data in incoming_edges): + continue + + line_out_edge_dict = get_line_out_edge_dict(cfg, insts) + if line_out_edge_dict is None: + continue + + # group up all the instructions in the line into a LineTemplate + line_template = LineTemplate(*[InstructionTemplate(inst) for inst in insts]) + cfg.remove_nodes_from(insts) + cfg.add_node(line_template) + cfg.add_edges_from((src, line_template, data) for src, dst, data in incoming_edges) + for edge_category, (target, edge_type) in line_out_edge_dict.items(): + cfg.add_edge(line_template, target, type=edge_type) + + +def condense_basic_blocks(cfg: nx.DiGraph) -> nx.DiGraph: + structured_cfg = cfg.copy() + for node in list(structured_cfg.nodes): + if node == "START": + continue + candidate_cfg = LinearSequenceTemplate.try_to_match_node(structured_cfg, node) + if candidate_cfg is not None: + structured_cfg = candidate_cfg + return structured_cfg + + +def structure_control_flow(cfg: nx.DiGraph, bytecode: EditableBytecode) -> ControlFlowTemplate: + # group lines with no weird control flow into LineTemplates + # currently reduces overall performance on 3.9 + # condense_lines(cfg, bytecode) + + # 1. wrap instructions globally + structured_cfg = InstructionTemplate.match_graph(cfg) + root_node = min([inst_template for inst_template in structured_cfg.nodes], key=lambda inst_template: inst_template.get_instructions()[0].offset) + structured_cfg.add_nodes_from(["START", "END"]) + structured_cfg.add_edge("START", root_node, type="meta") + structured_cfg.add_edges_from((inst_template, "END", {"type": "meta"}) for inst_template in structured_cfg.nodes if isinstance(inst_template, InstructionTemplate) and inst_template.instruction.opname in ["RETURN_VALUE", "RETURN_CONST"]) + + modification_counter = 0 + # 2. match linear sequences globally + structured_cfg = condense_basic_blocks(structured_cfg) + + # 3. repeat until the graph has no non-meta edges + # 3a. Check for matches on loop templates + # 3b. Check for matches on non-loop templates + # 3c. Check for matches on exception templates + visualize(structured_cfg, bytecode.name, modification_counter) + + def fully_structured(cfg: nx.DiGraph) -> bool: + # if there are any non-meta edges, the control flow is not fully structured + if any(edge_type != ControlFlowEdgeType.META.value for _, _, edge_type in structured_cfg.edges(data="type")): + return False + # if there is more than one node other than START and END, the control flow is not fully structured + if len(cfg) > 3: + return False + return True + + infinite_loop_detection_threshold = 50 + + while not fully_structured(structured_cfg): + modified = False + for acyclic_templates in get_acyclic_template_passes(version=bytecode.version.as_tuple()): + current_num_nodes = len(structured_cfg.nodes) + for node in nx.dfs_postorder_nodes(structured_cfg, source="START"): + # don't process the start node + if node in ["START", "END"]: + continue + + if new_cfg := structure_loop(structured_cfg, node): + structured_cfg = new_cfg + modified = True + modification_counter += 1 + visualize(structured_cfg, bytecode.name, modification_counter) + break + + # check acyclic patterns if no cyclic pattern was matched + for template in acyclic_templates: + candidate_cfg = template.try_to_match_node(structured_cfg, node) + if candidate_cfg is not None: + structured_cfg = candidate_cfg + modified = True + modification_counter += 1 + visualize(structured_cfg, bytecode.name, modification_counter) + break + + if modified: + break + + if modified: + break + + if not modified: + # if in debug mode and template is irreducible breakpoint to inspect cfg + if os.environ.get("DEBUG_CFLOW", None) == "1": + breakpoint() + return IrreduciblePlaceholderTemplate("irreducible") + else: + new_num_nodes = len(structured_cfg) + if new_num_nodes >= current_num_nodes: + infinite_loop_detection_threshold -= 1 + else: + infinite_loop_detection_threshold = 50 + + if infinite_loop_detection_threshold <= 0: + return IrreduciblePlaceholderTemplate("infinite grammar loop") + + structured_cfg.remove_nodes_from(["START", "END"]) + + return list(structured_cfg.nodes)[0] diff --git a/pylingual/decompiler.py b/pylingual/decompiler.py new file mode 100644 index 0000000..24d0783 --- /dev/null +++ b/pylingual/decompiler.py @@ -0,0 +1,499 @@ +from __future__ import annotations + +import datetime +import functools +import importlib.resources +import itertools +import keyword +import logging +import re +import tempfile +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from xdis.magics import magicint2version + +from pylingual.control_flow_reconstruction.cflow import bytecode_to_indented_source +from pylingual.control_flow_reconstruction.reconstruct_control_indentation import reconstruct_source +from pylingual.equivalence_check import TestResult, compare_pyc +from pylingual.models import CacheTranslator, load_models +from pylingual.utils.generate_bytecode import CompileError, compile_version +from pylingual.masking.model_disasm import create_global_masker, restore_masked_source_text +from pylingual.editable_bytecode import PYCFile +from pylingual.segmentation.segmentation_search_strategies import get_top_k_predictions, m_deep_top_k, naive_confidence_priority, filter_subwords +from pylingual.segmentation.sliding_window import merge, sliding_window +from pylingual.utils.lists import unflatten +from pylingual.utils.version import PythonVersion +from pylingual.utils.tracked_list import CFLOW_STEP, CORRECTION_STEP, SEGMENTATION_STEP, TrackedList, TrackedDataset + +if TYPE_CHECKING: + import transformers + from pylingual.editable_bytecode.Instruction import Inst + +logger = logging.getLogger(__name__) + +bytecode_separator = " " +lno_regex = re.compile(r"(?<=line )\d+") +def_regex = re.compile(r"(?<=def ).+?(?=\()") +class_regex = re.compile(r"(?<=class ).+?(?=:|\()") + + +def has_comp_error(results: list[TestResult]) -> bool: + return bool(results) and isinstance(results[0], Exception) + + +@dataclass +class DecompilerResult: + """ + Dataclass containing relevant results from decompiling a pyc + + :param equivalence_results: list of internal bytecode comparison results + :param original_pyc: path to original pyc + :param decompiled_source: path to decompiled source + :param out_dir: directory where decompiler output and internal steps are written + :param version: python version of pyc + """ + + equivalence_results: list[TestResult] + original_pyc: Path + decompiled_source: Path + out_dir: Path + version: PythonVersion + + def calculate_success_rate(self) -> float: + if not self.equivalence_results: + return 0 + return sum(1 for x in self.equivalence_results if x.success) / len(self.equivalence_results) * 100 + + +class Decompiler: + """ + You probably want to use decompile() instead. + + Decompiles a PYC file after masking bytecode, segmenting bytecode, and translating bytecode back into source statements, then reconstructs the control flow. + Additionally saves the decompiled file into the specified output directory. + + :param pyc: The PYCFile loaded into memory + :param out_dir: The output directory where decompilation results will be stored + :param segmenter: The loaded segmentation model + :param translator: The loaded translation model + :param version: The python version + :param top_k: Value of k to use for top k segmentation + :param trust_lnotab: Decides whether or not to use line number information + """ + + def __init__(self, pyc: PYCFile, out_dir: Path, segmenter: transformers.Pipeline, translator: CacheTranslator, version: PythonVersion, top_k=10, trust_lnotab=False): + self.pyc = pyc + self.file = pyc.pyc_path + self.out_dir = out_dir + self.segmenter = segmenter + self.translator = translator + self.version = version + self.out_dir.mkdir(parents=True, exist_ok=True) + + self.top_k = top_k + self.highest_k_used = 0 + + self.trust_lnotab = trust_lnotab + + self.header = "# Decompiled with PyLingual (https://pylingual.io)\n" + try: + self.header += ( + f"# Internal filename: {self.pyc.codeobj.co_filename}\n" + f"# Bytecode version: {magicint2version[self.pyc.magic]} ({self.pyc.magic})\n" + f"# Source timestamp: {datetime.datetime.fromtimestamp(self.pyc.timestamp, datetime.UTC).strftime('%Y-%m-%d %H:%M:%S UTC')} ({self.pyc.timestamp})\n\n" + ) + except: + pass + + self.decompile() + self.log_results() + + logger.info(f"Checking decompilation for {self.file.name}...") + if shutil.which("pyenv") is None and self.version != sys.version_info: + logger.warning(f"pyenv is not installed so equivalence check cannot be performed. Please install pyenv manually along with the required Python version ({self.version}) or run PyLingual again with the --init-pyenv flag") + self.result = DecompilerResult([TestResult(False, "Cannot compare equivalence without pyenv installed", bc.name, bc.name) for bc in self.pyc.iter_bytecodes()], self.file, self.candidate_source_path, self.out_dir, self.version) + return + + self.equivalence_results = self.check_reconstruction() + self.correct_failures() + + if has_comp_error(self.equivalence_results): + self.equivalence_results += self.purge_comp_errors() + + equivalence_report = self.out_dir / "equivalence_report.txt" + equivalence_report.write_text("\n".join(str(r) for r in self.equivalence_results)) + + self.result = DecompilerResult(self.equivalence_results, self.file, self.candidate_source_path, self.out_dir, self.version) + + def decompile(self): + self.mask_bytecode() + if self.trust_lnotab: + self.update_segmentation_from_lnotab() + else: + self.run_segmentation() + self.run_translation() + self.run_cflow_reconstruction() + self.reconstruct_source() + + def find_comp_error_cause(self, results: list[TestResult]): + # parse lno from exception + lno = int(lno_regex.search(str(results[0])).group(0)) - 1 + # adjust for lines added in postprocessing + lno -= sum(1 for x in (self.header + self.indented_source).split("\n")[: lno + 1] if x.endswith("# postinserted") or not x.strip() or x.strip().startswith("#")) + + # get offending codeobj + + bad_codeobj = self.blame[lno] + + bad_idx = next(i for i, e in enumerate(self.ordered_bytecodes) if e.codeobj == bad_codeobj) + return bad_idx + + def correct_failures(self): + changed = False + + try: + # fix compile errors + corrected_comp_errors = set() + while has_comp_error(self.equivalence_results): + bad_idx = self.find_comp_error_cause(self.equivalence_results) + # i don't think this will ever happen but better safe than sorry + if bad_idx in corrected_comp_errors: + return + if not self.correct_segmentation(bad_idx, from_comp_error=True): + return + changed = True + corrected_comp_errors.add(bad_idx) + failed = TrackedList(CORRECTION_STEP, [i for i, result in enumerate(self.equivalence_results) if not result.success]) + for i in failed: + if self.correct_segmentation(i): + changed = True + continue + # other fixes... + except Exception as e: + e.add_note("From error correction") + raise + finally: + if changed: + self.log_results() + + # get eq results after replacing all codeobjs with comp errors with pass, preserving nested codeobjs + def purge_comp_errors(self): + try: + equivalence_results = self.equivalence_results + + def replace_line(line): + line = line.strip() + x = def_regex.search(line) + if x is not None: + try: + x = x.group(0) + x = self.global_masker.unmask(x) if x.startswith("= 2: + bounds = "B" + "I" * (len(bc_line) - 2) + "E" + else: + raise ValueError("Unexpected amount of bytecodes segmented into a line") + boundaries.extend(list(bounds)) + + self.segmentation_results.append([{"entity": entity, "score": 1} for entity in boundaries]) + + self.update_starts_line() + + def run_translation(self): + logger.info(f"Translating statements for {self.file.name}...") + try: + translation_requests = [] + for instructions, boundary_predictions in zip(self.ordered_instructions, self.segmentation_results): + translation_requests.append(self.make_translation_request(instructions, boundary_predictions)) + flattened_translation_requests = list(itertools.chain.from_iterable(translation_requests)) + self.translation_results = self.translator(flattened_translation_requests) + unflatten(self.translation_results, translation_requests) + self.update_source_lines() + except Exception as e: + e.add_note("From translation") + raise + + def run_cflow_reconstruction(self): + logger.info(f"Reconstructing control flow for {self.file.name}...") + try: + self.cflow_results = {bc.codeobj: bytecode_to_indented_source(bc, self.source_lines) for bc in TrackedDataset(CFLOW_STEP, self.ordered_bytecodes)} + except Exception as e: + e.add_note("From control flow reconstruction") + raise + + # merge sources and unmask source + def reconstruct_source(self): + logger.info(f"Reconstructing source for {self.file.name}...") + # merge sources and postprocess results + try: + self.indented_masked_source, self.blame = reconstruct_source(self.pyc, {a: b for a, b in self.cflow_results.items()}) + except Exception as e: + e.add_note("From control flow reconstruction") + raise + # undo the masking + try: + self.indented_source = restore_masked_source_text(self.indented_masked_source, self.global_masker, python_version=self.version) + except Exception as e: + e.add_note("From unmasking source") + raise + + # write indented source and pyc + def log_results(self): + self.candidate_source_path = self.out_dir / self.file.with_suffix(".py").name + self.candidate_pyc_path = self.candidate_source_path.with_suffix(".pyc") + self.candidate_source_path.write_text(self.header + self.indented_source) + try: + compile_version(self.candidate_source_path, self.candidate_pyc_path, self.version) + except Exception: + pass # it's ok if the python doesn't compile + + # make a translation request from a segmentation result + def make_translation_request(self, instructions: list[Inst], boundary_predictions: list[dict]) -> list[str]: + translation_requests = [] + for inst, boundary_prediction in zip(instructions, boundary_predictions): + if boundary_prediction["entity"] == "B": + translation_requests.append(self.global_masker.get_model_view(inst)) + else: + translation_requests[-1] += bytecode_separator + self.global_masker.get_model_view(inst) + return translation_requests + + def update_source_lines(self): + self.source_lines = list(itertools.chain.from_iterable(self.translation_results)) + if self.version == (3, 12): + self.pyc.fix_while12(self.source_lines) + elif self.version >= (3, 10): + self.pyc.fix_while(self.source_lines) + + # compiles and compares result to original pyc + def check_reconstruction(self, write_source=False) -> list: + candidate_source_path = self.candidate_source_path + candidate_pyc_path = self.candidate_pyc_path + if write_source: + tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".py") + tmp.write(self.header + self.indented_source) + candidate_source_path = Path(tmp.name) + candidate_pyc_path = Path(tmp.name).with_suffix(".pyc") + + # compile source + try: + compile_version(candidate_source_path, candidate_pyc_path, self.version) + except CompileError as e: + return [e] + else: + return compare_pyc(self.file, candidate_pyc_path) + + # try to correct the segmentation of the ith code object + def correct_segmentation(self, i: int, from_comp_error=False) -> bool: + if not self.segmentation_results[i]: + return False + original_prediction = [r["entity"] for r in self.segmentation_results[i]] + strategy = functools.partial(m_deep_top_k, priority_function=naive_confidence_priority, m=2, k=self.top_k + 1) + # skip first prediction since it is the same as original + for k, prediction in enumerate(get_top_k_predictions(strategy, self.segmentation_results[i])[1:], start=1): + if prediction[0] != "B": + continue + # change segmentation to new prediction + for r, p in zip(self.segmentation_results[i], prediction): + r["entity"] = p + self.update_starts_line() + # retranslate affected bytecode + translation_request = self.make_translation_request(self.ordered_instructions[i], self.segmentation_results[i]) + try: + self.translation_results[i] = self.translator(translation_request) + self.update_source_lines() + except Exception as e: + e.add_note("From translation") + raise + # redo cflow of affected bytecode + try: + bc = self.ordered_bytecodes[i] + self.cflow_results[bc.codeobj] = bytecode_to_indented_source(bc, self.source_lines) + except Exception as e: + e.add_note("From control flow reconstruction") + raise + # check if new reconstruction is correct + previous_indented_masked_source, previous_blame, previous_indented_source = self.indented_masked_source, self.blame, self.indented_source + self.reconstruct_source() + equivalence_results = self.check_reconstruction(write_source=True) + if from_comp_error: + if not has_comp_error(equivalence_results) or self.find_comp_error_cause(equivalence_results) != i: + self.equivalence_results = equivalence_results + self.highest_k_used = max(self.highest_k_used, k) + return True + elif not has_comp_error(equivalence_results) and equivalence_results[i].success: + self.equivalence_results[i] = equivalence_results[i] + self.highest_k_used = max(self.highest_k_used, k) + return True + # correction failed, roll back changes to internal source code storage + self.indented_masked_source, self.blame, self.indented_source = previous_indented_masked_source, previous_blame, previous_indented_source + # revert to original segmentation + for r, p in zip(self.segmentation_results[i], original_prediction): + r["entity"] = p + return False + + # update starts_line of all instructions based on segmentation results + def update_starts_line(self): + line = 0 + for instructions, boundary_predictions in zip(self.ordered_instructions, self.segmentation_results): + for inst, boundary_prediction in zip(instructions, boundary_predictions): + if boundary_prediction["entity"] == "B": + line += 1 + inst.starts_line = line + else: + inst.starts_line = None + + +def decompile(file: Path, out_dir: Path, config_file: Path | None = None, version: PythonVersion | tuple[int, int] | str | None = None, top_k: int = 10, trust_lnotab: bool = False) -> DecompilerResult: + """ + Decompile a PYC file. + + :param file: path to pyc to decompile + :param out_dir: Path to save decompilation results and steps to. Defaults to ./decompiled_/ + :param config_file: Path to decompiler_config.yaml to load. recommended None, which loads the default pylingual config. + :param version: Loads the models corresponding to this python version. if None, automatically detects version based on input PYC file. + :param top_k: Max number of pyc segmentations to consider. + :param trust_lnotab: Trust the lnotab in the input PYC for segmentation, recommended False. + :return: DecompilerResult class including important information about decompilation + """ + logger.info(f"Loading {file}...") + pyc = PYCFile(file) + + # try to auto resolve version + if version is None: + try: + pversion = PythonVersion(pyc.version) + logger.info(f"Detected version as {pversion}") + except ValueError: + raise + except Exception as err: + raise TypeError("Error automatically parsing version from pyc") from err + else: + pversion = PythonVersion(version) + + # try auto load config file from package + if config_file is None: + pkg_path = importlib.resources.files("pylingual") + with importlib.resources.as_file(pkg_path.joinpath("decompiler_config.yaml")) as pylingual_config: + config_file = Path(pylingual_config) + + # check config exists + if not config_file.exists(): + raise FileNotFoundError(f"Decompiler config {config_file} not found") + + segmenter, translator = load_models(config_file, pversion) + + logger.info(f"Decompiling pyc {file.resolve()} to {out_dir.resolve()}") + result = Decompiler(pyc, out_dir, segmenter, translator, pversion, top_k, trust_lnotab).result + + logger.info("Decompilation complete") + logger.info(f"{round(result.calculate_success_rate(), 2)}% code object success rate") + logger.info(f"Result saved to {result.decompiled_source.resolve()}") + return result diff --git a/pylingual/decompiler_config.yaml b/pylingual/decompiler_config.yaml new file mode 100644 index 0000000..6b34438 --- /dev/null +++ b/pylingual/decompiler_config.yaml @@ -0,0 +1,87 @@ +v3.6: + SEGMENTATION_MODEL: + REPO: syssec-utd/py36-pylingual-v1-segmenter + REVISION: main + TOKENIZER: syssec-utd/py36-pylingual-v1-tokenizer + + STATEMENT_MODEL: + REPO: syssec-utd/py36-pylingual-v1-statement + REVISION: main + TOKENIZER: syssec-utd/py36-pylingual-v1-tok + +v3.7: + SEGMENTATION_MODEL: + REPO: syssec-utd/py37-pylingual-v1-segmenter + REVISION: main + TOKENIZER: syssec-utd/py37-pylingual-v1-tokenizer + + STATEMENT_MODEL: + REPO: syssec-utd/py37-pylingual-v1-statement + REVISION: main + TOKENIZER: syssec-utd/py37-pylingual-v1-tok + +v3.8: + SEGMENTATION_MODEL: + REPO: syssec-utd/py38-pylingual-v1-segmenter + REVISION: main + TOKENIZER: syssec-utd/py38-pylingual-v1-tokenizer + + STATEMENT_MODEL: + REPO: syssec-utd/py38-pylingual-v1-statement + REVISION: main + TOKENIZER: syssec-utd/py38-pylingual-v1-tok + +v3.9: + SEGMENTATION_MODEL: + REPO: syssec-utd/py39-pylingual-v1-segmenter + REVISION: main + TOKENIZER: syssec-utd/py39-pylingual-v1-tokenizer + + STATEMENT_MODEL: + REPO: syssec-utd/py39-pylingual-v1-statement + REVISION: main + TOKENIZER: syssec-utd/py39-pylingual-v1-tok + +v3.10: + SEGMENTATION_MODEL: + REPO: syssec-utd/py310-pylingual-v1-segmenter + REVISION: main + TOKENIZER: syssec-utd/py310-pylingual-v1-tokenizer + + STATEMENT_MODEL: + REPO: syssec-utd/py310-pylingual-v1-statement + REVISION: main + TOKENIZER: syssec-utd/py310-pylingual-v1-tok + +v3.11: + SEGMENTATION_MODEL: + REPO: syssec-utd/py311-pylingual-v1-segmenter + REVISION: main + TOKENIZER: syssec-utd/py311-pylingual-v1-tokenizer + + STATEMENT_MODEL: + REPO: syssec-utd/py311-pylingual-v1-statement + REVISION: main + TOKENIZER: syssec-utd/py311-pylingual-v1-tok + +v3.12: + SEGMENTATION_MODEL: + REPO: syssec-utd/py312-pylingual-v1-segmenter + REVISION: main + TOKENIZER: syssec-utd/py312-pylingual-v1-tokenizer + + STATEMENT_MODEL: + REPO: syssec-utd/py312-pylingual-v1-statement + REVISION: main + TOKENIZER: syssec-utd/py312-pylingual-v1-tok + +v3.13: + SEGMENTATION_MODEL: + REPO: syssec-utd/py313-pylingual-v1-segmenter + REVISION: main + TOKENIZER: syssec-utd/py313-pylingual-v1-tokenizer + + STATEMENT_MODEL: + REPO: syssec-utd/py313-pylingual-v1-statement + REVISION: main + TOKENIZER: syssec-utd/py313-pylingual-v1-tok diff --git a/pylingual/editable_bytecode/EditableBytecode.py b/pylingual/editable_bytecode/EditableBytecode.py new file mode 100644 index 0000000..2075ef8 --- /dev/null +++ b/pylingual/editable_bytecode/EditableBytecode.py @@ -0,0 +1,860 @@ +import copy as copy_module +import logging +import pprint +from collections import defaultdict +from typing import Any, Optional + +from xdis import Bytecode, iscode +from xdis.cross_dis import instruction_size, op_has_argument +from xdis.bytecode import parse_exception_table, _ExceptionTableEntry + +from pylingual.utils.version import PythonVersion + +from .Instruction import Inst +from .utils import codeobj_replace, comprehension_names, unwrap +from .control_flow_graph import bytecode_to_control_flow_graph + +from typing import Callable + + +class EditableBytecode: + """ + An editable representation of Pythonic bytecode. Many values in it, including its instructions, may be incorrect + until the final "bake," when it's converted back into raw bytecode. However, it supports arbitrary accessing, + insertion, deletion, and modification of instructions. It also annotates instructions and provides extra information + regarding the control flow they describe. + + Baking functions: to_code() (EditableBytecode -> code object) + to_bytecode() (EditableBytecode -> co_code [bytes]) + """ + + def __init__( + self, + codeobj, + opcode, + version, + name_prefix: Optional[str] = None, + parent=None, + ): + self.codeobj = codeobj + self.opcode = opcode + self.version = PythonVersion(version) + self.parent = parent + + self.co_consts = unwrap(list(codeobj.co_consts)) + self.co_names = unwrap(list(codeobj.co_names)) + self.co_varnames = unwrap(list(codeobj.co_varnames)) + + self.name_prefix = unwrap(name_prefix) + self.name = unwrap(codeobj.co_name) if name_prefix is None else unwrap(name_prefix) + "." + unwrap(codeobj.co_name) + + self._edited = False + + self.instructions = [Inst.from_instruction(self, inst) for inst in Bytecode(self.codeobj, self.opcode)] + + self.offsets = {inst.offset: inst for inst in self.instructions} + jump_targets = [inst.argval for inst in self.instructions if inst.is_jump] + for inst in self.instructions: + inst.is_jump_target = inst.offset in jump_targets + + # named exception table is a named tuple of the exception entries. + # should be defined when an exception_table exists for a codeobj in python versions >= (3,11) + self.named_exception_table = None + if hasattr(codeobj, "co_exceptiontable"): + self.named_exception_table = parse_exception_table(codeobj.co_exceptiontable) + + # maps starting offsets to ending offsets and target offsets + self.exception_table = {} + # maps ending offsets to the cause of the entry (i.e. with, finally, or except) + self.end_table = {} + + # check for globals and nonlocals + self.globals = set() + self.nonlocals = set() + for inst in self.instructions: + if inst.opname == "STORE_GLOBAL": + self.globals.add(inst.argval) + elif inst.opname == "STORE_DEREF": + self.nonlocals.add(inst.argval) + + if self.version >= (3, 11): + self.fix_format_string_lno() + + # Prepare "deep analysis" structures - cycle detection and annotation + self._preprocess_jumps() + + low_information_instruction_blacklist = ["RESUME", "EXTENDED_ARG", "CACHE", "PRECALL", "MAKE_CELL"] + self.remove_instructions({inst for inst in self.instructions if inst.opname in low_information_instruction_blacklist}) + + # updates attribute of instructions that contains information about the exception table + self._add_inst_exception_attrs() + + # Represents subsidiary EditableBytecode objects + self.child_bytecodes = [] + self.bytecode_lookup = {} + + # Initialize recursive structure + for i, const in enumerate(self.co_consts): + if iscode(const): + self.co_consts[i] = EditableBytecode(const, opcode, self.version, name_prefix=self.name, parent=self) + self.child_bytecodes.append(self.co_consts[i]) # Keeps it in order + self.bytecode_lookup[const.co_name] = self.co_consts[i] + + def get_recursive_length(self): + """Returns the recursive length of this bytecode and all its descendents""" + return len(self) + sum(bytecode.get_recursive_length() for bytecode in self.child_bytecodes) + + # + # INTERNAL PROCESSING + # + def _preprocess_jumps(self): + """Preprocesses jumps by setting the instruction's "target" attribute to the Inst object rather than the numeric + offset. O(N^2).""" + for inst in self.instructions: + target = self.get_jump_target_offset(inst) + + if target is not None: + try: + inst._target = self.get_by_offset(target) + except KeyError: + raise ValueError(f"Found invalid target when preprocessing jumps: offset {target} for instruction {inst}") + + def regenerate(self): + """Regenerates correct offsets for the instruction list and resets the _edited field. This is called + automatically, but calling it externally should do no harm. O(N).""" + if not self._edited: + return + + self.offsets = {} + offset = 0 + for inst in self.instructions: + inst.original_offset = offset + self.offsets[offset] = inst + + offset += inst.real_size + + self._edited = False + + def create_arg_inst(self, opname: str, arg, offset: int = -1): + opcode = getattr(self.opcode, opname) + return Inst(self, opname, opcode, None, instruction_size(opcode, self.opcode), arg, arg, repr(arg), True, offset, None, False, False) + + def _bake_jumps(self, add_extended=False): + """ + Bakes the correct jump target address into each jump's "arg" attribute and ensures the number of EXTENDED_ARGs preceeding it is + sufficient. This is automatically called by to_bytecode. O(N^2). + """ + + while True: + len_before = len(self) + changed = False + + i = 0 + while i < len(self): + instruction = self.instructions[i] + + if instruction.has_arg: + if instruction.is_jump: + # ensure that we jump to the start of the preceeding EXTENDED_ARG chain + current_jump_target_index = self.instructions.index(instruction.target) + for prev_instruction in self.instructions[current_jump_target_index - 1 :: -1]: + if prev_instruction.opcode == self.opcode.EXTENDED_ARG: + instruction.target = prev_instruction + changed = True + else: + # we have reached the end of the EXTENDED_ARG chain + break + + if instruction.is_rel_jump: + instruction.arg = instruction.target.offset - instruction.offset - instruction.real_size + if self.version >= (3, 10): + instruction.arg = int(instruction.arg / 2) + instruction.argval = instruction.target.offset + elif instruction.is_abs_jump: + if self.version >= (3, 10): + instruction.arg = int(instruction.target.offset / 2) + instruction.argval = instruction.target.offset + else: + instruction.arg = instruction.argval = instruction.target.offset + if add_extended: + # count the number of necessary preceeding EXTENDED_ARG instructions + n_extendeds_needed = (instruction.arg > 0xFF) + (instruction.arg > 0xFFFF) + (instruction.arg > 0xFFFFFF) + + # count the number of preceding EXTENDED_ARG instructions present in the bytecode + n_extendeds_found = 0 + for current_instruction in self.instructions[i - 1 :: -1]: + if current_instruction.opcode == self.opcode.EXTENDED_ARG: + n_extendeds_found += 1 + else: + break + + if n_extendeds_found > 3: + raise ValueError("Incorrect bytecode -- more than 3 extended args found before instruction " + repr(instruction)) + + if instruction.arg > 0xFFFFFFFF: + raise ValueError("Incorrect bytecode -- more than 3 extended args needed for instruction" + repr(instruction)) + + new_extendeds = [self.create_arg_inst("EXTENDED_ARG", 0) for _ in range(n_extendeds_needed - n_extendeds_found)] + + if new_extendeds: + self[i:i] = new_extendeds + i += len(new_extendeds) + + n_extendeds = max([n_extendeds_found, n_extendeds_needed]) + for j in range(n_extendeds): + self[i - j - 1].arg = ((0xFF << ((j + 1) * 8)) & instruction.arg) >> ((j + 1) * 8) + + instruction.has_extended_arg = True + + i += 1 + + if len(self) == len_before and not changed: + break + + self.regenerate() + + ## INSTRUCTION LOOKUP + + def get_offset(self, instruction): + """Gets the real, current offset of the instruction. O(N).""" + self.regenerate() + return instruction.original_offset # Prevent theoretically-impossible infinite loops + + def get_jump_target_offset(self, instruction): + """Returns the target initial offset of a jump instruction, or None if the instruction is not a jump. O(1).""" + if instruction.is_abs_jump: + # duct taping; arg is incorrect in 310 + return instruction.argval + + elif instruction.is_rel_jump: + # duct taping; arg is incorrect in 310 + return instruction.argval + + return None + + def get_by_offset(self, offset): + """Finds the instruction by its offset. O(N).""" + self.regenerate() + return self.offsets[offset] + + def _get_instruction_after(self, instruction): + """Gets the instruction directly following the specified instruction, or None if it's at the end of the bytecode. O(N).""" + next_offset = instruction.offset + instruction.real_size + + try: + return self.get_by_offset(next_offset) + except KeyError: + return None + + def resolve_namespace(self, value: Any) -> Any: + if not isinstance(value, str): + return value + + # true dunder methods (e.g., __init__, __myfunc__) don't get namespaced + if value.endswith("__"): + return value + + # try namespaces all the way up the tree + def namespace_generator(): + current = self + yield f"_{current.codeobj.co_name.lstrip('_')}" + while current.parent: + current = current.parent + yield f"_{current.codeobj.co_name.lstrip('_')}" + + for namespace in namespace_generator(): + if value.startswith(f"{namespace}__"): + return value[len(namespace) :] + + return value + + ## SUBSIDIARY OBJECTS + + def iter_bytecodes(self): + """Iterates through all EditableBytecode objects in the recursive object.""" + + yield self + + for bytecode in self.child_bytecodes: + for bc in bytecode.iter_bytecodes(): + yield bc + + def copy(self): + """Returns a copy of the current bytecode object, as it stands.""" + try: + copy = EditableBytecode( + self.to_code(), + self.opcode, + self.version, + self.name_prefix, + False, + ) + except IndexError: + # A sketchy workaround for what seems to be a strange bug with loading saved code objects + copy = copy_module.copy(self) + copy._edited = True + + instructions = [copy_module.copy(inst) for inst in copy.instructions] + for inst in instructions: + if inst.is_jump: + inst.target = instructions[copy.instructions.index(inst.target)] + copy.instructions = instructions + + copy.child_bytecodes = [] + for const, i in enumerate(copy.co_consts): + if isinstance(const, EditableBytecode): + new = const.copy() + new.parent = copy + + copy.co_consts[i] = new + copy.bytecode_lookup[const.codeobj.co_name] = new + copy.child_bytecodes.append(new) + + copy.regenerate() + + return copy + + ## BYTECODE PREPARATION + + def _gen_lines(self, line_start=None): + """Generates a line number table (co_lines) from the current instructions.""" + lines_table = [] + offset = 0 + last_line = line_start or getattr(self.codeobj, "co_firstlineno", None) + + for instruction in self: + if instruction.starts_line: + if offset: + if last_line is None: + lines_table.append(offset) + lines_table.append(-128) + else: + line_delta = instruction.starts_line - last_line + + while line_delta > 127: + lines_table.append(0) + lines_table.append(127) + line_delta -= 127 + + while line_delta < -127: + lines_table.append(0) + lines_table.append(-127) + line_delta += 127 + + lines_table.append(offset) + lines_table.append(line_delta) + + last_line = instruction.starts_line + + offset += instruction.real_size + + return bytes(lines_table) + + def to_bytecode(self): + """Converts the instruction list into valid bytecode, to be used with co_code. This was partially taken from the incomplete + list2bytecode function in xdis. O(N).""" + self._bake_jumps() + + bc = [] + for instruction in self: + bc.extend([instruction.opcode, (instruction.arg & 0xFF) if instruction.arg is not None else 0]) + + return bytes(bc) + + def to_code(self, no_lnotab=False): + """Returns a fully-functioning 'code' or xdis code object from this edited bytecode.""" + + # Here, we hotfix the potentially-edited EditableBytecode objects within this one's constant + # array. This used to be processed by recursive_fix, but it makes more sense to have the + # EditableBytecode objects do it directly. + co_consts = tuple((const.to_code(no_lnotab=no_lnotab) if isinstance(const, EditableBytecode) else const) for const in self.co_consts) + + replacement_args = { + "co_code": self.to_bytecode(), + "co_consts": tuple(co_consts), + "co_names": tuple(self.co_names), + "co_varnames": tuple(self.co_varnames), + } + + # co_lnotab deprecated in python >= 3.10 + if self.version < (3, 10): + replacement_args.update({"co_lnotab": b"\x00\x01" if no_lnotab else self._gen_lines()}) + + return codeobj_replace(self.codeobj, **replacement_args) + + ### INSTRUMENTATION FUNCTIONS + + def apply_patches(self, patch_functions: list[Callable[["EditableBytecode"], None]]) -> None: + for patch in patch_functions: + for bc in self.iter_bytecodes(): + patch(bc) + + def _change_jump_targets(self, from_inst: Inst, to_inst: Inst): + """Changes the targets of any instructions jumping to "from_inst" to "to_inst". + Before: + InstA --> InstB + After _change_jump_targets(InstB, InstC): + InstA --> !!InstC!! + """ + for i, inst in enumerate(self): + if inst.is_jump and inst.target == from_inst: + self[i]._target = to_inst + + def collapse_unconditional_jumps(self): + """Causes unnecessary unconditional jumps to "collapse" into a single jump.""" + + unconditional_jumps = set(instruction for instruction in self if instruction.is_uncond_jump and instruction.target.is_uncond_jump) + instructions_changed = set() + + while unconditional_jumps: + this_layer = set(jump for jump in unconditional_jumps if jump.target not in unconditional_jumps) + + if not this_layer: + logger = logging.getLogger("transform") + logger.warning("Two jumps are forming an infinite loop!\n" + pprint.pformat(unconditional_jumps)) + break + + for jump in this_layer: + jump._target = jump._target._target + instructions_changed.add(jump) + + unconditional_jumps.difference_update(this_layer) + + return len(instructions_changed) + + def remove_unreachable_instructions(self): + """Removes unreachable instructions from the bytecode as cleanly as possible.""" + + self.regenerate() + + cfg = bytecode_to_control_flow_graph(self) + unreachable_instructions = set(self.instructions) - set(cfg.nodes) + return self.remove_instructions(unreachable_instructions) + + def remove_useless_jumps(self): + """Removes jumps that just jump to the next instruction.""" + useless_jumps = {inst for inst in self if inst.is_jump and inst.target.offset == inst.offset + inst.real_size} + return self.remove_instructions(useless_jumps) + + def shrink(self): + """Shrinks and optimizes bytecode using various heuristics.""" + n = self.collapse_unconditional_jumps() + n += self.remove_unreachable_instructions() + n += self.remove_useless_jumps() + + return n + + def remove_instructions(self, to_remove): + """Removes every instruction in the provided list *fairly* gracefully, albeit not perfectly when ambiguity is involved.""" + if len(to_remove) == 0: + return 0 + + self.regenerate() + self._edited = True + + # store a list of all jumps to avoid repeatedly searching for them + jumps = [inst for inst in self.instructions if inst.is_jump] + + # store an instruction-based copy of the exception table to make offset fixing easier at the end + temp_exception_table = {self.get_by_offset(start): (self.get_by_offset(end), self.get_by_offset(target)) for start, (end, target) in self.exception_table.items()} + temp_named_exception_table = dict() + if self.named_exception_table: + temp_named_exception_table = [(self.get_by_offset(e.start), self.get_by_offset(e.end), self.get_by_offset(e.target), e.depth, e.lasti) for e in self.named_exception_table] + + # propagate jump targets backwards from the end of the bytecode and remove immediately + removed = set() + while self.instructions[-1] in to_remove: + # update jump targets + new_jump_target = self.instructions[-2] + for jump in jumps: + if jump.target is self.instructions[-1]: + jump._target = new_jump_target + new_jump_target.is_jump_target = True + + # update exception table entries + new_exception_target = self.instructions[-2] + for start, (end, target) in list(temp_exception_table.items()): + # move start back if deleted + if start is self.instructions[-1]: + temp_exception_table[new_exception_target] = (end, target) + del temp_exception_table[start] + start = new_exception_target + # move end back if deleted + if end is self.instructions[-1]: + temp_exception_table[start] = (new_exception_target, target) + end = new_exception_target + # move target back if deleted + if target is self.instructions[-1]: + temp_exception_table[start] = (end, new_exception_target) + + # update named exception table entries + for exception_index, (start, end, target, depth, lasti) in enumerate(list(temp_named_exception_table)): + if start is self.instructions[-1]: + temp_named_exception_table[exception_index] = (new_exception_target, end, target, depth, lasti) + start = new_exception_target + if end is self.instructions[-1]: + temp_named_exception_table[exception_index] = (start, new_exception_target, target, depth, lasti) + end = new_exception_target + if target is self.instructions[-1]: + temp_named_exception_table[exception_index] = (start, end, new_exception_target, depth, lasti) + + removed.add(self.instructions.pop()) + + for inst in sorted(to_remove, key=lambda x: x.offset): + if inst in removed: + continue + + next_instruction = self._get_instruction_after(inst) + + # update line starts + if next_instruction.starts_line is None: + next_instruction.starts_line = inst.starts_line + + # update jump targets + for jump in jumps: + if jump.target is inst: + jump._target = next_instruction + next_instruction.is_jump_target = True + + # update exception table entries + new_exception_target = next_instruction + for start, (end, target) in list(temp_exception_table.items()): + # move start back if deleted + if start is inst: + temp_exception_table[new_exception_target] = (end, target) + del temp_exception_table[start] + start = new_exception_target + # move end back if deleted + if end is inst: + temp_exception_table[start] = (new_exception_target, target) + end = new_exception_target + # move target back if deleted + if target is inst: + temp_exception_table[start] = (end, new_exception_target) + + # update named exception table entries + for exception_index, (start, end, target, depth, lasti) in enumerate(list(temp_named_exception_table)): + if start is inst: + temp_named_exception_table[exception_index] = (new_exception_target, end, target, depth, lasti) + start = new_exception_target + if end is inst: + temp_named_exception_table[exception_index] = (start, new_exception_target, target, depth, lasti) + end = new_exception_target + if target is inst: + temp_named_exception_table[exception_index] = (start, end, new_exception_target, depth, lasti) + + # Now, removing them this way will have no side effects. + self.instructions = [inst for inst in self.instructions if inst not in to_remove] + self._edited = True + self.regenerate() # recalculate offsets + + # fix jump target argval and argrepr + for jump in jumps: + jump.argval = jump.target.offset + jump.argrepr = f"to {jump.argval}" + + # fix exception table offsets + self.exception_table = {start.offset: (end.offset, target.offset) for start, (end, target) in temp_exception_table.items()} + if temp_named_exception_table: + self.named_exception_table = [_ExceptionTableEntry(start.offset, end.offset, target.offset, depth, lasti) for (start, end, target, depth, lasti) in temp_named_exception_table] + self._add_inst_exception_attrs() + + self._edited = True + + return len(to_remove) + + def new_instruction(self, *args, **kwargs): + """Creates a new instruction for use with this EditableBytecode object. This function does NOT automatically insert the instruction.""" + return Inst(self, *args, **kwargs) + + ## OTHER UTILS + + def disasm_view(self, b_iter_bytecodes: bool = True) -> str: + """Get a multi-line disassembled view of this bytecode obj + b_iter_bytecodes (bool) True : Iter through all bytecodes in self (child bytecodes)""" + disview = "" + for bytecode in self.iter_bytecodes() if b_iter_bytecodes else [self]: + for inst in bytecode: + if inst.starts_line: + disview += f"# Line {inst.starts_line}\n" + disview += inst.get_dis_view() + "\n" + disview += "\n" + return disview + + def _change_line_number_everywhere(self, original_line_number, new_line_number): + """ + Updates all instances of original_line_number in the line_starts of instructions to become new_line_number + """ + for bc in self.iter_bytecodes(): + for inst in bc: + if inst.starts_line == original_line_number: + inst.starts_line = new_line_number + + def _patch_dummy_decorator(self, dummy_decorator_name="dummy"): + """ + Exclusively used for python <= 3,7 + Removes "@dummy_decorator_name" decorators from bytecode. + """ + for bc in self.iter_bytecodes(): + for inst in bc.instructions: + if inst.opname.startswith("LOAD") and inst.argval == dummy_decorator_name: + # target found + target_idx = bc.instructions.index(inst) + for next_inst in bc.instructions[target_idx:]: + if (next_inst.opname, next_inst.arg) == ("CALL_FUNCTION", 1): + bc.remove_instructions([next_inst]) + break + self._change_line_number_everywhere(inst.starts_line, inst.starts_line + 1) + bc.remove_instructions([inst]) + return + + def fix_while(self, source_lines): + for i in range(len(source_lines)): + if source_lines[i].startswith("while ") and source_lines[i] != "while True:": + source_lines[i] = "if" + source_lines[i][5:] + for bc in self.iter_bytecodes(): + for inst in bc: + if inst.opname.startswith("POP_JUMP_BACKWARD_IF_") or inst.opname.startswith("POP_JUMP_IF_") and inst.argval < inst.offset: + prev = bc[inst.argval // 2 - 1] + while prev.starts_line is None: + if prev.offset == 0: + prev = None + break + else: + prev = bc[prev.offset // 2 - 1] + if prev is not None and source_lines[prev.starts_line - 1].startswith("if "): + source_lines[prev.starts_line - 1] = "while" + source_lines[prev.starts_line - 1][2:] + + def fix_while12(self, source_lines): + for i in range(len(source_lines)): + if source_lines[i].startswith("while ") and source_lines[i] != "while True:": + source_lines[i] = "if" + source_lines[i][5:] + for bc in self.iter_bytecodes(): + for inst in bc: + if inst.opname == "JUMP_BACKWARD": + prev = bc[inst.offset // 2 - 1] + if prev.opname.startswith("POP_JUMP_IF") and prev.argval == inst.offset + 2: + prev = bc[inst.argval // 2 - 1] + while prev.starts_line is None: + if prev.offset == 0: + prev = None + break + else: + prev = bc[prev.offset // 2 - 1] + if prev is not None and source_lines[prev.starts_line - 1].startswith("if "): + source_lines[prev.starts_line - 1] = "while" + source_lines[prev.starts_line - 1][2:] + + def fix_format_string_lno(self): + for a in range(len(self) - 1): + if self[a + 1].starts_line is not None: + if self[a].opname == "LOAD_CONST" and type(self[a].argval) == str: + self[a].starts_line = self[a + 1].starts_line + self[a + 1].starts_line = None + + def make_absolute(self, offset, target): + return [ + self.new_instruction( + "JUMP_ABSOLUTE", + self.opcode.JUMP_ABSOLUTE, + "jabs", + instruction_size(self.opcode.JUMP_ABSOLUTE, self.opcode), + target.offset, + target.offset, + repr(target.offset), + op_has_argument(self.opcode.JUMP_ABSOLUTE, self.opcode), + offset, + None, + False, + False, + target, + ) + ] + + def make_relative(self, offset, target): + return [ + self.new_instruction( + "JUMP_FORWARD", + self.opcode.JUMP_FORWARD, + "jrel", + instruction_size(self.opcode.JUMP_FORWARD, self.opcode), + target.offset - offset, + target.offset - offset, + repr(target.offset - offset), + op_has_argument(self.opcode.JUMP_FORWARD, self.opcode), + offset, + None, + False, + False, + target, + ) + ] + + def replace_duplicated_returns10(self, source_lines): + for bc in self.iter_bytecodes(): + offsets = defaultdict(int) + if bc.codeobj.co_name in comprehension_names: + continue + for inst in bc: + if inst.starts_line is not None: + offsets[inst.starts_line] = max(offsets[inst.starts_line], inst.offset) + for inst in bc: + if inst.starts_line is not None and inst.offset != offsets[inst.starts_line]: + line = source_lines[inst.starts_line - 1].strip() + if line == "return" or line.startswith("return "): + target = bc[offsets[inst.starts_line] // 2] + bc[inst.offset // 2] = self.make_absolute(inst.offset, target) + bc.remove_unreachable_instructions() + + def replace_duplicated_returns12(self, source_lines): + for bc in self.iter_bytecodes(): + offsets = defaultdict(int) + if bc.codeobj.co_name in comprehension_names: + continue + for inst in bc: + if inst.starts_line is not None: + offsets[inst.starts_line] = max(offsets[inst.starts_line], inst.offset) + for inst in bc: + if inst.starts_line is not None and inst.offset != offsets[inst.starts_line]: + insts = [inst] + i = bc[inst.offset // 2 + 1] + while i.starts_line is None: + insts.append(i) + i = bc[i.offset // 2 + 1] + if len(insts) <= 4 and insts[-1].opname in ("RETURN_VALUE", "RETURN_CONST"): + target = bc[offsets[inst.starts_line] // 2] + bc[inst.offset // 2] = self.make_relative(inst.offset, target) + bc.remove_unreachable_instructions() + + def get_lno_insts(self, hoist_comprehensions: bool = True, previously_seen_lines: set[int] = None) -> dict[int, list[Inst]]: + """Get a dictionary that maps line numbers to sequences of bytecodes""" + if hoist_comprehensions and self.is_comprehension: + return dict() + + if previously_seen_lines is None: + previously_seen_lines = set() + + # create a dict of line num : [bytecodes composing line] + lno_bytecodes = {} + seen_lnos = set(previously_seen_lines) + current_line_bytecodes = [] + current_lno = None + + for inst in self: + if inst.starts_line is not None and inst.starts_line not in seen_lnos: + if current_line_bytecodes: + lno_bytecodes.update({current_lno: current_line_bytecodes}) + current_line_bytecodes = [] + current_lno = inst.starts_line + seen_lnos.add(current_lno) + + current_line_bytecodes.append(inst) + + if hoist_comprehensions and inst.opname == "LOAD_CONST" and getattr(inst.argval, "co_name", None) in self.bytecode_lookup: + child_code = self.co_consts[inst.arg] + if child_code.is_comprehension: + for decendant_code in child_code.iter_bytecodes(): + current_line_bytecodes.extend(decendant_code.instructions) + + # update final list + if current_line_bytecodes: + lno_bytecodes.update({current_lno: current_line_bytecodes}) + + # resolve unallocated starting instructions + if None in lno_bytecodes and len(lno_bytecodes) > 1: + unallocated_insts = lno_bytecodes.pop(None) + earliest_line = min(lno_bytecodes.keys()) + lno_bytecodes[earliest_line] = unallocated_insts + lno_bytecodes[earliest_line] + + return lno_bytecodes + + def _add_inst_exception_attrs(self): + """ + Update instruction attributes to add additional context if they are entries in the exception table + for versions >= (3,11) + """ + if self.named_exception_table is None: + return + for entry in self.named_exception_table: + self.get_by_offset(entry.start).exception_start = entry.target + self.get_by_offset(entry.end).exception_end = True + self.get_by_offset(entry.target).exception_target = True + + ## PROPERTIES + + @property + def is_comprehension(self): + return self.codeobj.co_name in comprehension_names + + @property + def first_instruction(self): + return self.instructions[0] + + ## OVERLOADS + + def __iter__(self): + return self.instructions.__iter__() + + def __getitem__(self, i): + if isinstance(i, slice): + return self.instructions[i] + + return self.instructions[i] + + def __setitem__(self, i, value): + if not isinstance(i, slice): + i = slice(i, i + 1) + insts = self.instructions[i] + if isinstance(insts, Inst): + insts = [insts] + + instruction_before = self[i.start - 1] if i.start is not None and i.start > 0 else None + instruction_after = self[i.stop] if i.stop is not None and i.stop <= len(self) else None + + for j, inst in enumerate(insts): + if isinstance(value, (list, tuple)) and len(insts) == len(value): + self._change_jump_targets(inst, value[j]) + else: + new_target = instruction_before or instruction_after + if not new_target and len(value) > 0: + new_target = value[0] + elif len(value) == 0: + pass # They should have used __del__ + + if new_target: + self._change_jump_targets(inst, new_target) + + self.instructions[i] = value + self._edited = True + + def __delitem__(self, i): + if not isinstance(i, slice): + i = slice(i, i + 1) + insts = self.instructions[i] + if isinstance(insts, Inst): + insts = [insts] + + instruction_before = self[i.start - 1] if i.start is not None and i.start > 0 else None + instruction_after = self[i.stop] if i.stop is not None and i.stop <= len(self) else None + + for inst in insts: + new_target = instruction_before or instruction_after + + if new_target: + self._change_jump_targets(inst, new_target) + + del self.instructions[i] + self._edited = True + + def __hasitem__(self, value): + return value in self.instructions + + def __len__(self): + return len(self.instructions) + + def __str__(self): + name = "" if self.name is None else self.name + "," + return self.__class__.__name__ + "<" + name + "edited=" + repr(self._edited) + ">" + pprint.pformat(self.instructions) + + def __repr__(self): + name = "" if self.name is None else self.name + "," + return self.__class__.__name__ + "<" + name + "edited=" + repr(self._edited) + ">" + f"[{len(self.instructions)} instructions,{len(self.child_bytecodes)} code objects]" diff --git a/pylingual/editable_bytecode/Instruction.py b/pylingual/editable_bytecode/Instruction.py new file mode 100644 index 0000000..fe98dcf --- /dev/null +++ b/pylingual/editable_bytecode/Instruction.py @@ -0,0 +1,266 @@ +# seperate import for type checking to prevent circular reference with EditableBytecode +from typing import TYPE_CHECKING, Optional + +from xdis import iscode +from xdis.cross_dis import instruction_size, xstack_effect +from .utils import unwrap + +if TYPE_CHECKING: + from typing import Any + from .EditableBytecode import EditableBytecode + + +class Inst: + """This is a more object-oriented, editable version of the xdis Instruction class. For use with EditableBytecode.""" + + def __init__( + self, + bytecode: "EditableBytecode", + opname: str, + opcode, + optype: str, + inst_size: int, + arg: int, + argval: "Any", + argrepr: str, + has_arg: bool, + offset: int, + starts_line: int, + is_jump_target: bool, + has_extended_arg: bool, + target: Optional["Inst"] = None, + ): + self.bytecode = bytecode + + self.opname = opname + self.opcode = opcode + self.optype = optype + self.inst_size = inst_size + self.arg = unwrap(arg) + self.argval = unwrap(argval) + self.argrepr = unwrap(argrepr) + self.has_arg = has_arg + self.original_offset = offset # We typically calculate the offset on-the-fly and cache it as necessary + self.starts_line = starts_line + self.is_jump_target = is_jump_target + self.has_extended_arg = has_extended_arg + self._target = target # Will be set by the containing EditableBytecode object + + # exception table information used for model view, values are assigned during init of parent bytecode + # only set in versions >= 3,11 + self.exception_start: int | bool = False # will either be false + self.exception_end = False + self.exception_target = False + + # dependency graph for obfuscation + self.deps = [] + self.reqs = [] + self.stack = [] + self.pop = [] + + @property + def offset(self): + return self.bytecode.get_offset(self) + + @property + def real_size(self): + return instruction_size(self.opcode, self.bytecode.opcode) + + @property + def is_jump(self): + return self.optype in ("jabs", "jrel") or self.is_cond_jump + + @property + def is_abs_jump(self): + return self.optype == "jabs" + + @property + def is_rel_jump(self): + return self.optype == "jrel" + + @property + def is_cond_jump(self): + """ + It's technically inaccurate that SETUP_WITH and SETUP_FINALLY are jumps, but they cause jumps in subsequent + instructions, so this theoretically preserves the control flow + + @return: + @rtype: + """ + + return self.opname in ( + "POP_JUMP_IF_FALSE", + "POP_JUMP_IF_TRUE", + "POP_JUMP_FORWARD_IF_FALSE", + "POP_JUMP_FORWARD_IF_TRUE", + "POP_JUMP_BACKWARD_IF_FALSE", + "POP_JUMP_BACKWARD_IF_TRUE", + "JUMP_IF_TRUE_OR_POP", + "JUMP_IF_FALSE_OR_POP", + "JUMP_IF_TRUE", + "JUMP_IF_FALSE", + "FOR_ITER", + "POP_JUMP_IF_NONE", + "POP_JUMP_IF_NOT_NONE", + "POP_JUMP_BACKWARD_IF_NONE", + "POP_JUMP_BACKWARD_IF_NOT_NONE", + "POP_JUMP_FORWARD_IF_NONE", + "POP_JUMP_FORWARD_IF_NOT_NONE", + "JUMP_IF_NOT_EXC_MATCH", + "SETUP_WITH", + "SETUP_FINALLY", + "SEND", + ) + + @property + def is_uncond_jump(self): + """ + @return: + @rtype: + """ + + return self.opcode in self.bytecode.opcode.JUMP_UNCONDITONAL # His typo, not mine + + @property + def target(self): + if not self.is_jump: + raise AttributeError("Only jump instructions have target attributes, not " + repr(self)) + return self._target + + @target.setter + def target(self, value): + if not self.is_jump: + raise AttributeError("Only jump instructions have target attributes, not " + repr(self)) + self._target = value + + def add_reqs(self, *reqs): + for r in reqs: + if isinstance(r, tuple): + r = r[0] + if isinstance(r, Inst): + r.deps.append(self) + self.reqs.append(r) + + @property + def next_instructions(self): + """Returns the instruction(s) that follow this one, including a jump target (if any).""" + next = [] + if self.is_jump: + next.append(self.target) + if not self.is_uncond_jump and self.opname not in ("RETURN_VALUE", "RAISE_VARARGS", "RETURN_CONST"): + inst_after = self.bytecode._get_instruction_after(self) + if inst_after: + next.append(inst_after) + if self.offset in self.bytecode.exception_table: + target_offset = self.bytecode.exception_table[self.offset][1] + next.append(self.bytecode[target_offset // 2]) + + return next + + @property + def next_instructions_basic(self): + """Returns the next instruction(s), IGNORING control flow from exceptions""" + + if self.opname in ("SETUP_WITH", "SETUP_FINALLY"): + inst_after = self.bytecode._get_instruction_after(self) + return [inst_after] if inst_after else [] + + next = [] + if self.is_jump: + next.append(self.target) + if not self.is_uncond_jump: + inst_after = self.bytecode._get_instruction_after(self) + if inst_after: + next.append(inst_after) + + return next + + def get_next_instructions(self, follow_uncond_jumps: bool = False): + """ + A slightly more robust version of the property "next_instructions" that allows for following unconditional + jumps. + + @param follow_uncond_jumps: + @type follow_uncond_jumps: + @return: + @rtype: + """ + next_instructions = set(self.next_instructions) + last_next = next_instructions + + while follow_uncond_jumps: + new_next = set() + + for inst in last_next: + if (inst.is_jump and not inst.is_cond_jump) or inst.opcode == self.bytecode.opcode.EXTENDED_ARG: + new_next.update(inst.next_instructions) + + before = len(next_instructions) + next_instructions.update(new_next) + + if len(next_instructions) > before: + last_next = new_next + else: + break + + return next_instructions + + def get_stack_effect(self, jump: bool = True): + """Returns the "effect" this instruction will have on the stack. May occasionally return None?""" + return xstack_effect(self.opcode, self.bytecode.opcode, oparg=self.arg, jump=jump) + + ### DIS VIEWS ### + + @property + def jumped_to_from_insts(self) -> list["Inst"]: + if not self.is_jump_target: + return [] + + return [inst for inst in self.bytecode if hasattr(inst, "target") and self is inst.target] + + def get_dis_view(self) -> str: + """Get a dissassembled view of the instruction, similar to that of dis output""" + if iscode(self.argval): + argrepr = f"code object {self.argval.co_name}" + else: + argrepr = self.argrepr + return f"{self.offset} {self.opname}{' ' + str(self.arg) if self.has_arg else ''}{' (' + argrepr + ')' if argrepr else ''}" + + ##### + + def __repr__(self): + attr_list = ( + "opname", + "opcode", + "optype", + "real_size", + "arg", + "argval", + "argrepr", + "has_arg", + "offset", + "starts_line", + "is_jump_target", + "has_extended_arg", + ) + + return self.__class__.__name__ + "(" + ", ".join((attr + "=" + repr(getattr(self, attr))) for attr in attr_list) + ")" + + @classmethod + def from_instruction(self, bytecode, inst): + """Creates an Inst from an xdis Instruction object""" + return Inst( + bytecode, + inst.opname, + inst.opcode, + inst.optype, + inst.inst_size, + inst.arg, + inst.argval, + inst.argrepr, + inst.has_arg, + inst.offset, + inst.starts_line, + inst.is_jump_target, + inst.has_extended_arg, + ) diff --git a/pylingual/editable_bytecode/PYCFile.py b/pylingual/editable_bytecode/PYCFile.py new file mode 100644 index 0000000..fea304c --- /dev/null +++ b/pylingual/editable_bytecode/PYCFile.py @@ -0,0 +1,84 @@ +from pylingual.utils.version import PythonVersion +from .EditableBytecode import EditableBytecode +from .utils import write_pyc + +from xdis.load import load_module_from_file_object, load_module +import xdis.opcodes + +from io import BytesIO +import pathlib + + +class PYCFile(EditableBytecode): + """Represents a .pyc file. Upon creation, extracts all the bytecode from it.""" + + def __init__(self, source, name_prefix=None): + self.pyc_path = None + source_tuple = (None, None, None, None, None, None, None) + if isinstance(source, bytes): + source = BytesIO(source) + source_tuple = load_module_from_file_object(source) + elif isinstance(source, pathlib.Path): + source_tuple = load_module(str(source)) + self.pyc_path = source + elif source is not None: + source_tuple = load_module(source) + + ( + version, + self.timestamp, + self.magic, + self.code, + self.ispypy, + self.source_size, + self.sip_hash, + ) = source_tuple + + self.version = PythonVersion(version) + opcode = getattr(xdis.opcodes, f"opcode_{self.version[0]}{self.version[1]}") + + EditableBytecode.__init__( + self, + self.code, + opcode, + self.version, + name_prefix=name_prefix, + ) + + def copy(self): + try: + copy = PYCFile(None) + EditableBytecode.__init__(copy, self.to_code(), self.opcode, self.version, self.name_prefix, False) + except IndexError: + copy = EditableBytecode.copy(self) + + for attr in ( + "version", + "timestamp", + "magic", + "code", + "ispypy", + "source_size", + "sip_hash", + ): + setattr(copy, attr, getattr(self, attr)) + + return copy + + def save(self, file, should_close=True, no_lnotab=False): + """Saves the current recursive bytecode to the specified file.""" + if isinstance(file, str): + file = open(file, "wb") + + write_pyc( + file, + self.to_code(no_lnotab=no_lnotab), + self.version, + self.magic, + self.timestamp, + self.source_size, + ) + + if should_close: + file.close() + return file diff --git a/pylingual/editable_bytecode/__init__.py b/pylingual/editable_bytecode/__init__.py new file mode 100644 index 0000000..663ba8e --- /dev/null +++ b/pylingual/editable_bytecode/__init__.py @@ -0,0 +1,7 @@ +from .EditableBytecode import EditableBytecode +from .Instruction import Inst +from .PYCFile import PYCFile + +import pylingual.editable_bytecode.bytecode_patches + +__all__ = ["EditableBytecode", "Inst", "PYCFile"] diff --git a/pylingual/editable_bytecode/bytecode_patches/__init__.py b/pylingual/editable_bytecode/bytecode_patches/__init__.py new file mode 100644 index 0000000..4499de5 --- /dev/null +++ b/pylingual/editable_bytecode/bytecode_patches/__init__.py @@ -0,0 +1,5 @@ +from .fix_unreachable import fix_unreachable +from .remove_extended_arg import remove_extended_arg +from .remove_docstrings import remove_docstrings +from .remove_nop import remove_nop +from .fix_indirect_jump import fix_indirect_jump diff --git a/pylingual/editable_bytecode/bytecode_patches/fix_indirect_jump.py b/pylingual/editable_bytecode/bytecode_patches/fix_indirect_jump.py new file mode 100644 index 0000000..c18c319 --- /dev/null +++ b/pylingual/editable_bytecode/bytecode_patches/fix_indirect_jump.py @@ -0,0 +1,12 @@ +from ..EditableBytecode import EditableBytecode + + +def fix_indirect_jump(bytecode: EditableBytecode): + for i in bytecode: + if i.is_jump: + # avoid infinite loop + limit = 99 + while i.target.is_uncond_jump and limit: + i.argval = i.target.argval + i.target = i.target.target + limit -= 1 diff --git a/pylingual/editable_bytecode/bytecode_patches/fix_unreachable.py b/pylingual/editable_bytecode/bytecode_patches/fix_unreachable.py new file mode 100644 index 0000000..804ec30 --- /dev/null +++ b/pylingual/editable_bytecode/bytecode_patches/fix_unreachable.py @@ -0,0 +1,7 @@ +from ..EditableBytecode import EditableBytecode + + +def fix_unreachable(bytecode: EditableBytecode): + bytecode.remove_unreachable_instructions() + bytecode.remove_useless_jumps() + bytecode._bake_jumps() diff --git a/pylingual/editable_bytecode/bytecode_patches/remove_docstrings.py b/pylingual/editable_bytecode/bytecode_patches/remove_docstrings.py new file mode 100644 index 0000000..e2aa4a3 --- /dev/null +++ b/pylingual/editable_bytecode/bytecode_patches/remove_docstrings.py @@ -0,0 +1,9 @@ +from ..EditableBytecode import EditableBytecode + +import itertools + + +def remove_docstrings(bytecode: EditableBytecode): + to_remove = [(load_const, store_doc) for load_const, store_doc in itertools.pairwise(bytecode.instructions) if load_const.opname == "LOAD_CONST" and store_doc.opname == "STORE_NAME" and store_doc.argval == "__doc__"] + to_remove = list(itertools.chain.from_iterable(to_remove)) + bytecode.remove_instructions(to_remove) diff --git a/pylingual/editable_bytecode/bytecode_patches/remove_extended_arg.py b/pylingual/editable_bytecode/bytecode_patches/remove_extended_arg.py new file mode 100644 index 0000000..43d8c1b --- /dev/null +++ b/pylingual/editable_bytecode/bytecode_patches/remove_extended_arg.py @@ -0,0 +1,8 @@ +from ..EditableBytecode import EditableBytecode + + +def remove_extended_arg(bytecode: EditableBytecode): + for i in bytecode: + i.has_extended_arg = False + to_remove = [x for x in bytecode.instructions if x.opname == "EXTENDED_ARG"] + bytecode.remove_instructions(to_remove) diff --git a/pylingual/editable_bytecode/bytecode_patches/remove_nop.py b/pylingual/editable_bytecode/bytecode_patches/remove_nop.py new file mode 100644 index 0000000..ce3505b --- /dev/null +++ b/pylingual/editable_bytecode/bytecode_patches/remove_nop.py @@ -0,0 +1,6 @@ +from ..EditableBytecode import EditableBytecode + + +def remove_nop(bytecode: EditableBytecode): + to_remove = [x for x in bytecode.instructions if x.opname == "NOP"] + bytecode.remove_instructions(to_remove) diff --git a/pylingual/editable_bytecode/control_flow_graph.py b/pylingual/editable_bytecode/control_flow_graph.py new file mode 100644 index 0000000..c0287fc --- /dev/null +++ b/pylingual/editable_bytecode/control_flow_graph.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +from .Instruction import Inst + +import networkx as nx +from enum import Enum + + +class ControlFlowEdgeType(Enum): + NATURAL = "natural" # edge goes to the next instruction in sequence; this is the default + JUMP = "jump" # edge represents an unconditional jump + TRUE_JUMP = "true_jump" # edge represents a conditional jump that is taken when the condition is true + FALSE_JUMP = "false_jump" # edge represents a conditional jump that is taken when the condition is false + EXCEPTION = "exception" # edge goes to an exception handler + META = "meta" # used exclusively for the START and END meta-nodes that are added to the cfg + + +from typing import Any + + +def inst_to_node_attributes(inst: "Inst") -> dict: + return {"label": inst.get_dis_view()} + + +def inst_to_successors(inst: "Inst") -> list[tuple["Inst", ControlFlowEdgeType]]: + # premature exits + if inst.opname in ("RETURN_VALUE", "RETURN_CONST", "RAISE_VARARGS", "RERAISE"): + return [] + + successors = [] + # jump target + if inst.is_jump and inst.opname not in ("SETUP_WITH", "SETUP_ASYNC_WITH", "SETUP_FINALLY", "SETUP_CLEANUP", "SETUP_EXCEPT"): + edge_type = ControlFlowEdgeType.JUMP + if inst.is_cond_jump and any(s in inst.opname for s in ("IF_FALSE", "IF_NONE", "FOR_ITER", "IF_NOT_EXC_MATCH")): + edge_type = ControlFlowEdgeType.FALSE_JUMP + elif inst.is_cond_jump and ("IF_TRUE" in inst.opname or "IF_NOT_NONE" in inst.opname): + edge_type = ControlFlowEdgeType.TRUE_JUMP + elif inst.opname == "SEND": + edge_type = ControlFlowEdgeType.TRUE_JUMP + successors.append((inst.target, edge_type)) + + # regular follow-up instruction + if not inst.is_uncond_jump: + next_instruction = inst.bytecode._get_instruction_after(inst) + if next_instruction: + successors.append((next_instruction, ControlFlowEdgeType.NATURAL)) + + return successors + + +# returns a list of tuples of the form +# (source, destination, edge_properties, next_state) +def inst_to_edges_37(inst: "Inst", state: Any) -> list[tuple["Inst", "Inst", dict[str, ControlFlowEdgeType], Any]]: + if state is None: + block_stack = tuple() + else: + block_stack = state + + # update exception handler (pre-3.11 style) + if inst.opname in ("SETUP_FINALLY", "SETUP_WITH", "SETUP_ASYNC_WITH", "SETUP_CLEANUP", "SETUP_EXCEPT"): + # add new exception handler to the stack + next_block_stack = (*block_stack, (inst.target, "exception")) + elif inst.opname in ("POP_BLOCK",): + # remove an exception handler from the stack + next_block_stack = block_stack[:-1] + elif inst.opname in ("SETUP_LOOP",): + next_block_stack = (*block_stack, (inst.target, "loop")) + else: + # no change to the exception handler + next_block_stack = block_stack + + if inst.opname in ("BREAK_LOOP", "CONTINUE_LOOP"): + # get the highest loop block from the block stack; default to None + def is_loop_block(block: tuple["Inst", str]) -> bool: + return block[1] == "loop" + + break_target = next(filter(is_loop_block, reversed(block_stack)), None) + loop_block_index = block_stack.index(break_target) + block_stack = block_stack[:loop_block_index] + # after breaking out of the loop or continuing, pop off any exeption blocks leading up to the loop block + if inst.opname == "BREAK_LOOP": + edges = [((inst, break_target[0], ControlFlowEdgeType.JUMP), block_stack)] + elif inst.opname == "CONTINUE_LOOP": + # CONTINUE_LOOP has the offset of the loop instruction as the argval; go there to continue + edges = [((inst, inst.bytecode.get_by_offset(inst.argval), ControlFlowEdgeType.JUMP), block_stack)] + else: + # list of tuples [(successor, edge_type) ...] + successors = inst_to_successors(inst) + edges = [((inst, successor, edge_type), next_block_stack) for successor, edge_type in successors] + + # get the highest exception block from the block stack; default to None + def is_exception_block(block: tuple["Inst", str]) -> bool: + return block[1] == "exception" + + exception_target = next(filter(is_exception_block, reversed(block_stack)), None) + if exception_target: + exception_block_index = block_stack.index(exception_target) + edges.append(((inst, exception_target[0], ControlFlowEdgeType.EXCEPTION), block_stack[:exception_block_index])) + + return edges + + +def inst_to_edges_39(inst: "Inst", state: Any) -> list[tuple[tuple["Inst", "Inst", dict[str, ControlFlowEdgeType]], Any]]: + if state is None: + block_stack = tuple() + else: + block_stack = state + + exception_target = block_stack[-1] if block_stack else None + + # update exception handler (pre-3.11 style) + if inst.opname in ("SETUP_FINALLY", "SETUP_WITH", "SETUP_ASYNC_WITH", "SETUP_CLEANUP"): + # add new exception handler to the stack + next_block_stack = (*block_stack, inst.target) + elif inst.opname in ("POP_BLOCK"): + # remove an exception handler from the stack + next_block_stack = block_stack[:-1] + else: + # no change to the exception handler + next_block_stack = block_stack + + # list of tuples [(successor, edge_type) ...] + successors = inst_to_successors(inst) + edges = [((inst, successor, edge_type), next_block_stack) for successor, edge_type in successors] + + if exception_target: + edges.append(((inst, exception_target, ControlFlowEdgeType.EXCEPTION), block_stack[:-1])) + + return edges + + +def inst_to_edges_311(inst: "Inst", state: Any) -> list[tuple[tuple["Inst", "Inst", dict[str, ControlFlowEdgeType]], Any]]: + exception_table = inst.bytecode.named_exception_table + # search the exception table for any entry that applies to the current instruction + exception_target = None + for exception_range in exception_table: + if inst.offset >= exception_range.start and inst.offset < exception_range.end: + exception_target = inst.bytecode.get_by_offset(exception_range.target) + break + + # list of tuples [(successor, edge_type) ...] + successors = inst_to_successors(inst) + edges = [((inst, successor, edge_type), None) for successor, edge_type in successors] + + if exception_target: + edges.append(((inst, exception_target, ControlFlowEdgeType.EXCEPTION), None)) + + return edges + + +def bytecode_to_control_flow_graph(bytecode: "EditableBytecode") -> nx.DiGraph: + cfg = nx.DiGraph() + + # add one node for each instruction + cfg.add_nodes_from((inst, inst_to_node_attributes(inst)) for inst in bytecode) + + # depth first traversal of the bytecode + visited_instructions = set() + # dfs stack contains (instruction, inst_to_edges_state) + dfs_stack = [(bytecode.first_instruction, None)] + + inst_to_edges = inst_to_edges_311 + if bytecode.version < (3, 11): + inst_to_edges = inst_to_edges_39 + if bytecode.version < (3, 8): + inst_to_edges = inst_to_edges_37 + + while dfs_stack: + # boilerplate depth-first traversal + current_inst, current_state = dfs_stack.pop() + if current_inst in visited_instructions: + continue + visited_instructions.add(current_inst) + + edges = inst_to_edges(current_inst, current_state) + for (src, dst, edge_type), next_state in edges: + cfg.add_edge(src, dst, type=edge_type.value) + dfs_stack.append((dst, next_state)) + + # remove unreachable instructions + cfg.remove_nodes_from(set(bytecode) - visited_instructions) + + return cfg diff --git a/pylingual/editable_bytecode/utils.py b/pylingual/editable_bytecode/utils.py new file mode 100644 index 0000000..73566cf --- /dev/null +++ b/pylingual/editable_bytecode/utils.py @@ -0,0 +1,89 @@ +import re +from datetime import datetime +from struct import pack + +from xdis.cross_types import LongTypeForPython3, UnicodeForPython3 +import xdis.marsh as marshal +from xdis import iscode +from xdis.codetype import to_portable + +from pylingual.utils.version import PythonVersion + + +def unwrap(x): + """ + Turns xdis-specific types back into Python types + """ + if isinstance(x, UnicodeForPython3): + return str(x) + if isinstance(x, LongTypeForPython3): + return int(x) + if isinstance(x, tuple): + return tuple(unwrap(e) for e in x) + if isinstance(x, list): + return [unwrap(e) for e in x] + return x + + +def codeobj_replace(codeobj, **kwargs): + if hasattr(codeobj, "replace"): + return codeobj.replace(**kwargs) + else: + all_kwargs = {key: getattr(codeobj, key) for key in dir(codeobj) if key.startswith("co_")} + all_kwargs.update(kwargs) + + return to_portable(**all_kwargs) + + +def write_pyc(f, codeobj, version, magic_int, timestamp=None, filesize=0): + """ + Mostly taken from xdis.load's write_bytecode_file function. + Does not close the provided fileobject upon return. + """ + + version = PythonVersion(version) + if version >= (3, 0): + f.write(pack("= (3, 7): # pep552 bytes + f.write(pack("= (3, 3): + # In Python 3.3+, these 4 bytes are the size of the source code_obj file (mod 2^32) + f.write(pack("", + "", + "", + "", + "", +) diff --git a/pylingual/equivalence_check.py b/pylingual/equivalence_check.py new file mode 100644 index 0000000..45e8621 --- /dev/null +++ b/pylingual/equivalence_check.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import difflib +from dataclasses import dataclass +from pathlib import Path + +import networkx as nx +from pylingual.control_flow_reconstruction.structure_control_flow import condense_basic_blocks +from pylingual.editable_bytecode import EditableBytecode, Inst, PYCFile +from pylingual.editable_bytecode.bytecode_patches import fix_indirect_jump, fix_unreachable, remove_extended_arg, remove_nop +from pylingual.editable_bytecode.control_flow_graph import bytecode_to_control_flow_graph + + +def is_control_flow_equivalent(basic_block_graph_1: nx.DiGraph, basic_block_graph_2: nx.DiGraph) -> bool: + # for the graphs to be equal, they have to have the same number of nodes + + if len(basic_block_graph_1) != len(basic_block_graph_2): + return False + + # create a node mapping based on bytecode offset + # each node is (networkx node id, data_item); in this case, the data_item will be offset + ordered_nodes_1 = sorted(basic_block_graph_1.nodes(data="offset", default=(float("inf"),)), key=lambda node: min(node[1])) + ordered_nodes_2 = sorted(basic_block_graph_2.nodes(data="offset", default=(float("inf"),)), key=lambda node: min(node[1])) + + # map the node ids from graph 1 to those of graph 2 + node_id_mapping = {node1[0]: node2[0] for node1, node2 in zip(ordered_nodes_1, ordered_nodes_2)} + + for node1, edges1 in basic_block_graph_1.adjacency(): + # for each node in graph 1, find the corresponding node in graph 2 + node2 = node_id_mapping[node1] + edges2 = basic_block_graph_2[node2] + + # map the graph 1 edge destinations to use graph 2 node ids + # make a set of all the destinations for this node pair + mapped_destinations1 = set(node_id_mapping[dest] for dest in edges1.keys()) + destinations2 = set(edges2.keys()) + + # if the outgoing edges don't match, then the control flow is not equivalent + if mapped_destinations1 != destinations2: + return False + + # all checks passed; these graphs are equivalent + return True + + +def compare_instruction(inst_a: Inst, inst_b: Inst): + attr_list = ("opname", "opcode", "optype", "real_size", "has_arg", "has_extended_arg", "offset", "is_jump_target") + + # resolve different types of uncond jumps + if inst_a.is_uncond_jump and inst_b.is_uncond_jump: + return getattr(inst_a, "argval", None) == getattr(inst_b, "argval", None) + + if not all(getattr(inst_a, attr, None) == getattr(inst_b, attr, None) for attr in attr_list): + return False + + argval_a = getattr(inst_a, "argval", None) + argval_b = getattr(inst_b, "argval", None) + + if argval_a == argval_b: + return True + + if hasattr(argval_a, "co_code") and hasattr(argval_b, "co_code"): + # the co_code's will be checked in recursive call. so can be ignored here + return True + + return False + + +def compare_bytecode(pyc_a: EditableBytecode, pyc_b: EditableBytecode) -> bcComparisonResult: + """ + Directly Compares two pyc files by recursing through root bytecode and all child bytecodes of pyc files + Ignores some forensically irrelevant data such as: + white space / line differences, + Code_obj addresses + + :param pyc_a: First pyc to compare + :param pyc_b: Second pyc to compare + """ + + if len(pyc_a) != len(pyc_b): + return bcComparisonResult(False) + + if pyc_a.named_exception_table != pyc_b.named_exception_table: + return bcComparisonResult(False) + + # make sure all the instructions match at this node + + for inst_a, inst_b in zip(pyc_a, pyc_b): + if not compare_instruction(inst_a, inst_b): + # We purposefully check pyc_b as this is our candidate pyc in decompiler.py + fail_offset = inst_b.offset + insts_b = pyc_b.instructions + inst_idx = insts_b.index(inst_b) + try: + lno = next(inst.starts_line for inst in reversed(insts_b[:inst_idx]) if inst.starts_line is not None) + except StopIteration: + lno = None + return bcComparisonResult(False, lno, fail_offset) + return bcComparisonResult(True) + + +@dataclass(frozen=True) +class TestResult: + """ + This class stores the testing result of a code object, if the code object succeeds success will be true. + If the code object does not succeed success will be false, additionally the error message and line number is saved. + + :param success: Stores whether the code object succeeded or failed. + :param message: Error message if the code object failed + :param name_a: Code object name of bytecode a + :param name_b: Code object name of bytecode b + :param failed_line_number: The line number where the code object failed + :param failed offset: The offset where the code object failed + """ + + success: bool + message: str + name_a: str + name_b: str + failed_line_number: int | None = None + failed_offset: int | None = None + + def names(self): + if self.name_a == self.name_b: + return self.name_a + return f"{self.name_a}, {self.name_b}" + + def __str__(self): + if self.success: + return f"{self.names()}: Success: {self.message}" + lno_message = "" + if None not in (self.failed_line_number, self.failed_offset): + lno_message = f" detected at line number {self.failed_line_number} and instruction offset {self.failed_offset}" + return f"***{self.names()}: Failure{lno_message}: {self.message}" + + +@dataclass(frozen=True) +class bcComparisonResult: + result: bool + failed_line: int | None = None + failed_offset: int | None = None + + +def matching_iter(pyc_a, pyc_b): + """ + Matches bytecodes in pyc_a and pyc_b with the same name + """ + bc_a = list(pyc_a.iter_bytecodes()) + bc_b = list(pyc_b.iter_bytecodes()) + sm = difflib.SequenceMatcher(a=[x.name for x in bc_a], b=[x.name for x in bc_b]) + i_a = 0 + i_b = 0 + for block in sm.get_matching_blocks(): + while i_a < block.a: + yield bc_a[i_a], None + i_a += 1 + while i_b < block.b: + yield None, bc_b[i_b] + i_b += 1 + for i in range(block.size): + yield bc_a[i_a + i], bc_b[i_b + i] + i_a += block.size + i_b += block.size + while i_a < len(bc_a): + yield bc_a[i_a], None + i_a += 1 + while i_b < len(bc_b): + yield None, bc_b[i_b] + i_b += 1 + + +def compare_pyc(pyc_path_a: Path, pyc_path_b: Path) -> list[TestResult]: + """ + Tests the control flow of the two pyc files + Should not be imported as it relies on TestResult class. + + note: will always patch out unreachable code + + :param pyc_path_a: First pyc to compare + :param pyc_path_b: Second pyc to compare + """ + + pyc_a = PYCFile(pyc_path_a) + pyc_b = PYCFile(pyc_path_b) + + pyc_a.apply_patches([remove_extended_arg, remove_nop, fix_indirect_jump, fix_unreachable, remove_extended_arg]) + pyc_b.apply_patches([remove_extended_arg, remove_nop, fix_indirect_jump, fix_unreachable, remove_extended_arg]) + + results = [] + + for bytecode_a, bytecode_b in matching_iter(pyc_a, pyc_b): + if bytecode_a is None: + test_result = TestResult(False, "Extra bytecode", "None", bytecode_b.name) + results.append(test_result) + continue + if bytecode_b is None: + test_result = TestResult(False, "Missing bytecode", bytecode_a.name, "None") + results.append(test_result) + continue + cfg_a = bytecode_to_control_flow_graph(bytecode_a) + cfg_b = bytecode_to_control_flow_graph(bytecode_b) + block_graph_a = condense_basic_blocks(cfg_a) + block_graph_b = condense_basic_blocks(cfg_b) + if not is_control_flow_equivalent(block_graph_a, block_graph_b): + test_result = TestResult(False, "Different control flow", bytecode_a.name, bytecode_b.name) + results.append(test_result) + continue + + bytecode_result = compare_bytecode(bytecode_a, bytecode_b) + if not bytecode_result.result: + test_result = TestResult(False, "Different bytecode", bytecode_a.name, bytecode_b.name, bytecode_result.failed_line, bytecode_result.failed_offset) + results.append(test_result) + continue + + test_result = TestResult(True, "Equal", bytecode_a.name, bytecode_b.name) + results.append(test_result) + + return results diff --git a/pylingual/main.py b/pylingual/main.py new file mode 100644 index 0000000..d45a47f --- /dev/null +++ b/pylingual/main.py @@ -0,0 +1,164 @@ +from typing import TYPE_CHECKING +import click +import logging +import shutil +import platform +import subprocess +import os +from pathlib import Path + +import pylingual.utils.ascii_art as ascii_art +from pylingual.utils.generate_bytecode import CompileError +from pylingual.utils.version import PythonVersion, supported_versions +from pylingual.utils.tracked_list import TrackedList, SEGMENTATION_STEP, TRANSLATION_STEP, CFLOW_STEP, CORRECTION_STEP +from pylingual.utils.lazy import lazy_import +from pylingual.decompiler import DecompilerResult, decompile + +import rich +from rich.align import Align +from rich.console import Group +from rich.live import Live +from rich.logging import RichHandler +from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn, TimeElapsedColumn +from rich.rule import Rule +from rich.status import Status +from rich.theme import Theme +from rich.table import Table + +if TYPE_CHECKING: + import transformers +else: + lazy_import("transformers") + +logger = logging.getLogger(__name__) + + +def print_header(): + console = rich.get_console() + console.rule() + console.print(Align(ascii_art.PYLINGUAL_ART, "center"), style="royal_blue1", highlight=False) + console.print(ascii_art.PYLINGUAL_SUBHEADER, justify="center") + console.rule() + + +def print_result(file: str, result: DecompilerResult): + table = Table(title=f"Equivalence Results for {file}") + table.add_column("Code Object") + table.add_column("Success") + table.add_column("Message") + for r in result.equivalence_results: + if isinstance(r, CompileError): + continue + table.add_row(r.names(), "Success" if r.success else "Failure", r.message, style="red" if not r.success else "") + if table.rows: + rich.get_console().print(table, justify="center") + + +@click.command(help="End to end pipeline to decompile Python bytecode into source code.", context_settings={"help_option_names": ["-h", "--help"]}) +@click.argument("files", nargs=-1) +@click.option("-o", "--out-dir", default=None, type=Path, help="The directory to export results to.", metavar="PATH") +@click.option("-c", "--config-file", default=None, type=Path, help="Config file for model information.", metavar="PATH") +@click.option("-v", "--version", default=None, type=PythonVersion, help="Python version of the .pyc, default is auto detection.", metavar="VERSION") +@click.option("-k", "--top-k", default=10, type=int, help="Maximum number of additional segmentations to consider.", metavar="INT") +@click.option("-q", "--quiet", is_flag=True, default=False, help="Suppress console output.") +@click.option("--trust-lnotab", is_flag=True, default=False, help="Use the lnotab for segmentation instead of the segmentation model.") +@click.option("--init-pyenv", is_flag=True, default=False, help="Install pyenv before decompiling.") +def main(files: list[str], out_dir: Path | None, config_file: Path | None, version: PythonVersion | None, top_k: int, trust_lnotab: bool, init_pyenv: bool, quiet: bool): + rich.reconfigure(markup=False, emoji=False, quiet=quiet, theme=Theme({"logging.keyword": "yellow not bold"})) + console = rich.get_console() + log_handler = RichHandler(console=console, rich_tracebacks=True) + logging.basicConfig(level="INFO", format="%(message)s", datefmt="[%X]", handlers=[log_handler], force=True) + + if not init_pyenv and not files: + click.echo(click.get_current_context().get_help()) + return + + print_header() + + if init_pyenv and (not install_pyenv() or not files): + return + + if out_dir is not None: + out_dir.mkdir(parents=True, exist_ok=True) + + progress = Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TaskProgressColumn(), + TimeElapsedColumn(), + ) + status = Status("Initializing...") + + # extend TrackedList to update progress bars + def init(self): + self.task = next(x for x in progress.tasks if x.description == self.name) + self.task.total = len(self.x) + 1 + progress.start_task(self.task.id) + + TrackedList.init = init + TrackedList.progress = lambda self, i: progress.advance(self.task.id, i) + # the step is not done until the TrackedList is deleted + TrackedList.__del__ = lambda self: progress.advance(self.task.id, float("inf")) + + n = len(files) + with Live(Group(Rule(), status, progress), transient=True, console=console, refresh_per_second=12.5): + transformers.logging.disable_default_handler() + transformers.logging.add_handler(log_handler) + progress.add_task(SEGMENTATION_STEP, start=False) + progress.add_task(TRANSLATION_STEP, start=False) + progress.add_task(CFLOW_STEP, start=False) + progress.add_task(CORRECTION_STEP, start=False) + for i, file in enumerate(files): + for task in progress.tasks: + progress.reset(task.id, start=False) + pyc_path = Path(file) + log_handler.keywords = [file, pyc_path.name, pyc_path.with_suffix(".py").name] + status.update(f"Decompiling {pyc_path} ({i + 1} / {n})") + if not pyc_path.exists(): + raise FileNotFoundError(f"pyc file {pyc_path} does not exist") + + try: + result = decompile( + file=pyc_path, + out_dir=out_dir / f"decompiled_{pyc_path.stem}" if out_dir is not None else Path(f"decompiled_{pyc_path.stem}"), + config_file=Path(config_file) if config_file else None, + version=version, + top_k=top_k, + trust_lnotab=trust_lnotab, + ) + print_result(pyc_path.name, result) + except Exception: + logger.exception(f"Failed to decompile {pyc_path}") + console.rule() + + +def install_pyenv(): + if shutil.which("pyenv") is not None: + logger.warning("pyenv seems to already be installed, ignoring --init-pyenv...") + return True + if platform.system() not in ["Linux", "Darwin"] and not click.confirm("pyenv is probably not supported on your operating system. Continue?", default=False): + return False + cmd = "curl -fsSL https://pyenv.run | bash" + if not click.confirm(f"pyenv will be installed with the following command:\n\n\t{cmd}\n\nContinue?", default=True): + return False + if subprocess.run(cmd, shell=True).returncode != 0: + logger.error("pyenv install failed, exiting...") + return False + os.environ["PATH"] = f"{os.environ.get('PYENV_ROOT', os.path.expanduser('~/.pyenv'))}/bin:{os.environ['PATH']}" + if shutil.which("pyenv") is None: + logger.error("Could not find pyenv, exiting...") + return False + versions = click.prompt( + "Enter comma-separated Python versions to install (leave empty to install all supported versions)", + value_proc=lambda s: [PythonVersion(x) for x in s.split(",")] if isinstance(s, str) else s, + default=supported_versions, + show_default=False, + ) + if subprocess.run(["pyenv", "install", *map(str, versions)]).returncode != 0: + logger.error("Error installing Python versions, exiting...") + return False + return True + + +if __name__ == "__main__": + main() diff --git a/pylingual/masking/ast_masker.py b/pylingual/masking/ast_masker.py new file mode 100644 index 0000000..1e2fe8d --- /dev/null +++ b/pylingual/masking/ast_masker.py @@ -0,0 +1,327 @@ +import ast +import re +import copy +from pylingual.masking.global_masker import Masker +from pylingual.utils.version import PythonVersion + + +class customUnparser(ast._Unparser): + def __init__(self, masker: Masker, **kwargs): + ast._Unparser.__init__(self, **kwargs) + self.masker = masker + + def visit_Constant(self, node): + value = node.value + if isinstance(value, tuple): + with self.delimit("(", ")"): + self.items_view(self._write_constant, value) + elif value is ...: + self.write("...") + else: + if value in self.masker.global_tab.values(): + # get key from value in dictionary + key = self.masker.unmask(value) + if not isinstance(key, str): + self.write(value) + return + self._write_constant(value) + + def visit_FormattedValue(self, node): + def unparse_inner(inner): + unparser = type(self)(self.masker, _avoid_backslashes=True) + unparser.set_precedence(ast._Precedence.TEST.next(), inner) + return unparser.visit(inner) + + with self.delimit("{", "}"): + expr = unparse_inner(node.value) + if "\\" in expr: + raise ValueError("Unable to avoid backslash in f-string expression part") + if expr.startswith("{"): + # Separate pair of opening brackets as "{ {" + self.write(" ") + self.write(expr) + if node.conversion != -1: + self.write(f"!{chr(node.conversion)}") + if node.format_spec: + self.write(":") + self._write_fstring_inner(node.format_spec) + + +def custom_unparse(ast_obj, masker: Masker): + unparser = customUnparser(masker) + return unparser.visit(ast_obj) + + +def evaluate_binop_optimizations(node: ast.BinOp, version): + def eval_expr_binop(binop): + if ast.BinOp in (type(binop.left), type(binop.right)): + if type(binop.left) == ast.BinOp: + binop.left = eval_expr_binop(binop.left) + if type(binop.right) == ast.BinOp: + binop.right = eval_expr_binop(binop.right) + + if ast.UnaryOp in (type(binop.left), type(binop.right)): + if type(binop.left) == ast.UnaryOp: + binop.left = evaluate_unaryop_optimizations(binop.left, version) + if type(binop.right) == ast.UnaryOp: + binop.right = evaluate_unaryop_optimizations(binop.right, version) + + if type(binop.left) == type(binop.right) == ast.Constant: + # don't simplify if the calculation throws an error + try: + value = eval(compile(ast.fix_missing_locations(ast.Expression(binop)), "", "eval")) + except (TypeError, ZeroDivisionError): + return binop + # following two checks are for catching cases where the compiler will not premptively evaluate the expression + # see max sizes in cpython: https://github.com/python/cpython/blob/34e93d3998bab8acd651c50724eb1977f4860a08/Python/ast_opt.c#LL156C1-L156C1 + if type(value) in (str, bytes): + if len(value) > 20 and version == (3, 6) and type(binop.op) == ast.Mult: + return binop + if len(value) > 4096: + return binop + elif type(value) == int: + if len(bin(value)[2:]) > 128: + return binop + + # % style string formatting is not optimized by the compiler + if type(binop.left.value) == str and type(binop.right.value) == str and type(binop.op) == ast.Mod: + return binop + return ast.Constant(value=value) + + return binop + + return eval_expr_binop(node) + + +def evaluate_unaryop_optimizations(node: ast.UnaryOp, version): + def eval_expr_unaryop(unaryop: ast.UnaryOp): + if type(unaryop.operand) == ast.UnaryOp: + unaryop.operand = eval_expr_unaryop(unaryop.operand) + + if type(unaryop.operand) == ast.BinOp: + unaryop.operand = evaluate_binop_optimizations(unaryop.operand, version) + + if type(unaryop.operand) == ast.Constant: + # don't simplify if the calculation throws an error + try: + value = eval(ast.unparse(unaryop)) + except TypeError: + return unaryop + return ast.Constant(value=value) + + return unaryop + + return eval_expr_unaryop(node) + + +class RewriteMasks(ast.NodeTransformer): + def __init__(self, masker: Masker, line_offsets: dict, python_version: PythonVersion): + self.masker = masker + self.line_offsets = line_offsets + self.python_version = python_version + + def visit_Name(self, node): + if node.id in self.masker.global_tab: + node.id = self.masker.mask(node.id) + + self.generic_visit(node) + return node + + def visit_Constant(self, node): + """Replace constants as well as calculate line_offsets introduced by replacing multiline strings""" + # update lno for multiline + if node.value in self.masker.global_tab: + if hasattr(node, "lineno") and hasattr(node, "end_lineno"): + if node.end_lineno > node.lineno: + self.line_offsets.update({node.lineno: node.end_lineno - node.lineno}) + + node.value = self.masker.mask(node.value) + self.generic_visit(node) + return node + + def visit_Global(self, node): + # drop unused globals; mask all others + node.names = [self.masker.mask(name) for name in node.names if name in self.masker.global_tab] + self.generic_visit(node) + return node + + def visit_Nonlocal(self, node): + node.names = [self.masker.mask(name) for name in node.names] + self.generic_visit(node) + return node + + def visit_BinOp(self, node): + node = evaluate_binop_optimizations(node, self.masker.version) + + if isinstance(node, ast.Constant): + node = self.visit_Constant(node) + elif isinstance(node, ast.Name): + node = self.visit_Name(node) + + self.generic_visit(node) + + # handle printf-style string formatting in 3.11 + if isinstance(node, ast.BinOp) and self.python_version >= (3, 11): + if isinstance(node.op, ast.Mod) and isinstance(node.left, ast.Constant) and isinstance(node.left.value, str) and type(node.right) in (ast.Tuple, ast.List): + format_specifier_re = r"(%[\#0\- \+]*(?:\*|\d+)?(?:.(?:\*|\d+))?[diouxXeEfFgGcrsa%])" + string_fragments = re.split(format_specifier_re, node.left.value) + + # resolve "%%" -> "%"; this isn't a real format specifier + for i in range(1, len(string_fragments) - 1, 2): + while string_fragments[i] == "%%": + string_fragments[i - 1] += "%" + string_fragments[i + 1] + string_fragments[i - 1 :] = string_fragments[i + 1 :] + # mask each fragment individually + if len(string_fragments) > 1: + for i, fragment in enumerate(string_fragments): + # don't try to mask the format specifier + if i % 2 == 1 or not fragment: + continue + string_fragments[i] = self.masker.mask(fragment) + node.left.value = "".join(string_fragments) + + return node + + def visit_UnaryOp(self, node): + # simplify negative numbers + node = evaluate_unaryop_optimizations(node, self.masker.version) + + if isinstance(node, ast.Constant): + node = self.visit_Constant(node) + elif isinstance(node, ast.Name): + node = self.visit_Name(node) + + self.generic_visit(node) + return node + + def visit_FunctionDef(self, node): + node.name = self.masker.mask(node.name) + + if node.returns is not None and ast.unparse(node.returns) in self.masker.global_tab: + node.returns = ast.Name(id=ast.unparse(node.returns)) + + self.generic_visit(node) + return node + + def visit_AsyncFunctionDef(self, node): + # mask return annotation + node.name = self.masker.mask(node.name) + + if node.returns is not None and ast.unparse(node.returns) in self.masker.global_tab: + node.returns = ast.Name(id=ast.unparse(node.returns)) + + self.generic_visit(node) + return node + + def visit_ClassDef(self, node): + node.name = self.masker.mask(node.name) + self.generic_visit(node) + return node + + def visit_Attribute(self, node): + if node.attr in self.masker.global_tab: + node.attr = self.masker.mask(node.attr) + elif ast.unparse(node) in self.masker.global_tab: + node = ast.Name(id=self.masker.mask(ast.unparse(node))) + self.generic_visit(node) + return node + + def visit_AnnAssign(self, node): + if self.masker.future_annotations: + if ast.unparse(node.annotation) in self.masker.global_tab: + node.annotation = ast.Name(id=self.masker.mask(ast.unparse(node.annotation))) + elif node.value is not None: + # if we can't mask the annotation, just remove the annotation + node = ast.Assign(targets=[node.target], value=node.value, lineno=node.lineno) + self.generic_visit(node) + else: + self.generic_visit(node) + if node.value is not None and not re.match(r'["\'(\[]*<', ast.unparse(node.annotation)): + # if we can't mask the annotation, just remove the annotation + node = ast.Assign(targets=[node.target], value=node.value, lineno=node.lineno) + return node + + def visit_ImportFrom(self, node): + if node.module is None: # edge case for "from .. import func" + node.module = self.masker.mask("") + else: + node.module = self.masker.mask(node.module) + node.module = str(self.masker.mask(node.level)) + str(node.module or "") + node.level = 0 + self.generic_visit(node) + return node + + def visit_arg(self, node): + node.arg = self.masker.mask(node.arg) + + if node.annotation and ast.unparse(node.annotation) in self.masker.global_tab: + node.annotation = ast.Name(id=ast.unparse(node.annotation)) + + self.generic_visit(node) + return node + + def visit_alias(self, node): + if node.name not in "*": + node.name = self.masker.mask(node.name) + if node.asname is not None: + node.asname = self.masker.global_tab.get(node.asname, node.asname) + self.generic_visit(node) + return node + + def visit_keyword(self, node): + if node.arg is not None: + node.arg = self.masker.mask(node.arg) + self.generic_visit(node) + return node + + def visit_ExceptHandler(self, node): + if getattr(node, "name", None) is not None: + node.name = self.masker.mask(node.name) + self.generic_visit(node) + return node + + def visit_If(self, node): + # don't try to mask trivially unreachable code + if isinstance(node.test, ast.Constant): + # we have to use deepcopy due to how generic_visit masks nodes, we cannot pass it the individual body. + if node.test.value is False: + # only mask else body + masked_node = copy.deepcopy(node) + self.generic_visit(masked_node) + node.orelse = copy.deepcopy(masked_node.orelse) + return node + if node.test.value is True: + # only mask if body + masked_node = copy.deepcopy(node) + self.generic_visit(masked_node) + node.body = copy.deepcopy(masked_node.body) + return node + + self.generic_visit(node) + return node + + +DUMMY_DECORATOR = "PYLINGUAL_DUMMY_DECORATOR_finj3igh309jhasfjn2oihg20ni3" + + +def add_dummy_decorators(source: str) -> str: + tree = ast.parse(source) + dummy_deco_transformer().generic_visit(tree) + return ast.unparse(tree) + + +class dummy_deco_transformer(ast.NodeTransformer): + def visit_ClassDef(self, node): + node.decorator_list.append(ast.Name(id=DUMMY_DECORATOR)) + self.generic_visit(node) + return node + + def visit_AsyncFunctionDef(self, node): + node.decorator_list.append(ast.Name(id=DUMMY_DECORATOR)) + self.generic_visit(node) + return node + + def visit_FunctionDef(self, node): + node.decorator_list.append(ast.Name(id=DUMMY_DECORATOR)) + self.generic_visit(node) + return node diff --git a/pylingual/masking/global_masker.py b/pylingual/masking/global_masker.py new file mode 100644 index 0000000..188d85a --- /dev/null +++ b/pylingual/masking/global_masker.py @@ -0,0 +1,264 @@ +from collections.abc import MutableMapping +from copy import deepcopy +from xdis import iscode +from xdis.cross_types import UnicodeForPython3, LongTypeForPython3 + +from pylingual.editable_bytecode import Inst +from pylingual.editable_bytecode.utils import comprehension_names, find_loadconst_codeobj_from_makefunc + + +# added type-sensitivity to the dict to differentiate true/1 and false/0 +class TypeSensitiveDict(MutableMapping): + def __init__(self, *args, **kwargs): + self.store = dict() + self.update(dict(*args, **kwargs)) # use the free update to set keys + + def __getitem__(self, key): + return self.store[self._key_transform(key)] + + def __setitem__(self, key, value): + self.store[self._key_transform(key)] = value + + def __delitem__(self, key): + del self.store[self._key_transform(key)] + + def __iter__(self): + return iter(self.store) + + def __len__(self): + return len(self.store) + + def __contains__(self, key): + return self._key_transform(key) in self.store + + def keys(self) -> list: + return [self._key_restore(key) for key in self.store.keys()] + + def items(self): + return ((self._key_restore(key), value) for key, value in self.store.items()) + + def values(self): + return self.store.values() + + def _key_transform(self, key): + if type(key) == UnicodeForPython3: + return (key.value.decode("utf-8"), str) + if type(key) == LongTypeForPython3: + return (key.value, int) + return (key, type(key)) + + def _key_restore(self, key): + return key[0] + + +#### Main Masker +class Masker: + blacklist = [ + "__doc__", + "__annotations__", + "__qualname__", + "__class__", + "return", # for return annotations + True, + False, + None, + ] + + def __init__(self, global_table: TypeSensitiveDict | None = None): + self.global_tab = global_table if global_table is not None else TypeSensitiveDict() + + def mask(self, tok): + """Mask a token, must be in the global_table.""" + return self.global_tab[tok] if not any(tok == t and type(tok) == type(t) for t in self.blacklist) else tok + + def unmask(self, value): + """Unmask a token, value must be a metatoken value in the global_table; or this function will fail loudly""" + key = list(self.global_tab.keys())[list(self.global_tab.values()).index(value)] + return key + + def parse_MAKE_FUNCTION_info(self, inst: Inst) -> str: + """Parses out information about MAKE_FUNCTION, like args and flags + Used in get_model_view()""" + if inst.opcode != inst.bytecode.opcode.MAKE_FUNCTION: + raise ValueError("Inst is not type MAKE_FUNCTION") + + target_inst = find_loadconst_codeobj_from_makefunc(inst) + if target_inst is None: + raise ValueError("Could not find target LOAD_CONST codeobj from MAKE_FUNCTION") + target_co = target_inst.argval + + func_info = [] # list of info fields to use as argval for MAKE_FUNCTION + + flags_make_func = int(inst.argval) + if bool(flags_make_func & 0b0001): # b_default_vals + func_info.append("defaults") + if bool(flags_make_func & 0b0010): # b_kwonly_defaults + func_info.append("kwonly_defaults") + if bool(flags_make_func & 0b0100): # b_param_annotations + if self.future_annotations: + func_info.append("annotations-FUTURE") + else: + func_info.append("annotations") + if bool(flags_make_func & 0b1000): # b_free_vars + func_info.append("closures") + + # flags from the target code object + flags_co = int(target_co.co_flags) + if bool(flags_co & 0b10000000): # coroutine + func_info.append("coroutine") + if bool(flags_co & 0b1000000000): # async_generator + func_info.append("async_generator") + + posargcount = target_co.co_argcount + if not hasattr(target_co, "co_posonlyargcount"): + setattr(target_co, "co_posonlyargcount", 0) + argcount = posargcount + target_co.co_kwonlyargcount + + # parse args + if argcount: + args = [self.mask(inst.bytecode.resolve_namespace(arg)) for arg in target_co.co_varnames[:argcount]] + end = 0 + if posargcount: + begin, end = 0, posargcount + func_info.append(f"args: [{', '.join(args[begin:end])}]") + if target_co.co_posonlyargcount: + begin, end = end, end + target_co.co_posonlyargcount + func_info.append(f"posonly: [{', '.join(args[begin:end])}]") + if target_co.co_kwonlyargcount: + begin, end = end, end + target_co.co_kwonlyargcount + func_info.append(f"kwonly: [{', '.join(args[begin:end])}]") + + # kwargs and varargs + has_kwargs = bool(flags_co & 0b0100) + if has_kwargs: + func_info.append(f"kwarg: [*{self.mask(inst.bytecode.resolve_namespace(target_co.co_varnames[argcount]))}]") + if bool(flags_co & 0b1000): + func_info.append(f"vararg: [**{self.mask(inst.bytecode.resolve_namespace(target_co.co_varnames[argcount + int(has_kwargs)]))}]") + + return ", ".join(func_info) + + def get_model_view(self, inst: Inst) -> str: + """Get a dissassembled view of the instruction as the model will view it + - Replaces literals, varnames, consts etc with a mask defined in the global_table arg. + Will recursively replace items in list-like objects as well + - Brings necessary information for function definitions up from the child code obj + - Simplifies jump args + - Removes Offset notation except for jump_targets + + :param inst: Instruction we are trying to view + + """ + view = "" + + if inst.opcode == inst.bytecode.opcode.MAKE_FUNCTION: + # bring up necessary information from child bytecode for function def creation + view = f"{inst.opname} , {inst.arg}" + # parse for load const with our target bytecode + func_info = self.parse_MAKE_FUNCTION_info(inst) + if func_info: + view += f" ({func_info})" + + elif inst.optype in ("nargs", "vargs", "compare"): + view = f"{inst.opname} , {inst.argrepr if inst.argrepr else inst.argval}" + + elif inst.optype == "name": + view = f"{inst.opname} , {self.mask(inst.bytecode.resolve_namespace(inst.argval))}" + # additional import context to differ some import syntax + if inst.opname == "IMPORT_NAME" and "." in inst.argval: + view += "-DOT" + elif inst.opname == "LOAD_NAME" and inst.argval == "__annotations__" and self.future_annotations: + view += "-FUTURE" + + elif inst.optype in ("const", "local"): + if iscode(inst.argval): + if inst.argval.co_name in comprehension_names: + view = f"{inst.opname} , " + else: + view = f"{inst.opname} , " + + elif isinstance(inst.argval, str) and inst.argval in self.global_tab: # have to do this check incase string is varname of type annotation + view = f"{inst.opname} , {repr(self.mask(inst.argval))}" + + elif type(inst.argval) in (list, tuple, frozenset, set): + # do recursive in-place replacement of list elems if they are strs or bytestrs + + def replace_list(consts): + """recursive replacement of elements in arbitrary list-like objects""" + for idx, const in enumerate(consts): + if isinstance(const, str): + consts[idx] = repr(self.mask(inst.bytecode.resolve_namespace(const))) + elif const is None: + continue # don't mask None + elif type(const) in (list, tuple, frozenset): + consts[idx] = type(const)(replace_list(list(const))) + else: + consts[idx] = self.mask(const) + return consts + + consts = list(deepcopy(inst.argval)) + consts = replace_list(consts) + + if inst.bytecode.version < (3, 11): + # Format keyword argument list + # We left pad the list of kwargs so the model doesnt have to "look ahead" + next_insts = inst.next_instructions + next_inst = next_insts[0] if next_insts != [] else None + if next_inst is not None and inst.opname == "LOAD_CONST" and next_inst.opname == "CALL_FUNCTION_KW": + consts = [""] * (next_inst.argval - len(consts)) + consts + + # cast back to original type and print repr + # demote quotes one layer + arg_repr = repr(type(inst.argval)(consts)).replace("'", "").replace('"', "'") + view = f"{inst.opname} , {arg_repr}" + else: + view = f"{inst.opname} , {self.mask(inst.bytecode.resolve_namespace(inst.argval))}" + + # inst has other arg type, format + elif inst.has_arg: + # check for jump to use simplified jump format + if inst.is_jump: + jump_direction_indicator = "v~>" if inst.target.offset > inst.offset else "^~>" + view = f"{inst.opname} {inst.argrepr} {jump_direction_indicator}" + elif inst.optype is None or inst.optype == "??" or inst.optype == "encoded_arg": + # don't mask IS_OP args + view = f"{inst.opname} , {inst.argrepr if inst.argrepr else inst.argval}" + else: + if inst.argval in self.global_tab: + view = f"{inst.opname} , {self.mask(inst.argval)}" + else: + view = f"{inst.opname} , {inst.argrepr}" + # inst sets up annotations and __future__ annotations imported + elif inst.opname == "SETUP_ANNOTATIONS" and self.future_annotations: + view = "SETUP_ANNOTATIONS-FUTURE" + # no arg case + else: + view = f"{inst.opname}" + + # add offset if inst is a jump target + if inst.is_jump_target: + view = f"{inst.offset} {view}" + # create list of offsets greater than or less than inst, for all jump origins to this inst + jumps_greater_or_less = [inst.offset < inst.offset for inst in inst.jumped_to_from_insts] + if any(jumps_greater_or_less): # 1 in list + view = f"^~> {view}" + if not all(jumps_greater_or_less): # 0 in list + view = f"v~> {view}" + + # Add exception table information + if inst.bytecode.named_exception_table is not None: + if inst.exception_target: + view = f"E-> {view}" + if inst.exception_end: + view = f"{view} E-END" + + # check if next instruction is the start of entry in exception_table + next_inst = None + for inst in inst.next_instructions: + if inst.opname not in ("CACHE", "PRECALL"): + next_inst = inst + break + + if next_inst is not None and next_inst.exception_start: + view = f"{view} E-> {next_inst.exception_start}" + + return view diff --git a/pylingual/masking/model_disasm.py b/pylingual/masking/model_disasm.py new file mode 100644 index 0000000..5e74360 --- /dev/null +++ b/pylingual/masking/model_disasm.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import ast +import pathlib +import re +from copy import deepcopy +from typing import TYPE_CHECKING + +from pylingual.utils.use_escape_sequences import use_escape_sequences +from pylingual.utils.version import PythonVersion + +if TYPE_CHECKING: + from pylingual.editable_bytecode import EditableBytecode + +from pylingual.masking.ast_masker import RewriteMasks, custom_unparse +from pylingual.masking.global_masker import Masker + +mask_regex = re.compile(r"(?<=)") + + +def create_global_masker(bytecode: EditableBytecode) -> Masker: + """Creates four flat global tables of names, locals, consts, and freevars in given Bytecode for use with improved model view + Global Tables map local code obj tables: name, varname, const, and freevar to the global table + - For additional model context, values that match across the name and const tab will have their matching + partner in the other table appended to their token. ex: = + Access table in global dict via keywork of table optype""" + + global_masker = Masker() + global_tab = global_masker.global_tab + global_idx = 0 + + future_flag = 0x1000000 if bytecode.version > (3, 7) else 0x100000 + global_masker.future_annotations = bool(bytecode.codeobj.co_flags & future_flag) + global_masker.version = bytecode.version + + for bc in bytecode.iter_bytecodes(): + bc_co = bc.to_code(no_lnotab=True) + + # create consts + consts = list(deepcopy(bc_co.co_consts)) + while consts: + const = consts.pop(0) + # Don't mask None + if const is None: + continue + if type(const) in (list, tuple, frozenset, set): + consts.extend(const) + else: + global_tab.update({bc.resolve_namespace(const): f""}) + global_idx += 1 + + # create names + for name in bc_co.co_names: + if name in global_tab: + continue + global_tab.update({bc.resolve_namespace(name): f""}) + global_idx += 1 + + for free in bc_co.co_freevars: + if free in global_tab: + continue + global_tab.update({free: f""}) + global_idx += 1 + + for local in bc_co.co_varnames: + if local in global_tab: + continue + global_tab.update({bc.resolve_namespace(local): f""}) + global_idx += 1 + + global_tab.update({bc_co.co_name: f""}) + global_idx += 1 + + return global_masker + + +def mask_source(file_path: pathlib.Path, masker: Masker, python_version: PythonVersion) -> str: + """Replace source strings with tokens from keys in global_tab, + masks source and provides offsets incured by multiline string replacements""" + text = file_path.read_text() + tree = ast.parse(text, feature_version=python_version.as_tuple()) + + line_offsets = dict() + tree = RewriteMasks(masker, line_offsets, python_version).generic_visit(tree) + + source_text = custom_unparse(tree, masker) + source_lines = source_text.splitlines() + + re_added_lines = 0 + for line_offset, lines_to_add in line_offsets.items(): + insertion_target = line_offset + re_added_lines + source_lines[insertion_target:insertion_target] = [""] * lines_to_add + + return "\n".join(source_lines) + + +def restore_masked_source(file_path: pathlib.Path, masker: Masker, python_version: PythonVersion) -> str: + """Creates a large regex of all the tokens and their respective values + Replaces everything in file text in one pass.""" + return restore_masked_source_text(file_path.read_text(), masker, python_version) + + +def format_source_replacement(mask_value: str) -> str: + if mask_value is ...: + return "..." + if type(mask_value) in (int, float) and mask_value < 0: + return f"({mask_value})" + if type(mask_value) != str: + return str(mask_value) + + formatted_mask_value = use_escape_sequences(mask_value) + + return formatted_mask_value + + +def fix_jump_targets(disasm: str) -> str: + jump_target_re = r"to (\d+) ([\^v]~>)" + jump_target_map = {target: f"TARGET_{ind}" for ind, target in enumerate(sorted(set(match.group(1) for match in re.finditer(jump_target_re, disasm))))} + + # remove external jump entry points + incoming_jump_re = r"([\^v]~>) (\d+) " + result = re.sub(incoming_jump_re, lambda match: f"{match.group(1)} {jump_target_map[match.group(2)]} " if match.group(2) in jump_target_map else f"{match.group(1)} ", disasm) + + # only enforce order on external jumps + result = re.sub(jump_target_re, lambda match: f"to {jump_target_map[match.group(1)]} {match.group(2)}", result) + return result + + +def restore_masked_source_text(text: str, masker: Masker, python_version: PythonVersion) -> str: + """Creates a large regex of all the tokens and their respective values + Replaces everything in file text in one pass.""" + replacements = {re.escape(v): format_source_replacement(k) for k, v in masker.global_tab.items()} # we use encode + decode so multiline strings get replaced correctly + re_pattern = re.compile("|".join(replacements.keys())) + result = re_pattern.sub(lambda match: replacements[match.group()], text) + + # replace imports with a module starting with a number, with that number amount of dots for relative imports + re_rel_pattern = r"^(\s*)(import|from)\s*(\d+)(.*)" + result_rel_imports = re.sub(re_rel_pattern, lambda match: f"{match.group(1)}{match.group(2)} {'.' * int(match.group(3))}{match.group(4)}", result, 0, re.MULTILINE) + + # normalize with parse+unparse to catch replacement errors and simplify whitespace + try: + return ast.unparse(ast.parse(result_rel_imports, feature_version=python_version.as_tuple())) + except (SyntaxError, IndentationError): + return result_rel_imports + + +# replace mask values to start at 0 and count up +def normalize_masks(statement: str) -> tuple[str, list[str]]: + masks = mask_regex.findall(statement) + mask_order = [x for i, x in enumerate(masks) if masks.index(x) == i] + return mask_regex.sub(lambda x: str(mask_order.index(x.group(0))), statement), mask_order + + +# reverts masks to their original value +def restore_masks(translation: str, mask_order: list[str]): + def restore_mask(match): + i = int(match.group(0)) + # sometimes model adds random masks that did not appear in bytecode, + # so we have to check if mask is in bounds + return mask_order[i] if i < len(mask_order) else match.group(0) + + return mask_regex.sub(restore_mask, translation) diff --git a/pylingual/models.py b/pylingual/models.py new file mode 100644 index 0000000..ada1e21 --- /dev/null +++ b/pylingual/models.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import yaml +import logging + +from collections import OrderedDict +from pathlib import Path +from typing import TYPE_CHECKING + +from pylingual.masking.model_disasm import fix_jump_targets, normalize_masks, restore_masks +from pylingual.utils.lists import flatten +from pylingual.utils.tracked_list import TrackedDataset, TRANSLATION_STEP +from pylingual.utils.version import PythonVersion +from pylingual.utils.lazy import lazy_import + +if TYPE_CHECKING: + import torch + import transformers + import huggingface_hub +else: + lazy_import("torch") + lazy_import("transformers") + lazy_import("huggingface_hub") + +logger = logging.getLogger(__name__) + + +# translator with caching +class CacheTranslator: + """ + Adds cache support for statement translation + + :param translator : The loaded translation model + :param maxsize : The maximum amount of cached items + """ + + def __init__(self, translator: transformers.TranslationPipeline, maxsize=50000): + self.translator = translator + self.cache = OrderedDict() + self.maxsize = maxsize + + def __getitem__(self, item): + self.cache.move_to_end(item) + return self.cache[item] + + def _translate_and_decode(self, translation_requests: TrackedDataset | list[str], batch_size: int = 32, **kwargs) -> list[str]: + # return_tensors=True prevents standard postprocessing which skips special tokens + translation_result = self.translator(translation_requests, return_tensors=True, batch_size=batch_size, **kwargs) + decoded_results = [] + for result in flatten(translation_result): + # explicitly filter out the special tokens we want to skip: , , , , + filtered_tokens = [tok for tok in result["translation_token_ids"].tolist() if tok not in [0, 1, 2, 3, 4]] + # decode the remaining tokens + decoded_results.append(self.translator.tokenizer.decode(filtered_tokens, skip_special_tokens=False)) + + return decoded_results + + def _translate_with_backoff(self, translation_requests: TrackedDataset) -> list[str]: + try: + return self._translate_and_decode(translation_requests, batch_size=32) + except Exception as e: + logger.info(f"Lowering translation batch size ({e})") + # Try with batch_size = 1 if normal translation fails: + try: + return self._translate_and_decode(translation_requests, batch_size=1) + except Exception as e: + logger.info(f"Lowering translation batch size ({e})") + translation_results = [] + # try one-by-one if batch_size=1 fails + for request in translation_requests: + try: + translation_results.append(self._translate_and_decode([request], batch_size=1)[0]) + except Exception: + # last resort fallback + translation_results.append("'''Decompiler error: line too long for translation. Please decompile this statement manually.'''") + return translation_results + + def __call__(self, args: list, **_): + normalized_args = [normalize_masks(fix_jump_targets(x)) for x in args] + + # New are those not in the local cache + new = TrackedDataset(TRANSLATION_STEP, list({norm for norm, _ in normalized_args if norm not in self.cache})) + + # Now, "new" has been updated to those not in local + for arg, result in zip(new.x, self._translate_with_backoff(new)): + self.cache[arg] = result + + results = [restore_masks(self[norm], order) for norm, order in normalized_args] + while len(self.cache) > self.maxsize: + self.cache.popitem(last=False) + return results + + +def load_models(config_file: Path = Path("pylingual/decompiler_config.yaml"), version: PythonVersion = PythonVersion(3.9), token=False) -> tuple[transformers.Pipeline, CacheTranslator]: + logger.info(f"Loading models for {version}...") + with config_file.open() as f: + config = yaml.safe_load(f) + seg_config = config[f"v{version}"]["SEGMENTATION_MODEL"] + stmt_config = config[f"v{version}"]["STATEMENT_MODEL"] + + ################################# + # Segmentation model components # + ################################# + segmentation_model = transformers.AutoModelForTokenClassification.from_pretrained( + pretrained_model_name_or_path=seg_config["REPO"], + revision=seg_config["REVISION"], + token=token, + ) + segmentation_tokenizer_file = huggingface_hub.hf_hub_download(repo_id=seg_config["TOKENIZER"], filename="tokenizer.json", token=token) + segmentation_tokenizer = transformers.PreTrainedTokenizerFast( + tokenizer_file=segmentation_tokenizer_file, + unk_token="[UNK]", + pad_token="[PAD]", + cls_token="[CLS]", + sep_token="[SEP]", + mask_token="[MASK]", + ) + if torch.cuda.is_available(): + device = torch.device("cuda:0") + else: + logger.warning("Using CPU for models") + device = torch.device("cpu") + segmenter = transformers.pipeline("token-classification", model=segmentation_model, tokenizer=segmentation_tokenizer, aggregation_strategy="none", device=device) + ######################################### + # Sequence translation model components # + ######################################### + translation_model = transformers.T5ForConditionalGeneration.from_pretrained(stmt_config["REPO"], revision=stmt_config["REVISION"], token=token) + translation_tokenizer = transformers.RobertaTokenizer.from_pretrained(stmt_config["TOKENIZER"], token=token) + translator = transformers.TranslationPipeline(model=translation_model, tokenizer=translation_tokenizer, max_length=512, truncation=False, device=device) + + return segmenter, CacheTranslator(translator) diff --git a/pylingual/segmentation/segmentation_search_strategies.py b/pylingual/segmentation/segmentation_search_strategies.py new file mode 100644 index 0000000..93b7b70 --- /dev/null +++ b/pylingual/segmentation/segmentation_search_strategies.py @@ -0,0 +1,140 @@ +from __future__ import annotations +import heapq +import itertools + +from typing import TYPE_CHECKING, List, Tuple, Dict, Callable, Generator, Iterable +from pylingual.utils.lazy import lazy_import + +if TYPE_CHECKING: + import numpy as np + + BitMap = List[bool] + SearchStrategy = Callable[[List[float]], List[BitMap]] + PriorityFunction = Callable[[List[float]], np.array] +else: + lazy_import("numpy", "np") + +############# +# UTILITIES # +############# + + +def filter_subwords(results): + # Get results that are not associated with a subword + return [result for result in results if not result["word"].startswith("##")] + + +# incrementally generates the index-representation of bitstrings of length length and with num_ones ones +def bitstring_index_generator(length: int, num_ones: int) -> Generator[Tuple[int], None, None]: + if num_ones > length: + return + + if num_ones == 0: + yield tuple() + return + + # start by having ones in all the smallest positions + bit_indices = list(range(num_ones)) + yield tuple(bit_indices) + while bit_indices[0] < length - num_ones: + # increment the least significant bit and handle cascading effects + bit_indices[0] += 1 + for i in range(num_ones - 1): + # when you overlap with the next bit, increase it and set yourself back + if bit_indices[i] == bit_indices[i + 1]: + bit_indices[i + 1] += 1 + bit_indices[i] = bit_indices[i - 1] + 1 if i > 0 else 0 + yield tuple(bit_indices) + + +# convert bitstring indices to numeric representation +def indices_to_num(indices: List[int]) -> int: + return sum(2 ** int(digit) for digit in indices) + + +# https://docs.python.org/3/library/itertools.html#itertools.pairwise not introduced until 3.10 +# sliding_window('ABCDEFG') --> AB BC CD DE EF FG +def sliding_window(iterable: Iterable, window=2) -> Iterable: + if window < 1: + raise ValueError("Window size must be at least 1") + + iterators = [iterable] + + for _ in range(window - 1): + iterators[-1], next_window_member = itertools.tee(iterators[-1]) + next(next_window_member, None) + iterators.append(next_window_member) + + return zip(*iterators) + + +def bitmap_to_entities(bitmap: BitMap) -> List[str]: + # map of (current, next) pairs in the bitstring to segmentation entities + entity_map = { + (True, True): "B", + (True, False): "B", + (False, True): "E", + (False, False): "I", + } + + entities = [entity_map[pair] for pair in sliding_window(bitmap)] + entities += "B" if bitmap[-1] else "E" + return entities + + +def entities_to_bitmap(entities) -> BitMap: + return np.array([entity == "B" for entity in entities], dtype=bool) + + +######################## +# STRATEGY DEFINITIONS # +######################## + + +# a general m_deep_top_k implementation that takes the priority function as a parameter +def m_deep_top_k(scores: List[float], m: int, k: int, priority_function: PriorityFunction) -> List[BitMap]: + # flip_candidate_lists[i] is an iterator over all length l bitstrings with i ones; each list is sorted numerically + length = len(scores) + flip_candidate_lists = [bitstring_index_generator(length, num_flips) for num_flips in range(m + 1)] + + # generate a sorted list of the flip candidates (lazily evaluated) and collect the top k elements + sorted_flip_candidates = heapq.merge(*flip_candidate_lists, key=indices_to_num) + top_k_candidates = list(itertools.islice(sorted_flip_candidates, k)) + + # convert the feature-space indices into problem-space indices + # the priority map is a list of indices in order of their estimated uncertainty + priority_map = priority_function(scores) + flip_sets = [priority_map[list(candidate_flips)] for candidate_flips in top_k_candidates] + + # to match the formalization, build the error strings + error_strings = [] + for flip_set in flip_sets: + error_string = np.full(length, False) + error_string[flip_set] = True + error_strings.append(list(error_string)) + + return error_strings + + +def get_top_k_predictions(strategy: SearchStrategy, predicted_boundary: List[Dict]) -> List[List[str]]: + entities = [prediction["entity"] for prediction in predicted_boundary] + initial_segmentation = entities_to_bitmap(entities) + + scores = [prediction["score"] for prediction in predicted_boundary] + transformations = strategy(scores) + + candidate_segmentations = [np.logical_xor(initial_segmentation, transformation) for transformation in transformations] + + # convert to entities for compatibility with the pipeline + top_k_boundaries = [bitmap_to_entities(segmentation) for segmentation in candidate_segmentations] + return top_k_boundaries + + +###################### +# PRIORITY FUNCTIONS # +###################### + + +# this strategy simply flips predictions in order of least confidence +def naive_confidence_priority(scores: List[float]) -> np.array: + return np.argsort(scores) diff --git a/pylingual/segmentation/sliding_window.py b/pylingual/segmentation/sliding_window.py new file mode 100644 index 0000000..e3bc3c3 --- /dev/null +++ b/pylingual/segmentation/sliding_window.py @@ -0,0 +1,140 @@ +import itertools + + +# make windows based on the token sizes instrutions will be an instruction along with it's token size +def sliding_window(instructions: list, max_window_size: int, step_size: int = 1) -> tuple[list[str], list[int]]: + # go through each instruction in a code object and fill up windows + instructions_iter = enumerate(instructions) + + try: + while True: + # start a new window + instructions_iter, window_source = itertools.tee(instructions_iter) + + # take elements until it would overflow the window + window = [] + index_window = [] + window_len = 0 + + # this loop will raise stopIteration when we run out of instructions + while window_item := next(window_source): + inst_index, (inst, inst_len) = window_item + + # end the window if we would exceed the max_window_size + if inst_len + window_len > max_window_size: + break + + window.append(inst) + index_window.append(inst_index) + window_len += inst_len + + yield (window, index_window) + + # step forward step_size tokens + tokens_stepped = 0 + + while step_item := next(instructions_iter): + inst_index, (inst, inst_len) = step_item + tokens_stepped += inst_len + if tokens_stepped >= step_size: + break + except StopIteration: + # yield the final window (partially full) + yield (window, index_window) + + +# merges the bytecode together based on a point system given the division value for be results use an odd number and given the steps used in the sliding window +def merge(window_coords: list[tuple], window_segmentation_results: list[list[dict]], inst_index: list, window_size: int, step: int) -> list[list[dict]]: + # for each codeobject align and score + sorted_windows = align_segmentation_window_results(window_coords, window_segmentation_results, inst_index) + + weighted_segmentation_results = confidence_based_score(sorted_windows) + + # get ready for the merging + + entity_list = [] + for codeobj in weighted_segmentation_results: + codeobj_entities = [] + + for inst in codeobj: + highest = max(inst[1], key=inst[1].get) + final_choice = inst[0] + + if highest == "B_score": + final_choice["entity"] = "B" + elif highest == "I_score": + final_choice["entity"] = "I" + elif highest == "E_score": + final_choice["entity"] = "E" + codeobj_entities.append(final_choice) + + entity_list.append(codeobj_entities) + + return entity_list + + +# align windows to make scoring easier +def align_segmentation_window_results( + window_coords: list[tuple], + window_segmentation_results: list[list[dict]], + inst_index: list, +) -> dict[int, list[list[dict]]]: + # returns a dict of code object index -> list of instruction results, where each instruction result is a list of segmentation results for that instruction + segmentation_result_dict = dict() + for (codeobj_index, _), window_results, window_inst_indices in zip(window_coords, window_segmentation_results, inst_index): + if codeobj_index not in segmentation_result_dict: + segmentation_result_dict[codeobj_index] = dict() + + codeobj_result_dict = segmentation_result_dict[codeobj_index] + for inst_result, inst_index in zip(window_results, window_inst_indices): + if inst_index not in codeobj_result_dict: + codeobj_result_dict[inst_index] = list() + + codeobj_result_dict[inst_index].append(inst_result) + + segmentation_result_dict[codeobj_index] = codeobj_result_dict + + # bake codeobj_result dicts into lists + baked_result_dict = dict() + for codeobj_index, codeobj_result_dict in segmentation_result_dict.items(): + if bool(codeobj_result_dict): + codeobj_length = max(codeobj_result_dict.keys()) + 1 + codeobj_result_list = [None] * codeobj_length + + for instruction_index, instruction_result in codeobj_result_dict.items(): + codeobj_result_list[instruction_index] = instruction_result + + # make sure we didn't miss any indices! + assert None not in codeobj_result_list + + baked_result_dict[codeobj_index] = codeobj_result_list + + return baked_result_dict + + +# given a list of sorted windows for each codeobject we will score b,i,e for each instruction based on the segmentation models confidence level +def confidence_based_score(aligned_segmentation_results: list[list[dict]]) -> list[list[dict]]: + codeobj_scores = [] + for codeobj in range(len(aligned_segmentation_results)): + inst_list = [] + + for inst in aligned_segmentation_results[codeobj]: + scores = {"B_score": 0, "I_score": 0, "E_score": 0} + + # go through each result from the windows and give a score to b,i,e based on their offsets + for item in inst: + confidence_score = item["score"] + if item["entity"] == "B": + scores["B_score"] += confidence_score + elif item["entity"] == "I": + scores["I_score"] += confidence_score + elif item["entity"] == "E": + scores["E_score"] += confidence_score + + # scores for instructions + inst_list.append((inst[0], scores)) + + # scores for the instructions in each code object + codeobj_scores.append(inst_list) + + return codeobj_scores diff --git a/pylingual/utils/ascii_art.py b/pylingual/utils/ascii_art.py new file mode 100644 index 0000000..ac8292c --- /dev/null +++ b/pylingual/utils/ascii_art.py @@ -0,0 +1,25 @@ +import importlib.metadata + +PYLINGUAL_ART = r''' + ,ggggggggggg, ,gggg, +dP"""88""""""Y8, d8" "8I ,dPYb, +Yb, 88 `8b 88 ,dP IP'`Yb + `" 88 ,8P 8888888P" gg I8 8I + 88aaaad8P" 88 "" I8 8' + 88"""""gg gg 88 gg ,ggg,,ggg, ,gggg,gg gg gg ,gggg,gg I8 dP + 88 I8 8I ,aa,_88 88 ,8" "8P" "8, dP" "Y8I I8 8I dP" "Y8I I8dP + 88 I8, ,8IdP" "88P 88 I8 8I 8I i8' ,8I I8, ,8I i8' ,8I I8P + 88 ,d8b, ,d8IYb,_,d88b,,_ _,88,_,dP 8I Yb,,d8, ,d8I ,d8b, ,d8b,,d8, ,d8b,,d8b,_ + 88 P""Y88P"888"Y8P" "Y888888P""Y88P' 8I `Y8P"Y8888P"8888P'"Y88P"`Y8P"Y8888P"`Y88P'"Y88 + ,d8I' ,d8I' + ,dP'8I ,dP'8I + ,8" 8I ,8" 8I + I8 8I I8 8I + `8, ,8I `8, ,8I + `Y8P" `Y8P" +'''.strip("\n") + +PYLINGUAL_SUBHEADER = f""" +The University of Texas at Dallas, Syssec Lab +{importlib.metadata.version("pylingual")} - https://pylingual.io +""".rstrip() diff --git a/pylingual/utils/generate_bytecode.py b/pylingual/utils/generate_bytecode.py new file mode 100644 index 0000000..18585b9 --- /dev/null +++ b/pylingual/utils/generate_bytecode.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import subprocess +import sys +import shlex +import py_compile + +from pylingual.utils.version import PythonVersion + + +class CompileError(Exception): + success = False + + +def compile_version(py_file, out_file, version): + py_file = str(py_file) + out_file = str(out_file) + version = PythonVersion(version) + if version == sys.version_info: + try: + py_compile.compile(py_file, cfile=out_file, doraise=True, optimize=0) + except py_compile.PyCompileError as e: + raise CompileError(str(e)) + return + compile_cmd = f"import py_compile, sys; assert sys.version_info[:2] == {version.as_tuple()!r}; py_compile.compile({py_file!r}, cfile={out_file!r})" + cmd = f"env PYENV_VERSION={version.as_str()} PYTHONWARNINGS=ignore pyenv exec python -c {shlex.quote(compile_cmd)}" + output = subprocess.run(cmd, shell=True, capture_output=True, text=True) + if output.stderr: + raise CompileError(output.stderr) diff --git a/pylingual/utils/get_logger.py b/pylingual/utils/get_logger.py new file mode 100644 index 0000000..f2d18f6 --- /dev/null +++ b/pylingual/utils/get_logger.py @@ -0,0 +1,30 @@ +import datetime +import logging +import os + + +def get_logger(name: str) -> logging.Logger: + # Create a logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + # Create a file handler + current_datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + if not os.path.exists("logs"): # create a logging directory if not exists + os.makedirs("logs") + file_handler = logging.FileHandler(f"logs/{name}_{current_datetime}.log", mode="w") + file_handler.setLevel(logging.DEBUG) + + # Create a console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + + # Create a formatter and add it to the handlers + formatter = logging.Formatter("%(asctime)s- %(levelname)s - %(message)s") + file_handler.setFormatter(formatter) + console_handler.setFormatter(formatter) + + # Add the handlers to the logger + logger.addHandler(file_handler) + logger.addHandler(console_handler) + + return logger diff --git a/pylingual/utils/lazy.py b/pylingual/utils/lazy.py new file mode 100644 index 0000000..8eecab3 --- /dev/null +++ b/pylingual/utils/lazy.py @@ -0,0 +1,17 @@ +import inspect + + +class lazy_import: + """ + Delay importing a module until it is used + """ + + def __init__(self, name: str, as_name: str | None = None): + self.globals = inspect.currentframe().f_back.f_globals + self.as_name = as_name or name + self.globals[self.as_name] = self + self.name = name + + def __getattr__(self, attr): + self.globals[self.as_name] = module = __import__(self.name) + return getattr(module, attr) diff --git a/pylingual/utils/lists.py b/pylingual/utils/lists.py new file mode 100644 index 0000000..71313a4 --- /dev/null +++ b/pylingual/utils/lists.py @@ -0,0 +1,11 @@ +def unflatten(flattened: list, reference: list[list]): + for i, request in enumerate(reference): + flattened[i : i + len(request)] = [flattened[i : i + len(request)]] + + +def flatten(x): + for e in x: + if isinstance(e, list): + yield from flatten(e) + else: + yield e diff --git a/pylingual/utils/tracked_list.py b/pylingual/utils/tracked_list.py new file mode 100644 index 0000000..7acb2d0 --- /dev/null +++ b/pylingual/utils/tracked_list.py @@ -0,0 +1,63 @@ +from typing import TYPE_CHECKING +from .lazy import lazy_import + +if TYPE_CHECKING: + import transformers +else: + lazy_import("transformers") + +SEGMENTATION_STEP = "Segmentation" +TRANSLATION_STEP = "Translation" +CFLOW_STEP = "Control Flow" +CORRECTION_STEP = "Error Correction" + + +class TrackedList: + """ + List-like class that calls self.progress on each element access + Used to display progress bars when PyLingual is run as a script, does nothing otherwise + """ + + def __init__(self, name: str, x: list): + self.name = name + self.x = x + self.i = 0 + self.init() + + # overwritten when run as script + def init(self): + pass + + def __getitem__(self, i): + self.progress(i - self.i) + self.i = i + return self.x[i] + + def __len__(self): + return len(self.x) + + def __iter__(self): + return self + + def __next__(self): + try: + n = self.x[self.i] + except: + raise StopIteration() + self.progress(1) + self.i += 1 + return n + + # overwritten when run as script + def progress(self, i: int): + pass + + +class TrackedDataset(TrackedList): + """ + Like TrackedList, but inherits from Dataset + """ + + def __init__(self, name: str, x: list): + super().__init__(name, x) + TrackedDataset.__bases__ = (TrackedList, transformers.pipelines.base.Dataset) diff --git a/pylingual/utils/use_escape_sequences.py b/pylingual/utils/use_escape_sequences.py new file mode 100644 index 0000000..edd8a7f --- /dev/null +++ b/pylingual/utils/use_escape_sequences.py @@ -0,0 +1,17 @@ +def use_escape_sequences(s): + escapes = { + "\\": "\\\\", + "'": "\\'", + '"': '\\"', + "\a": "\\a", + "\b": "\\b", + "\f": "\\f", + "\n": "\\n", + "\r": "\\r", + "\t": "\\t", + "\v": "\\v", + "\x00": "\\x00", + } + for a, b in escapes.items(): + s = s.replace(a, b) + return s diff --git a/pylingual/utils/version.py b/pylingual/utils/version.py new file mode 100644 index 0000000..20fc4f1 --- /dev/null +++ b/pylingual/utils/version.py @@ -0,0 +1,75 @@ +supported_tuples = [(3, x) for x in range(6, 14)] +version_str = {f"{x[0]}{x[1]}": x for x in supported_tuples} | {f"{x[0]}.{x[1]}": x for x in supported_tuples} + + +class PythonVersion: + major: int + minor: int + _t: tuple + + @staticmethod + def normalize(x) -> tuple[int, int] | None: + if isinstance(x, PythonVersion): + return x._t + if isinstance(x, float): + if x == 3.1: + x = "3.10" + x = str(x) + if isinstance(x, str): + x = version_str.get(".".join(map(str.strip, x.split(".")[:2]))) + if isinstance(x, int): + x = (3, x) + if isinstance(x, tuple): + if len(x) >= 2 and isinstance(x[0], int) and isinstance(x[1], int): + return x[:2] + + def is_supported(self) -> bool: + return self.as_tuple() in supported_tuples + + def __init__(self, x): + v = PythonVersion.normalize(x) + if v is None: + raise ValueError(f"version {x} is invalid") + self.major, self.minor = v + self._t = v + + def __str__(self): + return f"{self.major}.{self.minor}" + + def __eq__(self, o): + return PythonVersion.normalize(o) == self._t + + def __ne__(self, o): + norm = PythonVersion.normalize(o) + return norm is not None and self._t != norm + + def __ge__(self, o): + norm = PythonVersion.normalize(o) + return norm is not None and self._t >= norm + + def __le__(self, o): + norm = PythonVersion.normalize(o) + return norm is not None and self._t <= norm + + def __gt__(self, o): + norm = PythonVersion.normalize(o) + return norm is not None and self._t > norm + + def __lt__(self, o): + norm = PythonVersion.normalize(o) + return norm is not None and self._t < norm + + def __getitem__(self, i): + return self._t[i] + + def as_str(self): + return str(self) + + def as_float(self): + return float(str(self)) + + def as_tuple(self): + return self._t + + +supported_versions = list(map(PythonVersion, supported_tuples)) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..eda4f9d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,96 @@ +[tool.poetry] +requires-poetry = ">=2.0" + +[project] +name = "pylingual" +version = "0.0.1" +description = "A Python bytecode decompilation tool, supporting versions 3.6 - 3.13" +authors = [ {name = "syssec-utd"} ] +keywords = ["python", "decompilation", "pylingual", "reversing", "decompiler", "bytecode"] +license = "GPL-3.0-only" + +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "asttokens", + "datasets", + "huggingface-hub", + "matplotlib", + "networkx", + "numpy", + "pydot", + "requests", + "tokenizers", + "torch", + "tqdm", + "rich", + "seqeval", + "transformers==4.46.1", + "transformers[torch]", + "xdis", + "click" +] + +[project.urls] +homepage = "https://pylingual.io" + +[project.scripts] +pylingual = "pylingual.main:main" + +[build-system] +requires = ["poetry-core>=2.0.0,<3.0.0"] +build-backend = "poetry.core.masonry.api" + +# linter and formatter +[tool.ruff] +# Exclude commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] +target-version = "py311" +line-length = 240 +indent-width = 4 + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] +ignore = [] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +line-ending = "auto" +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = true +docstring-code-line-length = "dynamic"