From 82aecbc4b693cf88ad2f0f072ed3e3489dc54bcc Mon Sep 17 00:00:00 2001 From: Keerthi Date: Tue, 3 Sep 2024 11:01:53 +0530 Subject: [PATCH 1/4] Add execution invariance test --- execution_invariance_test/__init__.py | 4 + .../analysis/LICENSE.txt | 9 ++ execution_invariance_test/analysis/README.md | 21 +++ .../analysis/pyproject.toml | 66 ++++++++ .../analysis/src/analysis/__about__.py | 4 + .../analysis/src/analysis/__init__.py | 3 + .../analysis/src/analysis/analysis.py | 8 + .../analysis/tests/__init__.py | 3 + .../projects/simple-test/LICENSE.txt | 9 ++ .../projects/simple-test/README.md | 21 +++ .../projects/simple-test/pyproject.toml | 61 +++++++ .../simple-test/src/simple_test/__about__.py | 5 + .../simple-test/src/simple_test/__init__.py | 4 + .../simple-test/src/simple_test/analysis.py | 22 +++ .../simple-test/src/simple_test/simple.py | 17 ++ .../projects/simple-test/tests/stack_test.py | 10 ++ .../projects/simple-test/tests/test_simple.py | 25 +++ execution_invariance_test/run_test.sh | 24 +++ execution_invariance_test/test.py | 153 ++++++++++++++++++ execution_invariance_test/test_folders.txt | 1 + pyproject.toml | 8 + 21 files changed, 478 insertions(+) create mode 100644 execution_invariance_test/__init__.py create mode 100644 execution_invariance_test/analysis/LICENSE.txt create mode 100644 execution_invariance_test/analysis/README.md create mode 100644 execution_invariance_test/analysis/pyproject.toml create mode 100644 execution_invariance_test/analysis/src/analysis/__about__.py create mode 100644 execution_invariance_test/analysis/src/analysis/__init__.py create mode 100644 execution_invariance_test/analysis/src/analysis/analysis.py create mode 100644 execution_invariance_test/analysis/tests/__init__.py create mode 100644 execution_invariance_test/projects/simple-test/LICENSE.txt create mode 100644 execution_invariance_test/projects/simple-test/README.md create mode 100644 execution_invariance_test/projects/simple-test/pyproject.toml create mode 100644 execution_invariance_test/projects/simple-test/src/simple_test/__about__.py create mode 100644 execution_invariance_test/projects/simple-test/src/simple_test/__init__.py create mode 100644 execution_invariance_test/projects/simple-test/src/simple_test/analysis.py create mode 100644 execution_invariance_test/projects/simple-test/src/simple_test/simple.py create mode 100644 execution_invariance_test/projects/simple-test/tests/stack_test.py create mode 100644 execution_invariance_test/projects/simple-test/tests/test_simple.py create mode 100755 execution_invariance_test/run_test.sh create mode 100644 execution_invariance_test/test.py create mode 100644 execution_invariance_test/test_folders.txt diff --git a/execution_invariance_test/__init__.py b/execution_invariance_test/__init__.py new file mode 100644 index 0000000..2d67744 --- /dev/null +++ b/execution_invariance_test/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2024-present Aryaz Eghbali +# +# SPDX-License-Identifier: MIT +# DYNAPYT: DO NOT INSTRUMENT diff --git a/execution_invariance_test/analysis/LICENSE.txt b/execution_invariance_test/analysis/LICENSE.txt new file mode 100644 index 0000000..2b52cdc --- /dev/null +++ b/execution_invariance_test/analysis/LICENSE.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024-present Keerthi + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/execution_invariance_test/analysis/README.md b/execution_invariance_test/analysis/README.md new file mode 100644 index 0000000..af12dc5 --- /dev/null +++ b/execution_invariance_test/analysis/README.md @@ -0,0 +1,21 @@ +# analysis + +[![PyPI - Version](https://img.shields.io/pypi/v/analysis.svg)](https://pypi.org/project/analysis) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/analysis.svg)](https://pypi.org/project/analysis) + +----- + +## Table of Contents + +- [Installation](#installation) +- [License](#license) + +## Installation + +```console +pip install analysis +``` + +## License + +`analysis` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license. diff --git a/execution_invariance_test/analysis/pyproject.toml b/execution_invariance_test/analysis/pyproject.toml new file mode 100644 index 0000000..e1e8f9c --- /dev/null +++ b/execution_invariance_test/analysis/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "analysis" +dynamic = ["version"] +description = '' +readme = "README.md" +requires-python = ">=3.8" +license = "MIT" +keywords = [] +authors = [ + { name = "Keerthi", email = "keerthivasudevan98@gmail.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [] + +[project.urls] +Documentation = "https://github.com/Keerthi/analysis#readme" +Issues = "https://github.com/Keerthi/analysis/issues" +Source = "https://github.com/Keerthi/analysis" + +[tool.hatch.version] +path = "src/analysis/__about__.py" + +[tool.hatch.envs.default] +dependencies = [ + "pytest-json-report", +] + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy>=1.0.0", +] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/analysis tests}" + +[tool.coverage.run] +source_pkgs = ["analysis", "tests"] +branch = true +parallel = true +omit = [ + "src/analysis/__about__.py", +] + +[tool.coverage.paths] +analysis = ["src/analysis", "*/analysis/src/analysis"] +tests = ["tests", "*/analysis/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/execution_invariance_test/analysis/src/analysis/__about__.py b/execution_invariance_test/analysis/src/analysis/__about__.py new file mode 100644 index 0000000..add216e --- /dev/null +++ b/execution_invariance_test/analysis/src/analysis/__about__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2024-present Keerthi +# +# SPDX-License-Identifier: MIT +__version__ = "0.0.1" diff --git a/execution_invariance_test/analysis/src/analysis/__init__.py b/execution_invariance_test/analysis/src/analysis/__init__.py new file mode 100644 index 0000000..f19df0c --- /dev/null +++ b/execution_invariance_test/analysis/src/analysis/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2024-present Keerthi +# +# SPDX-License-Identifier: MIT diff --git a/execution_invariance_test/analysis/src/analysis/analysis.py b/execution_invariance_test/analysis/src/analysis/analysis.py new file mode 100644 index 0000000..ac1ebe2 --- /dev/null +++ b/execution_invariance_test/analysis/src/analysis/analysis.py @@ -0,0 +1,8 @@ + +from dynapyt.analyses.BaseAnalysis import BaseAnalysis + + +class Analysis(BaseAnalysis): + + def runtime_event(self, dyn_ast: str, iid: int) -> None: + pass \ No newline at end of file diff --git a/execution_invariance_test/analysis/tests/__init__.py b/execution_invariance_test/analysis/tests/__init__.py new file mode 100644 index 0000000..f19df0c --- /dev/null +++ b/execution_invariance_test/analysis/tests/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2024-present Keerthi +# +# SPDX-License-Identifier: MIT diff --git a/execution_invariance_test/projects/simple-test/LICENSE.txt b/execution_invariance_test/projects/simple-test/LICENSE.txt new file mode 100644 index 0000000..2b52cdc --- /dev/null +++ b/execution_invariance_test/projects/simple-test/LICENSE.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024-present Keerthi + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/execution_invariance_test/projects/simple-test/README.md b/execution_invariance_test/projects/simple-test/README.md new file mode 100644 index 0000000..0662c9a --- /dev/null +++ b/execution_invariance_test/projects/simple-test/README.md @@ -0,0 +1,21 @@ +# simple-test + +[![PyPI - Version](https://img.shields.io/pypi/v/simple-test.svg)](https://pypi.org/project/simple-test) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/simple-test.svg)](https://pypi.org/project/simple-test) + +----- + +## Table of Contents + +- [Installation](#installation) +- [License](#license) + +## Installation + +```console +pip install simple-test +``` + +## License + +`simple-test` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license. diff --git a/execution_invariance_test/projects/simple-test/pyproject.toml b/execution_invariance_test/projects/simple-test/pyproject.toml new file mode 100644 index 0000000..c11434e --- /dev/null +++ b/execution_invariance_test/projects/simple-test/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "simple-test" +dynamic = ["version"] +description = '' +readme = "README.md" +requires-python = ">=3.8" +license = "MIT" +keywords = [] +authors = [ + { name = "Keerthi", email = "keerthivasudevan98@gmail.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [] + +[project.urls] +Documentation = "https://github.com/Keerthi/simple-test#readme" +Issues = "https://github.com/Keerthi/simple-test/issues" +Source = "https://github.com/Keerthi/simple-test" + +[tool.hatch.version] +path = "src/simple_test/__about__.py" + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy>=1.0.0", +] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/simple_test tests}" + +[tool.coverage.run] +source_pkgs = ["simple_test", "tests"] +branch = true +parallel = true +omit = [ + "src/simple_test/__about__.py", +] + +[tool.coverage.paths] +simple_test = ["src/simple_test", "*/simple-test/src/simple_test"] +tests = ["tests", "*/simple-test/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/execution_invariance_test/projects/simple-test/src/simple_test/__about__.py b/execution_invariance_test/projects/simple-test/src/simple_test/__about__.py new file mode 100644 index 0000000..a2b116e --- /dev/null +++ b/execution_invariance_test/projects/simple-test/src/simple_test/__about__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024-present Aryaz Eghbali +# +# SPDX-License-Identifier: MIT +# DYNAPYT: DO NOT INSTRUMENT +__version__ = "0.0.1" diff --git a/execution_invariance_test/projects/simple-test/src/simple_test/__init__.py b/execution_invariance_test/projects/simple-test/src/simple_test/__init__.py new file mode 100644 index 0000000..2d67744 --- /dev/null +++ b/execution_invariance_test/projects/simple-test/src/simple_test/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2024-present Aryaz Eghbali +# +# SPDX-License-Identifier: MIT +# DYNAPYT: DO NOT INSTRUMENT diff --git a/execution_invariance_test/projects/simple-test/src/simple_test/analysis.py b/execution_invariance_test/projects/simple-test/src/simple_test/analysis.py new file mode 100644 index 0000000..221c80a --- /dev/null +++ b/execution_invariance_test/projects/simple-test/src/simple_test/analysis.py @@ -0,0 +1,22 @@ +# DYNAPYT: DO NOT INSTRUMENT + +import json +from uuid import uuid4 +from dynapyt.analyses.BaseAnalysis import BaseAnalysis + + +class Analysis(BaseAnalysis): + def __init__(self, output_dir: str = None): + self.output_dir = output_dir + self.output = [] + + def begin_execution(self): + self.output.append("begin execution") + + def end_execution(self): + self.output.append("end execution") + with open(f"{self.output_dir}/output-{str(uuid4())}.json", "w") as f: + json.dump(self.output, f, indent=2) + + def binary_operation(self, dyn_ast, iid, op, left, right, result): + self.output.append(f"{left} {op} {right} = {result}") diff --git a/execution_invariance_test/projects/simple-test/src/simple_test/simple.py b/execution_invariance_test/projects/simple-test/src/simple_test/simple.py new file mode 100644 index 0000000..72fca01 --- /dev/null +++ b/execution_invariance_test/projects/simple-test/src/simple_test/simple.py @@ -0,0 +1,17 @@ +import traceback +import os.path + +def add_one(x: int) -> int: + return x + 1 + + +def multiply_two(x: int) -> int: + return x * 2 + + +def add_together(x: int, y: int) -> int: + return x + y + + +def get_stack(): + return traceback.extract_stack() \ No newline at end of file diff --git a/execution_invariance_test/projects/simple-test/tests/stack_test.py b/execution_invariance_test/projects/simple-test/tests/stack_test.py new file mode 100644 index 0000000..b55ffee --- /dev/null +++ b/execution_invariance_test/projects/simple-test/tests/stack_test.py @@ -0,0 +1,10 @@ +import traceback +from simple_test.simple import get_stack + +def test_stack(): + trace = get_stack() + trace_list = traceback.format_list(trace) + trace_length = len(trace_list) + expected_trace_str_length = 34 + assert trace_length == expected_trace_str_length + diff --git a/execution_invariance_test/projects/simple-test/tests/test_simple.py b/execution_invariance_test/projects/simple-test/tests/test_simple.py new file mode 100644 index 0000000..40bcc3a --- /dev/null +++ b/execution_invariance_test/projects/simple-test/tests/test_simple.py @@ -0,0 +1,25 @@ +from simple_test.simple import add_one, multiply_two, add_together + + +def test_add_one_1(): + assert add_one(1) == 2 + + +def test_add_one_2(): + assert add_one(2) == 3 + + +def test_multiply_two_1(): + assert multiply_two(1) == 2 + + +def test_multiply_two_2(): + assert multiply_two(2) == 4 + + +def test_add_together_1_2(): + assert add_together(1, 2) == 3 + + +def test_add_together_2_3(): + assert add_together(2, 3) == 5 diff --git a/execution_invariance_test/run_test.sh b/execution_invariance_test/run_test.sh new file mode 100755 index 0000000..53f597e --- /dev/null +++ b/execution_invariance_test/run_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +python3 -m venv venv +source venv/bin/activate +pip install -U pip setuptools +SCRIPTS_DIR=$(dirname $(realpath "$0")) +PARENT_DIR=$(dirname $SCRIPTS_DIR) +ANALYSIS_DIR=$SCRIPTS_DIR/analysis +pip install -e $ANALYSIS_DIR +pip install pytest-json-report +pip install $PARENT_DIR +uniqueID=$(python -c "import uuid; print(uuid.uuid4())") +export DYNAPYT_SESSION_ID=$uniqueID +echo "DYNAPYT_SESSION_ID=$DYNAPYT_SESSION_ID" +temp_dir="${TMPDIR:-/tmp}" +file_path=$temp_dir/dynapyt_analyses-$uniqueID.txt +touch $file_path +echo "analysis.analysis.Analysis" > $file_path +cat $file_path +python $SCRIPTS_DIR/test.py +deactivate +rm $temp_dir/dynapyt_analyses-$uniqueID.txt +rm -rf venv + diff --git a/execution_invariance_test/test.py b/execution_invariance_test/test.py new file mode 100644 index 0000000..c6a15bb --- /dev/null +++ b/execution_invariance_test/test.py @@ -0,0 +1,153 @@ +import json +from pathlib import Path +import pytest +from dynapyt.analyses.BaseAnalysis import BaseAnalysis +from dynapyt.instrument.instrument import instrument_file +from dynapyt.utils.hooks import get_hooks_from_analysis +import subprocess +import sys + + +def run_project(project: Path): + project_dir = project.resolve() + json_report_file = project_dir / "result.json" + result = pytest.main(["-v", "--cache-clear", "--json-report", "--json-report-file="+str(json_report_file), str(project_dir/"tests"), ]) + with open(json_report_file, "r") as f: + result = f.read() + + result_json = json.loads(result) + test_result_map = {} + for item in result_json["tests"]: + test_result_map[item["nodeid"]] = item["outcome"] + json_report_file.unlink() + return test_result_map + + +def run_instrumented_project(project): + selected_hooks = get_hooks_from_analysis(["analysis.analysis.Analysis"]) + project_dir = project.resolve() + project_src_dir = project_dir / "src" + for code_file in project_src_dir.rglob("*.py"): + instrument_file(str(project_dir / code_file), selected_hooks) + + install_project(project) + json_report_file = project_dir / "instrumented_result.json" + result = pytest.main(["-v", "--cache-clear", "--json-report", "--json-report-file="+str(json_report_file), str(project_dir/"tests"), ]) + with open(json_report_file, "r") as f: + result = f.read() + + result_json = json.loads(result) + test_result_map = {} + for item in result_json["tests"]: + test_result_map[item["nodeid"]] = item["outcome"] + + # Clean up the project + for code_file in project_dir.rglob("*.py.orig"): + metadata_file = ( + project_dir + / Path(*(code_file.parts[:-1])) + / (code_file.name[:-8] + "-dynapyt.json") + ) + metadata_file.unlink() + correct_file = ( + project_dir + / Path(*(code_file.parts[:-1])) + / (code_file.name[:-8] + ".py") + ) + code_file.rename(correct_file) + json_report_file.unlink() + + return test_result_map + + +def remove_related_modules(project): + project_module = project.name.replace("-", "_") + + project_src_dir = project / "src" + for code_file in project_src_dir.rglob("*.py"): + module_name = project_module + "." + code_file.stem + if module_name in sys.modules: + print(f"Deleting module {module_name}") + del sys.modules[module_name] + + project_test_dir = project / "tests" + for code_file in project_test_dir.rglob("*.py"): + module_name = code_file.stem + if module_name in sys.modules: + print(f"Deleting module {module_name}") + del sys.modules[module_name] + + +def run_tests(projects_dir): + invariance_test_node_id = "tests/stack_test.py::test_stack" + failed = False + # Run the tests with and without instrumentation + for project in projects_dir.iterdir(): + print(f"Installing project {project.name}") + install_project(project) + print(f"Running project {project.name}") + test_result = run_project(project) + remove_related_modules(project) + instrumented_test_result = run_instrumented_project(project) + + for test, result in test_result.items(): + if (test not in instrumented_test_result): + print(f"Test {test} not found in instrumented test results") + failed = True + break + if test == invariance_test_node_id: + if (result == instrumented_test_result[test]): + print(f"Test {test} results match for invariance test") + failed = True + break + else: + continue + if (result != instrumented_test_result[test]): + print(f"Test {test} results do not match") + failed = True + break + sys.path.remove(str(project / "src")) + sys.path.remove(str(project.parent)) + + if failed: + print("Test results do not match before and after instrumentation") + return False + + print(f"All tests passed for projects in folder {projects_dir}") + + return True + + +def install_project(projects_dir): + subprocess.run(["pip", "install", "-e", str(projects_dir)]) + requirements_file = projects_dir / "requirements.txt" + if requirements_file.exists(): + subprocess.run(["pip", "install", "-r", str(requirements_file)]) + sys.path.append(str(projects_dir / "src")) + sys.path.append(str(projects_dir.parent)) + + +def run_test_on_projects(): + is_successful = True + dynapyt_dir = Path(__file__).parent.parent + execution_invariance_test_dir = Path(__file__).parent + test_folders_file = execution_invariance_test_dir / "test_folders.txt" + with open(test_folders_file, "r") as f: + while True: + line = f.readline() + if not line: + break + projects_dir = dynapyt_dir / line.strip() + if not run_tests(projects_dir): + is_successful = False + break + + if is_successful: + print("All tests passed for all projects") + else: + print("Some tests failed") + return is_successful + + +if __name__ == "__main__": + run_test_on_projects() \ No newline at end of file diff --git a/execution_invariance_test/test_folders.txt b/execution_invariance_test/test_folders.txt new file mode 100644 index 0000000..3891447 --- /dev/null +++ b/execution_invariance_test/test_folders.txt @@ -0,0 +1 @@ +execution_invariance_test/projects diff --git a/pyproject.toml b/pyproject.toml index 1472d6f..821c40c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,3 +57,11 @@ dependencies = [ [tool.hatch.envs.end2end.scripts] run = "bash end2end_tests/scripts/run_single_project.sh simple-with-pytest simple_with_pytest tests" cli = "bash end2end_tests/scripts/run_cli.sh simple-with-pytest simple_with_pytest \"pytest -n 6 tests\"" + +[tool.hatch.envs.exec_invariance_test] +dependencies = [ + "pytest-json-report", +] +[tool.hatch.envs.exec_invariance_test.scripts] +run = "bash execution_invariance_test/run_test.sh" + From 8673ba762fdfd0ae67be2487fc7e9c5ebd43ddea Mon Sep 17 00:00:00 2001 From: Keerthi Date: Tue, 3 Sep 2024 16:31:06 +0530 Subject: [PATCH 2/4] Added yml file in github workflows for execution invariance tests --- .../workflows/execution-invariance-tests.yml | 31 +++++++++++++++++++ execution_invariance_test/test.py | 26 ++++++++-------- execution_invariance_test/test_folders.txt | 3 +- 3 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/execution-invariance-tests.yml diff --git a/.github/workflows/execution-invariance-tests.yml b/.github/workflows/execution-invariance-tests.yml new file mode 100644 index 0000000..307188f --- /dev/null +++ b/.github/workflows/execution-invariance-tests.yml @@ -0,0 +1,31 @@ +name: execution invariance tests + +on: [push] + +jobs: + test: + + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + os: ["ubuntu-latest"] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch + pip install pytest-json-report + - name: Install DynaPyt + run: | + pip install -e . + - name: Run the test script + run: | + hatch run exec_invariance_test:run \ No newline at end of file diff --git a/execution_invariance_test/test.py b/execution_invariance_test/test.py index c6a15bb..7afdb59 100644 --- a/execution_invariance_test/test.py +++ b/execution_invariance_test/test.py @@ -60,9 +60,7 @@ def run_instrumented_project(project): return test_result_map -def remove_related_modules(project): - project_module = project.name.replace("-", "_") - +def remove_related_modules(project, project_module): project_src_dir = project / "src" for code_file in project_src_dir.rglob("*.py"): module_name = project_module + "." + code_file.stem @@ -78,16 +76,16 @@ def remove_related_modules(project): del sys.modules[module_name] -def run_tests(projects_dir): +def run_tests(projects, project_module_map): invariance_test_node_id = "tests/stack_test.py::test_stack" failed = False # Run the tests with and without instrumentation - for project in projects_dir.iterdir(): + for project in projects: print(f"Installing project {project.name}") install_project(project) print(f"Running project {project.name}") test_result = run_project(project) - remove_related_modules(project) + remove_related_modules(project, project_module_map[project.name]) instrumented_test_result = run_instrumented_project(project) for test, result in test_result.items(): @@ -106,6 +104,7 @@ def run_tests(projects_dir): print(f"Test {test} results do not match") failed = True break + print("All tests passed for project", project.name) sys.path.remove(str(project / "src")) sys.path.remove(str(project.parent)) @@ -113,8 +112,6 @@ def run_tests(projects_dir): print("Test results do not match before and after instrumentation") return False - print(f"All tests passed for projects in folder {projects_dir}") - return True @@ -132,20 +129,23 @@ def run_test_on_projects(): dynapyt_dir = Path(__file__).parent.parent execution_invariance_test_dir = Path(__file__).parent test_folders_file = execution_invariance_test_dir / "test_folders.txt" + projects = [] + project_module_map = {} with open(test_folders_file, "r") as f: while True: line = f.readline() if not line: break - projects_dir = dynapyt_dir / line.strip() - if not run_tests(projects_dir): - is_successful = False - break + project_dir, project_module = line.strip().split() + project = dynapyt_dir / project_dir + projects.append(project) + project_module_map[project.name] = project_module - if is_successful: + if run_tests(projects, project_module_map): print("All tests passed for all projects") else: print("Some tests failed") + return is_successful diff --git a/execution_invariance_test/test_folders.txt b/execution_invariance_test/test_folders.txt index 3891447..62f71a0 100644 --- a/execution_invariance_test/test_folders.txt +++ b/execution_invariance_test/test_folders.txt @@ -1 +1,2 @@ -execution_invariance_test/projects +execution_invariance_test/projects/simple-test simple_test +end2end_tests/projects/simple-with-pytest simple_with_pytest From fcb4d4ffa2262bb6d9556d30e3d000b4b114741e Mon Sep 17 00:00:00 2001 From: Keerthi Date: Tue, 3 Sep 2024 16:45:13 +0530 Subject: [PATCH 3/4] Code cleanup --- execution_invariance_test/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/execution_invariance_test/test.py b/execution_invariance_test/test.py index 7afdb59..631362e 100644 --- a/execution_invariance_test/test.py +++ b/execution_invariance_test/test.py @@ -146,7 +146,8 @@ def run_test_on_projects(): else: print("Some tests failed") - return is_successful + if not is_successful: + raise ValueError("Some tests failed") if __name__ == "__main__": From fe4a66d3c86c7df8e6b635873fab15d5b34c93d3 Mon Sep 17 00:00:00 2001 From: Keerthi Date: Wed, 4 Sep 2024 17:52:15 +0530 Subject: [PATCH 4/4] Review rework --- .../simple-test/src/simple_test/analysis.py | 22 ------------------- execution_invariance_test/test.py | 4 ++-- execution_invariance_test/test_folders.txt | 1 - 3 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 execution_invariance_test/projects/simple-test/src/simple_test/analysis.py diff --git a/execution_invariance_test/projects/simple-test/src/simple_test/analysis.py b/execution_invariance_test/projects/simple-test/src/simple_test/analysis.py deleted file mode 100644 index 221c80a..0000000 --- a/execution_invariance_test/projects/simple-test/src/simple_test/analysis.py +++ /dev/null @@ -1,22 +0,0 @@ -# DYNAPYT: DO NOT INSTRUMENT - -import json -from uuid import uuid4 -from dynapyt.analyses.BaseAnalysis import BaseAnalysis - - -class Analysis(BaseAnalysis): - def __init__(self, output_dir: str = None): - self.output_dir = output_dir - self.output = [] - - def begin_execution(self): - self.output.append("begin execution") - - def end_execution(self): - self.output.append("end execution") - with open(f"{self.output_dir}/output-{str(uuid4())}.json", "w") as f: - json.dump(self.output, f, indent=2) - - def binary_operation(self, dyn_ast, iid, op, left, right, result): - self.output.append(f"{left} {op} {right} = {result}") diff --git a/execution_invariance_test/test.py b/execution_invariance_test/test.py index 631362e..a8679ac 100644 --- a/execution_invariance_test/test.py +++ b/execution_invariance_test/test.py @@ -95,7 +95,7 @@ def run_tests(projects, project_module_map): break if test == invariance_test_node_id: if (result == instrumented_test_result[test]): - print(f"Test {test} results match for invariance test") + print(f"Test {test} results are not expected to match for instrumented and uninstrumented code") failed = True break else: @@ -147,7 +147,7 @@ def run_test_on_projects(): print("Some tests failed") if not is_successful: - raise ValueError("Some tests failed") + sys.exit(1) if __name__ == "__main__": diff --git a/execution_invariance_test/test_folders.txt b/execution_invariance_test/test_folders.txt index 62f71a0..8702dd2 100644 --- a/execution_invariance_test/test_folders.txt +++ b/execution_invariance_test/test_folders.txt @@ -1,2 +1 @@ execution_invariance_test/projects/simple-test simple_test -end2end_tests/projects/simple-with-pytest simple_with_pytest