Run tests in the order specified in the replay file (#53)

Fix #52

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 repos:
--   repo: https://github.com/psf/black
+-   repo: https://github.com/psf/black-pre-commit-mirror
     rev: 23.12.1
     hooks:
     -   id: black

CHANGELOG.rst

@@ -1,3 +1,10 @@
+UNRELEASED
+==========
+
+* Test execution order using ``--replay`` now follows the recorded order, not the collection order, as was always intended (`#52`_).
+
+.. _`#52`: https://github.com/ESSS/pytest-replay/pull/53
+
 1.4.0 (2021-06-09)
 ==================
 

src/pytest_replay/__init__.py

@@ -106,22 +106,26 @@ def pytest_collection_modifyitems(self, items, config):
 
         with open(replay_file, "r", encoding="UTF-8") as f:
             all_lines = f.readlines()
-            nodeids = {
+            # Use a dict to deduplicate the node ids while keeping the order.
+            nodeids = dict.fromkeys(
                 json.loads(line)["nodeid"]
                 for line in all_lines
                 if not line.strip().startswith(("#", "//"))
-            }
+            )
+
+        items_dict = {item.nodeid: item for item in items}
         remaining = []
-        deselected = []
-        for item in items:
-            if item.nodeid in nodeids:
+        # Make sure to respect the order from the JSON file (#52).
+        for nodeid in nodeids:
+            item = items_dict.pop(nodeid)
+            if item:
                 remaining.append(item)
-            else:
-                deselected.append(item)
+        deselected = list(items_dict.values())
 
         if deselected:
             config.hook.pytest_deselected(items=deselected)
-            items[:] = remaining
+
+        items[:] = remaining
 
     def append_test_to_script(self, nodeid, line):
         suffix = "-" + self.xdist_worker_name if self.xdist_worker_name else ""

tests/test_replay.py

@@ -1,3 +1,4 @@
+import itertools as it
 import json
 
 import pytest
@@ -216,3 +217,56 @@ def test_2():
     expected = {"test_cwd_changed.py::test_1", "test_cwd_changed.py::test_2"}
     assert contents == expected
     assert result.ret == 0
+
+
+@pytest.mark.usefixtures("suite")
+def test_execution_different_order(testdir):
+    """Ensure tests execute in the order defined by the JSON file, not collection (#52)."""
+    dir = testdir.tmpdir / "replay"
+    options = [f"--replay-record-dir={dir}"]
+    result = testdir.runpytest(*options)
+
+    replay_file = dir / ".pytest-replay.txt"
+
+    with replay_file.open("r+") as f:
+        content = f.readlines()
+
+        # pairwise shuffle of replay file
+        pairs = [(content[i], content[i + 1]) for i in range(0, len(content), 2)]
+        pairs = [pairs[2], pairs[0], pairs[3], pairs[1]]
+        content = list(it.chain.from_iterable(pairs))
+
+        f.seek(0)
+        f.writelines(content)
+
+    result = testdir.runpytest(f"--replay={replay_file}", "-v")
+    assert result.ret == 0
+    result.stdout.fnmatch_lines(
+        [
+            "test_2.py::test_zz*25%*",
+            "test_1.py::test_foo*50%*",
+            "test_3.py::test_foobar*75%*",
+            "test_1.py::test_bar*100%*",
+        ],
+        consecutive=True,
+    )
+
+
+@pytest.mark.usefixtures("suite")
+def test_filter_out_tests_not_in_file(testdir):
+    """Tests not found in the JSON file should not run."""
+    dir = testdir.tmpdir / "replay"
+    options = [f"--replay-record-dir={dir}", "-k", "foo"]
+    result = testdir.runpytest(*options)
+
+    replay_file = dir / ".pytest-replay.txt"
+
+    result = testdir.runpytest(f"--replay={replay_file}", "-v")
+    assert result.ret == 0
+    result.stdout.fnmatch_lines(
+        [
+            "test_1.py::test_foo*50%*",
+            "test_3.py::test_foobar*100%*",
+        ],
+        consecutive=True,
+    )