Parallelize user vs operator

.buildkite/it/run_serverless.sh

@@ -12,20 +12,22 @@ export DEBIAN_FRONTEND=noninteractive
 sudo mkdir -p /etc/needrestart
 echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
 
+PYTHON_VERSION="$1"
+TEST_NAME="$2"
+
 echo "--- System dependencies"
 
-PYTHON_VERSION="$1"
 retry 5 sudo add-apt-repository --yes ppa:deadsnakes/ppa
 retry 5 sudo apt-get update
 retry 5 sudo apt-get install -y \
     "python${PYTHON_VERSION}" "python${PYTHON_VERSION}-dev" "python${PYTHON_VERSION}-venv" \
     dnsutils  # provides nslookup
 
-echo "--- Run IT test :pytest:"
+echo "--- Run IT test \"$TEST_NAME\" :pytest:"
 
 "python${PYTHON_VERSION}" -m venv .venv
 source .venv/bin/activate
 
 python -m pip install .[develop]
-hatch -v -e it_serverless run test
+hatch -v -e it_serverless run $TEST_NAME
 

.buildkite/it/serverless-pipeline.yml

@@ -3,18 +3,33 @@ agents:
   image: family/core-ubuntu-2204
 
 steps:
-  - label: "Run IT Serverless tests"
+  - label: "Run IT serverless tests with user privileges"
     plugins:
       - elastic/vault-secrets#v0.0.2:
-          path: "secret/ci/elastic-rally/employees/cloud/it-serverless"
+          path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
           field: "base_url"
           env_var: "RALLY_IT_SERVERLESS_BASE_URL"
       - elastic/vault-secrets#v0.0.2:
-          path: "secret/ci/elastic-rally/employees/cloud/it-serverless"
+          path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
           field: "get_credentials_endpoint"
           env_var: "RALLY_IT_SERVERLESS_GET_CREDENTIALS_ENDPOINT"
       - elastic/vault-secrets#v0.0.2:
-          path: "secret/ci/elastic-rally/employees/cloud/it-serverless"
+          path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
           field: "api_key"
           env_var: "RALLY_IT_SERVERLESS_API_KEY"
-    command: bash .buildkite/it/run_serverless.sh 3.11
+    command: bash .buildkite/it/run_serverless.sh 3.11 test_user
+  - label: "Run IT Serverless tests with operator privileges"
+    plugins:
+      - elastic/vault-secrets#v0.0.2:
+          path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
+          field: "base_url"
+          env_var: "RALLY_IT_SERVERLESS_BASE_URL"
+      - elastic/vault-secrets#v0.0.2:
+          path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
+          field: "get_credentials_endpoint"
+          env_var: "RALLY_IT_SERVERLESS_GET_CREDENTIALS_ENDPOINT"
+      - elastic/vault-secrets#v0.0.2:
+          path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
+          field: "api_key"
+          env_var: "RALLY_IT_SERVERLESS_API_KEY"
+    command: bash .buildkite/it/run_serverless.sh 3.11 test_operator

it_serverless/conftest.py

@@ -160,4 +160,15 @@ def write_options_file(config: ServerlessProjectConfig, operator, tmp_path):
     options_file = tmp_path / "client-options.json"
     with options_file.open("w") as f:
         json.dump(client_options(client_auth), fp=f)
-    return options_file
+    return options_file
+
+
+def pytest_addoption(parser):
+    parser.addoption("--operator", action="store_true", help="run as operator")
+
+
+def pytest_generate_tests(metafunc):
+    if "operator" in metafunc.fixturenames:
+        operator = metafunc.config.getoption("operator")
+        label = "operator" if operator else "user"
+        metafunc.parametrize("operator", [operator], ids=[label])

it_serverless/test_selected_tracks_and_challenges.py

@@ -45,7 +45,6 @@ class TestTrackRepository:
         # TODO requires a fix after https://github.com/elastic/rally-tracks/pull/513
         # "tsdb_k8s_queries",
     ]
-    #include_tracks = ["geonames"]
 
     skip_challenges = {
         "nyc_taxis": ["esql"],
@@ -59,7 +58,6 @@ class TestTrackRepository:
         "nyc_taxis": ["update-aggs-only"],
     }
 
-    @pytest.mark.parametrize("operator", [False, True], ids=["User", "Operator"])
     def test_autogenerated(self, operator, tmp_path, rally, track, challenge, rally_options, serverless_project_config: ServerlessProjectConfig):
         client_options_file = write_options_file(serverless_project_config, operator, tmp_path)
         track_params = {"number_of_replicas": 1}

pyproject.toml

@@ -49,7 +49,8 @@ test = "pytest"
 test = "pytest it --log-cli-level=INFO"
 
 [tool.hatch.envs.it_serverless.scripts]
-test = "pytest -s it_serverless --log-cli-level=INFO"
+test_user = "pytest -s it_serverless --log-cli-level=INFO"
+test_operator = "pytest -s it_serverless --log-cli-level=INFO --operator"
 
 [tool.pytest.ini_options]
 # set to true for more verbose output of tests