Add IT tests for serverless (#529)

Adds IT tests for serverless for both public user and operator.

.buildkite/it/run_serverless.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+source .buildkite/retry.sh
+
+export TERM=dumb
+export LC_ALL=en_US.UTF-8
+export TZ=Etc/UTC
+export DEBIAN_FRONTEND=noninteractive
+# https://askubuntu.com/questions/1367139/apt-get-upgrade-auto-restart-services
+sudo mkdir -p /etc/needrestart
+echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
+
+PYTHON_VERSION="$1"
+TEST_NAME="$2"
+
+echo "--- System dependencies"
+
+retry 5 sudo add-apt-repository --yes ppa:deadsnakes/ppa
+retry 5 sudo apt-get update
+retry 5 sudo apt-get install -y \
+    "python${PYTHON_VERSION}" "python${PYTHON_VERSION}-dev" "python${PYTHON_VERSION}-venv" \
+    dnsutils  # provides nslookup
+
+echo "--- Python modules"
+
+"python${PYTHON_VERSION}" -m venv .venv
+source .venv/bin/activate
+python -m pip install .[develop]
+
+echo "--- Run IT serverless test \"$TEST_NAME\" :pytest:"
+
+hatch -v -e it_serverless run $TEST_NAME
+

.buildkite/it/serverless-pipeline.yml

@@ -1,6 +1,32 @@
+common:
+  plugins:
+    - elastic/vault-secrets#v0.0.2: &vault-base_url
+        path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
+        field: "base_url"
+        env_var: "RALLY_IT_SERVERLESS_BASE_URL"
+    - elastic/vault-secrets#v0.0.2: &vault-get_credentials_endpoint
+        path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
+        field: "get_credentials_endpoint"
+        env_var: "RALLY_IT_SERVERLESS_GET_CREDENTIALS_ENDPOINT"
+    - elastic/vault-secrets#v0.0.2: &vault-api_key
+        path: "secret/ci/elastic-rally-tracks/employees/cloud/it-serverless"
+        field: "api_key"
+        env_var: "RALLY_IT_SERVERLESS_API_KEY"
+
 agents:
-  image: "docker.elastic.co/ci-agent-images/es-perf/buildkite-agent-python"
+  provider: "gcp"
+  image: family/core-ubuntu-2204
 
 steps:
-  - label: "Run IT Serverless Test"
-    command: echo "TODO"
+  - label: "Run IT serverless tests with user privileges"
+    plugins:
+      - elastic/vault-secrets#v0.0.2: *vault-base_url
+      - elastic/vault-secrets#v0.0.2: *vault-get_credentials_endpoint
+      - elastic/vault-secrets#v0.0.2: *vault-api_key
+    command: bash .buildkite/it/run_serverless.sh 3.11 test_user
+  - label: "Run IT Serverless tests with operator privileges"
+    plugins:
+      - elastic/vault-secrets#v0.0.2: *vault-base_url
+      - elastic/vault-secrets#v0.0.2: *vault-get_credentials_endpoint
+      - elastic/vault-secrets#v0.0.2: *vault-api_key
+    command: bash .buildkite/it/run_serverless.sh 3.11 test_operator

.buildkite/retry.sh

@@ -0,0 +1,31 @@
+# based on https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746?permalink_comment_id=4155247#gistcomment-4155247
+function retry {
+  local retries=$1
+  shift
+  local cmd=($@)
+  local cmd_string="${@}"
+  local count=0
+
+  # be lenient with non-zero exit codes, to allow retries
+  set +o errexit
+  set +o pipefail
+  until "${cmd[@]}"; do
+    retcode=$?
+    wait=$(( 2 ** count ))
+    count=$(( count + 1))
+    if [[ $count -le $retries ]]; then
+      printf "Command [%s] failed. Retry [%d/%d] in [%d] seconds.\n" "$cmd_string" $count $retries $wait
+      sleep $wait
+    else
+      printf "Exhausted all [%s] retries for command [%s]. Exiting.\n" "$cmd_string" $retries
+      # restore settings to fail immediately on error
+      set -o errexit
+      set -o pipefail
+      return $retcode
+    fi
+  done
+  # restore settings to fail immediately on error
+  set -o errexit
+  set -o pipefail
+  return 0
+}

it_serverless/conftest.py

@@ -0,0 +1,183 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import contextlib
+import json
+import os
+import subprocess
+import time
+import urllib.parse
+from dataclasses import dataclass
+
+import pytest
+import requests
+from elasticsearch import Elasticsearch
+
+BASE_URL = os.environ["RALLY_IT_SERVERLESS_BASE_URL"]
+API_KEY = os.environ["RALLY_IT_SERVERLESS_API_KEY"]
+GET_CREDENTIALS_ENDPOINT = os.environ["RALLY_IT_SERVERLESS_GET_CREDENTIALS_ENDPOINT"]
+
+
+@dataclass
+class ServerlessProjectConfig:
+    target_host: str
+    username: str
+    password: str
+    api_key: str
+    user_client_options_file: str = None
+    operator_client_options_file: str = None
+
+    def get_client_options_file(self, operator) -> str:
+        return self.operator_client_options_file if operator else self.user_client_options_file
+
+    @staticmethod
+    def _client_options(client_auth):
+        return {
+            "default": {
+                "verify_certs": False,
+                "use_ssl": True,
+                "timeout": 240,
+                **client_auth,
+            }
+        }
+
+    def prepare_client_options_files(self, tmpdir_factory):
+        tmp_path = tmpdir_factory.mktemp("client-options")
+
+        client_auth = {
+            "basic_auth_user": self.username,
+            "basic_auth_password": self.password,
+        }
+        self.operator_client_options_file = tmp_path / "operator.json"
+        with self.operator_client_options_file.open("w") as f:
+            json.dump(self._client_options(client_auth), fp=f)
+
+        client_auth = {"api_key": self.api_key}
+        self.user_client_options_file = tmp_path / "user.json"
+        with self.user_client_options_file.open("w") as f:
+            json.dump(self._client_options(client_auth), fp=f)
+
+
+def serverless_api(method, endpoint, json=None):
+    resp = requests.request(
+        method,
+        BASE_URL + endpoint,
+        headers={
+            "Authorization": f"ApiKey {API_KEY}",
+            "Content-Type": "application/json",
+        },
+        json=json,
+        timeout=60,
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+@pytest.fixture(scope="module")
+def project():
+    print("\nCreating project")
+    created_project = serverless_api(
+        "POST",
+        "/api/v1/serverless/projects/elasticsearch",
+        json={
+            "name": "rally-track-it",
+            "region_id": "aws-eu-west-1",
+        },
+    )
+
+    yield created_project
+
+    print("Deleting project")
+    serverless_api("DELETE", f"/api/v1/serverless/projects/elasticsearch/{created_project['id']}")
+
+
+@pytest.fixture(scope="module")
+def project_config(project, tmpdir_factory):
+    credentials = serverless_api(
+        "POST",
+        f"/api/v1/serverless/projects/elasticsearch/{project['id']}{GET_CREDENTIALS_ENDPOINT}",
+    )
+
+    es_endpoint = project["endpoints"]["elasticsearch"]
+    es_hostname = urllib.parse.urlparse(es_endpoint).hostname
+    rally_target_host = f"{es_hostname}:443"
+
+    print("Waiting for DNS propagation")
+    for _ in range(6):
+        time.sleep(30)
+        with contextlib.suppress(subprocess.CalledProcessError):
+            subprocess.run(["nslookup", es_hostname, "8.8.8.8"], check=True)
+            break
+    else:
+        raise ValueError("Timed out waiting for DNS propagation")
+
+    print("Waiting for Elasticsearch")
+    for _ in range(18):
+        try:
+            es = Elasticsearch(
+                f"https://{rally_target_host}",
+                basic_auth=(
+                    credentials["username"],
+                    credentials["password"],
+                ),
+                request_timeout=10,
+            )
+            info = es.info()
+            print("GET /")
+            print(json.dumps(info.body, indent=2))
+
+            authenticate = es.perform_request(method="GET", path="/_security/_authenticate")
+            print("GET /_security/_authenticate")
+            print(json.dumps(authenticate.body, indent=2))
+
+            break
+        except Exception as e:
+            print(f"GET / Failed with {type(e)}")
+            time.sleep(10)
+    else:
+        raise ValueError("Timed out waiting for Elasticsearch")
+
+    # Create API key to test Rally with a public user
+    for _ in range(3):
+        try:
+            api_key = es.security.create_api_key(name="public-api-key")
+            break
+        except Exception as e:
+            print(f"API create failed with {type(e)}")
+            time.sleep(10)
+    else:
+        raise ValueError("Timed out waiting for API key")
+
+    project_config = ServerlessProjectConfig(
+        rally_target_host,
+        credentials["username"],
+        credentials["password"],
+        api_key.body["encoded"],
+    )
+    project_config.prepare_client_options_files(tmpdir_factory)
+    yield project_config
+
+
+def pytest_addoption(parser):
+    parser.addoption("--operator", action="store_true", help="run as operator")
+
+
+def pytest_generate_tests(metafunc):
+    if "operator" in metafunc.fixturenames:
+        operator = metafunc.config.getoption("operator")
+        label = "operator" if operator else "user"
+        metafunc.parametrize("operator", [operator], ids=[label])