From 64f430fd86d5da7f2a327b6fa45760ae59982c6b Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 26 Jan 2024 14:52:28 +0100 Subject: [PATCH 1/4] add no_checks for setUpValidatorProcesses --- tests/test_server.py | 19 +++++++++++++- .../coordinator/server.py | 25 ++++++++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/tests/test_server.py b/tests/test_server.py index 4ffc86d..21cbba2 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,6 +1,10 @@ import logging +import os import sys -from uptime_service_validation.coordinator.server import try_get_hostname_ip +from uptime_service_validation.coordinator.server import ( + bool_env_var_set, + try_get_hostname_ip, +) def test_try_get_hostname_ip(): @@ -14,3 +18,16 @@ def test_try_get_hostname_ip(): try_get_hostname_ip("wronglocalhost", logging, max_retries=3, initial_wait=0.1) == "wronglocalhost" ) + + +def test_bool_env_var_set(): + os.environ["TEST_VAR"] = "True" + assert bool_env_var_set("TEST_VAR") == True + os.environ["TEST_VAR"] = "1" + assert bool_env_var_set("TEST_VAR") == True + os.environ["TEST_VAR"] = "False" + assert bool_env_var_set("TEST_VAR") == False + os.environ["TEST_VAR"] = "0" + assert bool_env_var_set("TEST_VAR") == False + + assert bool_env_var_set("TEST_VAR_THAT_IS_NOT_SET") == False diff --git a/uptime_service_validation/coordinator/server.py b/uptime_service_validation/coordinator/server.py index 5d22e94..4e89103 100644 --- a/uptime_service_validation/coordinator/server.py +++ b/uptime_service_validation/coordinator/server.py @@ -9,6 +9,17 @@ import socket +def bool_env_var_set(env_var_name): + """ + Checks if an environment variable is set and is set to a truthy value. + + :param env_var_name: The name of the environment variable. + :return: True if the environment variable is set and is set to a truthy value. + """ + env_var = os.environ.get(env_var_name) + return env_var is not None and env_var.lower() in ["true", "1"] + + def try_get_hostname_ip(hostname, logger, max_retries=5, initial_wait=0.2): """ Attempts to resolve a hostname to an IP address with retries. @@ -41,6 +52,7 @@ def try_get_hostname_ip(hostname, logger, max_retries=5, initial_wait=0.2): def datetime_formatter(dt): return dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-5] + "+0000" + def setUpValidatorPods(time_intervals, logging, worker_image, worker_tag): # Configuring Kubernetes client config.load_incluster_config() @@ -97,10 +109,7 @@ def setUpValidatorPods(time_intervals, logging, worker_image, worker_tag): name="CASSANDRA_PORT", value=os.environ.get("CASSANDRA_PORT"), ), - client.V1EnvVar( - name="CASSANDRA_USE_SSL", - value="1" - ), + client.V1EnvVar(name="CASSANDRA_USE_SSL", value="1"), client.V1EnvVar( name="SSL_CERTFILE", value="/root/.cassandra/sf-class2-root.crt", @@ -233,9 +242,12 @@ def setUpValidatorPods(time_intervals, logging, worker_image, worker_tag): time.sleep(10) logging.info("All jobs have been processed.") - + + def setUpValidatorProcesses(time_intervals, logging, worker_image, worker_tag): processes = [] + if bool_env_var_set("NO_CHECKS"): + logging.info("stateless-verifier will run with --no-checks flag") for index, mini_batch in enumerate(time_intervals): process_name = ( f"local-validator-{datetime.now().strftime('%y-%m-%d-%H-%M')}-{index}" @@ -277,8 +289,9 @@ def setUpValidatorProcesses(time_intervals, logging, worker_image, worker_tag): os.environ.get("AWS_KEYSPACE"), f"{datetime_formatter(mini_batch[0])}", f"{datetime_formatter(mini_batch[1])}", - "--no-check", ] + if bool_env_var_set("NO_CHECKS"): + command.append("--no-checks") cmd_str = " ".join(command) # Set up environment variables for the process From 963fc27af408006d8e02e42be764aa95cf082573 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 26 Jan 2024 14:52:40 +0100 Subject: [PATCH 2/4] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2b97faf..a853949 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ The Coordinator program runs the `stateless-verification-tool` for validation ag - `WORKER_IMAGE` - Docker image name for the stateless verifier (e.g., `mina-delegation-verify`). - `WORKER_TAG` - Specific tag of the Docker image, indicating the version or build. +- `NO_CHECKS` - if set to `1`, stateless verifier will run with `--no-checks` flag ### Slack Alerts From 4e5ca82e6b8fc07987089d7f2efb7b9a9de12e79 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 26 Jan 2024 14:53:22 +0100 Subject: [PATCH 3/4] use bool_env_var_set in coordinator for checking TEST_ENV --- uptime_service_validation/coordinator/coordinator.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/uptime_service_validation/coordinator/coordinator.py b/uptime_service_validation/coordinator/coordinator.py index e5e0800..f2bff16 100644 --- a/uptime_service_validation/coordinator/coordinator.py +++ b/uptime_service_validation/coordinator/coordinator.py @@ -9,6 +9,7 @@ from dataclasses import asdict from uptime_service_validation.coordinator.helper import * from uptime_service_validation.coordinator.server import ( + bool_env_var_set, setUpValidatorPods, setUpValidatorProcesses, ) @@ -21,14 +22,6 @@ sys.path.insert(0, project_root) -def test_env(): - test = os.environ.get("TEST_ENV") - if test == "1": - return True - else: - return False - - def main(): process_loop_count = 0 load_dotenv() @@ -86,7 +79,7 @@ def main(): worker_image = os.environ["WORKER_IMAGE"] worker_tag = os.environ["WORKER_TAG"] start = time() - if test_env(): + if bool_env_var_set("TEST_ENV"): logging.warning("running in test environment") setUpValidatorProcesses( time_intervals, logging, worker_image, worker_tag From 6850d5630c228b31bcab701984edcbf27c48af81 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 26 Jan 2024 14:54:23 +0100 Subject: [PATCH 4/4] add log to setUpValidatorPods if no_checks is set --- uptime_service_validation/coordinator/server.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uptime_service_validation/coordinator/server.py b/uptime_service_validation/coordinator/server.py index 4e89103..c7950f3 100644 --- a/uptime_service_validation/coordinator/server.py +++ b/uptime_service_validation/coordinator/server.py @@ -77,7 +77,8 @@ def setUpValidatorPods(time_intervals, logging, worker_image, worker_tag): # List to keep track of job names jobs = [] cassandra_ip = try_get_hostname_ip(os.environ.get("CASSANDRA_HOST"), logging) - + if bool_env_var_set("NO_CHECKS"): + logging.info("stateless-verifier will run with --no-checks flag") for index, mini_batch in enumerate(time_intervals): # Define the environment variables env_vars = [ @@ -141,9 +142,7 @@ def setUpValidatorPods(time_intervals, logging, worker_image, worker_tag): ] # Entrypoint configmap name - entrypoint_configmap_name = ( - f"delegation-verify-coordinator-worker" - ) + entrypoint_configmap_name = f"delegation-verify-coordinator-worker" # Define the volumes auth_volume = client.V1Volume( @@ -179,7 +178,9 @@ def setUpValidatorPods(time_intervals, logging, worker_image, worker_tag): container = client.V1Container( name="delegation-verify", image=f"{worker_image}:{worker_tag}", - command=["/bin/entrypoint/entrypoint-worker.sh"], # The entrypoint script is in the cluster as a configmap. The script can be found in the helm chart of coordinator + command=[ + "/bin/entrypoint/entrypoint-worker.sh" + ], # The entrypoint script is in the cluster as a configmap. The script can be found in the helm chart of coordinator resources=resource_requirements_container, env=env_vars, image_pull_policy=os.environ.get("IMAGE_PULL_POLICY", "IfNotPresent"),