Abu UID2 4880 fix e2e (#1412)

* Change URL update to entire URL
IABTechLab · Feb 24, 2025 · 8a39bce · 8a39bce
1 parent 25f07da
commit 8a39bce
Show file tree

Hide file tree

Showing 13 changed files with 76 additions and 103 deletions.
diff --git a/Makefile.eif b/Makefile.eif
@@ -28,11 +28,14 @@ euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/s
 
 .PHONY: build_configs
 
-build_configs: build/conf/default-config.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml
+build_configs: build/conf/default-config.json build/conf/feat-flag.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml
 
 build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json
 	cp ./scripts/aws/conf/default-config.json ./build/conf/
 
+build/conf/feat-flag.json: build_artifacts ./scripts/aws/conf/feat-flag/feat-flag.json
+	cp ./scripts/aws/conf/feat-flag/feat-flag.json ./build/conf/
+
 build/conf/euid-integ-config.json: build_artifacts ./scripts/aws/conf/euid-integ-config.json
 	cp ./scripts/aws/conf/euid-integ-config.json ./build/conf/
 

diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-operator</artifactId>
-    <version>5.47.47</version>
+    <version>5.47.73-alpha-180-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -22,7 +22,7 @@
         <enclave-aws.version>2.1.0</enclave-aws.version>
         <enclave-azure.version>2.1.0</enclave-azure.version>
         <enclave-gcp.version>2.1.0</enclave-gcp.version>
-        <uid2-shared.version>8.0.32</uid2-shared.version>
+        <uid2-shared.version>8.1.10</uid2-shared.version>
         <image.version>${project.version}</image.version>
         <maven.compiler.source>21</maven.compiler.source>
         <maven.compiler.target>21</maven.compiler.target>

diff --git a/scripts/aws/Dockerfile b/scripts/aws/Dockerfile
@@ -37,7 +37,7 @@ COPY ./conf/default-config.json    /app/conf/
 COPY ./conf/*.json  /app/conf/
 COPY ./conf/*.xml                  /app/conf/
 COPY ./syslog-ng-client.conf       /etc/syslog-ng/syslog-ng.conf
-COPY ./conf/feat-flag/feat-flag.json    /app/conf/feat-flag/
+COPY ./conf/feat-flag.json    /app/conf/feat-flag/
 
 RUN chmod +x /app/vsockpx && chmod +x /app/entrypoint.sh
 

diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json
@@ -30,9 +30,12 @@
   "service_links_metadata_path": "service_links/metadata.json",
   "optout_metadata_path": null,
   "optout_inmem_cache": false,
-  "enclave_platform": null,
+  "enclave_platform": "aws-nitro",
   "failure_shutdown_wait_hours": 120,
   "sharing_token_expiry_seconds": 2592000,
   "validate_service_links": false,
+  "identity_token_expires_after_seconds": 86400,
+  "refresh_token_expires_after_seconds": 2592000,
+  "refresh_identity_token_after_seconds": 3600,
   "operator_type": "private"
-}
+}
diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json
@@ -11,12 +11,5 @@
   "core_attest_url": "https://core.integ.euid.eu/attest",
   "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate",
   "optout_s3_folder": "optout/",
-  "allow_legacy_api": false,
-  "runtime_config_store": {
-    "type": "http",
-    "config" : {
-      "url": "https://core.integ.euid.eu/operator/config"
-    },
-    "config_scan_period_ms": 300000
-  }
+  "allow_legacy_api": false
 }
diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json
@@ -1,22 +1,14 @@
 {
+  "core_attest_url": "https://core-integ.uidapi.com/attest",
+  "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate",
   "sites_metadata_path": "https://core-integ.uidapi.com/sites/refresh",
   "clients_metadata_path": "https://core-integ.uidapi.com/clients/refresh",
+  "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh",
   "keysets_metadata_path": "https://core-integ.uidapi.com/key/keyset/refresh",
   "keyset_keys_metadata_path": "https://core-integ.uidapi.com/key/keyset-keys/refresh",
-  "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh",
   "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh",
   "services_metadata_path": "https://core-integ.uidapi.com/services/refresh",
   "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh",
   "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh",
-  "core_attest_url": "https://core-integ.uidapi.com/attest",
-  "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate",
-  "optout_s3_folder": "uid-optout-integ/",
-  "allow_legacy_api": false,
-  "runtime_config_store": {
-    "type": "http",
-    "config" : {
-      "url": "https://core-integ.uidapi.com/operator/config"
-    },
-    "config_scan_period_ms": 300000
-  }
-}
+  "optout_s3_folder": "uid-optout-integ/"
+}
diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py
@@ -15,7 +15,7 @@
 import sys
 import time
 import yaml
-
+logging.basicConfig(level=logging.INFO)
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, InstanceProfileMissingError, OperatorKeyNotFoundError, ConfigurationValueError, ConfidentialComputeStartupError
 

diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh
@@ -10,21 +10,18 @@ exec &> >(tee -a "$LOG_FILE")
 PARAMETERIZED_CONFIG="/app/conf/config-overrides.json"
 OPERATOR_CONFIG="/tmp/final-config.json"
 
-setup_auxiliaries() {
-  set -o pipefail
-  ulimit -n 65536
+set -o pipefail
+ulimit -n 65536
 
-  # -- setup loopback device
-  echo "Setting up loopback device..."
-  ifconfig lo 127.0.0.1
+# -- setup loopback device
+echo "Setting up loopback device..."
+ifconfig lo 127.0.0.1
 
-  # -- start vsock proxy
-  echo "Starting vsock proxy..."
-  /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3
-
-  /usr/sbin/syslog-ng --verbose
-}
+# -- start vsock proxy
+echo "Starting vsock proxy..."
+/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3
 
+/usr/sbin/syslog-ng --verbose
 
 build_parameterized_config() {
   curl -s -f -o "${PARAMETERIZED_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig
@@ -46,9 +43,7 @@ build_parameterized_config() {
 
 build_operator_config() {
   CORE_BASE_URL=$(jq -r ".core_base_url" < "${PARAMETERIZED_CONFIG}")
-  CORE_BASE_URL=$(echo "$CORE_BASE_URL" | sed -E 's#^(https?://)?([^/]+).*#\2#')
   OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${PARAMETERIZED_CONFIG}")
-  OPTOUT_BASE_URL=$(echo "$OPTOUT_BASE_URL" | sed -E 's#^(https?://)?([^/]+).*#\2#')
   DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${PARAMETERIZED_CONFIG}")
   DEBUG_MODE=$(jq -r ".debug_mode" < "${PARAMETERIZED_CONFIG}")
 
@@ -69,22 +64,22 @@ build_operator_config() {
 
   if [[ "$DEPLOYMENT_ENVIRONMENT" != "prod" ]]; then
     #Allow override of base URL in non-prod environments
-    CORE_PATTERN="core.*uidapi.com"
-    OPTOUT_PATTERN="optout.*uidapi.com"
-    if [[ "$DEPLOYMENT_ENVIRONMENT" == "euid" ]]; then
-      CORE_PATTERN="core.*euid.eu"
-      OPTOUT_PATTERN="optout.*euid.eu"
+    CORE_PATTERN="https://core.*uidapi.com"
+    OPTOUT_PATTERN="https://optout.*uidapi.com"
+    if [[ "$IDENTITY_SCOPE_LOWER" == "euid" ]]; then
+      CORE_PATTERN="https://core.*euid.eu"
+      OPTOUT_PATTERN="https://optout.*euid.eu"
     fi
     sed -i "s#${CORE_PATTERN}#${CORE_BASE_URL}#g" "${OPERATOR_CONFIG}"
     sed -i "s#${OPTOUT_PATTERN}#${OPTOUT_BASE_URL}#g" "${OPERATOR_CONFIG}"
   fi
 
 }
 
-setup_auxiliaries
 build_parameterized_config
 build_operator_config
 
+
 DEBUG_MODE=$(jq -r ".debug_mode" < "${OPERATOR_CONFIG}")
 LOGBACK_CONF="./conf/logback.xml"
 
@@ -107,3 +102,4 @@ java \
   -Dlogback.configurationFile=${LOGBACK_CONF} \
   -Dhttp_proxy=socks5://127.0.0.1:3305 \
   -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar
+
diff --git a/scripts/azure-cc/azureEntryPoint.py b/scripts/azure-cc/azureEntryPoint.py
@@ -8,7 +8,6 @@
 import shutil
 import requests
 import logging
-from urllib.parse import urlparse
 from confidential_compute import ConfidentialCompute, ConfigurationMissingError, OperatorKeyPermissionError, OperatorKeyNotFoundError, ConfidentialComputeStartupError
 from azure.keyvault.secrets import SecretClient
 from azure.identity import DefaultAzureCredential, CredentialUnavailableError
@@ -22,6 +21,8 @@ class AzureEntryPoint(ConfidentialCompute):
     env_name = os.getenv("DEPLOYMENT_ENVIRONMENT")
     jar_name = os.getenv("JAR_NAME", "default-jar-name")
     jar_version = os.getenv("JAR_VERSION", "default-jar-version")
+    default_core_endpoint = f"https://core-{env_name}.uidapi.com".lower()
+    default_optout_endpoint = f"https://optout-{env_name}.uidapi.com".lower()
 
     FINAL_CONFIG = "/tmp/final-config.json"
 
@@ -50,37 +51,25 @@ def __create_final_config(self):
         except IOError as e:
             logging.error(f"Failed to create {AzureEntryPoint.FINAL_CONFIG} with error: {e}")
             sys.exit(1)
-
-        CORE_BASE_URL = os.getenv("CORE_BASE_URL")
-        OPTOUT_BASE_URL = os.getenv("OPTOUT_BASE_URL")
 
-        if CORE_BASE_URL and OPTOUT_BASE_URL and AzureEntryPoint.env_name != 'prod':
-            logging.info(f"-- replacing URLs by {CORE_BASE_URL} and {OPTOUT_BASE_URL}")
-            with open(AzureEntryPoint.FINAL_CONFIG, "r") as file:
-                config = file.read()
-
-            config = config.replace("core-integ.uidapi.com", urlparse(CORE_BASE_URL).netloc)
-            config = config.replace("optout-integ.uidapi.com", urlparse(OPTOUT_BASE_URL).netloc)
+        logging.info(f"-- replacing URLs by {self.configs["core_base_url"]} and {self.configs["optout_base_url"]}")
+        with open(AzureEntryPoint.FINAL_CONFIG, "r") as file:
+            config = file.read()
 
-            with open(AzureEntryPoint.FINAL_CONFIG, "w") as file:
-                file.write(config)
+        config = config.replace("https://core.uidapi.com", self.configs["core_base_url"])
+        config = config.replace("https://optout.uidapi.com", self.configs["optout_base_url"])
+        with open(AzureEntryPoint.FINAL_CONFIG, "w") as file:
+            file.write(config)
 
         with open(AzureEntryPoint.FINAL_CONFIG, "r") as file:
             logging.info(file.read())
 
-    def __set_base_urls(self):
-        with open(AzureEntryPoint.FINAL_CONFIG, "r") as file:
-            jdata = json.load(file)
-            self.configs["core_base_url"] = jdata["core_attest_url"]
-            self.configs["optout_base_url"] = jdata["optout_api_uri"]
-
     def __set_operator_key(self):
         try:
             credential = DefaultAzureCredential()
             kv_URL = f"https://{AzureEntryPoint.kv_name}.vault.azure.net"
             secret_client = SecretClient(vault_url=kv_URL, credential=credential)
             secret = secret_client.get_secret(AzureEntryPoint.secret_name)
-            # print(f"Secret Value: {secret.value}")
             self.configs["operator_key"] = secret.value
 
         except (CredentialUnavailableError, ClientAuthenticationError) as auth_error:
@@ -92,14 +81,13 @@ def __set_operator_key(self):
 
 
     def _set_confidential_config(self, secret_identifier: str = None):
+        """Builds and sets ConfidentialComputeConfig"""
         self.configs["skip_validations"] = os.getenv("SKIP_VALIDATIONS", "false").lower() == "true"
         self.configs["debug_mode"] = os.getenv("DEBUG_MODE", "false").lower() == "true"
         self.configs["environment"] = AzureEntryPoint.env_name
-
-        # set self.configs["operator_key"]
+        self.configs["core_base_url"] = os.getenv("CORE_BASE_URL") if os.getenv("CORE_BASE_URL") and AzureEntryPoint.env_name == "integ" else AzureEntryPoint.default_core_endpoint
+        self.configs["optout_base_url"] = os.getenv("OPTOUT_BASE_URL")  if os.getenv("OPTOUT_BASE_URL") and AzureEntryPoint.env_name == "integ" else AzureEntryPoint.default_optout_endpoint
         self.__set_operator_key()
-        # set base urls from final config file
-        self.__set_base_urls()
 
     def __run_operator(self):
 
@@ -150,10 +138,10 @@ def _validate_auxiliaries(self):
     def run_compute(self) -> None:
         """Main execution flow for confidential compute."""
         self.__check_env_variables()
-        self.__create_final_config()
         self._set_confidential_config()
         if not self.configs.get("skip_validations"):
             self.validate_configuration()
+        self.__create_final_config()
         self._setup_auxiliaries()
         self.__run_operator()
 

diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json
@@ -1,20 +1,20 @@
 {
-  "sites_metadata_path": "https://core-integ.uidapi.com/sites/refresh",
-  "clients_metadata_path": "https://core-integ.uidapi.com/clients/refresh",
-  "keysets_metadata_path": "https://core-integ.uidapi.com/key/keyset/refresh",
-  "keyset_keys_metadata_path": "https://core-integ.uidapi.com/key/keyset-keys/refresh",
-  "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh",
-  "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh",
-  "services_metadata_path": "https://core-integ.uidapi.com/services/refresh",
-  "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh",
-  "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh",
-  "core_attest_url": "https://core-integ.uidapi.com/attest",
-  "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate",
+  "sites_metadata_path": "https://core.uidapi.com/sites/refresh",
+  "clients_metadata_path": "https://core.uidapi.com/clients/refresh",
+  "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh",
+  "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh",
+  "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh",
+  "salts_metadata_path": "https://core.uidapi.com/salt/refresh",
+  "services_metadata_path": "https://core.uidapi.com/services/refresh",
+  "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh",
+  "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
+  "core_attest_url": "https://core.uidapi.com/attest",
+  "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
   "optout_s3_folder": "uid-optout-integ/",
   "runtime_config_store": {
     "type": "http",
     "config" : {
-      "url": "https://core-integ.uidapi.com/operator/config"
+      "url": "https://core.uidapi.com/operator/config"
     },
     "config_scan_period_ms": 300000
   }

diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json
@@ -1,21 +1,21 @@
 {
-  "sites_metadata_path": "https://core-prod.uidapi.com/sites/refresh",
-  "clients_metadata_path": "https://core-prod.uidapi.com/clients/refresh",
-  "keysets_metadata_path": "https://core-prod.uidapi.com/key/keyset/refresh",
-  "keyset_keys_metadata_path": "https://core-prod.uidapi.com/key/keyset-keys/refresh",
-  "client_side_keypairs_metadata_path": "https://core-prod.uidapi.com/client_side_keypairs/refresh",
-  "salts_metadata_path": "https://core-prod.uidapi.com/salt/refresh",
-  "services_metadata_path": "https://core-prod.uidapi.com/services/refresh",
-  "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh",
-  "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh",
-  "core_attest_url": "https://core-prod.uidapi.com/attest",
-  "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate",
+  "sites_metadata_path": "https://core.uidapi.com/sites/refresh",
+  "clients_metadata_path": "https://core.uidapi.com/clients/refresh",
+  "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh",
+  "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh",
+  "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh",
+  "salts_metadata_path": "https://core.uidapi.com/salt/refresh",
+  "services_metadata_path": "https://core.uidapi.com/services/refresh",
+  "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh",
+  "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
+  "core_attest_url": "https://core.uidapi.com/attest",
+  "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
   "optout_s3_folder": "optout-v2/",
   "identity_token_expires_after_seconds": 259200,
   "runtime_config_store": {
     "type": "http",
     "config" : {
-      "url": "https://core-prod.uidapi.com/operator/config"
+      "url": "https://core.uidapi.com/operator/config"
     },
     "config_scan_period_ms": 300000
   }

diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py
@@ -98,7 +98,7 @@ def validate_connectivity() -> None:
 
         type_hints = get_type_hints(ConfidentialComputeConfig, include_extras=True)
         required_keys = [field for field, hint in type_hints.items() if "NotRequired" not in str(hint)]
-        missing_keys = [key for key in required_keys if key not in self.configs]
+        missing_keys = [key for key in required_keys if key not in self.configs or self.configs[key] == None]
         if missing_keys:
             raise ConfigurationMissingError(self.__class__.__name__, missing_keys)
 
@@ -144,7 +144,7 @@ def run_command(command, separate_process=False):
             if separate_process:
                 subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
             else:
-                subprocess.run(command,check=True)
+                subprocess.run(command,check=True,text=True)
         except Exception as e:
             logging.error(f"Failed to run command: {e}", exc_info=True)
             raise RuntimeError (f"Failed to start {' '.join(command)} ")
diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py
@@ -6,10 +6,8 @@
 import sys
 import logging
 from google.cloud import secretmanager
-from google.auth import default
 from google.auth.exceptions import DefaultCredentialsError
 from google.api_core.exceptions import PermissionDenied, NotFound
-from urllib.parse import urlparse
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, ConfigurationMissingError, OperatorKeyNotFoundError, OperatorKeyPermissionError, ConfidentialComputeStartupError
 
@@ -50,8 +48,8 @@ def __populate_operator_config(self, destination):
         shutil.copy(target_config, destination)
         with open(destination, 'r') as file:
             config = file.read()
-        config = config.replace("core.uidapi.com", urlparse(self.configs.get("core_base_url")).netloc)
-        config = config.replace("optout.uidapi.com", urlparse(self.configs.get("optout_base_url")).netloc)
+        config = config.replace("https://core.uidapi.com", self.configs.get("core_base_url"))
+        config = config.replace("https://optout.uidapi.com", self.configs.get("optout_base_url"))
         with open(destination, 'w') as file:
             file.write(config)