diff --git a/Makefile.eif b/Makefile.eif index 38e47c13c..2b65069f5 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -28,11 +28,14 @@ euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/s .PHONY: build_configs -build_configs: build/conf/default-config.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml +build_configs: build/conf/default-config.json build/conf/feat-flag.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json cp ./scripts/aws/conf/default-config.json ./build/conf/ +build/conf/feat-flag.json: build_artifacts ./scripts/aws/conf/feat-flag/feat-flag.json + cp ./scripts/aws/conf/feat-flag/feat-flag.json ./build/conf/ + build/conf/euid-integ-config.json: build_artifacts ./scripts/aws/conf/euid-integ-config.json cp ./scripts/aws/conf/euid-integ-config.json ./build/conf/ diff --git a/pom.xml b/pom.xml index ad02103be..00fac30af 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.47 + 5.47.73-alpha-180-SNAPSHOT UTF-8 @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.32 + 8.1.10 ${project.version} 21 21 diff --git a/scripts/aws/Dockerfile b/scripts/aws/Dockerfile index b0d6a10ff..88e1a2242 100644 --- a/scripts/aws/Dockerfile +++ b/scripts/aws/Dockerfile @@ -37,7 +37,7 @@ COPY ./conf/default-config.json /app/conf/ COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ COPY ./syslog-ng-client.conf /etc/syslog-ng/syslog-ng.conf -COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ +COPY ./conf/feat-flag.json /app/conf/feat-flag/ RUN chmod +x /app/vsockpx && chmod +x /app/entrypoint.sh diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index 35c3be58c..8f4477336 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -30,9 +30,12 @@ "service_links_metadata_path": "service_links/metadata.json", "optout_metadata_path": null, "optout_inmem_cache": false, - "enclave_platform": null, + "enclave_platform": "aws-nitro", "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, + "identity_token_expires_after_seconds": 86400, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, "operator_type": "private" -} +} \ No newline at end of file diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json index ca7c331e3..9e82cee9d 100644 --- a/scripts/aws/conf/euid-integ-config.json +++ b/scripts/aws/conf/euid-integ-config.json @@ -11,12 +11,5 @@ "core_attest_url": "https://core.integ.euid.eu/attest", "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", "optout_s3_folder": "optout/", - "allow_legacy_api": false, - "runtime_config_store": { - "type": "http", - "config" : { - "url": "https://core.integ.euid.eu/operator/config" - }, - "config_scan_period_ms": 300000 - } + "allow_legacy_api": false } \ No newline at end of file diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json index 8f0252082..8ab5b3e7b 100644 --- a/scripts/aws/conf/uid2-integ-config.json +++ b/scripts/aws/conf/uid2-integ-config.json @@ -1,22 +1,14 @@ { + "core_attest_url": "https://core-integ.uidapi.com/attest", + "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "sites_metadata_path": "https://core-integ.uidapi.com/sites/refresh", "clients_metadata_path": "https://core-integ.uidapi.com/clients/refresh", + "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", "keysets_metadata_path": "https://core-integ.uidapi.com/key/keyset/refresh", "keyset_keys_metadata_path": "https://core-integ.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "core_attest_url": "https://core-integ.uidapi.com/attest", - "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", - "optout_s3_folder": "uid-optout-integ/", - "allow_legacy_api": false, - "runtime_config_store": { - "type": "http", - "config" : { - "url": "https://core-integ.uidapi.com/operator/config" - }, - "config_scan_period_ms": 300000 - } -} \ No newline at end of file + "optout_s3_folder": "uid-optout-integ/" +} diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index fc9243fc2..627a6c88c 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -15,7 +15,7 @@ import sys import time import yaml - +logging.basicConfig(level=logging.INFO) sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, InstanceProfileMissingError, OperatorKeyNotFoundError, ConfigurationValueError, ConfidentialComputeStartupError diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 5f324ce20..4d67a14fa 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -10,21 +10,18 @@ exec &> >(tee -a "$LOG_FILE") PARAMETERIZED_CONFIG="/app/conf/config-overrides.json" OPERATOR_CONFIG="/tmp/final-config.json" -setup_auxiliaries() { - set -o pipefail - ulimit -n 65536 +set -o pipefail +ulimit -n 65536 - # -- setup loopback device - echo "Setting up loopback device..." - ifconfig lo 127.0.0.1 +# -- setup loopback device +echo "Setting up loopback device..." +ifconfig lo 127.0.0.1 - # -- start vsock proxy - echo "Starting vsock proxy..." - /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 - - /usr/sbin/syslog-ng --verbose -} +# -- start vsock proxy +echo "Starting vsock proxy..." +/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 +/usr/sbin/syslog-ng --verbose build_parameterized_config() { curl -s -f -o "${PARAMETERIZED_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig @@ -46,9 +43,7 @@ build_parameterized_config() { build_operator_config() { CORE_BASE_URL=$(jq -r ".core_base_url" < "${PARAMETERIZED_CONFIG}") - CORE_BASE_URL=$(echo "$CORE_BASE_URL" | sed -E 's#^(https?://)?([^/]+).*#\2#') OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${PARAMETERIZED_CONFIG}") - OPTOUT_BASE_URL=$(echo "$OPTOUT_BASE_URL" | sed -E 's#^(https?://)?([^/]+).*#\2#') DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${PARAMETERIZED_CONFIG}") DEBUG_MODE=$(jq -r ".debug_mode" < "${PARAMETERIZED_CONFIG}") @@ -69,11 +64,11 @@ build_operator_config() { if [[ "$DEPLOYMENT_ENVIRONMENT" != "prod" ]]; then #Allow override of base URL in non-prod environments - CORE_PATTERN="core.*uidapi.com" - OPTOUT_PATTERN="optout.*uidapi.com" - if [[ "$DEPLOYMENT_ENVIRONMENT" == "euid" ]]; then - CORE_PATTERN="core.*euid.eu" - OPTOUT_PATTERN="optout.*euid.eu" + CORE_PATTERN="https://core.*uidapi.com" + OPTOUT_PATTERN="https://optout.*uidapi.com" + if [[ "$IDENTITY_SCOPE_LOWER" == "euid" ]]; then + CORE_PATTERN="https://core.*euid.eu" + OPTOUT_PATTERN="https://optout.*euid.eu" fi sed -i "s#${CORE_PATTERN}#${CORE_BASE_URL}#g" "${OPERATOR_CONFIG}" sed -i "s#${OPTOUT_PATTERN}#${OPTOUT_BASE_URL}#g" "${OPERATOR_CONFIG}" @@ -81,10 +76,10 @@ build_operator_config() { } -setup_auxiliaries build_parameterized_config build_operator_config + DEBUG_MODE=$(jq -r ".debug_mode" < "${OPERATOR_CONFIG}") LOGBACK_CONF="./conf/logback.xml" @@ -107,3 +102,4 @@ java \ -Dlogback.configurationFile=${LOGBACK_CONF} \ -Dhttp_proxy=socks5://127.0.0.1:3305 \ -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar + diff --git a/scripts/azure-cc/azureEntryPoint.py b/scripts/azure-cc/azureEntryPoint.py index 25348e9fd..ffa49b3c7 100644 --- a/scripts/azure-cc/azureEntryPoint.py +++ b/scripts/azure-cc/azureEntryPoint.py @@ -8,7 +8,6 @@ import shutil import requests import logging -from urllib.parse import urlparse from confidential_compute import ConfidentialCompute, ConfigurationMissingError, OperatorKeyPermissionError, OperatorKeyNotFoundError, ConfidentialComputeStartupError from azure.keyvault.secrets import SecretClient from azure.identity import DefaultAzureCredential, CredentialUnavailableError @@ -22,6 +21,8 @@ class AzureEntryPoint(ConfidentialCompute): env_name = os.getenv("DEPLOYMENT_ENVIRONMENT") jar_name = os.getenv("JAR_NAME", "default-jar-name") jar_version = os.getenv("JAR_VERSION", "default-jar-version") + default_core_endpoint = f"https://core-{env_name}.uidapi.com".lower() + default_optout_endpoint = f"https://optout-{env_name}.uidapi.com".lower() FINAL_CONFIG = "/tmp/final-config.json" @@ -50,37 +51,25 @@ def __create_final_config(self): except IOError as e: logging.error(f"Failed to create {AzureEntryPoint.FINAL_CONFIG} with error: {e}") sys.exit(1) - - CORE_BASE_URL = os.getenv("CORE_BASE_URL") - OPTOUT_BASE_URL = os.getenv("OPTOUT_BASE_URL") - if CORE_BASE_URL and OPTOUT_BASE_URL and AzureEntryPoint.env_name != 'prod': - logging.info(f"-- replacing URLs by {CORE_BASE_URL} and {OPTOUT_BASE_URL}") - with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: - config = file.read() - - config = config.replace("core-integ.uidapi.com", urlparse(CORE_BASE_URL).netloc) - config = config.replace("optout-integ.uidapi.com", urlparse(OPTOUT_BASE_URL).netloc) + logging.info(f"-- replacing URLs by {self.configs["core_base_url"]} and {self.configs["optout_base_url"]}") + with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: + config = file.read() - with open(AzureEntryPoint.FINAL_CONFIG, "w") as file: - file.write(config) + config = config.replace("https://core.uidapi.com", self.configs["core_base_url"]) + config = config.replace("https://optout.uidapi.com", self.configs["optout_base_url"]) + with open(AzureEntryPoint.FINAL_CONFIG, "w") as file: + file.write(config) with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: logging.info(file.read()) - def __set_base_urls(self): - with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: - jdata = json.load(file) - self.configs["core_base_url"] = jdata["core_attest_url"] - self.configs["optout_base_url"] = jdata["optout_api_uri"] - def __set_operator_key(self): try: credential = DefaultAzureCredential() kv_URL = f"https://{AzureEntryPoint.kv_name}.vault.azure.net" secret_client = SecretClient(vault_url=kv_URL, credential=credential) secret = secret_client.get_secret(AzureEntryPoint.secret_name) - # print(f"Secret Value: {secret.value}") self.configs["operator_key"] = secret.value except (CredentialUnavailableError, ClientAuthenticationError) as auth_error: @@ -92,14 +81,13 @@ def __set_operator_key(self): def _set_confidential_config(self, secret_identifier: str = None): + """Builds and sets ConfidentialComputeConfig""" self.configs["skip_validations"] = os.getenv("SKIP_VALIDATIONS", "false").lower() == "true" self.configs["debug_mode"] = os.getenv("DEBUG_MODE", "false").lower() == "true" self.configs["environment"] = AzureEntryPoint.env_name - - # set self.configs["operator_key"] + self.configs["core_base_url"] = os.getenv("CORE_BASE_URL") if os.getenv("CORE_BASE_URL") and AzureEntryPoint.env_name == "integ" else AzureEntryPoint.default_core_endpoint + self.configs["optout_base_url"] = os.getenv("OPTOUT_BASE_URL") if os.getenv("OPTOUT_BASE_URL") and AzureEntryPoint.env_name == "integ" else AzureEntryPoint.default_optout_endpoint self.__set_operator_key() - # set base urls from final config file - self.__set_base_urls() def __run_operator(self): @@ -150,10 +138,10 @@ def _validate_auxiliaries(self): def run_compute(self) -> None: """Main execution flow for confidential compute.""" self.__check_env_variables() - self.__create_final_config() self._set_confidential_config() if not self.configs.get("skip_validations"): self.validate_configuration() + self.__create_final_config() self._setup_auxiliaries() self.__run_operator() diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index e3d34d846..569304e35 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -1,20 +1,20 @@ { - "sites_metadata_path": "https://core-integ.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core-integ.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core-integ.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core-integ.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", - "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "core_attest_url": "https://core-integ.uidapi.com/attest", - "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", "runtime_config_store": { "type": "http", "config" : { - "url": "https://core-integ.uidapi.com/operator/config" + "url": "https://core.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index bccbc2444..9c4eba18a 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -1,21 +1,21 @@ { - "sites_metadata_path": "https://core-prod.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core-prod.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core-prod.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core-prod.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-prod.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core-prod.uidapi.com/salt/refresh", - "services_metadata_path": "https://core-prod.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", - "core_attest_url": "https://core-prod.uidapi.com/attest", - "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", "identity_token_expires_after_seconds": 259200, "runtime_config_store": { "type": "http", "config" : { - "url": "https://core-prod.uidapi.com/operator/config" + "url": "https://core.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py index dd8f2d5bc..bb77fb39a 100644 --- a/scripts/confidential_compute.py +++ b/scripts/confidential_compute.py @@ -98,7 +98,7 @@ def validate_connectivity() -> None: type_hints = get_type_hints(ConfidentialComputeConfig, include_extras=True) required_keys = [field for field, hint in type_hints.items() if "NotRequired" not in str(hint)] - missing_keys = [key for key in required_keys if key not in self.configs] + missing_keys = [key for key in required_keys if key not in self.configs or self.configs[key] == None] if missing_keys: raise ConfigurationMissingError(self.__class__.__name__, missing_keys) @@ -144,7 +144,7 @@ def run_command(command, separate_process=False): if separate_process: subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: - subprocess.run(command,check=True) + subprocess.run(command,check=True,text=True) except Exception as e: logging.error(f"Failed to run command: {e}", exc_info=True) raise RuntimeError (f"Failed to start {' '.join(command)} ") \ No newline at end of file diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py index c2704c7ed..ce91c9f32 100644 --- a/scripts/gcp-oidc/gcp.py +++ b/scripts/gcp-oidc/gcp.py @@ -6,10 +6,8 @@ import sys import logging from google.cloud import secretmanager -from google.auth import default from google.auth.exceptions import DefaultCredentialsError from google.api_core.exceptions import PermissionDenied, NotFound -from urllib.parse import urlparse sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, ConfigurationMissingError, OperatorKeyNotFoundError, OperatorKeyPermissionError, ConfidentialComputeStartupError @@ -50,8 +48,8 @@ def __populate_operator_config(self, destination): shutil.copy(target_config, destination) with open(destination, 'r') as file: config = file.read() - config = config.replace("core.uidapi.com", urlparse(self.configs.get("core_base_url")).netloc) - config = config.replace("optout.uidapi.com", urlparse(self.configs.get("optout_base_url")).netloc) + config = config.replace("https://core.uidapi.com", self.configs.get("core_base_url")) + config = config.replace("https://optout.uidapi.com", self.configs.get("optout_base_url")) with open(destination, 'w') as file: file.write(config)