From de2d61d56dc88aeb0524596cf8699772ac07aefe Mon Sep 17 00:00:00 2001 From: huangxu17 Date: Mon, 8 Apr 2024 20:53:04 +0800 Subject: [PATCH 1/5] Meituan HDFS Access Without Proxy for Kerberos Authentication --- deploy/scripts/sgx/run_trainer_worker_sgx.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/scripts/sgx/run_trainer_worker_sgx.sh b/deploy/scripts/sgx/run_trainer_worker_sgx.sh index dc4c5543c..1771f75fc 100755 --- a/deploy/scripts/sgx/run_trainer_worker_sgx.sh +++ b/deploy/scripts/sgx/run_trainer_worker_sgx.sh @@ -32,6 +32,7 @@ fi echo $PROXY_LOCAL_PORT > /pod-data/proxy_local_port cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./ +unset HTTPS_PROXY https_proxy http_proxy ftp_proxy source /app/deploy/scripts/hdfs_common.sh || true source /app/deploy/scripts/pre_start_hook.sh || true source /app/deploy/scripts/env_to_args.sh @@ -48,8 +49,6 @@ cp /app/sgx/gramine/CI-Examples/tensorflow_io.py /gramine/follower/ cp /app/sgx/gramine/CI-Examples/tensorflow_io.py /gramine/leader/ source /app/deploy/scripts/sgx/enclave_env.sh worker -unset HTTPS_PROXY https_proxy http_proxy ftp_proxy - mode=$(normalize_env_to_args "--mode" "$MODE") sparse_estimator=$(normalize_env_to_args "--sparse-estimator" "$SPARSE_ESTIMATOR") batch_size=$(normalize_env_to_args "--batch-size" "$BATCH_SIZE") From d839cc9d3ce45a3b8efec697244219b353a3f592 Mon Sep 17 00:00:00 2001 From: huangxu17 Date: Tue, 9 Apr 2024 13:16:30 +0800 Subject: [PATCH 2/5] Meituan HDFS Access Without Proxy for Kerberos Authentication --- deploy/scripts/sgx/run_trainer_ps_sgx.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/scripts/sgx/run_trainer_ps_sgx.sh b/deploy/scripts/sgx/run_trainer_ps_sgx.sh index 84450ad10..de156bf6b 100755 --- a/deploy/scripts/sgx/run_trainer_ps_sgx.sh +++ b/deploy/scripts/sgx/run_trainer_ps_sgx.sh @@ -18,6 +18,7 @@ set -ex source ~/.env export CUDA_VISIBLE_DEVICES= cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./ +unset HTTPS_PROXY https_proxy http_proxy ftp_proxy source /app/deploy/scripts/hdfs_common.sh || true source /app/deploy/scripts/pre_start_hook.sh || true source /app/deploy/scripts/env_to_args.sh @@ -48,4 +49,4 @@ if [[ -z "${END_CPU_SN}" ]]; then END_CPU_SN=3 fi -taskset -c $START_CPU_SN-$END_CPU_SN stdbuf -o0 gramine-sgx python -m fedlearner.trainer.parameter_server $POD_IP:${LISTEN_PORT} \ No newline at end of file +taskset -c $START_CPU_SN-$END_CPU_SN stdbuf -o0 gramine-sgx python -m fedlearner.trainer.parameter_server $POD_IP:${LISTEN_PORT} From 5cc16a8ce8ac9857a55cf4c58b8cfb6493a05ce0 Mon Sep 17 00:00:00 2001 From: huangxu17 Date: Thu, 11 Apr 2024 23:00:53 +0800 Subject: [PATCH 3/5] 1. FedLearner Framework and Core Dependency RA-TLS Configuration To ensure code consistency and implement hash signature-based remote attestation for the FedLearner framework and its core dependencies, an environment variable has been introduced in Gramine that prevents the generation of .pyc files. 2. Gramine Template Configuration for FedLearner The Gramine template configuration has been updated to include the code locations for the FedLearner framework and some essential dependencies. This addition facilitates bidirectional remote attestation between parties. 3. Meituan HDFS File Path Management Optimization The code responsible for handling file paths in Meituan's Hadoop Distributed File System (HDFS) has been migrated from the main entry point to be processed by the master node. This change aims to prevent file read and write conflicts that could arise from multiple workers operating simultaneously. --- fedlearner-sgx-dev.dockerfile | 31 +++++--- fedlearner/trainer/trainer_worker.py | 54 ++++++------- .../generate-token/python.manifest.template | 76 +++++++++++++++++-- 3 files changed, 117 insertions(+), 44 deletions(-) diff --git a/fedlearner-sgx-dev.dockerfile b/fedlearner-sgx-dev.dockerfile index 56e6a9d8e..915c70c09 100644 --- a/fedlearner-sgx-dev.dockerfile +++ b/fedlearner-sgx-dev.dockerfile @@ -6,6 +6,8 @@ ENV DEBIAN_FRONTEND=noninteractive ENV INSTALL_PREFIX=/usr/local ENV LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:${INSTALL_PREFIX}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH} ENV PATH=${INSTALL_PREFIX}/bin:${LD_LIBRARY_PATH}:${PATH} +# For Gramine RA-TLS +ENV PYTHONDONTWRITEBYTECODE=1 # Add steps here to set up common dependencies RUN apt-get update \ @@ -60,8 +62,8 @@ RUN apt-get install -y libcurl4-openssl-dev libprotobuf-c-dev python3-protobuf p RUN apt-get install -y libgmp-dev libmpfr-dev libmpc-dev libisl-dev nasm protobuf-compiler RUN ln -s /usr/bin/python3 /usr/bin/python \ - && pip3 install --upgrade pip \ - && pip3 install toml meson pyelftools + && pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \ + && pip3 install --no-compile toml meson pyelftools -i https://mirrors.aliyun.com/pypi/simple/ RUN git clone https://github.com/analytics-zoo/gramine ${GRAMINEDIR} \ && cd ${GRAMINEDIR} \ @@ -101,8 +103,8 @@ ENV GRPC_VERSION=v1.38.1 RUN git clone --recurse-submodules -b ${GRPC_VERSION} https://github.com/grpc/grpc ${GRPC_PATH} -RUN pip3 install --upgrade pip \ - && pip3 install -r ${GRPC_PATH}/requirements.txt +RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \ + && pip3 install --no-compile -r ${GRPC_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ # Tensorflow dependencies ENV BAZEL_VERSION=3.1.0 @@ -110,8 +112,8 @@ ENV TF_VERSION=v2.4.2 ENV TF_BUILD_PATH=/tf/src ENV TF_BUILD_OUTPUT=/tf/output -RUN pip3 install --upgrade pip \ - && pip3 install numpy keras_preprocessing +RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \ + && pip3 install --no-compile numpy keras_preprocessing -i https://mirrors.aliyun.com/pypi/simple/ RUN wget "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb" \ && dpkg -i bazel_*.deb @@ -127,7 +129,7 @@ RUN apt-get install -y libmysqlclient-dev COPY sgx/grpc/common ${GRPC_PATH} COPY sgx/grpc/v1.38.1 ${GRPC_PATH} -RUN pip3 install 'cython==0.29.36' +RUN pip3 install --no-compile 'cython==0.29.36' -i https://mirrors.aliyun.com/pypi/simple/ RUN ${GRPC_PATH}/build_python.sh # Build tensorflow @@ -150,17 +152,17 @@ RUN if [ -f ${FEDLEARNER_PATH}/docker/hadoop-mt-2.7.0.tar.gz ]; then mkdir -p /o # For meituan hadoop auth RUN apt-get install -y libkrb5-dev openjdk-8-jdk -RUN pip3 install --upgrade pip \ - && pip3 install -r ${FEDLEARNER_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ +RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \ + && pip3 install --no-compile -r ${FEDLEARNER_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ RUN cd ${FEDLEARNER_PATH} \ && make protobuf \ && python3 setup.py bdist_wheel \ - && pip3 install ./dist/*.whl + && pip3 install --no-compile ./dist/*.whl # Re-install tensorflow, uninstall tensorflow_io, mock it RUN pip3 uninstall -y tensorflow tensorflow-io \ - && pip3 install ${TF_BUILD_OUTPUT}/*.whl + && pip3 install --no-compile ${TF_BUILD_OUTPUT}/*.whl # Re-install fedlearner plugin RUN cd ${FEDLEARNER_PATH} \ @@ -170,7 +172,7 @@ RUN cd ${FEDLEARNER_PATH} \ # Re-install grpcio RUN pip3 uninstall -y grpcio \ - && pip3 install ${GRPC_PATH}/dist/grpcio*.whl + && pip3 install --no-compile ${GRPC_PATH}/dist/grpcio*.whl # For debug RUN apt-get install -y strace gdb ctags vim @@ -182,6 +184,11 @@ COPY sgx/configs / RUN echo "enabled=0" > /etc/default/apport RUN echo "exit 0" > /usr/sbin/policy-rc.d +# For gramine ra-tls +RUN dpkg --remove --force-depends libgtk2.0-0 \ + && pip3 uninstall -y numpy keras_preprocessing protobuf \ + && pip3 install --no-compile numpy keras_preprocessing protobuf -i https://mirrors.aliyun.com/pypi/simple/ + # Clean tmp files RUN apt-get clean all \ && rm -rf /var/lib/apt/lists/* \ diff --git a/fedlearner/trainer/trainer_worker.py b/fedlearner/trainer/trainer_worker.py index 4827272b6..caa5bf292 100644 --- a/fedlearner/trainer/trainer_worker.py +++ b/fedlearner/trainer/trainer_worker.py @@ -232,6 +232,33 @@ def _run_master(role, cluster_server = ClusterServer(cluster_spec, "master", server_port=args.server_port) + # use Meituan hadoop + # first：convert Meituan HDFS path to local storage path, if local exit psi result file, user local file + # second：if local not exit psi result file，from Meituan HDFS download to local + if args.using_mt_hadoop: + data_path = args.data_path + if data_path: + local_data_path = get_local_temp_path(data_path) + if not exists(local_data_path): + data_path = mt_hadoop_download(data_path) + else: + data_path = local_data_path + args.data_path = data_path + + checkpoint_path = args.checkpoint_path + if checkpoint_path: + args.checkpoint_path = get_local_temp_path(checkpoint_path) + + load_checkpoint_path = args.load_checkpoint_path + if load_checkpoint_path: + args.load_checkpoint_path = get_local_temp_path(load_checkpoint_path) + if not exists(args.load_checkpoint_path): + mt_hadoop_download(load_checkpoint_path) + + export_path = args.export_path + if export_path: + args.export_path = get_local_temp_path(export_path) + checkpoint_filename_with_path = _get_checkpoint_filename_with_path(args) data_visitor = _create_data_visitor(args) master_factory = LeaderTrainerMaster \ @@ -479,33 +506,6 @@ def train(role, if not isinstance(role, str) or role.lower() not in (LEADER, FOLLOER): raise ValueError("--role must set one of %s or %s"%(LEADER, FOLLOER)) - # use Meituan hadoop - # first：convert Meituan HDFS path to local storage path, if local exit psi result file, user local file - # second：if local not exit psi result file，from Meituan HDFS download to local - if args.using_mt_hadoop: - data_path = args.data_path - if data_path: - local_data_path = get_local_temp_path(data_path) - if not exists(local_data_path): - data_path = mt_hadoop_download(data_path) - else: - data_path = local_data_path - args.data_path = data_path - - checkpoint_path = args.checkpoint_path - if checkpoint_path: - args.checkpoint_path = get_local_temp_path(checkpoint_path) - - load_checkpoint_path = args.load_checkpoint_path - if load_checkpoint_path: - args.load_checkpoint_path = get_local_temp_path(load_checkpoint_path) - if not exists(args.load_checkpoint_path): - mt_hadoop_download(load_checkpoint_path) - - export_path = args.export_path - if export_path: - args.export_path = get_local_temp_path(export_path) - if args.loglevel: fl_logging.set_level(args.loglevel) diff --git a/sgx/gramine/CI-Examples/generate-token/python.manifest.template b/sgx/gramine/CI-Examples/generate-token/python.manifest.template index 573bbd4c0..185e75f55 100644 --- a/sgx/gramine/CI-Examples/generate-token/python.manifest.template +++ b/sgx/gramine/CI-Examples/generate-token/python.manifest.template @@ -67,12 +67,75 @@ sgx.trusted_files = [ "file:/usr/{{ arch_libdir }}/", "file:/etc/ssl/certs/ca-certificates.crt", "file:/etc/default/apport", - "file:/usr/local/lib/", - "file:{{ python.stdlib }}/", - "file:{{ python.distlib }}/", "file:/etc/mime.types", "file:/gramine/leader/", - "file:/gramine/follower/" + "file:/gramine/follower/", + "file:/usr/local/lib/x86_64-linux-gnu/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/libtensorflow_framework.so.2", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__init__.py", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__internal__/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__operators__/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/audio/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/autodiff/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/autograph/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/bitwise/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/compat/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/config/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/data/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/debugging/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/distribute/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/dtypes/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/errors/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/experimental/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/feature_column/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/graph_util/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/image/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/io/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/linalg/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/lite/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/lookup/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/math/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/mixed_precision/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/mlir/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/nest/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/nn/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/profiler/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/quantization/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/queue/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/ragged/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/random/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/raw_ops/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/saved_model/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sets/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/signal/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sparse/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/strings/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/summary/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sysconfig/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/test/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/tpu/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/train/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/types/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/version/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/xla/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/compiler/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/core/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/include/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/keras/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/python/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/tools/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/experimental/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/toco/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/xla_aot_runtime_src/", + "file:/usr/local/lib/python3.6/dist-packages/pyspark/", + "file:/usr/local/lib/python3.6/dist-packages/Cython/", + "file:/usr/local/lib/python3.6/dist-packages/numpy/", + "file:/usr/local/lib/python3.6/dist-packages/grpc/", + "file:/usr/local/lib/python3.6/dist-packages/google/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/", + "file:/usr/local/lib/python3.6/dist-packages/sklearn/", + "file:/usr/local/lib/python3.6/dist-packages/fedlearner/", + "file:/usr/local/lib/python3.6/dist-packages/pandas/" ] sgx.allowed_files = [ @@ -81,6 +144,9 @@ sgx.allowed_files = [ "file:/opt/meituan/", "file:/usr/lib/ssl/openssl.cnf", "file:/usr/lib/gcc", + "file:/usr/local/lib/", + "file:{{ python.stdlib }}/", + "file:{{ python.distlib }}/", "file:/etc/ethers", "file:/etc/hosts", "file:/etc/group", @@ -103,4 +169,4 @@ sgx.allowed_files = [ "file:/lib/", "file:/bin/", "file:/data/", -] \ No newline at end of file +] From a39baec6fedb2712cd355cc3f705dd337c768c32 Mon Sep 17 00:00:00 2001 From: huangxu17 Date: Thu, 11 Apr 2024 23:00:53 +0800 Subject: [PATCH 4/5] Fix gramine allowed files lib risk, add sub path to allowed files --- fedlearner-sgx-dev.dockerfile | 31 +- fedlearner/trainer/trainer_worker.py | 54 +-- setup.py | 3 +- .../generate-token/python.manifest.template | 409 +++++++++++++++++- 4 files changed, 451 insertions(+), 46 deletions(-) diff --git a/fedlearner-sgx-dev.dockerfile b/fedlearner-sgx-dev.dockerfile index 56e6a9d8e..915c70c09 100644 --- a/fedlearner-sgx-dev.dockerfile +++ b/fedlearner-sgx-dev.dockerfile @@ -6,6 +6,8 @@ ENV DEBIAN_FRONTEND=noninteractive ENV INSTALL_PREFIX=/usr/local ENV LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:${INSTALL_PREFIX}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH} ENV PATH=${INSTALL_PREFIX}/bin:${LD_LIBRARY_PATH}:${PATH} +# For Gramine RA-TLS +ENV PYTHONDONTWRITEBYTECODE=1 # Add steps here to set up common dependencies RUN apt-get update \ @@ -60,8 +62,8 @@ RUN apt-get install -y libcurl4-openssl-dev libprotobuf-c-dev python3-protobuf p RUN apt-get install -y libgmp-dev libmpfr-dev libmpc-dev libisl-dev nasm protobuf-compiler RUN ln -s /usr/bin/python3 /usr/bin/python \ - && pip3 install --upgrade pip \ - && pip3 install toml meson pyelftools + && pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \ + && pip3 install --no-compile toml meson pyelftools -i https://mirrors.aliyun.com/pypi/simple/ RUN git clone https://github.com/analytics-zoo/gramine ${GRAMINEDIR} \ && cd ${GRAMINEDIR} \ @@ -101,8 +103,8 @@ ENV GRPC_VERSION=v1.38.1 RUN git clone --recurse-submodules -b ${GRPC_VERSION} https://github.com/grpc/grpc ${GRPC_PATH} -RUN pip3 install --upgrade pip \ - && pip3 install -r ${GRPC_PATH}/requirements.txt +RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \ + && pip3 install --no-compile -r ${GRPC_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ # Tensorflow dependencies ENV BAZEL_VERSION=3.1.0 @@ -110,8 +112,8 @@ ENV TF_VERSION=v2.4.2 ENV TF_BUILD_PATH=/tf/src ENV TF_BUILD_OUTPUT=/tf/output -RUN pip3 install --upgrade pip \ - && pip3 install numpy keras_preprocessing +RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \ + && pip3 install --no-compile numpy keras_preprocessing -i https://mirrors.aliyun.com/pypi/simple/ RUN wget "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb" \ && dpkg -i bazel_*.deb @@ -127,7 +129,7 @@ RUN apt-get install -y libmysqlclient-dev COPY sgx/grpc/common ${GRPC_PATH} COPY sgx/grpc/v1.38.1 ${GRPC_PATH} -RUN pip3 install 'cython==0.29.36' +RUN pip3 install --no-compile 'cython==0.29.36' -i https://mirrors.aliyun.com/pypi/simple/ RUN ${GRPC_PATH}/build_python.sh # Build tensorflow @@ -150,17 +152,17 @@ RUN if [ -f ${FEDLEARNER_PATH}/docker/hadoop-mt-2.7.0.tar.gz ]; then mkdir -p /o # For meituan hadoop auth RUN apt-get install -y libkrb5-dev openjdk-8-jdk -RUN pip3 install --upgrade pip \ - && pip3 install -r ${FEDLEARNER_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ +RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \ + && pip3 install --no-compile -r ${FEDLEARNER_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ RUN cd ${FEDLEARNER_PATH} \ && make protobuf \ && python3 setup.py bdist_wheel \ - && pip3 install ./dist/*.whl + && pip3 install --no-compile ./dist/*.whl # Re-install tensorflow, uninstall tensorflow_io, mock it RUN pip3 uninstall -y tensorflow tensorflow-io \ - && pip3 install ${TF_BUILD_OUTPUT}/*.whl + && pip3 install --no-compile ${TF_BUILD_OUTPUT}/*.whl # Re-install fedlearner plugin RUN cd ${FEDLEARNER_PATH} \ @@ -170,7 +172,7 @@ RUN cd ${FEDLEARNER_PATH} \ # Re-install grpcio RUN pip3 uninstall -y grpcio \ - && pip3 install ${GRPC_PATH}/dist/grpcio*.whl + && pip3 install --no-compile ${GRPC_PATH}/dist/grpcio*.whl # For debug RUN apt-get install -y strace gdb ctags vim @@ -182,6 +184,11 @@ COPY sgx/configs / RUN echo "enabled=0" > /etc/default/apport RUN echo "exit 0" > /usr/sbin/policy-rc.d +# For gramine ra-tls +RUN dpkg --remove --force-depends libgtk2.0-0 \ + && pip3 uninstall -y numpy keras_preprocessing protobuf \ + && pip3 install --no-compile numpy keras_preprocessing protobuf -i https://mirrors.aliyun.com/pypi/simple/ + # Clean tmp files RUN apt-get clean all \ && rm -rf /var/lib/apt/lists/* \ diff --git a/fedlearner/trainer/trainer_worker.py b/fedlearner/trainer/trainer_worker.py index 4827272b6..caa5bf292 100644 --- a/fedlearner/trainer/trainer_worker.py +++ b/fedlearner/trainer/trainer_worker.py @@ -232,6 +232,33 @@ def _run_master(role, cluster_server = ClusterServer(cluster_spec, "master", server_port=args.server_port) + # use Meituan hadoop + # first：convert Meituan HDFS path to local storage path, if local exit psi result file, user local file + # second：if local not exit psi result file，from Meituan HDFS download to local + if args.using_mt_hadoop: + data_path = args.data_path + if data_path: + local_data_path = get_local_temp_path(data_path) + if not exists(local_data_path): + data_path = mt_hadoop_download(data_path) + else: + data_path = local_data_path + args.data_path = data_path + + checkpoint_path = args.checkpoint_path + if checkpoint_path: + args.checkpoint_path = get_local_temp_path(checkpoint_path) + + load_checkpoint_path = args.load_checkpoint_path + if load_checkpoint_path: + args.load_checkpoint_path = get_local_temp_path(load_checkpoint_path) + if not exists(args.load_checkpoint_path): + mt_hadoop_download(load_checkpoint_path) + + export_path = args.export_path + if export_path: + args.export_path = get_local_temp_path(export_path) + checkpoint_filename_with_path = _get_checkpoint_filename_with_path(args) data_visitor = _create_data_visitor(args) master_factory = LeaderTrainerMaster \ @@ -479,33 +506,6 @@ def train(role, if not isinstance(role, str) or role.lower() not in (LEADER, FOLLOER): raise ValueError("--role must set one of %s or %s"%(LEADER, FOLLOER)) - # use Meituan hadoop - # first：convert Meituan HDFS path to local storage path, if local exit psi result file, user local file - # second：if local not exit psi result file，from Meituan HDFS download to local - if args.using_mt_hadoop: - data_path = args.data_path - if data_path: - local_data_path = get_local_temp_path(data_path) - if not exists(local_data_path): - data_path = mt_hadoop_download(data_path) - else: - data_path = local_data_path - args.data_path = data_path - - checkpoint_path = args.checkpoint_path - if checkpoint_path: - args.checkpoint_path = get_local_temp_path(checkpoint_path) - - load_checkpoint_path = args.load_checkpoint_path - if load_checkpoint_path: - args.load_checkpoint_path = get_local_temp_path(load_checkpoint_path) - if not exists(args.load_checkpoint_path): - mt_hadoop_download(load_checkpoint_path) - - export_path = args.export_path - if export_path: - args.export_path = get_local_temp_path(export_path) - if args.loglevel: fl_logging.set_level(args.loglevel) diff --git a/setup.py b/setup.py index dceb3c304..fc7f33313 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,7 @@ def get_version(): base = "1.5" - day = time.strftime('%Y%m%d', time.localtime()) - return '%s-dev%s'%(base, day) + return '%s-dev' % base setup( name='fedlearner', diff --git a/sgx/gramine/CI-Examples/generate-token/python.manifest.template b/sgx/gramine/CI-Examples/generate-token/python.manifest.template index 573bbd4c0..c654adedc 100644 --- a/sgx/gramine/CI-Examples/generate-token/python.manifest.template +++ b/sgx/gramine/CI-Examples/generate-token/python.manifest.template @@ -67,12 +67,286 @@ sgx.trusted_files = [ "file:/usr/{{ arch_libdir }}/", "file:/etc/ssl/certs/ca-certificates.crt", "file:/etc/default/apport", - "file:/usr/local/lib/", - "file:{{ python.stdlib }}/", - "file:{{ python.distlib }}/", "file:/etc/mime.types", "file:/gramine/leader/", - "file:/gramine/follower/" + "file:/gramine/follower/", + "file:/usr/local/lib/cmake/", + "file:/usr/local/lib/libabsl_bad_any_cast_impl.a", + "file:/usr/local/lib/libabsl_bad_optional_access.a", + "file:/usr/local/lib/libabsl_bad_variant_access.a", + "file:/usr/local/lib/libabsl_base.a", + "file:/usr/local/lib/libabsl_city.a", + "file:/usr/local/lib/libabsl_civil_time.a", + "file:/usr/local/lib/libabsl_cord.a", + "file:/usr/local/lib/libabsl_debugging_internal.a", + "file:/usr/local/lib/libabsl_demangle_internal.a", + "file:/usr/local/lib/libabsl_examine_stack.a", + "file:/usr/local/lib/libabsl_exponential_biased.a", + "file:/usr/local/lib/libabsl_failure_signal_handler.a", + "file:/usr/local/lib/libabsl_flags.a", + "file:/usr/local/lib/libabsl_flags_commandlineflag.a", + "file:/usr/local/lib/libabsl_flags_commandlineflag_internal.a", + "file:/usr/local/lib/libabsl_flags_config.a", + "file:/usr/local/lib/libabsl_flags_internal.a", + "file:/usr/local/lib/libabsl_flags_marshalling.a", + "file:/usr/local/lib/libabsl_flags_parse.a", + "file:/usr/local/lib/libabsl_flags_private_handle_accessor.a", + "file:/usr/local/lib/libabsl_flags_program_name.a", + "file:/usr/local/lib/libabsl_flags_reflection.a", + "file:/usr/local/lib/libabsl_flags_usage.a", + "file:/usr/local/lib/libabsl_flags_usage_internal.a", + "file:/usr/local/lib/libabsl_graphcycles_internal.a", + "file:/usr/local/lib/libabsl_hash.a", + "file:/usr/local/lib/libabsl_hashtablez_sampler.a", + "file:/usr/local/lib/libabsl_int128.a", + "file:/usr/local/lib/libabsl_leak_check.a", + "file:/usr/local/lib/libabsl_leak_check_disable.a", + "file:/usr/local/lib/libabsl_log_severity.a", + "file:/usr/local/lib/libabsl_malloc_internal.a", + "file:/usr/local/lib/libabsl_periodic_sampler.a", + "file:/usr/local/lib/libabsl_random_distributions.a", + "file:/usr/local/lib/libabsl_random_internal_distribution_test_util.a", + "file:/usr/local/lib/libabsl_random_internal_platform.a", + "file:/usr/local/lib/libabsl_random_internal_pool_urbg.a", + "file:/usr/local/lib/libabsl_random_internal_randen.a", + "file:/usr/local/lib/libabsl_random_internal_randen_hwaes.a", + "file:/usr/local/lib/libabsl_random_internal_randen_hwaes_impl.a", + "file:/usr/local/lib/libabsl_random_internal_randen_slow.a", + "file:/usr/local/lib/libabsl_random_internal_seed_material.a", + "file:/usr/local/lib/libabsl_random_seed_gen_exception.a", + "file:/usr/local/lib/libabsl_random_seed_sequences.a", + "file:/usr/local/lib/libabsl_raw_hash_set.a", + "file:/usr/local/lib/libabsl_raw_logging_internal.a", + "file:/usr/local/lib/libabsl_scoped_set_env.a", + "file:/usr/local/lib/libabsl_spinlock_wait.a", + "file:/usr/local/lib/libabsl_stacktrace.a", + "file:/usr/local/lib/libabsl_status.a", + "file:/usr/local/lib/libabsl_statusor.a", + "file:/usr/local/lib/libabsl_str_format_internal.a", + "file:/usr/local/lib/libabsl_strerror.a", + "file:/usr/local/lib/libabsl_strings.a", + "file:/usr/local/lib/libabsl_strings_internal.a", + "file:/usr/local/lib/libabsl_symbolize.a", + "file:/usr/local/lib/libabsl_synchronization.a", + "file:/usr/local/lib/libabsl_throw_delegate.a", + "file:/usr/local/lib/libabsl_time.a", + "file:/usr/local/lib/libabsl_time_zone.a", + "file:/usr/local/lib/libabsl_wyhash.a", + "file:/usr/local/lib/libaddress_sorting.a", + "file:/usr/local/lib/libcares.a", + "file:/usr/local/lib/libcjson.a", + "file:/usr/local/lib/libcjson_utils.a", + "file:/usr/local/lib/libcrypto.a", + "file:/usr/local/lib/libgpr.a", + "file:/usr/local/lib/libgrpc++.a", + "file:/usr/local/lib/libgrpc++_alts.a", + "file:/usr/local/lib/libgrpc++_error_details.a", + "file:/usr/local/lib/libgrpc++_reflection.a", + "file:/usr/local/lib/libgrpc++_unsecure.a", + "file:/usr/local/lib/libgrpc.a", + "file:/usr/local/lib/libgrpc_plugin_support.a", + "file:/usr/local/lib/libgrpc_unsecure.a", + "file:/usr/local/lib/libgrpcpp_channelz.a", + "file:/usr/local/lib/libmbedcrypto_gramine.a", + "file:/usr/local/lib/libmbedtls_gramine.a", + "file:/usr/local/lib/libmbedx509_gramine.a", + "file:/usr/local/lib/libprotobuf-lite.a", + "file:/usr/local/lib/libprotobuf.a", + "file:/usr/local/lib/libprotoc.a", + "file:/usr/local/lib/libre2.a", + "file:/usr/local/lib/libssl.a", + "file:/usr/local/lib/libupb.a", + "file:/usr/local/lib/libz.a", + "file:/usr/local/lib/libz.so", + "file:/usr/local/lib/libz.so.1", + "file:/usr/local/lib/libz.so.1.2.11", + "file:/usr/local/lib/pkgconfig/", + "file:/usr/local/lib/x86_64-linux-gnu/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/libtensorflow_framework.so.2", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/__init__.py", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__init__.py", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__internal__/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__operators__/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/audio/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/autodiff/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/autograph/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/bitwise/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/compat/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/config/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/data/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/debugging/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/distribute/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/dtypes/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/errors/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/experimental/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/feature_column/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/graph_util/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/image/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/io/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/linalg/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/lite/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/lookup/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/math/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/mixed_precision/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/mlir/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/nest/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/nn/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/profiler/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/quantization/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/queue/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/ragged/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/random/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/raw_ops/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/saved_model/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sets/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/signal/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sparse/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/strings/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/summary/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sysconfig/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/test/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/tpu/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/train/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/types/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/version/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/xla/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/compiler/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/core/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/include/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/keras/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/python/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/tools/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/experimental/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/toco/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/__init__.py", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/xla_aot_runtime_src/", + "file:/usr/local/lib/python3.6/dist-packages/pyspark/", + "file:/usr/local/lib/python3.6/dist-packages/Cython/", + "file:/usr/local/lib/python3.6/dist-packages/numpy/", + "file:/usr/local/lib/python3.6/dist-packages/grpc/", + "file:/usr/local/lib/python3.6/dist-packages/google/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/", + "file:/usr/local/lib/python3.6/dist-packages/sklearn/", + "file:/usr/local/lib/python3.6/dist-packages/fedlearner/", + "file:/usr/local/lib/python3.6/dist-packages/pandas/", + "file:/usr/local/lib/python3.6/dist-packages/Cryptodome/", + "file:/usr/local/lib/python3.6/dist-packages/PIL/", + "file:/usr/local/lib/python3.6/dist-packages/Pillow.libs/", + "file:/usr/local/lib/python3.6/dist-packages/_cffi_backend.cpython-36m-x86_64-linux-gnu.so", + "file:/usr/local/lib/python3.6/dist-packages/_yaml/", + "file:/usr/local/lib/python3.6/dist-packages/absl/", + "file:/usr/local/lib/python3.6/dist-packages/aiocontextvars.py", + "file:/usr/local/lib/python3.6/dist-packages/apsw/", + "file:/usr/local/lib/python3.6/dist-packages/astroid/", + "file:/usr/local/lib/python3.6/dist-packages/astunparse/", + "file:/usr/local/lib/python3.6/dist-packages/backoff/", + "file:/usr/local/lib/python3.6/dist-packages/backports/", + "file:/usr/local/lib/python3.6/dist-packages/cachetools/", + "file:/usr/local/lib/python3.6/dist-packages/cc/", + "file:/usr/local/lib/python3.6/dist-packages/certifi/", + "file:/usr/local/lib/python3.6/dist-packages/cffi/", + "file:/usr/local/lib/python3.6/dist-packages/chardet/", + "file:/usr/local/lib/python3.6/dist-packages/cityhash.cpython-36m-x86_64-linux-gnu.so", + "file:/usr/local/lib/python3.6/dist-packages/cityhashcrc.cpython-36m-x86_64-linux-gnu.so", + "file:/usr/local/lib/python3.6/dist-packages/configparser.py", + "file:/usr/local/lib/python3.6/dist-packages/contextvars/", + "file:/usr/local/lib/python3.6/dist-packages/coverage/", + "file:/usr/local/lib/python3.6/dist-packages/cryptography/", + "file:/usr/local/lib/python3.6/dist-packages/cycler.py", + "file:/usr/local/lib/python3.6/dist-packages/cython.py", + "file:/usr/local/lib/python3.6/dist-packages/dataclasses.py", + "file:/usr/local/lib/python3.6/dist-packages/dateutil/", + "file:/usr/local/lib/python3.6/dist-packages/decorator.py", + "file:/usr/local/lib/python3.6/dist-packages/deprecated/", + "file:/usr/local/lib/python3.6/dist-packages/docopt.py", + "file:/usr/local/lib/python3.6/dist-packages/easy_install.py", + "file:/usr/local/lib/python3.6/dist-packages/elasticsearch/", + "file:/usr/local/lib/python3.6/dist-packages/elasticsearch6/", + "file:/usr/local/lib/python3.6/dist-packages/elftools/", + "file:/usr/local/lib/python3.6/dist-packages/etcd3/", + "file:/usr/local/lib/python3.6/dist-packages/farmhash.cpython-36m-x86_64-linux-gnu.so", + "file:/usr/local/lib/python3.6/dist-packages/flatbuffers/", + "file:/usr/local/lib/python3.6/dist-packages/flatten_dict/", + "file:/usr/local/lib/python3.6/dist-packages/fsspec/", + "file:/usr/local/lib/python3.6/dist-packages/gast/", + "file:/usr/local/lib/python3.6/dist-packages/gmpy2/", + "file:/usr/local/lib/python3.6/dist-packages/gmpy2.libs/", + "file:/usr/local/lib/python3.6/dist-packages/google_auth_oauthlib/", + "file:/usr/local/lib/python3.6/dist-packages/googleapis_common_protos-1.56.3-py3.10-nspkg.pth", + "file:/usr/local/lib/python3.6/dist-packages/graminelibos/", + "file:/usr/local/lib/python3.6/dist-packages/grpc_tools/", + "file:/usr/local/lib/python3.6/dist-packages/guppy/", + "file:/usr/local/lib/python3.6/dist-packages/h5py/", + "file:/usr/local/lib/python3.6/dist-packages/hdfs/", + "file:/usr/local/lib/python3.6/dist-packages/immutables/", + "file:/usr/local/lib/python3.6/dist-packages/importlib_metadata/", + "file:/usr/local/lib/python3.6/dist-packages/influxdb/", + "file:/usr/local/lib/python3.6/dist-packages/isort/", + "file:/usr/local/lib/python3.6/dist-packages/joblib/", + "file:/usr/local/lib/python3.6/dist-packages/keras_preprocessing/", + "file:/usr/local/lib/python3.6/dist-packages/kiwisolver.cpython-36m-x86_64-linux-gnu.so", + "file:/usr/local/lib/python3.6/dist-packages/kubernetes/", + "file:/usr/local/lib/python3.6/dist-packages/lazy_object_proxy/", + "file:/usr/local/lib/python3.6/dist-packages/markdown/", + "file:/usr/local/lib/python3.6/dist-packages/matplotlib/", + "file:/usr/local/lib/python3.6/dist-packages/matplotlib-3.3.4-py3.6-nspkg.pth", + "file:/usr/local/lib/python3.6/dist-packages/mccabe.py", + "file:/usr/local/lib/python3.6/dist-packages/mesonbuild/", + "file:/usr/local/lib/python3.6/dist-packages/mpl_toolkits/", + "file:/usr/local/lib/python3.6/dist-packages/msgpack/", + "file:/usr/local/lib/python3.6/dist-packages/numpy/", + "file:/usr/local/lib/python3.6/dist-packages/numpy.libs/", + "file:/usr/local/lib/python3.6/dist-packages/oauthlib/", + "file:/usr/local/lib/python3.6/dist-packages/opentelemetry/", + "file:/usr/local/lib/python3.6/dist-packages/opt_einsum/", + "file:/usr/local/lib/python3.6/dist-packages/pasta/", + "file:/usr/local/lib/python3.6/dist-packages/peewee.py", + "file:/usr/local/lib/python3.6/dist-packages/pip/", + "file:/usr/local/lib/python3.6/dist-packages/pkg_resources/", + "file:/usr/local/lib/python3.6/dist-packages/playhouse/", + "file:/usr/local/lib/python3.6/dist-packages/prettytable/", + "file:/usr/local/lib/python3.6/dist-packages/prison/", + "file:/usr/local/lib/python3.6/dist-packages/protobuf-3.19.6-py3.6-nspkg.pth", + "file:/usr/local/lib/python3.6/dist-packages/psutil/", + "file:/usr/local/lib/python3.6/dist-packages/pwiz.py", + "file:/usr/local/lib/python3.6/dist-packages/py4j/", + "file:/usr/local/lib/python3.6/dist-packages/pyasn1/", + "file:/usr/local/lib/python3.6/dist-packages/pyasn1_modules/", + "file:/usr/local/lib/python3.6/dist-packages/pycparser/", + "file:/usr/local/lib/python3.6/dist-packages/pylab.py", + "file:/usr/local/lib/python3.6/dist-packages/pylint/", + "file:/usr/local/lib/python3.6/dist-packages/pyparsing/", + "file:/usr/local/lib/python3.6/dist-packages/pytz/", + "file:/usr/local/lib/python3.6/dist-packages/pyximport/", + "file:/usr/local/lib/python3.6/dist-packages/requests/", + "file:/usr/local/lib/python3.6/dist-packages/requests_kerberos/", + "file:/usr/local/lib/python3.6/dist-packages/requests_oauthlib/", + "file:/usr/local/lib/python3.6/dist-packages/rsa/", + "file:/usr/local/lib/python3.6/dist-packages/scikit_learn.libs/", + "file:/usr/local/lib/python3.6/dist-packages/scipy/", + "file:/usr/local/lib/python3.6/dist-packages/scipy.libs/", + "file:/usr/local/lib/python3.6/dist-packages/setuptools/", + "file:/usr/local/lib/python3.6/dist-packages/six.py", + "file:/usr/local/lib/python3.6/dist-packages/spnego/", + "file:/usr/local/lib/python3.6/dist-packages/src/", + "file:/usr/local/lib/python3.6/dist-packages/tenacity/", + "file:/usr/local/lib/python3.6/dist-packages/tensorboard/", + "file:/usr/local/lib/python3.6/dist-packages/tensorboard_data_server/", + "file:/usr/local/lib/python3.6/dist-packages/tensorboard_plugin_wit/", + "file:/usr/local/lib/python3.6/dist-packages/termcolor.py", + "file:/usr/local/lib/python3.6/dist-packages/test/", + "file:/usr/local/lib/python3.6/dist-packages/tests/", + "file:/usr/local/lib/python3.6/dist-packages/threadpoolctl.py", + "file:/usr/local/lib/python3.6/dist-packages/toml/", + "file:/usr/local/lib/python3.6/dist-packages/typed_ast/", + "file:/usr/local/lib/python3.6/dist-packages/typing_extensions.py", + "file:/usr/local/lib/python3.6/dist-packages/urllib3/", + "file:/usr/local/lib/python3.6/dist-packages/wcwidth/", + "file:/usr/local/lib/python3.6/dist-packages/websocket/", + "file:/usr/local/lib/python3.6/dist-packages/werkzeug/", + "file:/usr/local/lib/python3.6/dist-packages/wheel/", + "file:/usr/local/lib/python3.6/dist-packages/yaml/", + "file:/usr/local/lib/python3.6/dist-packages/zipp.py" ] sgx.allowed_files = [ @@ -81,6 +355,131 @@ sgx.allowed_files = [ "file:/opt/meituan/", "file:/usr/lib/ssl/openssl.cnf", "file:/usr/lib/gcc", + "file:/usr/local/lib/python3.6/dist-packages/MySQLdb/", + "file:/usr/local/lib/python3.6/dist-packages/gssapi/", + "file:/usr/local/lib/python3.6/dist-packages/krb5/", + "file:/usr/local/lib/python3.6/dist-packages/leveldb.cpython-36m-x86_64-linux-gnu.so", + "file:/usr/local/lib/python3.6/dist-packages/leveldb-0.201.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/mysqlclient-2.1.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/wrapt-1.12.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/sqlalchemy/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/__init__.py", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/python/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/", + "file:/usr/local/lib/python3.6/dist-packages/wrapt/", + "file:/usr/local/lib/python3.6/dist-packages/Cython-0.29.36.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/Deprecated-1.2.14.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/Keras_Preprocessing-1.1.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/Markdown-3.3.7.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/Pillow-8.4.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/PyYAML-6.0.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/Werkzeug-2.0.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/absl_py-0.15.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/aiocontextvars-0.2.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/apsw-3.43.0.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/astroid-2.3.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/astunparse-1.6.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/backoff-1.10.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/cachetools-4.2.4.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/certifi-2024.2.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/cffi-1.15.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/chardet-4.0.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/cityhash-0.4.7.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/configparser-5.2.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/contextvars-2.4.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/coverage-6.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/cryptography-40.0.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/cycler-0.11.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/dataclasses-0.8.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/decorator-5.1.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/docopt-0.6.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/elasticsearch-7.11.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/elasticsearch6-6.8.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/etcd3-0.12.0-py3.6.egg-info/", + "file:/usr/local/lib/python3.6/dist-packages/fedlearner-1.5.dev0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/flatbuffers-1.12.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/flatten_dict-0.4.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/fsspec-2022.1.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/gast-0.3.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/gmpy2-2.1.5.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/google_auth-2.22.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/google_auth_oauthlib-0.4.6.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/google_pasta-0.2.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/googleapis_common_protos-1.56.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/grpcio-1.38.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/grpcio_tools-1.48.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/gssapi-1.7.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/guppy3-3.1.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/h5py-2.10.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/hdfs-2.7.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/immutables-0.19.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/importlib_metadata-4.8.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/influxdb-5.3.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/isort-4.3.21.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/joblib-1.1.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/kiwisolver-1.3.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/krb5-0.3.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/kubernetes-29.0.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/lazy_object_proxy-1.4.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/matplotlib-3.3.4.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/mccabe-0.6.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/meson-0.61.5.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/msgpack-1.0.5.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/numpy-1.19.5.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/oauthlib-3.2.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/opentelemetry_api-1.10.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/opentelemetry_exporter_otlp-1.10.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/opentelemetry_exporter_otlp_proto_grpc-1.10.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/opentelemetry_exporter_otlp_proto_http-1.10.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/opentelemetry_proto-1.10.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/opentelemetry_sdk-1.10.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/opentelemetry_semantic_conventions-0.29b0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/opt_einsum-3.3.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pandas-1.1.5.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/peewee-3.17.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pip-21.3.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/prettytable-2.5.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/prison-0.1.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/protobuf-3.19.6.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/psutil-5.9.8.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/py4j-0.10.9.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pyasn1-0.5.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pyasn1_modules-0.3.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pycparser-2.21.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pycryptodomex-3.20.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pyelftools-0.31.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pylint-2.4.4.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pyparsing-3.1.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pyspark-3.0.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pyspnego-0.5.4.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/python_dateutil-2.9.0.post0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/pytz-2021.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/requests-2.25.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/requests_kerberos-0.14.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/requests_oauthlib-2.0.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/rsa-4.9.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/scikit_learn-0.24.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/scipy-1.5.4.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/setuptools-41.0.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/six-1.15.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/tenacity-8.2.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/tensorboard-2.10.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/tensorboard_data_server-0.6.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/tensorboard_plugin_wit-1.8.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow-2.4.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow_estimator-2.4.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/termcolor-1.1.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/threadpoolctl-3.1.0.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/toml-0.10.2.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/typed_ast-1.4.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/typing_extensions-3.7.4.3.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/urllib3-1.26.18.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/wcwidth-0.2.13.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/websocket_client-1.3.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/wheel-0.37.1.dist-info/", + "file:/usr/local/lib/python3.6/dist-packages/zipp-3.6.0.dist-info/", + "file:{{ python.stdlib }}/", + "file:{{ python.distlib }}/", "file:/etc/ethers", "file:/etc/hosts", "file:/etc/group", @@ -103,4 +502,4 @@ sgx.allowed_files = [ "file:/lib/", "file:/bin/", "file:/data/", -] \ No newline at end of file +] From 4c943f93c1d5a073c6d7518d0fc97913230c35f6 Mon Sep 17 00:00:00 2001 From: huangxu17 Date: Mon, 15 Apr 2024 17:38:13 +0800 Subject: [PATCH 5/5] Regarding the TensorFlow API, it allows the addition of specific files that change with each compilation to the Gramine configuration, rather than the entire API folder. --- sgx/gramine/CI-Examples/generate-token/python.manifest.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sgx/gramine/CI-Examples/generate-token/python.manifest.template b/sgx/gramine/CI-Examples/generate-token/python.manifest.template index c654adedc..e328a59bf 100644 --- a/sgx/gramine/CI-Examples/generate-token/python.manifest.template +++ b/sgx/gramine/CI-Examples/generate-token/python.manifest.template @@ -365,7 +365,7 @@ sgx.allowed_files = [ "file:/usr/local/lib/python3.6/dist-packages/sqlalchemy/", "file:/usr/local/lib/python3.6/dist-packages/tensorflow/__init__.py", "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/python/", - "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/", + "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/v2.py", "file:/usr/local/lib/python3.6/dist-packages/wrapt/", "file:/usr/local/lib/python3.6/dist-packages/Cython-0.29.36.dist-info/", "file:/usr/local/lib/python3.6/dist-packages/Deprecated-1.2.14.dist-info/",