From 71d4e8f649af869e4871c874bd08e970ba546b82 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 19 Dec 2023 18:20:22 +0800
Subject: [PATCH 01/65] run on sco

---
 .github/workflows/main.yml | 96 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9b3f4cff4..e2d4a4207 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -78,6 +78,8 @@ jobs:
           ssh ${CLUSTER_ASCEND_910B} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main"
           rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/
           rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/
+          rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ SCO:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ || echo "ignore sco error"
+          rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ SCO:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "ignore sco error"
 
   Build-Camb:
     name: Build-dipu-camb
@@ -487,3 +489,97 @@ jobs:
           source scripts/ci/ascend/ci_ascend_env.sh
           bash tests/run_ascend_tests.sh && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Ascend-Latest-Target \
           || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Ascend-Latest-Target && exit 1 )
+
+
+
+
+  Build-V100:
+    name: Build-dipu-v100
+    needs: checkout_code
+    runs-on: tps-sco-ci
+    steps:
+      - name: Build v100
+        run: |
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB} \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
+
+  Test-V100:
+    name: Test-dipu-v100
+    needs: [Build-V100]
+    runs-on: tps-sco-ci
+    steps:
+      - name: Run-test
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && sh tests/run_nv_tests.sh"
+
+  Test-One-Iter_V100:
+    name: Test-one-iter-v100
+    needs: [Build-V100]
+    runs-on: tps-sco-ci
+    steps:
+      - name: build some env
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c"cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && ../scripts/ci/ci_one_iter.sh build_cuda
+      - name: run-one-iter-for-tradition
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && rm -rf one_iter_data \
+          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
+          """
+      - name: run-one-iter-for-llm
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && rm -rf one_iter_data \
+          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
+          """
+      - name: Perform cleanup one iter data
+        if: always()
+        run: |
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          rm -rf one_iter_data
+          touch one_iter_data   #用于占位，防止创建新的one_iter_data文件夹
+      - name: Check for failure
+        if: ${{ failure() }}
+        run: exit 1
+
+
+  Build-V100-Latest-Target:
+    name: Build-dipu-v100-latest-target
+    needs: [Rsync]
+    runs-on: tps-sco-ci
+    steps:
+      - name: Build dipu diopi-latest-target
+        run: |
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB} \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
+
+
+  Test-V100-Latest-Target:
+    name: Test-dipu-v100-latest-target
+    needs: [Build-V100-Latest-Target]
+    runs-on: tps-sco-ci
+    steps:
+      - name: Run-test
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Latest-Target/dipu \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && sh tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target \
+          || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )

From 7a5efb46890dd4b0e67507a0486c64ba4b5d5c16 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 19 Dec 2023 18:28:57 +0800
Subject: [PATCH 02/65] run on sco

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index e2d4a4207..4b17ac4e6 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -495,7 +495,7 @@ jobs:
 
   Build-V100:
     name: Build-dipu-v100
-    needs: checkout_code
+    needs: Rsync
     runs-on: tps-sco-ci
     steps:
       - name: Build v100

From 8e1ce7999d01c5296a0db21b8a3b4e21ca543654 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 19 Dec 2023 18:33:45 +0800
Subject: [PATCH 03/65] run on sco

---
 .github/workflows/main.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 4b17ac4e6..b7db01211 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -78,6 +78,7 @@ jobs:
           ssh ${CLUSTER_ASCEND_910B} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main"
           rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/
           rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/
+          ssh SCO "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main"
           rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ SCO:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ || echo "ignore sco error"
           rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ SCO:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "ignore sco error"
 
@@ -491,8 +492,6 @@ jobs:
           || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Ascend-Latest-Target && exit 1 )
 
 
-
-
   Build-V100:
     name: Build-dipu-v100
     needs: Rsync
@@ -556,7 +555,6 @@ jobs:
         if: ${{ failure() }}
         run: exit 1
 
-
   Build-V100-Latest-Target:
     name: Build-dipu-v100-latest-target
     needs: [Rsync]
@@ -570,7 +568,6 @@ jobs:
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 
-
   Test-V100-Latest-Target:
     name: Test-dipu-v100-latest-target
     needs: [Build-V100-Latest-Target]

From d9b8af3b4eff5302218d429d613c14e7b00a4641 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 19 Dec 2023 18:44:47 +0800
Subject: [PATCH 04/65] run on sco

---
 .github/workflows/main.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b7db01211..4e578f320 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -500,8 +500,8 @@ jobs:
       - name: Build v100
         run: |
           set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB} \
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 
@@ -525,9 +525,10 @@ jobs:
       - name: build some env
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c"cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && ../scripts/ci/ci_one_iter.sh build_cuda
+          """
       - name: run-one-iter-for-tradition
         run: |
           set -e
@@ -563,8 +564,8 @@ jobs:
       - name: Build dipu diopi-latest-target
         run: |
           set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB} \
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 

From 7fb7bdb0e1fcd1154d2f69b0d2bfae8aa33eb5d4 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 10:26:22 +0800
Subject: [PATCH 05/65] Update main.yml

---
 .github/workflows/main.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 4e578f320..8dbc557d3 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -513,7 +513,7 @@ jobs:
       - name: Run-test
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu \
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && sh tests/run_nv_tests.sh"
 
@@ -525,14 +525,14 @@ jobs:
       - name: build some env
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && ../scripts/ci/ci_one_iter.sh build_cuda
           """
       - name: run-one-iter-for-tradition
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
           && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
@@ -540,7 +540,7 @@ jobs:
       - name: run-one-iter-for-llm
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
           && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
@@ -549,7 +549,7 @@ jobs:
         if: always()
         run: |
           set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
           rm -rf one_iter_data
           touch one_iter_data   #用于占位，防止创建新的one_iter_data文件夹
       - name: Check for failure
@@ -577,7 +577,7 @@ jobs:
       - name: Run-test
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Latest-Target/dipu \
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100-Latest-Target/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && sh tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target \
+          && sh tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-V100-Latest-Target \
           || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )

From c905efcff94cf497ce974a6061c4073fff447ef5 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 11:01:34 +0800
Subject: [PATCH 06/65] Update main.yml

---
 .github/workflows/main.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 8dbc557d3..eebccb1aa 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -515,7 +515,7 @@ jobs:
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && sh tests/run_nv_tests.sh"
+          && bash tests/run_nv_tests.sh"
 
   Test-One-Iter_V100:
     name: Test-one-iter-v100
@@ -527,7 +527,7 @@ jobs:
           set -e
           srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && ../scripts/ci/ci_one_iter.sh build_cuda
+          && bash ../scripts/ci/ci_one_iter.sh build_cuda
           """
       - name: run-one-iter-for-tradition
         run: |
@@ -579,5 +579,5 @@ jobs:
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100-Latest-Target/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && sh tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-V100-Latest-Target \
+          && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-V100-Latest-Target \
           || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )

From 142ebbc8c5cd22bf4af49f823ac4f2e34f634d78 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 17:16:33 +0800
Subject: [PATCH 07/65] change v100 to sco

---
 .github/workflows/main.yml | 43 +++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index eebccb1aa..5835c43b8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -492,47 +492,48 @@ jobs:
           || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Ascend-Latest-Target && exit 1 )
 
 
-  Build-V100:
-    name: Build-dipu-v100
+  Build-Sco:
+    name: Build-dipu-sco
     needs: Rsync
     runs-on: tps-sco-ci
     steps:
-      - name: Build v100
+      - name: Build sco
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
           srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ \
           && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 
-  Test-V100:
-    name: Test-dipu-v100
-    needs: [Build-V100]
+  Test-Sco:
+    name: Test-dipu-sco
+    needs: [Build-Sco]
     runs-on: tps-sco-ci
     steps:
       - name: Run-test
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu \
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && bash tests/run_nv_tests.sh"
 
-  Test-One-Iter_V100:
-    name: Test-one-iter-v100
-    needs: [Build-V100]
+  Test-One-Iter_Sco:
+    name: Test-one-iter-sco
+    needs: [Build-Sco]
     runs-on: tps-sco-ci
     steps:
       - name: build some env
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu/mmlab_pack
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && bash ../scripts/ci/ci_one_iter.sh build_cuda
           """
       - name: run-one-iter-for-tradition
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu/mmlab_pack
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
           && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
@@ -540,7 +541,7 @@ jobs:
       - name: run-one-iter-for-llm
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu/mmlab_pack
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
           && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
@@ -549,15 +550,15 @@ jobs:
         if: always()
         run: |
           set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100/dipu/mmlab_pack
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu/mmlab_pack
           rm -rf one_iter_data
           touch one_iter_data   #用于占位，防止创建新的one_iter_data文件夹
       - name: Check for failure
         if: ${{ failure() }}
         run: exit 1
 
-  Build-V100-Latest-Target:
-    name: Build-dipu-v100-latest-target
+  Build-Sco-Latest-Target:
+    name: Build-dipu-sco-latest-target
     needs: [Rsync]
     runs-on: tps-sco-ci
     steps:
@@ -569,15 +570,15 @@ jobs:
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 
-  Test-V100-Latest-Target:
-    name: Test-dipu-v100-latest-target
-    needs: [Build-V100-Latest-Target]
+  Test-Sco-Latest-Target:
+    name: Test-dipu-sco-latest-target
+    needs: [Build-Sco-Latest-Target]
     runs-on: tps-sco-ci
     steps:
       - name: Run-test
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-V100-Latest-Target/dipu \
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco-Latest-Target/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-V100-Latest-Target \
+          && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Sco-Latest-Target \
           || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )

From 9cb78537b13522a9803c3d151ea52cac45adc103 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 17:57:41 +0800
Subject: [PATCH 08/65] add run on sco

---
 .github/workflows/runs-on-sco.yml | 129 ++++++++++++++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 .github/workflows/runs-on-sco.yml

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
new file mode 100644
index 000000000..679c984d0
--- /dev/null
+++ b/.github/workflows/runs-on-sco.yml
@@ -0,0 +1,129 @@
+name: runs on sco
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - ".github/ISSUE_TEMPLATE/**"
+      - ".git*"
+      - "CODE_OF_CONDUCT**"
+
+concurrency:
+  group: sco-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}'
+  ALL_COVERAGE: ${{ (contains( github.ref, 'main') || startsWith(github.ref, 'refs/heads/v') || startsWith(github.ref, 'refs/heads/dev')) && 'ON' || 'OFF' }}
+  REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '40' }}
+
+jobs:
+  checkout_code:
+    name: checkout code
+    runs-on: tps-sco-ci
+    steps:
+      - name: Checkout Code
+        uses: .github/actions/checkout-code@main
+      - name: add mmlab_pack
+        run: |
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}  && cp -R source source_DIOPI
+          cd source/dipu && bash /home/autolink/rsync/sourcecode/update_code.sh
+          rsync -a /home/autolink/rsync/sourcecode/mmlab_pack . && cd mmlab_pack
+          bash ../scripts/ci/ci_one_iter.sh clone
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source_DIOPI/dipu/third_party && rm -rf DIOPI && git clone https://github.com/DeepLink-org/DIOPI.git
+          
+  Build-Cuda:
+    name: Build-dipu-cuda
+    needs: checkout_code
+    runs-on: tps-sco-ci
+    steps:
+      - name: Build cuda
+        run: |
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ \
+          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
+
+  Test-Cuda:
+    name: Test-dipu-cuda
+    needs: [Build-Cuda]
+    runs-on: tps-sco-ci
+    steps:
+      - name: Run-test
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && bash tests/run_nv_tests.sh"
+
+  Test-One-Iter_Cuda:
+    name: Test-one-iter-cuda
+    needs: [Build-Cuda]
+    runs-on: tps-sco-ci
+    steps:
+      - name: build some env
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && bash ../scripts/ci/ci_one_iter.sh build_cuda
+          """
+      - name: run-one-iter-for-tradition
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && rm -rf one_iter_data \
+          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
+          """
+      - name: run-one-iter-for-llm
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && rm -rf one_iter_data \
+          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
+          """
+      - name: Perform cleanup one iter data
+        if: always()
+        run: |
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
+          rm -rf one_iter_data
+          touch one_iter_data   #用于占位，防止创建新的one_iter_data文件夹
+      - name: Check for failure
+        if: ${{ failure() }}
+        run: exit 1
+
+  Build-Cuda-Latest-Target:
+    name: Build-dipu-cuda-latest-target
+    needs: [checkout_code]
+    runs-on: tps-sco-ci
+    steps:
+      - name: Build dipu diopi-latest-target
+        run: |
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
+
+  Test-Cuda-Latest-Target:
+    name: Test-dipu-cuda-latest-target
+    needs: [Build-Cuda-Latest-Target]
+    runs-on: tps-sco-ci
+    steps:
+      - name: Run-test
+        run: |
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Latest-Target/dipu \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target \
+          || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )

From f2f0ae1848594e480697df7d583d76e066b2d744 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 18:01:27 +0800
Subject: [PATCH 09/65] Update main.yml

---
 .github/workflows/main.yml | 95 --------------------------------------
 1 file changed, 95 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5835c43b8..9b3f4cff4 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -78,9 +78,6 @@ jobs:
           ssh ${CLUSTER_ASCEND_910B} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main"
           rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/
           rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/
-          ssh SCO "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main"
-          rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ SCO:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ || echo "ignore sco error"
-          rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ SCO:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "ignore sco error"
 
   Build-Camb:
     name: Build-dipu-camb
@@ -490,95 +487,3 @@ jobs:
           source scripts/ci/ascend/ci_ascend_env.sh
           bash tests/run_ascend_tests.sh && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Ascend-Latest-Target \
           || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Ascend-Latest-Target && exit 1 )
-
-
-  Build-Sco:
-    name: Build-dipu-sco
-    needs: Rsync
-    runs-on: tps-sco-ci
-    steps:
-      - name: Build sco
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ \
-          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
-
-  Test-Sco:
-    name: Test-dipu-sco
-    needs: [Build-Sco]
-    runs-on: tps-sco-ci
-    steps:
-      - name: Run-test
-        run: |
-          set -e
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && bash tests/run_nv_tests.sh"
-
-  Test-One-Iter_Sco:
-    name: Test-one-iter-sco
-    needs: [Build-Sco]
-    runs-on: tps-sco-ci
-    steps:
-      - name: build some env
-        run: |
-          set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu/mmlab_pack
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && bash ../scripts/ci/ci_one_iter.sh build_cuda
-          """
-      - name: run-one-iter-for-tradition
-        run: |
-          set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu/mmlab_pack
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && rm -rf one_iter_data \
-          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
-          """
-      - name: run-one-iter-for-llm
-        run: |
-          set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu/mmlab_pack
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && rm -rf one_iter_data \
-          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
-          """
-      - name: Perform cleanup one iter data
-        if: always()
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco/dipu/mmlab_pack
-          rm -rf one_iter_data
-          touch one_iter_data   #用于占位，防止创建新的one_iter_data文件夹
-      - name: Check for failure
-        if: ${{ failure() }}
-        run: exit 1
-
-  Build-Sco-Latest-Target:
-    name: Build-dipu-sco-latest-target
-    needs: [Rsync]
-    runs-on: tps-sco-ci
-    steps:
-      - name: Build dipu diopi-latest-target
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
-
-  Test-Sco-Latest-Target:
-    name: Test-dipu-sco-latest-target
-    needs: [Build-Sco-Latest-Target]
-    runs-on: tps-sco-ci
-    steps:
-      - name: Run-test
-        run: |
-          set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Sco-Latest-Target/dipu \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Sco-Latest-Target \
-          || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )

From 629d1a8214630558af616868ac11686a0785a2cf Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 18:04:23 +0800
Subject: [PATCH 10/65] update runs-on-sco.yml

---
 .github/workflows/runs-on-sco.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 679c984d0..29a7d056e 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -27,7 +27,7 @@ jobs:
     runs-on: tps-sco-ci
     steps:
       - name: Checkout Code
-        uses: .github/actions/checkout-code@main
+        uses: DeepLink-org/deeplink.framework/.github/actions/checkout-code@main
       - name: add mmlab_pack
         run: |
           set -e

From f59fd3b9aa56df9f9c61797b017012f2ed890fa5 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 18:14:27 +0800
Subject: [PATCH 11/65] Update runs-on-sco.yml

---
 .github/workflows/runs-on-sco.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 29a7d056e..9282d22c0 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -31,11 +31,11 @@ jobs:
       - name: add mmlab_pack
         run: |
           set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}  && cp -R source source_DIOPI
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}  && cp -R source source_main
           cd source/dipu && bash /home/autolink/rsync/sourcecode/update_code.sh
           rsync -a /home/autolink/rsync/sourcecode/mmlab_pack . && cd mmlab_pack
-          bash ../scripts/ci/ci_one_iter.sh clone
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source_DIOPI/dipu/third_party && rm -rf DIOPI && git clone https://github.com/DeepLink-org/DIOPI.git
+          bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/dipu/scripts/ci/ci_one_iter.sh clone
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source_main/dipu/third_party && rm -rf DIOPI && git clone https://github.com/DeepLink-org/DIOPI.git
           
   Build-Cuda:
     name: Build-dipu-cuda

From 40e3d170f6f5647c6ef65145d1632535b13dcc5e Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 18:19:31 +0800
Subject: [PATCH 12/65] update source-main

---
 .github/workflows/runs-on-sco.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 9282d22c0..4fcfbd042 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -31,11 +31,11 @@ jobs:
       - name: add mmlab_pack
         run: |
           set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}  && cp -R source source_main
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}  && cp -R source source-main
           cd source/dipu && bash /home/autolink/rsync/sourcecode/update_code.sh
           rsync -a /home/autolink/rsync/sourcecode/mmlab_pack . && cd mmlab_pack
           bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/dipu/scripts/ci/ci_one_iter.sh clone
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source_main/dipu/third_party && rm -rf DIOPI && git clone https://github.com/DeepLink-org/DIOPI.git
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party && rm -rf DIOPI && git clone https://github.com/DeepLink-org/DIOPI.git
           
   Build-Cuda:
     name: Build-dipu-cuda

From 35ca3f8036222505d7c23e9c0d21161f19e29f4d Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 19:36:19 +0800
Subject: [PATCH 13/65] Update runs-on-sco.yml

---
 .github/workflows/runs-on-sco.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 4fcfbd042..8ff26f185 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -71,7 +71,7 @@ jobs:
       - name: build some env
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && bash ../scripts/ci/ci_one_iter.sh build_cuda
           """

From a35efae2d618762904d648e469e1e89dfa2a1525 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 19:55:24 +0800
Subject: [PATCH 14/65] Update runs-on-sco.yml

---
 .github/workflows/runs-on-sco.yml | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 8ff26f185..734d9c0d0 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -71,26 +71,23 @@ jobs:
       - name: build some env
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && bash ../scripts/ci/ci_one_iter.sh build_cuda
-          """
+          && bash ../scripts/ci/ci_one_iter.sh build_cuda"
       - name: run-one-iter-for-tradition
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
-          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
-          """
+          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
       - name: run-one-iter-for-llm
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c """ cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
-          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
-          """
+          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
       - name: Perform cleanup one iter data
         if: always()
         run: |

From 683fcad60b24304ea600071e4e95bfc517af9310 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 20:04:48 +0800
Subject: [PATCH 15/65] Update path

---
 .github/workflows/runs-on-sco.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 734d9c0d0..d23ac4382 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -73,21 +73,21 @@ jobs:
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
-          && bash ../scripts/ci/ci_one_iter.sh build_cuda"
+          && bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_one_iter.sh build_cuda"
       - name: run-one-iter-for-tradition
         run: |
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
-          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
       - name: run-one-iter-for-llm
         run: |
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
-          && python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
       - name: Perform cleanup one iter data
         if: always()
         run: |

From 443883aae439620a0604eabeac8faa257d40231f Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 20:32:18 +0800
Subject: [PATCH 16/65] add sco

---
 dipu/scripts/ci/ci_run_one_iter.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 373856629..a5499a2af 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -99,6 +99,17 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         else:
             cmd_run_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=40 sh SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}"
             cmd_cp_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=30 sh SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}"
+    elif device == 'sco':
+        if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
+            cmd_run_one_iter = f"srun --job-name={job_name} bash mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh"
+            cmd_cp_one_iter = ""
+        # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
+        elif ('infer' in p2 and 'infer' in p3):
+            cmd_run_one_iter = f"srun --job-name={job_name} python {train_path}"
+            cmd_cp_one_iter = ""
+        else:
+            cmd_run_one_iter = f"srun --job-name={job_name} bash SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}"
+            cmd_cp_one_iter = f"srun --job-name={job_name} bash SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}"
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 21d2699e1e450236ac8ec1e57cdd033c76809f06 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 20:34:24 +0800
Subject: [PATCH 17/65] modify one iter

---
 .github/workflows/runs-on-sco.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index d23ac4382..4f2d7f286 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -80,14 +80,14 @@ jobs:
           srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
-          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
       - name: run-one-iter-for-llm
         run: |
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && rm -rf one_iter_data \
-          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
       - name: Perform cleanup one iter data
         if: always()
         run: |

From e8f2526f401e73891491d892c5057823a8abab31 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 21:19:04 +0800
Subject: [PATCH 18/65] update diopi path

---
 .github/workflows/runs-on-sco.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 4f2d7f286..cae9bfc56 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -71,21 +71,21 @@ jobs:
       - name: build some env
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_one_iter.sh build_cuda"
       - name: run-one-iter-for-tradition
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
       - name: run-one-iter-for-llm
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
       - name: Perform cleanup one iter data

From be33fd112ad8e05eeda04a897d385fd78435bb53 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 20 Dec 2023 21:26:47 +0800
Subject: [PATCH 19/65] update sco

---
 dipu/scripts/ci/ci_run_one_iter.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index a5499a2af..183120c63 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -67,12 +67,7 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
     if not os.path.exists(storage_path):
         os.makedirs(storage_path)
 
-    if device == 'camb':
-        base_data_src = '/mnt/lustre/share/parrotsci/github/model_baseline_data'
-    elif device == 'cuda':
-        base_data_src = '/mnt/cache/share/parrotsci/github/model_baseline_data'
-    elif device == "ascend":
-        base_data_src = "/mnt/cache/share/deeplinkci/github/model_baseline_data"
+    base_data_src = '/mnt/cache/share/parrotsci/github/model_baseline_data'
     src = f'{base_data_src}/{p3}/baseline'
     if not os.path.exists(src):
         os.makedirs(src)
@@ -195,7 +190,11 @@ def print_file(file_name):
     curPath = os.path.dirname(os.path.realpath(__file__))
     yamlPath = os.path.join(curPath, selected_model_list)
     with open(yamlPath, 'r', encoding='utf-8') as f:
-        original_list = yaml.safe_load(f.read()).get(device, None)
+        if device == 'sco':
+            original_list = yaml.safe_load(f.read()).get("cuda", None)
+        else:
+            original_list = yaml.safe_load(f.read()).get(device, None)
+
         if not original_list:
             logging.warning(f"Device type: {device} is not supported!")
             exit(0)

From d547472140aec0a531b510218ef7d631f27bd5bb Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Thu, 21 Dec 2023 11:41:52 +0800
Subject: [PATCH 20/65] add sco

---
 dipu/scripts/ci/ci_run_one_iter.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 183120c63..e74f3a834 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -11,9 +11,9 @@
 log_format = '%(asctime)s - %(levelname)s: %(message)s'
 logging.basicConfig(level=logging.INFO, format=log_format, datefmt='%Y-%m-%d %H:%M:%S')
 
-
+env = os.environ
 def run_cmd(cmd: str) -> None:
-    cp = sp.run(cmd, shell=True, encoding="utf-8")
+    cp = sp.run(cmd, shell=True, encoding="utf-8", env=env)
     if cp.returncode != 0:
         error = f"Some thing wrong has happened when running command [{cmd}]:{cp.stderr}"
         raise Exception(error)
@@ -95,16 +95,16 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
             cmd_run_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=40 sh SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}"
             cmd_cp_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=30 sh SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}"
     elif device == 'sco':
+        current_path = os.getcwd()
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"srun --job-name={job_name} bash mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh"
-            cmd_cp_one_iter = ""
-        # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
+            cmd_run_one_iter_sco = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_cp_one_iter_sco = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"srun --job-name={job_name} python {train_path}"
-            cmd_cp_one_iter = ""
+            cmd_run_one_iter_sco = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {current_path}/mmlab_pack/{train_path}" """
+            cmd_cp_one_iter_sco = ""
         else:
-            cmd_run_one_iter = f"srun --job-name={job_name} bash SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}"
-            cmd_cp_one_iter = f"srun --job-name={job_name} bash SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}"
+            cmd_run_one_iter_sco = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter_sco = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 61a104326c4344320ba86b7c1511ad3748e09c3f Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Thu, 21 Dec 2023 12:12:13 +0800
Subject: [PATCH 21/65] add sco

---
 dipu/scripts/ci/ci_run_one_iter.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index e74f3a834..cb608cd70 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -97,14 +97,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
     elif device == 'sco':
         current_path = os.getcwd()
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter_sco = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
-            cmd_cp_one_iter_sco = ""
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter_sco = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {current_path}/mmlab_pack/{train_path}" """
-            cmd_cp_one_iter_sco = ""
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {current_path}/mmlab_pack/{train_path}" """
+            cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter_sco = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter_sco = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 2fdc34a3819a2aced480a4cc3f66477aa5a10fec Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Thu, 21 Dec 2023 13:55:17 +0800
Subject: [PATCH 22/65] update sco

---
 .github/workflows/runs-on-sco.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index cae9bfc56..de1bb44e2 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -77,17 +77,17 @@ jobs:
       - name: run-one-iter-for-tradition
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
-          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
-          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)"
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data
         if: always()
         run: |

From 56728860deaf2b51127f54a3d992b805b8ab9f64 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Thu, 21 Dec 2023 14:52:07 +0800
Subject: [PATCH 23/65] add sco

---
 .github/workflows/runs-on-sco.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index de1bb44e2..7c7b6eafe 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -80,14 +80,14 @@ jobs:
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
-          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
-          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} gpu:${GPU_REQUESTS} sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data
         if: always()
         run: |

From 6b5f41c3c030bbca23e75c35c6063eb18d3daea9 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Thu, 21 Dec 2023 16:47:43 +0800
Subject: [PATCH 24/65] add sco

---
 .github/workflows/runs-on-sco.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 7c7b6eafe..fd8867927 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -31,7 +31,7 @@ jobs:
       - name: add mmlab_pack
         run: |
           set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}  && cp -R source source-main
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}  &&rm -rf source-main && cp -R source source-main
           cd source/dipu && bash /home/autolink/rsync/sourcecode/update_code.sh
           rsync -a /home/autolink/rsync/sourcecode/mmlab_pack . && cd mmlab_pack
           bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/dipu/scripts/ci/ci_one_iter.sh clone

From d3531f8f3327cb1fc639b8b1fc1925d2234737e8 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Thu, 21 Dec 2023 17:54:54 +0800
Subject: [PATCH 25/65] run on sco

---
 .github/workflows/runs-on-sco.yml  | 4 ++--
 dipu/scripts/ci/ci_run_one_iter.py | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index fd8867927..22f3b5f72 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -79,14 +79,14 @@ jobs:
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
-          && rm -rf one_iter_data \
+          && rm -rf one_iter_data && cd .. \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
-          && rm -rf one_iter_data \
+          && rm -rf one_iter_data && cd .. \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data
         if: always()
diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index cb608cd70..812c6e1b7 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -11,9 +11,8 @@
 log_format = '%(asctime)s - %(levelname)s: %(message)s'
 logging.basicConfig(level=logging.INFO, format=log_format, datefmt='%Y-%m-%d %H:%M:%S')
 
-env = os.environ
 def run_cmd(cmd: str) -> None:
-    cp = sp.run(cmd, shell=True, encoding="utf-8", env=env)
+    cp = sp.run(cmd, shell=True, encoding="utf-8")
     if cp.returncode != 0:
         error = f"Some thing wrong has happened when running command [{cmd}]:{cp.stderr}"
         raise Exception(error)

From 1a8e52bdc7f0fe7d6736ed7c769bbce4b9a6398a Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Fri, 22 Dec 2023 13:32:29 +0800
Subject: [PATCH 26/65] run on sco

---
 .github/workflows/runs-on-sco.yml  | 4 ++--
 dipu/scripts/ci/ci_run_one_iter.py | 9 +++++----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 22f3b5f72..fd8867927 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -79,14 +79,14 @@ jobs:
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
-          && rm -rf one_iter_data && cd .. \
+          && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
-          && rm -rf one_iter_data && cd .. \
+          && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data
         if: always()
diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 812c6e1b7..0f8a83e9f 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -95,15 +95,16 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
             cmd_cp_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=30 sh SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}"
     elif device == 'sco':
         current_path = os.getcwd()
+        parent_directory = os.path.dirname(current_path)  # 获取上一级目录
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {parent_directory}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {current_path}/mmlab_pack/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {parent_directory}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path}/mmlab_pack && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmlab_pack/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {parent_directory}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {parent_directory}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 204edaafc08b5c17dcbf2fc670f75a7d933728ad Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Fri, 22 Dec 2023 14:04:29 +0800
Subject: [PATCH 27/65] run on sco

---
 dipu/scripts/ci/ci_run_one_iter.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 0f8a83e9f..d4c3197ce 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -95,16 +95,16 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
             cmd_cp_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=30 sh SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}"
     elif device == 'sco':
         current_path = os.getcwd()
-        parent_directory = os.path.dirname(current_path)  # 获取上一级目录
+        parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {parent_directory}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {parent_directory}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {parent_directory}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {parent_directory}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From a4a8f74abddc769cba3ba895d61194f97b645a43 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:06:09 +0800
Subject: [PATCH 28/65] test model

---
 .../ci/test_one_iter_traditional_model_list.yaml   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/dipu/scripts/ci/test_one_iter_traditional_model_list.yaml b/dipu/scripts/ci/test_one_iter_traditional_model_list.yaml
index 431b0d380..300832bc8 100644
--- a/dipu/scripts/ci/test_one_iter_traditional_model_list.yaml
+++ b/dipu/scripts/ci/test_one_iter_traditional_model_list.yaml
@@ -55,13 +55,13 @@ cuda:
     # # mmpretrain
     - model_cfg: "mmpretrain resnet/resnet50_8xb32_in1k.py workdirs_resnet"
     - model_cfg: "mmpretrain swin_transformer/swin-base_16xb64_in1k.py workdirs_swin_transformer"
-    - model_cfg: "mmpretrain vision_transformer/vit-base-p16_32xb128-mae_in1k.py workdirs_vision_transformer"
+    #- model_cfg: "mmpretrain vision_transformer/vit-base-p16_32xb128-mae_in1k.py workdirs_vision_transformer"
     - model_cfg: "mmpretrain efficientnet/efficientnet-b2_8xb32_in1k.py workdirs_efficientnet"
     - model_cfg: "mmpretrain mobilenet_v3/mobilenet-v3-large_8xb128_in1k.py workdirs_mobilenetv3"
     - model_cfg: "mmpretrain mobilenet_v2/mobilenet-v2_8xb32_in1k.py workdirs_mobilenetv2"
     - model_cfg: "mmpretrain convnext/convnext-small_32xb128_in1k.py workdirs_convnext"
-    - model_cfg: "mmpretrain shufflenet_v2/shufflenet-v2-1x_16xb64_in1k_256.py workdirs_shufflenetv2"
-      precision: {atol: 0.015, metric: 0.015, rtol: 0.01}
+    #- model_cfg: "mmpretrain shufflenet_v2/shufflenet-v2-1x_16xb64_in1k_256.py workdirs_shufflenetv2"
+    #  precision: {atol: 0.015, metric: 0.015, rtol: 0.01}
     # mmdetection
     - model_cfg: "mmdetection detr/detr_r50_8xb2-150e_coco.py workdirs_detr"
     - model_cfg: "mmdetection yolo/yolov3_d53_8xb8-320-273e_coco.py workdirs_yolov3"
@@ -77,11 +77,11 @@ cuda:
     - model_cfg: "mmaction2 recognition/tsn/tsn_imagenet-pretrained-r50_8xb32-1x1x3-100e_kinetics400-rgb.py workdirs_tsn"
     # mmocr
     - model_cfg: "mmocr textrecog/crnn/crnn_mini-vgg_5e_mj.py workdirs_crnn"
-    - model_cfg: "mmocr textdet/dbnet/dbnet_resnet50-dcnv2_fpnc_1200e_icdar2015.py workdirs_dbnet"
+    #- model_cfg: "mmocr textdet/dbnet/dbnet_resnet50-dcnv2_fpnc_1200e_icdar2015.py workdirs_dbnet"
     # mmsegmentation
     - model_cfg: "mmsegmentation deeplabv3/deeplabv3_r50-d8_4xb2-40k_cityscapes-512x1024.py workdirs_deeplabv3"
-    - model_cfg: "mmsegmentation deeplabv3plus/deeplabv3plus_r50-d8_4xb2-40k_cityscapes-512x1024.py workdirs_deeplabv3plus"
-    - model_cfg: "mmsegmentation unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py workdirs_unet"
+    #- model_cfg: "mmsegmentation deeplabv3plus/deeplabv3plus_r50-d8_4xb2-40k_cityscapes-512x1024.py workdirs_deeplabv3plus"
+    #- model_cfg: "mmsegmentation unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py workdirs_unet"
     - model_cfg: "mmsegmentation pspnet/pspnet_r50-d8_4xb2-40k_cityscapes-512x1024.py workdirs_pspnet"
     # mmyolo
     - model_cfg: "mmyolo yolov5/yolov5_s-v61_syncbn_fast_8xb16-300e_coco.py workdirs_yolov5_fast"
@@ -93,7 +93,7 @@ cuda:
     # mmagic
     # - model_cfg: "mmagic stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py workdirs_stable_diffusion"
     # transformers
-    - model_cfg: "transformers examples/pytorch/question-answering/run_bert_qa.py workdirs_bert"
+    #- model_cfg: "transformers examples/pytorch/question-answering/run_bert_qa.py workdirs_bert"
 
 ascend:
     # mmsegmentation

From 5dfcede6914509a32954b5ef428e50fe3d230687 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 27 Dec 2023 17:13:23 +0800
Subject: [PATCH 29/65] rm cancel for test

---
 .github/workflows/runs-on-sco.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index fd8867927..bfd8cd416 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -12,9 +12,9 @@ on:
       - ".git*"
       - "CODE_OF_CONDUCT**"
 
-concurrency:
-  group: sco-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+#concurrency:
+#  group: sco-${{ github.head_ref || github.ref }}
+#  cancel-in-progress: true
 
 env:
   DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}'

From 1b799d439869c1955a6408ab104b6231dece3a88 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Fri, 29 Dec 2023 15:40:03 +0800
Subject: [PATCH 30/65] echo env

---
 dipu/scripts/ci/ci_run_one_iter.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index d4c3197ce..23f4c2e6f 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -81,7 +81,10 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
     rtol = precision.get('rtol', 1e-4)
     metric = precision.get('metric', 1e-2)
     logging.info(f'Using pricision: atol-{atol}, rtol-{rtol}, metric-{metric}')
-
+    env_vars = os.environ
+    print(env_vars)
+    for key, value in env_vars.items():
+    print(f"{key}: {value}")
     if device == 'cuda':
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
             cmd_run_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=40 sh mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh"

From fe5b2d232c5a1d7da1e0bf0f855835b3b4d6dab3 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 10:38:45 +0800
Subject: [PATCH 31/65] add sco ci

---
 .github/workflows/runs-on-sco.yml  |  8 ++++++--
 dipu/scripts/ci/ci_run_one_iter.py | 13 ++++---------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index bfd8cd416..fd4992530 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -78,14 +78,18 @@ jobs:
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env \
+          && basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack \
+          && source scripts/ci/ci_one_iter.sh export_pythonpath_camb \${basic_path} && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env \
+          && basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack \
+          && source scripts/ci/ci_one_iter.sh export_pythonpath_camb \${basic_path} && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data
diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 23f4c2e6f..ad33674fc 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -81,10 +81,6 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
     rtol = precision.get('rtol', 1e-4)
     metric = precision.get('metric', 1e-2)
     logging.info(f'Using pricision: atol-{atol}, rtol-{rtol}, metric-{metric}')
-    env_vars = os.environ
-    print(env_vars)
-    for key, value in env_vars.items():
-    print(f"{key}: {value}")
     if device == 'cuda':
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
             cmd_run_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=40 sh mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh"
@@ -98,16 +94,15 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
             cmd_cp_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=30 sh SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}"
     elif device == 'sco':
         current_path = os.getcwd()
-        parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 3f1d12ad089a4880d21d14de8c71a109de066152 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 11:32:05 +0800
Subject: [PATCH 32/65] add sco ci

---
 .github/workflows/runs-on-sco.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index fd4992530..ba42871ab 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -80,7 +80,7 @@ jobs:
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env \
           && basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack \
-          && source scripts/ci/ci_one_iter.sh export_pythonpath_camb \${basic_path} && cd mmlab_pack \
+          && source scripts/ci/ci_one_iter.sh export_pythonpath_camb ${basic_path} && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
@@ -89,7 +89,7 @@ jobs:
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
           && source /mnt/cache/share/deeplinkci/github/dipu_env \
           && basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack \
-          && source scripts/ci/ci_one_iter.sh export_pythonpath_camb \${basic_path} && cd mmlab_pack \
+          && source scripts/ci/ci_one_iter.sh export_pythonpath_camb ${basic_path} && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data

From 16633274653dc8c9bdbaeb8eef8e6e1b1ea42db0 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 12:22:59 +0800
Subject: [PATCH 33/65] add sco ci

---
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index ad33674fc..3e46d4cab 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -95,14 +95,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
     elif device == 'sco':
         current_path = os.getcwd()
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && cd .. && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && cd .. && source /mnt/cache/share/deeplinkci/github/dipu_env && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && cd .. && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && cd .. && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 7f67964fae67ebb0552accd4515ede04b9d6a577 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 12:29:34 +0800
Subject: [PATCH 34/65] add sco ci

---
 dipu/scripts/ci/ci_run_one_iter.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 3e46d4cab..028d81e7a 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -94,15 +94,16 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
             cmd_cp_one_iter = f"srun --job-name={job_name} --partition={partition}  --gres={gpu_requests} --cpus-per-task=5 --mem=16G --time=30 sh SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}"
     elif device == 'sco':
         current_path = os.getcwd()
+        parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && cd .. && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && mmlab_pack && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && cd .. && source /mnt/cache/share/deeplinkci/github/dipu_env && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && mmlab_pack && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && cd .. && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {current_path} && source environment_exported && cd .. && source /mnt/cache/share/deeplinkci/github/dipu_env && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From fa0aa7628292b208fc6ed57c688316f9f7d5cf2e Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 12:32:17 +0800
Subject: [PATCH 35/65] add sco ci

---
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 028d81e7a..19e824f20 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -96,14 +96,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && mmlab_pack && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && mmlab_pack && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 397f58357b2b8884daf5b218db1b356df62392b8 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 12:43:31 +0800
Subject: [PATCH 36/65] add sco ci

---
 .github/workflows/runs-on-sco.yml  | 8 ++------
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index ba42871ab..bfd8cd416 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -78,18 +78,14 @@ jobs:
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env \
-          && basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack \
-          && source scripts/ci/ci_one_iter.sh export_pythonpath_camb ${basic_path} && cd mmlab_pack \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env \
-          && basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack \
-          && source scripts/ci/ci_one_iter.sh export_pythonpath_camb ${basic_path} && cd mmlab_pack \
+          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data
diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 19e824f20..676163cfd 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -96,14 +96,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source mmlab_pack/environment_exported && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From c6d9506bac6fd11265b2a68df33605b7975935e2 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 13:13:47 +0800
Subject: [PATCH 37/65] add sco ci

---
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 676163cfd..04f4ed639 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -96,14 +96,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From d6ff2bfa416e524b475f3f9b683aa51e1902b3a9 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 13:54:59 +0800
Subject: [PATCH 38/65] add sco ci

---
 dipu/scripts/ci/ci_run_one_iter.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 04f4ed639..aa9b12f0c 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -17,7 +17,6 @@ def run_cmd(cmd: str) -> None:
         error = f"Some thing wrong has happened when running command [{cmd}]:{cp.stderr}"
         raise Exception(error)
 
-
 def process_one_iter(log_file, clear_log, model_info: dict) -> None:
     begin_time = time.time()
 
@@ -188,6 +187,16 @@ def print_file(file_name):
     # os.environ['ONE_ITER_TOOL_IOSAVE_RATIO'] = "1.0"  # 0.2 by default
     curPath = os.path.dirname(os.path.realpath(__file__))
     yamlPath = os.path.join(curPath, selected_model_list)
+    file_path = os.path.join(curPath, "environment_exported")
+    env_variables = os.environ
+    keywords_to_filter = ['DIPU', 'ONE_ITER']
+    if os.path.exists(file_path):
+        os.remove(file_path)
+    with open("environment_exported", "w") as file:
+        file.write("pwd\n")
+        for key, value in env_variables.items():
+            if any(keyword in key for keyword in keywords_to_filter):
+                file.write(f'export {key}="{value}"\n')
     with open(yamlPath, 'r', encoding='utf-8') as f:
         if device == 'sco':
             original_list = yaml.safe_load(f.read()).get("cuda", None)

From 4e7bfc4aab530ee436fcf95f3b1072457ff4e39e Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 2 Jan 2024 15:22:17 +0800
Subject: [PATCH 39/65] add sco ci

---
 .github/workflows/runs-on-sco.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index bfd8cd416..ac165e14e 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -92,7 +92,7 @@ jobs:
         if: always()
         run: |
           set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack \
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
           rm -rf one_iter_data
           touch one_iter_data   #用于占位，防止创建新的one_iter_data文件夹
       - name: Check for failure

From e3692643d8f91777e5c3bf00f698a6610875ce56 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 3 Jan 2024 13:57:52 +0800
Subject: [PATCH 40/65] rm bash

---
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index aa9b12f0c..58231e72f 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -95,14 +95,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 8d71c105f17671d1f97d9fb7b40b9acca4d9cf2d Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Wed, 3 Jan 2024 14:15:36 +0800
Subject: [PATCH 41/65] add bash

---
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 58231e72f..aa9b12f0c 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -95,14 +95,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 30a29812e8abae64997918887b4bec72aa207383 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Mon, 8 Jan 2024 16:05:33 +0800
Subject: [PATCH 42/65] add other sco ci

---
 .github/workflows/runs-on-sco.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index ac165e14e..a629f6ccb 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -19,7 +19,7 @@ on:
 env:
   DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}'
   ALL_COVERAGE: ${{ (contains( github.ref, 'main') || startsWith(github.ref, 'refs/heads/v') || startsWith(github.ref, 'refs/heads/dev')) && 'ON' || 'OFF' }}
-  REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '40' }}
+  REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '0' }}
 
 jobs:
   checkout_code:
@@ -62,6 +62,17 @@ jobs:
           srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu \
           && source /mnt/cache/share/deeplinkci/github/dipu_env  \
           && bash tests/run_nv_tests.sh"
+          if [ "${ALL_COVERAGE}" = "ON" ]; then
+          bash /mnt/cache/share/platform/dep/sonar/coverage_DIPU_nv.sh ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
+          fi
+      - name: increment coverage check
+        if: ${{ contains( github.event_name, 'pull_request' ) && contains( github.base_ref, 'main' ) }}
+        run: |
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda
+          ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
+          source ${ENV_PATH}/pt2.0_diopi
+          bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
 
   Test-One-Iter_Cuda:
     name: Test-one-iter-cuda

From 316034f57790a1fb38fa3a2372966a00e777850b Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Mon, 8 Jan 2024 16:42:15 +0800
Subject: [PATCH 43/65] update env path

---
 .github/workflows/runs-on-sco.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index a629f6ccb..c7385ee2e 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -71,7 +71,7 @@ jobs:
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda
           ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
-          source ${ENV_PATH}/pt2.0_diopi
+          source /mnt/cache/share/platform/env/pt2.0_diopi
           bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
 
   Test-One-Iter_Cuda:

From d615103188449246947f7f273d727d5c6c80b289 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Mon, 8 Jan 2024 16:46:28 +0800
Subject: [PATCH 44/65] update ENV_PATH

---
 .github/workflows/runs-on-sco.yml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index c7385ee2e..5878bf85f 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -20,6 +20,7 @@ env:
   DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}'
   ALL_COVERAGE: ${{ (contains( github.ref, 'main') || startsWith(github.ref, 'refs/heads/v') || startsWith(github.ref, 'refs/heads/dev')) && 'ON' || 'OFF' }}
   REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '0' }}
+  ENV_PATH: '/mnt/cache/share/deeplinkci/github'
 
 jobs:
   checkout_code:
@@ -47,7 +48,7 @@ jobs:
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
           srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && source ${ENV_PATH}/dipu_env  \
           && rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ \
           && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 
@@ -60,7 +61,7 @@ jobs:
         run: |
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && source ${ENV_PATH}/dipu_env  \
           && bash tests/run_nv_tests.sh"
           if [ "${ALL_COVERAGE}" = "ON" ]; then
           bash /mnt/cache/share/platform/dep/sonar/coverage_DIPU_nv.sh ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
@@ -71,7 +72,7 @@ jobs:
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda
           ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
-          source /mnt/cache/share/platform/env/pt2.0_diopi
+          source ${ENV_PATH}/dipu_env
           bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
 
   Test-One-Iter_Cuda:
@@ -83,20 +84,20 @@ jobs:
         run: |
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
+          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
           && bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_one_iter.sh build_cuda"
       - name: run-one-iter-for-tradition
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
+          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack \
+          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
           && rm -rf one_iter_data \
           && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data
@@ -120,7 +121,7 @@ jobs:
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}
           srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && source ${ENV_PATH}/dipu_env  \
           && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 
   Test-Cuda-Latest-Target:
@@ -132,6 +133,6 @@ jobs:
         run: |
           set -e
           srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Latest-Target/dipu \
-          && source /mnt/cache/share/deeplinkci/github/dipu_env  \
+          && source ${ENV_PATH}/dipu_env  \
           && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target \
           || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )

From 169f1145a2789d87b6a796220e633c516f9fab82 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Tue, 9 Jan 2024 18:32:48 +0800
Subject: [PATCH 45/65] add sco ci

---
 .../ci/test_one_iter_traditional_model_list.yaml   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/dipu/scripts/ci/test_one_iter_traditional_model_list.yaml b/dipu/scripts/ci/test_one_iter_traditional_model_list.yaml
index 300832bc8..431b0d380 100644
--- a/dipu/scripts/ci/test_one_iter_traditional_model_list.yaml
+++ b/dipu/scripts/ci/test_one_iter_traditional_model_list.yaml
@@ -55,13 +55,13 @@ cuda:
     # # mmpretrain
     - model_cfg: "mmpretrain resnet/resnet50_8xb32_in1k.py workdirs_resnet"
     - model_cfg: "mmpretrain swin_transformer/swin-base_16xb64_in1k.py workdirs_swin_transformer"
-    #- model_cfg: "mmpretrain vision_transformer/vit-base-p16_32xb128-mae_in1k.py workdirs_vision_transformer"
+    - model_cfg: "mmpretrain vision_transformer/vit-base-p16_32xb128-mae_in1k.py workdirs_vision_transformer"
     - model_cfg: "mmpretrain efficientnet/efficientnet-b2_8xb32_in1k.py workdirs_efficientnet"
     - model_cfg: "mmpretrain mobilenet_v3/mobilenet-v3-large_8xb128_in1k.py workdirs_mobilenetv3"
     - model_cfg: "mmpretrain mobilenet_v2/mobilenet-v2_8xb32_in1k.py workdirs_mobilenetv2"
     - model_cfg: "mmpretrain convnext/convnext-small_32xb128_in1k.py workdirs_convnext"
-    #- model_cfg: "mmpretrain shufflenet_v2/shufflenet-v2-1x_16xb64_in1k_256.py workdirs_shufflenetv2"
-    #  precision: {atol: 0.015, metric: 0.015, rtol: 0.01}
+    - model_cfg: "mmpretrain shufflenet_v2/shufflenet-v2-1x_16xb64_in1k_256.py workdirs_shufflenetv2"
+      precision: {atol: 0.015, metric: 0.015, rtol: 0.01}
     # mmdetection
     - model_cfg: "mmdetection detr/detr_r50_8xb2-150e_coco.py workdirs_detr"
     - model_cfg: "mmdetection yolo/yolov3_d53_8xb8-320-273e_coco.py workdirs_yolov3"
@@ -77,11 +77,11 @@ cuda:
     - model_cfg: "mmaction2 recognition/tsn/tsn_imagenet-pretrained-r50_8xb32-1x1x3-100e_kinetics400-rgb.py workdirs_tsn"
     # mmocr
     - model_cfg: "mmocr textrecog/crnn/crnn_mini-vgg_5e_mj.py workdirs_crnn"
-    #- model_cfg: "mmocr textdet/dbnet/dbnet_resnet50-dcnv2_fpnc_1200e_icdar2015.py workdirs_dbnet"
+    - model_cfg: "mmocr textdet/dbnet/dbnet_resnet50-dcnv2_fpnc_1200e_icdar2015.py workdirs_dbnet"
     # mmsegmentation
     - model_cfg: "mmsegmentation deeplabv3/deeplabv3_r50-d8_4xb2-40k_cityscapes-512x1024.py workdirs_deeplabv3"
-    #- model_cfg: "mmsegmentation deeplabv3plus/deeplabv3plus_r50-d8_4xb2-40k_cityscapes-512x1024.py workdirs_deeplabv3plus"
-    #- model_cfg: "mmsegmentation unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py workdirs_unet"
+    - model_cfg: "mmsegmentation deeplabv3plus/deeplabv3plus_r50-d8_4xb2-40k_cityscapes-512x1024.py workdirs_deeplabv3plus"
+    - model_cfg: "mmsegmentation unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py workdirs_unet"
     - model_cfg: "mmsegmentation pspnet/pspnet_r50-d8_4xb2-40k_cityscapes-512x1024.py workdirs_pspnet"
     # mmyolo
     - model_cfg: "mmyolo yolov5/yolov5_s-v61_syncbn_fast_8xb16-300e_coco.py workdirs_yolov5_fast"
@@ -93,7 +93,7 @@ cuda:
     # mmagic
     # - model_cfg: "mmagic stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py workdirs_stable_diffusion"
     # transformers
-    #- model_cfg: "transformers examples/pytorch/question-answering/run_bert_qa.py workdirs_bert"
+    - model_cfg: "transformers examples/pytorch/question-answering/run_bert_qa.py workdirs_bert"
 
 ascend:
     # mmsegmentation

From e0442f81a797961bdd970fe7e8c78de7ccaef76c Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Mon, 15 Jan 2024 11:46:51 +0800
Subject: [PATCH 46/65] run on sco

---
 .github/workflows/main.yml         | 144 ++++++++++-------------------
 .github/workflows/runs-on-sco.yml  |  11 +--
 dipu/scripts/ci/ci_run_one_iter.py |  10 +-
 dipu/scripts/ci/nv/ci_nv_tidy.sh   |   4 +-
 4 files changed, 58 insertions(+), 111 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f72aa9190..994e749a5 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -244,123 +244,84 @@ jobs:
   Build-Cuda:
     name: Build-dipu-cuda
     needs: [Rsync]
-    runs-on: github-poc-ci
-    env:
-      GPU_REQUESTS: 1
+    runs-on: tps-sco-ci
     steps:
-      - name: Build dipu
+      - name: Build cuda
         run: |
-          ssh ${CUDA_CLUSTER} """
           set -e
-          export USE_COVERAGE=ON
-          cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu
-          source scripts/ci/nv/ci_nv_env.sh
-          rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/
-          srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:${GPU_REQUESTS} --time=30 bash scripts/ci/nv/ci_nv_script.sh build_dipu \
-          || ( cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
-          """
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
+          && source ${ENV_PATH}/dipu_env  \
+          && rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ \
+          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 
   Tidy-Cuda:
     name: Run tidy (cuda)
     needs: [Build-Cuda]
-    runs-on: github-poc-ci
+    runs-on: tps-sco-ci
     steps:
       - name: clang-tidy
         run: |
-          ssh $CUDA_CLUSTER """
           set -eo pipefail
-          source ~/.bashrc
-          (bash $CUDA_CI_PATH/$GITHUB_RUN_NUMBER/Build-Cuda/dipu/scripts/ci/nv/ci_nv_tidy.sh srun --job-name=${GITHUB_RUN_NUMBER}_$GITHUB_JOB --partition=$CUDA_PARTATION --time=20) ||\
+          srun --job-name=${GITHUB_JOB} bash -c "bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/nv/ci_nv_tidy.sh " || \
           (rm -rf "$CUDA_CI_PATH/$GITHUB_RUN_NUMBER/Build-Cuda" && exit 1)
-          """
 
   Test-Cuda:
     name: Test-dipu-cuda
     needs: [Build-Cuda, Tidy-Cuda]
-    runs-on: github-poc-ci
-    env:
-      GPU_REQUESTS: 1
+    runs-on: tps-sco-ci
     steps:
       - name: Run-test
         run: |
-          ssh ${CUDA_CLUSTER} """
-          set -ex
-          export USE_COVERAGE=ON
-          cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda/dipu
-          source scripts/ci/nv/ci_nv_env.sh
-          srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:${GPU_REQUESTS} --cpus-per-task=5 --mem=16G --time=70 sh tests/run_nv_tests.sh
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu \
+          && source ${ENV_PATH}/dipu_env  \
+          && bash tests/run_nv_tests.sh"
           if [ "${ALL_COVERAGE}" = "ON" ]; then
-          bash /mnt/cache/share/platform/dep/sonar/coverage_DIPU_nv.sh ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
+          bash /mnt/cache/share/platform/dep/sonar/coverage_DIPU_nv.sh ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
           fi
-          """
       - name: increment coverage check
         if: ${{ contains( github.event_name, 'pull_request' ) && contains( github.base_ref, 'main' ) }}
         run: |
-          ssh ${CUDA_CLUSTER} """
           set -e
-          cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/
-          rm -rf scripts
-          ln -s ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
-          source /mnt/cache/share/platform/env/pt2.0_diopi
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda
+          ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
+          source ${ENV_PATH}/dipu_env
           bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
           """
 
   Test-One-Iter_Cuda:
     name: Test-one-iter-cuda
     needs: [Build-Cuda, Tidy-Cuda]
-    runs-on: github-poc-ci
-    env:
-      GPU_REQUESTS: 1
+    runs-on: tps-sco-ci
     steps:
       - name: build some env
         run: |
-          ssh ${CUDA_CLUSTER} """
-          set -ex
-          cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda/dipu
-          source scripts/ci/nv/ci_nv_env.sh
-          basic_path=${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
-          export PYTHONPATH=\${basic_path}/mmengine:\$PYTHONPATH
-          export PYTHONPATH=\${basic_path}/mmcv:\$PYTHONPATH
-          export PYTHONPATH=\$(pwd):\$PYTHONPATH
-          cd mmlab_pack
-          srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:${GPU_REQUESTS} --time=20 bash ../scripts/ci/ci_one_iter.sh build_cuda
-          """
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
+          && bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_one_iter.sh build_cuda"
       - name: run-one-iter-for-tradition
         run: |
-          ssh ${CUDA_CLUSTER} """
-            cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda/dipu
-            source scripts/ci/nv/ci_nv_env.sh
-            basic_path=${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
-            source scripts/ci/ci_one_iter.sh export_pythonpath_cuda \${basic_path}
-            export PYTHONPATH=\$(pwd):\$PYTHONPATH
-            cd mmlab_pack
-            rm -rf one_iter_data
-            python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
-          """
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
+          && rm -rf one_iter_data \
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: run-one-iter-for-llm
         run: |
-          ssh ${CUDA_CLUSTER} """
-            cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda/dipu
-            source scripts/ci/nv/ci_nv_env.sh
-            basic_path=${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
-            source scripts/ci/ci_one_iter.sh export_pythonpath_cuda \${basic_path}
-            export PYTHONPATH=\$(pwd):\$PYTHONPATH
-            cd mmlab_pack
-            rm -rf one_iter_data
-            python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:${GPU_REQUESTS}" \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
-          """
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
+          && rm -rf one_iter_data \
+          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
       - name: Perform cleanup one iter data
         if: always()
         run: |
-          ssh ${CUDA_CLUSTER} """
-              set -ex
-              echo "${GITHUB_RUN_NUMBER}_${GITHUB_JOB}"
-              scancel -n "${GITHUB_RUN_NUMBER}_${GITHUB_JOB}"
-              cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
-              rm -rf one_iter_data
-              touch one_iter_data # 用于占位，防止创建新的 one_iter_data 文件夹
-          """
-
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
+          rm -rf one_iter_data
+          touch one_iter_data   #用于占位，防止创建新的one_iter_data文件夹
       - name: Check for failure
         if: ${{ failure() }}
         run: exit 1
@@ -368,36 +329,29 @@ jobs:
   Build-Cuda-Latest-Target:
     name: Build-dipu-cuda-latest-target
     needs: [Tidy-Cuda]
-    runs-on: github-poc-ci
-    env:
-      GPU_REQUESTS: 1
+    runs-on: tps-sco-ci
     steps:
       - name: Build dipu diopi-latest-target
         run: |
-          ssh ${CUDA_CLUSTER} """
-          set -ex
-          cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}  && cd ${GITHUB_JOB}/dipu
-          source scripts/ci/nv/ci_nv_env.sh
-          srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:${GPU_REQUESTS} --cpus-per-task=5 --mem=16G --time=30 bash scripts/ci/nv/ci_nv_script.sh build_dipu \
-          || ( cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
+          set -e
+          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}
+          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
+          && source ${ENV_PATH}/dipu_env  \
+          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
           """
 
   Test-Cuda-Latest-Target:
     name: Test-dipu-cuda-latest-target
     needs: [Build-Cuda-Latest-Target]
-    runs-on: github-poc-ci
-    env:
-      GPU_REQUESTS: 1
+    runs-on: tps-sco-ci
     steps:
       - name: Run-test
         run: |
-          ssh ${CUDA_CLUSTER} """
-          set -ex
-          cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda-Latest-Target/dipu
-          source scripts/ci/nv/ci_nv_env.sh
-          srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:${GPU_REQUESTS} --cpus-per-task=5 --mem=16G --time=60 sh tests/run_nv_tests.sh && cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target \
-          || ( cd ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
-          """
+          set -e
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Latest-Target/dipu \
+          && source ${ENV_PATH}/dipu_env  \
+          && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target \
+          || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
 
   Build-PyTorch-For-Ascend-910b:
     name: Build-dipu-pytorch-for-ascend-910b
diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
index 5878bf85f..fe4fb32cc 100644
--- a/.github/workflows/runs-on-sco.yml
+++ b/.github/workflows/runs-on-sco.yml
@@ -1,16 +1,7 @@
-name: runs on sco
+name: runs on 1424
 
 on:
   workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - "**.md"
-      - ".github/ISSUE_TEMPLATE/**"
-      - ".git*"
-      - "CODE_OF_CONDUCT**"
 
 #concurrency:
 #  group: sco-${{ github.head_ref || github.ref }}
diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index b45938c91..0bd194a7d 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -100,14 +100,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):
@@ -162,6 +162,8 @@ def print_file(file_name):
     args = parser.parse_args()
 
     device = args.device
+    if device == 'sco':
+        max_parall = 5
     job_name = args.job_name
     gpu_requests = args.gpu_requests
     partition = args.partition_arg
diff --git a/dipu/scripts/ci/nv/ci_nv_tidy.sh b/dipu/scripts/ci/nv/ci_nv_tidy.sh
index dd9bd0228..5421cfc48 100644
--- a/dipu/scripts/ci/nv/ci_nv_tidy.sh
+++ b/dipu/scripts/ci/nv/ci_nv_tidy.sh
@@ -15,12 +15,12 @@ repo=$(cd $self && git rev-parse --show-toplevel)
 # Try finding clangd and libstdc++.so.6 on 1988.
 # Note: ":+:" is used to handle unbound variable.
 [ -d /mnt/lustre/share/platform/dep/clang-16/bin ] &&
-    export PATH=/mnt/lustre/share/platform/dep/clang-16/bin${PATH:+:$PATH}
+    export PATH=/mnt/cache/share/platform/dep/clang-16/bin${PATH:+:$PATH}
 [ -d /mnt/cache/share/platform/env/miniconda3.10/envs/pt2.0_diopi/lib ] &&
     export LD_LIBRARY_PATH=/mnt/cache/share/platform/env/miniconda3.10/envs/pt2.0_diopi/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
 
 # Forward srun commands.
 # e.g. you can use: bash scripts/ci/nv/ci_nv_tidy.sh srun -p pat_rd
 (cd "$repo/dipu" &&
-    ($@ find torch_dipu ! -path '*/vendor/*' ! -name AutoGenedKernels.cpp \( -name '*.cpp' -o -name '*.h' -o -name '*.hpp' \) |
+    (find torch_dipu ! -path '*/vendor/*' ! -name AutoGenedKernels.cpp \( -name '*.cpp' -o -name '*.h' -o -name '*.hpp' \) |
         xargs $self/clangd-tidy/clangd-tidy -j4))

From 3fba918107e10e884293f88bc6d570a6495e5a23 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Mon, 15 Jan 2024 12:37:34 +0800
Subject: [PATCH 47/65] run on sco

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 994e749a5..c53aa3b8b 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -16,7 +16,7 @@ env:
   CAMB_TORCH_BASE_DIR: '/mnt/lustre/share/parrotsci/github/cibuild/pytorchbase'
   CUDA_CI_PATH: '/mnt/cache/share/parrotsci/github/cibuild/${{ github.repository }}'
   CUDA_PARTATION: ${{ vars.SH1988_SLURM_PAR != '' && vars.SH1988_SLURM_PAR || 'pat_rd' }}
-  CUDA_CLUSTER: SH1988
+  CUDA_CLUSTER: SCO
   DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}'
   ASCEND_CLUSTER: ASCEND
   CLUSTER_ASCEND_910B: ASCEND-910B

From 762072e375148e8e109ed71bfbdb84db739dbff7 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Mon, 15 Jan 2024 12:40:27 +0800
Subject: [PATCH 48/65] run on sco

---
 .github/workflows/main.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c53aa3b8b..8e61dcfe5 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -73,9 +73,9 @@ jobs:
           ssh ${CAMB_CLUSTER} "mkdir -p ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
           && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source/ \
           && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to camb"
-          ssh ${CUDA_CLUSTER} "mkdir -p ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/source ${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${CUDA_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda"
+          ssh ${CUDA_CLUSTER} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
+          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda"
           ssh ${CLUSTER_ASCEND_910B} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
           && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
           && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to ascend"

From f06e6fd7531b06076091b8acf927c6d26917baca Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Mon, 15 Jan 2024 13:25:41 +0800
Subject: [PATCH 49/65] run on sco

---
 .github/workflows/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 8e61dcfe5..3b9dfc1a1 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -18,6 +18,7 @@ env:
   CUDA_PARTATION: ${{ vars.SH1988_SLURM_PAR != '' && vars.SH1988_SLURM_PAR || 'pat_rd' }}
   CUDA_CLUSTER: SCO
   DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}'
+  ENV_PATH: '/mnt/cache/share/deeplinkci/github'
   ASCEND_CLUSTER: ASCEND
   CLUSTER_ASCEND_910B: ASCEND-910B
   CLUSTER_KLX: KUNLUNXIN

From e826e084f9c70861c7f2a186f5a1a1ec960eb728 Mon Sep 17 00:00:00 2001
From: wugeshui <106943115+wugeshui@users.noreply.github.com>
Date: Mon, 15 Jan 2024 21:15:52 +0800
Subject: [PATCH 50/65] Update ci_nv_tidy.sh

---
 dipu/scripts/ci/nv/ci_nv_tidy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dipu/scripts/ci/nv/ci_nv_tidy.sh b/dipu/scripts/ci/nv/ci_nv_tidy.sh
index 5421cfc48..f930a1609 100644
--- a/dipu/scripts/ci/nv/ci_nv_tidy.sh
+++ b/dipu/scripts/ci/nv/ci_nv_tidy.sh
@@ -14,7 +14,7 @@ repo=$(cd $self && git rev-parse --show-toplevel)
 
 # Try finding clangd and libstdc++.so.6 on 1988.
 # Note: ":+:" is used to handle unbound variable.
-[ -d /mnt/lustre/share/platform/dep/clang-16/bin ] &&
+[ -d /mnt/cache/share/platform/dep/clang-16/bin ] &&
     export PATH=/mnt/cache/share/platform/dep/clang-16/bin${PATH:+:$PATH}
 [ -d /mnt/cache/share/platform/env/miniconda3.10/envs/pt2.0_diopi/lib ] &&
     export LD_LIBRARY_PATH=/mnt/cache/share/platform/env/miniconda3.10/envs/pt2.0_diopi/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}

From f052a2f9aef6eda62de26533c1eed0d64be816a1 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 16 Jan 2024 10:40:03 +0800
Subject: [PATCH 51/65] run on sco

---
 .github/workflows/main.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3b9dfc1a1..ac299d486 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -264,8 +264,9 @@ jobs:
       - name: clang-tidy
         run: |
           set -eo pipefail
+          echo """
           srun --job-name=${GITHUB_JOB} bash -c "bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/nv/ci_nv_tidy.sh " || \
-          (rm -rf "$CUDA_CI_PATH/$GITHUB_RUN_NUMBER/Build-Cuda" && exit 1)
+          (rm -rf "$CUDA_CI_PATH/$GITHUB_RUN_NUMBER/Build-Cuda" && exit 1) """
 
   Test-Cuda:
     name: Test-dipu-cuda

From e82dbc70ba8f91bd5ed84203264b0a0c5d69fd78 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 16 Jan 2024 11:26:27 +0800
Subject: [PATCH 52/65] run on sco

---
 .github/workflows/main.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index ac299d486..0adb12270 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -290,7 +290,6 @@ jobs:
           ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
           source ${ENV_PATH}/dipu_env
           bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
-          """
 
   Test-One-Iter_Cuda:
     name: Test-one-iter-cuda
@@ -340,7 +339,6 @@ jobs:
           srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
           && source ${ENV_PATH}/dipu_env  \
           && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
-          """
 
   Test-Cuda-Latest-Target:
     name: Test-dipu-cuda-latest-target

From 04472cb63af84a0a06a38a1868b8240a7a2a3794 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 16 Jan 2024 12:45:25 +0800
Subject: [PATCH 53/65] run on sco

---
 .github/workflows/main.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0adb12270..ab5e88876 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -247,7 +247,7 @@ jobs:
     needs: [Rsync]
     runs-on: tps-sco-ci
     steps:
-      - name: Build cuda
+      - name: Build dipu
         run: |
           set -e
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
@@ -264,9 +264,8 @@ jobs:
       - name: clang-tidy
         run: |
           set -eo pipefail
-          echo """
           srun --job-name=${GITHUB_JOB} bash -c "bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/nv/ci_nv_tidy.sh " || \
-          (rm -rf "$CUDA_CI_PATH/$GITHUB_RUN_NUMBER/Build-Cuda" && exit 1) """
+          (rm -rf "$CUDA_CI_PATH/$GITHUB_RUN_NUMBER/Build-Cuda" && exit 1)
 
   Test-Cuda:
     name: Test-dipu-cuda

From 1066562b5506823d6959c61f89d35541a87475ea Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 16 Jan 2024 13:43:06 +0800
Subject: [PATCH 54/65] run on sco

---
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 0bd194a7d..5e6ce29e3 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -100,14 +100,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From 9b140e1ab9334f2035b20fef0e44ca8046ea75bd Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 16 Jan 2024 22:17:10 +0800
Subject: [PATCH 55/65] add  sco ci

---
 .github/workflows/main.yml         | 16 ++++++++--------
 dipu/scripts/ci/ci_run_one_iter.py |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index ab5e88876..0fe16d6e1 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -72,17 +72,17 @@ jobs:
       - name: Rsync to Server
         run: |
           ssh ${CAMB_CLUSTER} "mkdir -p ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to camb"
+          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to camb"
           ssh ${CUDA_CLUSTER} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda"
+          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda"
           ssh ${CLUSTER_ASCEND_910B} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to ascend"
+          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to ascend"
           ssh ${CLUSTER_KLX} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to kunlunxin"
+          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to kunlunxin"
 
   Build-Camb:
     name: Build-dipu-camb
diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 5e6ce29e3..f6f31783d 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -163,7 +163,7 @@ def print_file(file_name):
 
     device = args.device
     if device == 'sco':
-        max_parall = 5
+        max_parall = 4
     job_name = args.job_name
     gpu_requests = args.gpu_requests
     partition = args.partition_arg

From ba9d1b6599ba19ad3e91e54db2e4e97162f151e2 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 16 Jan 2024 22:19:10 +0800
Subject: [PATCH 56/65] add  sco ci

---
 .github/workflows/runs-on-sco.yml | 129 ------------------------------
 1 file changed, 129 deletions(-)
 delete mode 100644 .github/workflows/runs-on-sco.yml

diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml
deleted file mode 100644
index fe4fb32cc..000000000
--- a/.github/workflows/runs-on-sco.yml
+++ /dev/null
@@ -1,129 +0,0 @@
-name: runs on 1424
-
-on:
-  workflow_dispatch:
-
-#concurrency:
-#  group: sco-${{ github.head_ref || github.ref }}
-#  cancel-in-progress: true
-
-env:
-  DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}'
-  ALL_COVERAGE: ${{ (contains( github.ref, 'main') || startsWith(github.ref, 'refs/heads/v') || startsWith(github.ref, 'refs/heads/dev')) && 'ON' || 'OFF' }}
-  REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '0' }}
-  ENV_PATH: '/mnt/cache/share/deeplinkci/github'
-
-jobs:
-  checkout_code:
-    name: checkout code
-    runs-on: tps-sco-ci
-    steps:
-      - name: Checkout Code
-        uses: DeepLink-org/deeplink.framework/.github/actions/checkout-code@main
-      - name: add mmlab_pack
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}  &&rm -rf source-main && cp -R source source-main
-          cd source/dipu && bash /home/autolink/rsync/sourcecode/update_code.sh
-          rsync -a /home/autolink/rsync/sourcecode/mmlab_pack . && cd mmlab_pack
-          bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/dipu/scripts/ci/ci_one_iter.sh clone
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party && rm -rf DIOPI && git clone https://github.com/DeepLink-org/DIOPI.git
-          
-  Build-Cuda:
-    name: Build-dipu-cuda
-    needs: checkout_code
-    runs-on: tps-sco-ci
-    steps:
-      - name: Build cuda
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
-          && source ${ENV_PATH}/dipu_env  \
-          && rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ \
-          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
-
-  Test-Cuda:
-    name: Test-dipu-cuda
-    needs: [Build-Cuda]
-    runs-on: tps-sco-ci
-    steps:
-      - name: Run-test
-        run: |
-          set -e
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu \
-          && source ${ENV_PATH}/dipu_env  \
-          && bash tests/run_nv_tests.sh"
-          if [ "${ALL_COVERAGE}" = "ON" ]; then
-          bash /mnt/cache/share/platform/dep/sonar/coverage_DIPU_nv.sh ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
-          fi
-      - name: increment coverage check
-        if: ${{ contains( github.event_name, 'pull_request' ) && contains( github.base_ref, 'main' ) }}
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda
-          ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
-          source ${ENV_PATH}/dipu_env
-          bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
-
-  Test-One-Iter_Cuda:
-    name: Test-one-iter-cuda
-    needs: [Build-Cuda]
-    runs-on: tps-sco-ci
-    steps:
-      - name: build some env
-        run: |
-          set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
-          && bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_one_iter.sh build_cuda"
-      - name: run-one-iter-for-tradition
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
-          && rm -rf one_iter_data \
-          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
-      - name: run-one-iter-for-llm
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
-          && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
-          && rm -rf one_iter_data \
-          && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
-      - name: Perform cleanup one iter data
-        if: always()
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
-          rm -rf one_iter_data
-          touch one_iter_data   #用于占位，防止创建新的one_iter_data文件夹
-      - name: Check for failure
-        if: ${{ failure() }}
-        run: exit 1
-
-  Build-Cuda-Latest-Target:
-    name: Build-dipu-cuda-latest-target
-    needs: [checkout_code]
-    runs-on: tps-sco-ci
-    steps:
-      - name: Build dipu diopi-latest-target
-        run: |
-          set -e
-          cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}
-          srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
-          && source ${ENV_PATH}/dipu_env  \
-          && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
-
-  Test-Cuda-Latest-Target:
-    name: Test-dipu-cuda-latest-target
-    needs: [Build-Cuda-Latest-Target]
-    runs-on: tps-sco-ci
-    steps:
-      - name: Run-test
-        run: |
-          set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Latest-Target/dipu \
-          && source ${ENV_PATH}/dipu_env  \
-          && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target \
-          || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )

From 56e83fa70d3ae5643440999a5655d0ea7550963a Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Wed, 17 Jan 2024 10:52:25 +0800
Subject: [PATCH 57/65] run on sco

---
 .github/workflows/main.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0fe16d6e1..1187c558d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -75,14 +75,14 @@ jobs:
           && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source/ \
           && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to camb"
           ssh ${CUDA_CLUSTER} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda"
+          && timeout 200 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && timeout 200 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda"
           ssh ${CLUSTER_ASCEND_910B} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
           && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
           && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to ascend"
           ssh ${CLUSTER_KLX} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to kunlunxin"
+          && timeout 200 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && timeout 200 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to kunlunxin"
 
   Build-Camb:
     name: Build-dipu-camb
@@ -264,8 +264,7 @@ jobs:
       - name: clang-tidy
         run: |
           set -eo pipefail
-          srun --job-name=${GITHUB_JOB} bash -c "bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/nv/ci_nv_tidy.sh " || \
-          (rm -rf "$CUDA_CI_PATH/$GITHUB_RUN_NUMBER/Build-Cuda" && exit 1)
+          srun --job-name=${GITHUB_JOB} bash -c "bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/nv/ci_nv_tidy.sh "
 
   Test-Cuda:
     name: Test-dipu-cuda

From 0361e4ff497e1b6a519595853f209479a2f4604b Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Wed, 17 Jan 2024 11:03:21 +0800
Subject: [PATCH 58/65] run on sco

---
 .github/workflows/main.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 1187c558d..1aeaf7986 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -72,17 +72,17 @@ jobs:
       - name: Rsync to Server
         run: |
           ssh ${CAMB_CLUSTER} "mkdir -p ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to camb"
+          && rsync -a --delete --timeout=60 ${GITHUB_WORKSPACE}/${REPO}/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && rsync -a --delete --timeout=60 ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to camb"
           ssh ${CUDA_CLUSTER} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && timeout 200 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && timeout 200 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda"
+          && rsync -a --delete --timeout=60 ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && rsync -a --delete --timeout=60 ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda"
           ssh ${CLUSTER_ASCEND_910B} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && timeout 100 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to ascend"
+          && rsync -a --delete --timeout=60 ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && rsync -a --delete --timeout=60 ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to ascend"
           ssh ${CLUSTER_KLX} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \
-          && timeout 200 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
-          && timeout 200 rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to kunlunxin"
+          && rsync -a --delete --timeout=60 ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \
+          && rsync -a --delete --timeout=60 ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_KLX}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to kunlunxin"
 
   Build-Camb:
     name: Build-dipu-camb

From 37a27304b033991ebfe2ed111708cc2b4cb3c1ba Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Wed, 17 Jan 2024 11:42:23 +0800
Subject: [PATCH 59/65] add bashrc

---
 dipu/scripts/ci/nv/ci_nv_tidy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dipu/scripts/ci/nv/ci_nv_tidy.sh b/dipu/scripts/ci/nv/ci_nv_tidy.sh
index f930a1609..5772c9908 100644
--- a/dipu/scripts/ci/nv/ci_nv_tidy.sh
+++ b/dipu/scripts/ci/nv/ci_nv_tidy.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 set -euo pipefail
-
+source ~/.bashrc
 # Require Git.
 [ -x "$(command -v git)" ] || (echo "missing git tool" && exit 1)
 

From b02bcc7e9396b48cb4830b595628c00685f091a6 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Wed, 17 Jan 2024 12:15:57 +0800
Subject: [PATCH 60/65] add bashrc

---
 dipu/scripts/ci/nv/ci_nv_tidy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dipu/scripts/ci/nv/ci_nv_tidy.sh b/dipu/scripts/ci/nv/ci_nv_tidy.sh
index 5772c9908..f930a1609 100644
--- a/dipu/scripts/ci/nv/ci_nv_tidy.sh
+++ b/dipu/scripts/ci/nv/ci_nv_tidy.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 set -euo pipefail
-source ~/.bashrc
+
 # Require Git.
 [ -x "$(command -v git)" ] || (echo "missing git tool" && exit 1)
 

From 8351804f074eea7a5e096a1eac08a3bdb4d90d63 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Wed, 17 Jan 2024 12:22:37 +0800
Subject: [PATCH 61/65] add proxy

---
 dipu/scripts/ci/nv/ci_nv_tidy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dipu/scripts/ci/nv/ci_nv_tidy.sh b/dipu/scripts/ci/nv/ci_nv_tidy.sh
index f930a1609..3401d819d 100644
--- a/dipu/scripts/ci/nv/ci_nv_tidy.sh
+++ b/dipu/scripts/ci/nv/ci_nv_tidy.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 set -euo pipefail
-
+source /mnt/cache/share/deeplinkci/github/proxy_on
 # Require Git.
 [ -x "$(command -v git)" ] || (echo "missing git tool" && exit 1)
 

From 06f82f693c405e8d63c8140469077ba7ad2dc5e8 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Wed, 17 Jan 2024 20:43:16 +0800
Subject: [PATCH 62/65] add  sco ci

---
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index f6f31783d..0ffc8282a 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -100,14 +100,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From eb261bf337cf6fdc3c6dcdfb2fe652ae4ad16a9d Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Wed, 17 Jan 2024 22:20:14 +0800
Subject: [PATCH 63/65] add  sco ci

---
 dipu/scripts/ci/ci_run_one_iter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 2dd48ae5e..0c6e63c01 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -100,14 +100,14 @@ def process_one_iter(log_file, clear_log, model_info: dict) -> None:
         current_path = os.getcwd()
         parent_directory = os.path.dirname(current_path)
         if (p2 == "stable_diffusion/stable-diffusion_ddim_denoisingunet_infer.py"):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/mmagic/configs/stable_diffusion/stable-diffusion_ddim_denoisingunet_one_iter.sh" """
             cmd_cp_one_iter = ""
         elif ('infer' in p2 and 'infer' in p3):
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source /mnt/cache/share/deeplinkci/github/dipu_env && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && python {current_path}/{train_path}" """
             cmd_cp_one_iter = ""
         else:
-            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
-            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory}  && source /mnt/cache/share/deeplinkci/github/dipu_env && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && cd mmlab_pack && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
+            cmd_run_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/run_one_iter.sh {train_path} {config_path} {work_dir} {opt_arg}" """
+            cmd_cp_one_iter = f"""srun --job-name={job_name} bash -c "cd {parent_directory} && source scripts/ci/ci_one_iter.sh export_pythonpath_cuda {current_path} && source /mnt/cache/share/deeplinkci/github/dipu_env && cd mmlab_pack && source environment_exported && export ONE_ITER_TOOL_STORAGE_PATH={storage_path} && bash {current_path}/SMART/tools/one_iter_tool/compare_one_iter.sh {package_name}" """
     elif device == "camb" :
         # For the inference of large language models, simply compare the inference results on the current device directly with the results generated on the GPU
         if ('infer' in p2 and 'infer' in p3):

From ef0fb0eb719d17b5e2b5e1eb39c45f87e48a6a0f Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Thu, 18 Jan 2024 10:56:57 +0800
Subject: [PATCH 64/65] add  sco ci

---
 dipu/scripts/ci/ci_run_one_iter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dipu/scripts/ci/ci_run_one_iter.py b/dipu/scripts/ci/ci_run_one_iter.py
index 0c6e63c01..3fad912de 100644
--- a/dipu/scripts/ci/ci_run_one_iter.py
+++ b/dipu/scripts/ci/ci_run_one_iter.py
@@ -166,7 +166,7 @@ def print_file(file_name):
 
     device = args.device
     if device == 'sco':
-        max_parall = 4
+        max_parall = 3
     job_name = args.job_name
     gpu_requests = args.gpu_requests
     partition = args.partition_arg

From 548b29f87923eafe89b95ab8d8c25083434b75c7 Mon Sep 17 00:00:00 2001
From: wugeshui <wugeshui@sensetime.com>
Date: Tue, 23 Jan 2024 15:37:40 +0800
Subject: [PATCH 65/65] add  sco ci

---
 .github/workflows/main.yml                                 | 7 +++++--
 .../ci/test_one_iter_large_language_model_list.yaml        | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f04dc3fe7..5bac3bc61 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -250,6 +250,7 @@ jobs:
       - name: Build dipu
         run: |
           set -e
+#          cd ${DEEPLINK_PATH}/ && ls -al && find ${DEEPLINK_PATH}/ -maxdepth 1 -mmin +300 -type d |xargs rm -rf
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
           srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
           && source ${ENV_PATH}/dipu_env  \
@@ -297,7 +298,9 @@ jobs:
       - name: build some env
         run: |
           set -e
-          srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
+          export basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/
+          srun --job-name=${GITHUB_JOB} bash -c "cd ${basic_path} \
+          && export PYTHONPATH=$(basic_path)/mmlab_pack:$(basic_path)/mmlab_pack/mmengine:$(basic_path)/mmlab_pack/mmcv:$PYTHONPATH \
           && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
           && bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_one_iter.sh build_cuda"
       - name: run-one-iter-for-tradition
@@ -393,7 +396,7 @@ jobs:
           cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu
           source scripts/ci/ascend/ci_ascend_env.sh
           bash tests/run_ascend_tests.sh
-      
+
   Test-One-Iter-Ascend-910b:
     name: Test-one-iter-ascend-910b
     needs: [Build-Ascend-910b]
diff --git a/dipu/scripts/ci/test_one_iter_large_language_model_list.yaml b/dipu/scripts/ci/test_one_iter_large_language_model_list.yaml
index 33c12be37..f0cbccc9d 100644
--- a/dipu/scripts/ci/test_one_iter_large_language_model_list.yaml
+++ b/dipu/scripts/ci/test_one_iter_large_language_model_list.yaml
@@ -5,7 +5,7 @@ cuda:
     - model_cfg: "transformers examples/pytorch/language-modeling/llama_7b_infer.py workdirs_transformers_llama_infer"
     - model_cfg: "transformers examples/pytorch/language-modeling/internlm_7b_infer.py workdirs_transformers_internlm_infer"
     # lightllm
-    - model_cfg: "lightllm llama_7b_via_lightllm_infer.py workdirs_lightllm_llama_infer"
+    # - model_cfg: "lightllm llama_7b_via_lightllm_infer.py workdirs_lightllm_llama_infer"
 
 
 camb: