.gitlab-ci.yml

variables:
  GITLAB_REPO: git@g.a-bug.org:maix_sw/k230_linux_sdk.git
  GITHUB_REPO: git@github.com:kendryte/k230_linux_sdk.git
  GITEE_REPO: git@gitee.com:kendryte/k230_linux_sdk.git
  DL_SITE: https://ai.b-bug.org/~/wangjianxin/dl/
  HW_VER: v0.1
  TFTP_BASE: /data1/tftp_server
  NFS_SERVER: 10.10.1.94
  NFS_BASE: /data/nfs_server
  TEST_CONFIGS: >
    k230_canmv_defconfig
    k230d_canmv_defconfig

default:
  image: ai.b-bug.org:5000/k230_sdk:latest
  tags:
    - k230_sdk

stages:
  - build_setup
  - build_src
  - smoke_test
  - release

.show_vars: &show_vars
  - echo "${JOB_TYPE}"
  - echo "${DST_BASE}"
  - echo "${SUB_BASE}"
  - echo "${image_url}"
  - echo "${image_path}"
  - echo "${BUILD}"

.get_job_type: &get_job_type
  - echo "----------get dst dir with job type----------"
  - >
    echo "check job from MR or tag or schedule or web";
    if [[ $CI_PIPELINE_SOURCE == "merge_request_event" ]]; then
      echo "current job is MR";
      JOB_TYPE="merge_request";
      DST_BASE="/data1/k230/gitlab-ci/images/${CI_PROJECT_NAME}";
    elif [[ $CI_PIPELINE_SOURCE == "schedule" ]]; then
      echo "current job is daily build schedule job";
      JOB_TYPE="daily_build";
      DST_BASE="/data1/k230/dev-release";
    elif [[ $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_TAG =~ ^v[0-9]+\.[0-9]+.*$ ]]; then
      echo "current job is pre release job";
      JOB_TYPE="pre_release";
      DST_BASE="/data1/k230/pre-release";
    elif [[ $CI_PIPELINE_SOURCE == "web" ]]; then
      echo "current job is manual test job";
      JOB_TYPE="manual_test";
      DST_BASE="/data1/k230/gitlab-ci/images/${CI_PROJECT_NAME}";
    elif [[ $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ ^v[0-9]+\.[0-9]+.*$ ]]; then
      echo "current job is release job";
      JOB_TYPE="release";
      DST_BASE="/data1/k230/release";
    else
      echo "current job is not define, EXIT with ERROR";
      exit 1;
    fi
  - echo ${JOB_TYPE}
  - echo ${DST_BASE}
  - echo "----------get dst dir job done----------"

.latest_version: &latest_version
  - echo "check latest commit id for daily build"
  - >
    if [[ "${JOB_TYPE}" == "daily_build" ]]; then
      echo "current job is daily_build, check commit id";
      cur_commitid=$(git rev-parse --short HEAD);
      echo "get latest commit id";
      if [ -d "${DST_BASE}/sdk/latest/" ]; then
        latest_file=$(ls "${DST_BASE}/sdk/latest/");
        echo "latest_file $latest_file";
        finished_commitid=$(echo $latest_file | sed "s/.*-gitlab-runner-\(.*\)\.tar\.gz/\1/g");
      else
        echo "latest_file not exist";
        finished_commitid="";
      fi
      echo "current commit_id $cur_commitid";
      echo "finished commit_id $finished_commitid";
      if [[ "$cur_commitid" == "$finished_commitid" ]]; then
        echo "WARNNING current commit id is equal to finished commit id";
        echo "There is no any code changes,  SKIP and EXIT";
        echo "BUILD=False" > build.env;
        exit 0;
      else
        echo "current commit id is NOT equal to finished commit id, will continue to run build job";
      fi
      echo "temp disable commit id check at 2023.12.18, and enable duplicate/same commit id build job";
    else
        echo "current job is NOT daily build, will continue to run build job";
    fi

.generate_version: &generate_version
  - echo "----------generate version----------"
  - echo "generate version with tag or commit id"
  - >
    if [[ $CI_COMMIT_TAG =~ ^v[0-9]+\.[0-9]+.*$ ]];
    then
      echo "tag exist, version should be tag";
      new_ver=$CI_COMMIT_TAG;
      echo "tag is ${new_ver}";
    else
      echo "tag is null, version should be commit id";
      commitid="unkonwn";
      latest_tag="unkonwn";
      git rev-parse --short HEAD  &&  commitid=$(git rev-parse --short HEAD);
      git describe --tags `git rev-list --tags --max-count=1` && latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`);
      cur_date=$(date "+%Y%m%d-%H%M%S") || exit 1;
      new_ver="${latest_tag}-${cur_date}-$(whoami)-$(hostname)-${commitid}" || exit 1;
      echo "ver is ${new_ver}";
    fi
  - echo ${new_ver} > version || exit 1
  - cat version || exit 1

.check_build_result: &check_build_result
  - date
  - >
    if [[ "${BUILD}" == "False" ]];
    then
      echo "WARNNING: current commit id is equal to finished commit id";
      echo "There is no any code changes,  SKIP test job and quit";
      exit 0;
    else
      echo "----------start to run test job----------";
    fi

before_script:
  - echo '----------Build ENV Prepare----------'
  - echo 'Add SSH KEY for Multiple repo'
  - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
  - eval $(ssh-agent -s)
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh
  - echo "$SSH_PRIVATE_KEY" | ssh-add - > ~/.ssh/id_ed25519 || exit 1
  - '[[ -f /.dockerenv ]] && echo -e "Host *\n    StrictHostKeyChecking no\n" > ~/.ssh/config'
  - echo $SHELL
  - echo "increate ssh timeout for github"
  - echo "    ServerAliveInterval 30" >> ~/.ssh/config
  - echo "    ServerAliveCountMax 60" >> ~/.ssh/config
  - echo "    TCPKeepAlive yes" >> ~/.ssh/config
  - cat ~/.ssh/config
  - whoami
  - uptime
  - pwd
  - uname -a
  - cat /etc/issue
  - echo $CI_PROJECT_DIR
  - echo $CI_PROJECT_NAME
  - echo '----------set git config ----------'
  - echo "${GITLAB_USER_EMAIL}"
  - echo "${GITLAB_USER_NAME}"
  - git config --global user.email "${GITLAB_USER_EMAIL}"
  - git config --global user.name "${GITLAB_USER_NAME}"
  - git config --global --add safe.directory $CI_PROJECT_DIR
  - echo '----------fetch all tags----------'
  - timeout 3m git fetch --tags || { timeout 3m git fetch --tags || { timeout 3m git fetch --tags || exit 1; }; }
  - HW_TYPE=$(echo $CONF | awk -F '_'  '{print $1}')
  - HW_MODEL=$(echo $CONF | awk -F "_"  '{if ($3 == "v2") print $2"_"$3; else print $2}')
  - echo "HW_TYPE ${HW_TYPE}, HW_MODEL ${HW_MODEL}"
  - ONLY_LINUX=True
  - ONLY_RTT=False
  - echo "ONLY_RTT is ${ONLY_RTT}"
  - echo "ONLY_LINUX is ${ONLY_LINUX}"

.github_to_gitlab_gitee:
  timeout: 30m
  retry: 2
  script:
    - echo $GITHUB_PUSH
    - cd $CI_PROJECT_DIR
    - ls -alht
    - rm -rf ./k230_linux_sdk/
    - echo "set http proxy for git clone"
    - git config --global http.proxy ${GITHUB_PROXY}
    - git config --global https.proxy ${GITHUB_PROXY}
    - timeout 3m git clone ${GITHUB_REPO} k230_linux_sdk || { timeout 3m git clone ${GITHUB_REPO} k230_linux_sdk || { timeout 3m git clone ${GITHUB_REPO} k230_linux_sdk || exit 1; }; }
    - cd k230_linux_sdk || exit 1
    - pwd
    - git checkout ${BRANCH}
    - git branch -a
    - git status
    - echo '---pull latest ${BRANCH} branch---'
    - timeout 3m git pull origin ${BRANCH} || { timeout 3m git pull origin ${BRANCH} || { timeout 3m git pull origin ${BRANCH} || exit 1; }; }
    - echo '---fetch all tags---'
    - timeout 3m git fetch --tags || { timeout 3m git fetch --tags || { timeout 3m git fetch --tags || exit 1; }; }
    - git remote add gitlab ${GITLAB_REPO}
    - git remote add gitee ${GITEE_REPO}
    - git remote -v
    - git branch -a
    - git status
    - echo "---push to gitee---"
    - timeout 3m git push --atomic --tags -u gitee ${BRANCH} || { timeout 3m git push --atomic --tags -u gitee ${BRANCH} || { timeout 3m git push --atomic --tags -u gitee ${BRANCH} || exit 1; }; }
    - echo "---push to gitlab---"
    - timeout 3m git push --atomic --tags -u gitlab ${BRANCH} || { timeout 3m git push --atomic --tags -u gitlab ${BRANCH} || { timeout 3m git push --atomic --tags -u gitlab ${BRANCH} || exit 1; }; }

.gitlab_to_github_gitee:
  timeout: 30m
  retry: 2
  script:
    - cd $CI_PROJECT_DIR
    - ls -alht
    - rm -rf ./k230_linux_sdk/
    - timeout 3m git clone ${GITLAB_REPO} k230_linux_sdk || { timeout 3m git clone ${GITLAB_REPO} k230_linux_sdk || { timeout 3m git clone ${GITLAB_REPO} k230_linux_sdk || exit 1; }; }
    - cd k230_linux_sdk || exit 1
    - pwd
    - git checkout ${BRANCH}
    - git branch -a
    - git status
    - echo '---pull latest ${BRANCH} branch---'
    - timeout 3m git pull origin ${BRANCH} || { timeout 3m git pull origin ${BRANCH} || { timeout 3m git pull origin ${BRANCH} || exit 1; }; }
    - echo '---fetch all tags---'
    - timeout 3m git fetch --tags || { timeout 3m git fetch --tags || { timeout 3m git fetch --tags || exit 1; }; }
    - git remote add github ${GITHUB_REPO}
    - git remote add gitee ${GITEE_REPO}
    - git remote -v
    - git branch -a
    - git status
    - echo "---push to gitee---"
    - timeout 3m git push --atomic --tags -u gitee ${BRANCH} || { timeout 3m git push --atomic --tags -u gitee ${BRANCH} || { timeout 3m git push --atomic --tags -u gitee ${BRANCH} || exit 1; }; }
    - echo "---push to github---"
    - timeout 3m git push --atomic --tags -u github ${BRANCH} || { timeout 3m git push --atomic --tags -u github ${BRANCH} || { timeout 3m git push --atomic --tags -u github ${BRANCH} || exit 1; }; }

.release_download_dir:
  timeout: 60m
  retry: 2
  tags:
    - k230_sdk
  script:
    - cd $CI_PROJECT_DIR
    - ls -alht
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/release/release_download_dir.sh -O ./release_download_dir.sh
    - chmod +x ./release_download_dir.sh
    - time ./release_download_dir.sh || time ./release_download_dir.sh
    - echo "all file synced"

.sync_release_image:
  timeout: 60m
  retry: 2
  tags:
    - k230_sdk
  script:
    - cd $CI_PROJECT_DIR
    - ls -alht
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/release/release_k230_linux_sdk_image.sh -O ./release_k230_linux_sdk_image.sh
    - chmod +x ./release_k230_linux_sdk_image.sh
    - echo $CI_COMMIT_TAG || exit 1
    - echo $CONF || exit 1
    - time ./release_k230_linux_sdk_image.sh $CI_COMMIT_TAG $CONF || time ./release_k230_linux_sdk_image.sh $CI_COMMIT_TAG $CONF
    - echo "all release file synced"

.update_version: &update_version
  - echo "---curren version ${SUB_BASE}"
  - >
    if [[ $CI_COMMIT_TAG =~ ^v[0-9]+\.[0-9]+.*$ ]];
    then
      echo "release tag exist, will replace version dir with tag";
      SUB_BASE=$CI_COMMIT_TAG;
      echo "tag is ${SUB_BASE}";
    else
      echo "tag is null, skip ver overwrite";
      echo "ver is ${SUB_BASE}";
    fi
  - echo "---update version ${SUB_BASE}"

.build_job: &build_job
  - pwd
  - echo "----------start to build linux sdk image----------"
  - time make CONF=${CONF}  BR2_PRIMARY_SITE=${DL_SITE} || exit 1
  - echo "----------show linux sdk image build output----------"
  - pwd
  - du -h -d 2 output/${CONF}/ || exit 1
  - ls -alht output/${CONF}/images || exit 1
  - echo "----------remove img----------"
  - rm -rf output/${CONF}/images/sysimage-sdcard.img
  - echo "----------build sdk image done----------"

.common:
  build_image:
    - cd $CI_PROJECT_DIR
    - echo "----------build image----------"
    - *build_job
    - echo "----------set test-image flag----------"
    - echo "SKIP=False" > build.env
  save_image:
    - pwd
    - echo "----------save image----------"
    - echo ${DST_BASE}
    - echo "set DST_DIR with different type based on docs/images/src"
    - DST_DIR="${DST_BASE}/linux_sdk_images"
    - echo ${DST_DIR}
    - echo "---create repo dir---"
    - sudo mkdir -p ${DST_DIR}
    - echo "----------Save build to external path----------"
    - *update_version
    - SUB_DIR="${SUB_BASE}/${CONF}";
    - echo "---create current image version dir---"
    - sudo mkdir -p ${DST_DIR}/${SUB_DIR}/ || exit 1
    - echo "---save sdk build output---"
    - sudo cp -rf --sparse=always -L ${SRC_DIR}/linux_${HW_TYPE}_${HW_MODEL}_*.img.gz ${DST_DIR}/${SUB_DIR}/
    - echo "${DST_DIR}/${SUB_DIR}/"
    - ls "${DST_DIR}/${SUB_DIR}/"
    - echo "add latest link for current build"
    - test -h ${DST_DIR}/latest && sudo rm ${DST_DIR}/latest
    - sudo ln -s ${DST_DIR}/${SUB_BASE} ${DST_DIR}/latest || exit 1
    - ls ${DST_DIR}/latest
    - echo "----------output URL----------"
    - echo "${DST_DIR}/${SUB_DIR}/" | sed "s/\/data1/https:\/\/ai\.b-bug\.org/g"
    - echo "----------save image done----------"
  save_tftp:
    - pwd
    - echo "----------save image to tftp----------"
    - >
      for CONFIG in $TEST_CONFIGS
        do
          if [[ "${CONF}" == "${CONFIG}" ]];
          then
            echo "start to save test image for default config: ${CONF} ------";
            echo ${TFTP_BASE};
            echo "set DST_DIR with different type based on docs/images/src";
            DST_DIR="${TFTP_BASE}/${HW_TYPE}/${HW_MODEL}_${HW_VER}";
            echo ${DST_DIR};
            echo "---create tftp dir---";
            sudo mkdir -p ${DST_DIR} || exit 1;
            echo "generate sub dir for MR/Pre-releae/relese job";
            echo "check job from MR or tag or schedule or web";
            if [[ ${CI_MERGE_REQUEST_IID} ]]; then
                echo "current job is MR, skip and use CI_MERGE_REQUEST_IID";
            elif [[ ${CI_COMMIT_TAG} ]]; then
                echo "current job is release job, use CI_COMMIT_TAG";
                CI_MERGE_REQUEST_IID=${CI_COMMIT_TAG};
            elif [[ $CI_PIPELINE_SOURCE == "schedule" ]]; then
                echo "current job is daily build schedule job, use CI_PIPELINE_SOURCE";
                CI_MERGE_REQUEST_IID=$CI_PIPELINE_SOURCE;
            else
                echo "current job is not match, use CI_PIPELINE_SOURCE";
                CI_MERGE_REQUEST_IID=$CI_PIPELINE_SOURCE;
            fi
            echo "current MR ID ${CI_MERGE_REQUEST_IID}";
            SUB_DIR="${CI_MERGE_REQUEST_IID}_${CI_PIPELINE_ID}/${CONF}";
            sudo mkdir -p ${DST_DIR}/${SUB_DIR}/ || exit 1;
            src_file="${SRC_DIR}/linux_${HW_TYPE}_${HW_MODEL}_*.img.gz";
            echo ${SRC_DIR};
            echo ${src_file};
            ls -alht output/${CONF}/images/ || exit 1;
            ls -alht -L ${src_file} || exit 1;
            sudo cp -rf --sparse=always -L ${src_file} ${DST_DIR}/${SUB_DIR}/sysimage-sdcard.img.gz;
            echo "${DST_DIR}/${SUB_DIR}/";
            ls -alht "${DST_DIR}/${SUB_DIR}/" || exit 1;
            sysimage_md5=$(md5sum ${DST_DIR}/${SUB_DIR}/sysimage-sdcard.img.gz | awk '{print $1}');
            echo "sysimage md5 is ${sysimage_md5}";
            echo "release job works in release_sdk sub dir, use full path";
            echo "sysimage_path=${DST_DIR}/${SUB_DIR}/sysimage-sdcard.img.gz" >> $CI_PROJECT_DIR/build.env;
            echo "sysimage_md5=${sysimage_md5}" >> $CI_PROJECT_DIR/build.env;
            ls;
            cat $CI_PROJECT_DIR/build.env;
          else
            echo "SKIP save test image as current config ${CONF} not match build config ${CONFIG} ------";
          fi
        done
    - echo "----------save image to tftp done----------"

.test:
  test_setup:
    - echo "----------Step 1. get available test devices----------"
    - |
      echo "check job from MR or tag or schedule or web";
      if [[ ${CI_MERGE_REQUEST_IID} ]]; then
          echo "current job is MR, skip and use CI_MERGE_REQUEST_IID";
      elif [[ ${CI_COMMIT_TAG} ]]; then
          echo "current job is release job, use CI_COMMIT_TAG";
          CI_MERGE_REQUEST_IID=${CI_COMMIT_TAG};
      elif [[ $CI_PIPELINE_SOURCE == "schedule" ]]; then
          echo "current job is daily build schedule job, use CI_PIPELINE_SOURCE";
          CI_MERGE_REQUEST_IID=$CI_PIPELINE_SOURCE;
      else
          echo "current job is not match, use CI_PIPELINE_SOURCE";
          CI_MERGE_REQUEST_IID=$CI_PIPELINE_SOURCE;
        fi
    - echo "MR ID ${CI_MERGE_REQUEST_IID}"
    - echo "Pipeline ID ${CI_PIPELINE_ID}"
    - test -d bin || mkdir ./bin
    - test -f ./bin/ailab && rm -rf ./bin/ailab
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/tools/ailab -O ./bin/ailab
    - chmod +x ./bin/ailab
    - |
      echo "for loop to get available test device"
      for i in {1..6}; do
        if [ $i -eq 6 ];
        then
          echo "ERROR: No Available DUT after 5 times retry, Please rerun curent job to check it again";
          exit 1;
        fi
        starttime=$(date +'%Y-%m-%d %H:%M:%S');
        echo "---loop $i start time: $starttime";
        echo "---start to get available test device in loop $i---";
        available=$(./bin/ailab show --dest available --domain ${DOMAIN} --hw_type ${HW_TYPE} --hw_model ${HW_MODEL} --hw_ver ${HW_VER} --format args)
        echo ${available}
        if [[ $available =~ "k230" ]];
        then
          echo "---Get Available DUT pass in loop $i---";
          echo "----------get available test devices done----------";
          echo "----------Step 2. reserve test device----------";
          reserved=$(./bin/ailab add ${available} --time 15m --site ${CI_PROJECT_NAME} --source ${CI_MERGE_REQUEST_IID} --job ${CI_PIPELINE_ID} --format args);
          echo ${reserved};
          if [[ ${reserved} =~ "gitlab" ]];
          then
            echo "Reserve DUT pass in loop $i, break loop and continue the next job";
            # exit for loop with break once device avaiable and reserved pass
            break;
          else
            echo "ERROR: Reserve DUT failed";
            if [ $i -eq 6 ];
            then
              echo "ERROR: No Available DUT after 5 times retry, Please rerun curent job to check it again";
              exit 1;
            else
              echo "---sleep 30 seconds and auto rerun in next loop...";
              sleep 30
            fi
          fi
        else
          echo "ERROR: Get available test device failed in loop $i, ignore Error and try again...";
          echo "---sleep for 30seconds and start to rerun in next loop...";
          sleep 30;
        fi
      done
    - echo "----------Step 3. save reserved/available for after_script----------"
    - echo "${available}" > available
    - echo "${reserved}" > reserved
    - echo "${HW_MODEL}" > HW_MODEL
    - echo "----------Step 4. power on/reset test device before test start----------"
    - power_off="./bin/ailab power --type=off ${available}";
    - power_on="./bin/ailab power --type=on ${available}";
    - power_reset="./bin/ailab power --type=cycle ${available}";
    - echo $power_off
    - echo $power_on
    - echo $power_reset
    - |
      echo "---start to run device power job before test start...";
      if [[ "$HW_MODEL" =~ "canmv_v2" ]]
      then
        echo "---canmv v2 board will be power reset at test start ---";
        $power_reset;
        sleep 2;
        echo "canmv_v2 board power reset done. continue to run test job";
      elif [[ "$HW_MODEL" == "canmv" ]]
      then
        echo "---canmv board will be power reset at test start ---";
        $power_reset;
        sleep 2;
        echo "canmv board power reset done. continue to run test job";
      else
        echo "---evb board will be power off and power on at test start---";
        $power_off;
        sleep 2;
        $power_on;
        sleep 2;
        echo "evb board power off and power on done. continue in next loop";
      fi
      echo "---device power action done, start to run test job...";
    - echo "----------reserve test device and power on/reset test device done----------"
  test_teardown:
    - echo "----------release test device----------"
    - echo "get variables from previous steps"
    - reserved=$(cat reserved)
    - available=$(cat available)
    - HW_MODEL=$(cat HW_MODEL)
    - echo "Release DUT start"
    - test -d bin || mkdir ./bin
    - test -f ./bin/ailab && rm -rf ./bin/ailab
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/tools/ailab -O ./bin/ailab
    - chmod +x ./bin/ailab
    - result=$(./bin/ailab update --dest=reserved ${reserved})
    - echo $result
    - |
      if [[ $result =~ "True" ]]; then
        echo "Release DUT pass";
      else
        echo "ERROR: Release DUT failed";
        exit 1;
      fi
    - power_off="./bin/ailab power --type=off ${available}";
    - power_on="./bin/ailab power --type=on ${available}";
    - power_reset="./bin/ailab power --type=cycle ${available}";
    - echo $power_off
    - echo $power_on
    - echo $power_reset
    - |
      echo "---start to run device power job after test done...";
      if [[ "$HW_MODEL" =~ "canmv_v2" ]]
      then
        echo "---canmv v2 board will skip power job at test start ---";
        # $power_reset;
        # sleep 2;
        # echo "canmv_v2 board power reset done. continue to run test job";
      elif [[ "$HW_MODEL" == "canmv" ]]
      then
        echo "---canmv board will skip power job after test done---";
        # $power_reset;
        # sleep 2;
        # echo "canmv board power reset done. continue to run test job";
      else
        echo "---evb board will be power off after test done---";
        $power_off;
        # sleep 2;
        # $power_on;
        # sleep 2;
        echo "evb board power off and power on done. continue in next loop";
      fi
      echo "---device power job done---";
    - echo "----------release test device done----------"
  copy_test_resource:
    - echo "----------Step 5. copy test resource----------"
    - echo "get build step result"
    - echo $image_path
    - echo $SUB_BASE
    - echo $HW_TYPE
    - echo $HW_MODEL
    - echo $HW_VER
    - echo "current test device config is ${CONF}"
    - pwd
    - echo "----------Step 6. save test resource to nfs case folder----------"
    - echo "---get nfs case folder---"
    - |
      echo ${NFS_BASE};
      echo "set NFS_DST_DIR with different type based on docs/images/src";
      NFS_DST_DIR="${NFS_BASE}/${HW_TYPE}/${HW_MODEL}_${HW_VER}";
      echo ${NFS_DST_DIR};
      echo "---create NFS dir---";
      echo "generate sub dir for MR/Pre-releae/relese job";
      echo "check job from MR or tag or schedule or web";
      if [[ ${CI_MERGE_REQUEST_IID} ]]; then
          echo "current job is MR, skip and use CI_MERGE_REQUEST_IID";
      elif [[ ${CI_COMMIT_TAG} ]]; then
          echo "current job is release job, use CI_COMMIT_TAG";
          CI_MERGE_REQUEST_IID=${CI_COMMIT_TAG};
      elif [[ $CI_PIPELINE_SOURCE == "schedule" ]]; then
          echo "current job is daily build schedule job, use CI_PIPELINE_SOURCE";
          CI_MERGE_REQUEST_IID=$CI_PIPELINE_SOURCE;
      else
          echo "current job is not match, use CI_PIPELINE_SOURCE";
          CI_MERGE_REQUEST_IID=$CI_PIPELINE_SOURCE;
      fi
      echo "current MR ID ${CI_MERGE_REQUEST_IID}";
      SUB_DIR="${CI_MERGE_REQUEST_IID}_${CI_PIPELINE_ID}";
      NFS_CASE_FOLDER="${NFS_DST_DIR}/${SUB_DIR}/${CONF}/${DOMAIN}";
      echo "NFS_CASE_FOLDER: $NFS_CASE_FOLDER";
    - echo "create $NFS_CASE_FOLDER"
    - sudo mkdir -p ${NFS_CASE_FOLDER} || exit 1;
    - sudo chmod -R 777 ${NFS_CASE_FOLDER} || exit 1;
    - echo "----------Step 7. generate test device yml file for test job----------"
    - echo "run command ./bin/ailab convert --dest=script --format yaml --file_name=${DOMAIN}.yml --nfs_server_ip $NFS_SERVER --nfs_case_folder $NFS_CASE_FOLDER --only_rtt ${ONLY_RTT} --only_linux ${ONLY_LINUX}--conf ${CONF} --pr_id ${CI_MERGE_REQUEST_IID} --job_id ${CI_PIPELINE_ID} --sysimage_md5 ${sysimage_md5} ${available}"
    - ./bin/ailab convert --dest=script --format yaml --file_name=${DOMAIN}.yml --nfs_server_ip $NFS_SERVER --nfs_case_folder $NFS_CASE_FOLDER --only_rtt ${ONLY_RTT} --only_linux ${ONLY_LINUX} --conf ${CONF} --pr_id ${CI_MERGE_REQUEST_IID} --job_id ${CI_PIPELINE_ID} --sysimage_md5 ${sysimage_md5} ${available}
    - test -f ${DOMAIN}.yml || exit 1
    - cat ${DOMAIN}.yml || exit 1
    - echo "cp ${DOMAIN}.yml to nfs_case_folder"
    - cp -rf --sparse=always ${DOMAIN}.yml $NFS_CASE_FOLDER/ || exit 1
    - test -f $NFS_CASE_FOLDER/${DOMAIN}.yml || exit 1
    - echo "show test resource in nfs_server"
    - echo $NFS_CASE_FOLDER
    - ls -alht $NFS_CASE_FOLDER
  load_image:
    - echo "----------load image to test device----------"
    - echo "sysimage-sdcard.img.gz md5 ${sysimage_md5}"
    - test -d bin || mkdir ./bin
    - test -f ./bin/aiload && rm -rf ./bin/aiload
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/tools/k230load -O ./bin/aiload
    - chmod +x ./bin/aiload
    - echo "MR ID ${CI_MERGE_REQUEST_IID}"
    - echo "Pipeline ID ${CI_PIPELINE_ID}"
    - |
      echo "---start to load image in for loop";
      stime=$(date +'%Y-%m-%d %H:%M:%S');
      echo "---load image start time: $stime";
      load_cmd="./bin/aiload --file_name=$NFS_CASE_FOLDER/${DOMAIN}.yml";
      echo "${load_cmd}";
      for i in {1..6}; do
        if [ $i -eq 6 ]
        then
            echo "ERROR: Max retries reached with load image";
            exit 1;
        fi
        starttime=$(date +'%Y-%m-%d %H:%M:%S');
        echo "---loop $i start time: $starttime";
        echo "---start to load image in loop $i---";
        if eval $load_cmd
        then
          echo "---loop $i load image pass---";
          break;
        else
          echo "ERROR:loop $i load image failed, ignore Error and retry again...";
          echo "---start to run device power job after failed job...";
          if [[ "$HW_MODEL" == "canmv_v2" ]]
          then
            echo "---canmv_v2 board will be power reset at the end of loop $i ---";
            $power_reset;
            sleep 15;
            echo "canmv_v2 board power reset done. continue in next loop";
          elif [[ "$HW_MODEL" == "canmv" ]]
          then
            echo "---canmv board will be power reset at the end of loop $i ---";
            $power_reset;
            sleep 15;
            echo "canmv board power reset done. continue in next loop";
          else
            echo "---evb board will be power off and power on at the end of loop $i---";
            $power_off;
            sleep 3;
            $power_on;
            sleep 15;
            echo "evb board power off and power on done. continue in next loop";
          fi
          echo "---device power action, start to rerun in next loop...";
        fi
        endtime=$(date +'%Y-%m-%d %H:%M:%S');
        start_seconds=$(date -d "$starttime" +%s);
        end_seconds=$(date -d "$endtime" +%s);
        echo "---loop $i end time: $endtime";
        echo "---loop $i total cost time：$((end_seconds - start_seconds)) s";
      done
      etime=$(date +'%Y-%m-%d %H:%M:%S');
      s_seconds=$(date -d "$stime" +%s);
      e_seconds=$(date -d "$etime" +%s);
      echo "---load image end time: $etime";
      echo "---load image total cost time：$((e_seconds - s_seconds)) s";
      echo "---finished load image in loop $i";
    - echo "---load image pass"
    - echo "----------load image to test device done----------"
  smoke_test:
    - echo "----------Run Smoke Test start----------"
    - test -d bin || mkdir ./bin
    - test -f ./bin/airobot && rm -rf ./bin/airobot
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/tools/airobot -O ./bin/airobot
    - chmod +x ./bin/airobot
    - echo "---add smoke test script"
    - git clone ${TEST_SCRIPTS_REPO} k230_testscripts || exit 1
    - cd k230_testscripts || exit 1
    - git fetch origin dev || { sleep 30; git fetch origin dev || { sleep 30; git fetch origin dev || exit 1; } }
    - git checkout dev || exit 1
    - cd ../ || exit 1
    - echo "---download evb board smoke test script in loop $i---";
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/testscripts/k230_sdk_smoke_test.sh -O ./k230_sdk_smoke_test.sh;
    - chmod +x ./k230_sdk_smoke_test.sh
    - |
      echo "---start to run k230 smoke test in for loop";
      stime=$(date +'%Y-%m-%d %H:%M:%S');
      echo "---run k230 smoke test start time: $stime";
      test_cmd="./k230_sdk_smoke_test.sh $NFS_CASE_FOLDER/${DOMAIN}.yml";
      for i in {1..3}; do
        if [ $i -eq 3 ];
        then
          echo "ERROR: Max retries reached with run k230 smoke test";
          exit 1;
        fi
        starttime=$(date +'%Y-%m-%d %H:%M:%S');
        echo "---loop $i start time: $starttime";
        echo "---start to run k230 smoke test in loop $i---";
        if eval $test_cmd;
        then
          echo "---loop $i run k230 smoke test pass---";
          break;
        else
          echo "ERROR:loop $i run k230 smoke test failed, ignore Error and try again...";
          echo "---start to rerun in next loop...";
          sleep 3
        fi
        endtime=$(date +'%Y-%m-%d %H:%M:%S');
        start_seconds=$(date -d "$starttime" +%s);
        end_seconds=$(date -d "$endtime" +%s);
        echo "---loop $i end time: $endtime";
        echo "---loop $i total cost time：$((end_seconds - start_seconds)) s";
      done
      etime=$(date +'%Y-%m-%d %H:%M:%S');
      s_seconds=$(date -d "$stime" +%s);
      e_seconds=$(date -d "$etime" +%s);
      echo "---run k230 smoke test end time: $etime";
      echo "---run k230 smoke test total cost time：$((e_seconds - s_seconds)) s";
      echo "---finished run k230 smoke test in loop $i";
    - echo "----------Run Smoke Test pass----------"
  daily_test:
    - echo "----------Run Daily Test start----------"
    - test -d bin || mkdir ./bin
    - test -f ./bin/airobot && rm -rf ./bin/airobot
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/tools/airobot -O ./bin/airobot
    - chmod +x ./bin/airobot
    - echo "---get daily test scripts"
    - git clone ${TEST_SCRIPTS_REPO} k230_testscripts || exit 1
    - cd k230_testscripts || exit 1
    - git fetch origin dev || { sleep 30; git fetch origin dev || { sleep 30; git fetch origin dev || exit 1; } }
    - git checkout dev || exit 1
    - cd ../ || exit 1
    - wget -qc https://ai.b-bug.org/k230/test_resources/ci/testscripts/k230_sdk_daily_test.sh -O ./k230_sdk_daily_test.sh
    - chmod +x ./k230_sdk_daily_test.sh
    - |
      echo "---start to run k230 daily test in for loop";
      stime=$(date +'%Y-%m-%d %H:%M:%S');
      echo "---run k230 daily test start time: $stime";
      test_cmd="./k230_sdk_daily_test.sh $NFS_CASE_FOLDER/${DOMAIN}.yml";
      for i in {1..3}; do
        if [ $i -eq 3 ];
        then
          echo "ERROR: Max retries reached with run k230 daily test";
          exit 1;
        fi
        starttime=$(date +'%Y-%m-%d %H:%M:%S');
        echo "---loop $i start time: $starttime";
        echo "---start to run k230 daily test in loop $i---";
        if eval $test_cmd;
        then
          echo "---loop $i run k230 daily test pass---";
          break;
        else
          echo "ERROR:loop $i run k230 daily test failed, ignore Error and try again...";
          echo "---start to rerun in next loop...";
          sleep 3;
        fi
        endtime=$(date +'%Y-%m-%d %H:%M:%S');
        start_seconds=$(date -d "$starttime" +%s);
        end_seconds=$(date -d "$endtime" +%s);
        echo "---loop $i end time: $endtime";
        echo "---loop $i total cost time：$((end_seconds - start_seconds)) s";
      done
      etime=$(date +'%Y-%m-%d %H:%M:%S');
      s_seconds=$(date -d "$stime" +%s);
      e_seconds=$(date -d "$etime" +%s);
      echo "---run k230 daily test end time: $etime";
      echo "---run k230 daily test total cost time：$((e_seconds - s_seconds)) s";
      echo "---finished run k230 daily test in loop $i";
    - echo "----------Run Daily Test pass----------"

# download dir publish with tag
download_dir_release:
  stage: release
  rules:
   - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+.*$/
  extends:
    - .release_download_dir

# sdk zip and image publish with tag
release_image_publish:
  stage: release
  rules:
   - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+.*$/
  parallel:
    matrix:
      - CONF:
        - k230_canmv_defconfig
        - k230d_canmv_defconfig

  extends:
    - .sync_release_image

# main branch
main_release:
  stage: release
  rules:
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"
  variables:
    BRANCH: main
  extends:
    - .gitlab_to_github_gitee

# tag push
tag_release:
  stage: release
  rules:
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+.*$/
  variables:
    BRANCH: main
  extends:
    - .gitlab_to_github_gitee

# schedule daily build sync
gitlab_to_github_gitee:
  stage: release
  rules:
    # daily build
    - if: $CI_PIPELINE_SOURCE == "schedule"
    # merge release
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"
  variables:
    BRANCH: dev
  extends:
    - .gitlab_to_github_gitee

# manual sync
manual_sync:
  stage: release
  rules:
    - if: '$CI_PIPELINE_SOURCE == "web"'
      when: always
  variables:
    BRANCH: dev
  extends:
    - .gitlab_to_github_gitee

# github webhook trigger sync
github_to_gitlab:
  stage: release
  rules:
    # trigger with args
    - if: '$CI_PIPELINE_SOURCE == "trigger" && $GITHUB_PUSH == "true"'
    # trigger without args
    - if: '$CI_PIPELINE_SOURCE == "trigger"'
  variables:
    BRANCH: dev
  extends:
    - .github_to_gitlab_gitee

# build setup for all jobs
build_setup:
  stage: build_setup
  rules:
    # MR job
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    # release job
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+.*$/
    # pre-release job
    - if: $CI_PIPELINE_SOURCE == "web"
    # daily build
    - if: $CI_PIPELINE_SOURCE == "schedule"
  timeout: 5m
  script:
    - echo "build env setup"
    - *get_job_type
    - *latest_version
    - *generate_version
    - echo "JOB_TYPE=${JOB_TYPE}" >> build.env
    - echo "DST_BASE=${DST_BASE}" >> build.env
    - echo "SUB_BASE=${new_ver}" >> build.env
  artifacts:
    reports:
      dotenv: build.env

build-image:
  stage: build_src
  needs: [build_setup]
  parallel:
    matrix:
      - CONF:
        - k230_canmv_defconfig
        - k230d_canmv_defconfig
  variables:
    DST_BASE: /data1/k230/gitlab-ci/images/${CI_PROJECT_NAME}
    SRC_DIR: ./output/${CONF}/images
  rules:
    # MR open
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    # release job
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+.*$/
    # pre-release job
    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_TAG =~ /^v\d+\.\d+.*$/
    # manual job
    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_TAG !~ /^v\d+\.\d+.*$/
    # daily build
    - if: $CI_PIPELINE_SOURCE == "schedule"
  timeout: 45m
  script:
    - *show_vars
    - *check_build_result
    - !reference [.common, build_image]
    - !reference [.common, save_image]
    - !reference [.common, save_tftp]
    - cat $CI_PROJECT_DIR/build.env
  artifacts:
    reports:
      dotenv: build.env

load_image_smoke_test:
  image: ai.b-bug.org:5000/k230_testscripts:latest
  stage: smoke_test
  variables:
    DOMAIN: ai
  parallel:
    matrix:
      - CONF:
        - k230_canmv_defconfig
        - k230d_canmv_defconfig
  rules:
    # MR open
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    # release job
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+.*$/
    # pre-release job
    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_TAG =~ /^v\d+\.\d+.*$/
    # manual job
    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_TAG !~ /^v\d+\.\d+.*$/
    # daily build
    - if: $CI_PIPELINE_SOURCE == "schedule"
  timeout: 30m
  script:
    - *show_vars
    - *check_build_result
    - !reference [.test, test_setup]
    - !reference [.test, copy_test_resource]
    - !reference [.test, load_image]
    #- !reference [.test, smoke_test]
  after_script:
    - !reference [.test, test_teardown]