feat(packages): support extract from zip file

- support zip type url, oci artifacts, or local file. - update etcd url for ctl pkg in range [v8.4.0, ) Signed-off-by: wuhuizuo <wuhuizuo@126.com>
PingCAP-QE · Sep 11, 2024 · d84f2fb · d84f2fb
1 parent 46911bd
commit d84f2fb
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 7 deletions.
diff --git a/packages/packages.yaml.tmpl b/packages/packages.yaml.tmpl
@@ -48,16 +48,73 @@ components:
     builders:
       - image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
     routers:
-      - description: interpret versions according to semantic version spec.
-        if: {{ semver.CheckConstraint ">= 6.1.0-0" .Release.version }}
+      - description: For range [v8.4.0, )
+        if: {{ semver.CheckConstraint ">= 8.4.0-0" .Release.version }}
         os: [linux, darwin]
         arch: [amd64, arm64]
         profile: [release, experiment]
+        artifacts:
+          - name: "ctl-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+            files:
+              - name: cdc
+                src:
+                  type: oci
+                  url: "{{ .Release.registry }}/pingcap/tiflow/package:{{ .Git.ref }}_{{ .Release.os }}_{{ .Release.arch }}"
+                  path: "cdc-.*-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+                  extract: true
+                  extract_inner_path: cdc
+              - name: ctl
+                src:
+                  type: http
+                  url: "https://github.com/pingcap/tiup/releases/download/v1.8.1/tiup-v1.8.1-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+                  extract: true
+                  extract_inner_path: bin/tiup-ctl
+              - name: etcdctl
+                src:
+                  type: http                  
+                  url: "https://github.com/etcd-io/etcd/releases/download/v3.5.15/etcd-v3.5.15-{{ .Release.os }}-{{ .Release.arch }}{{ ternary ".zip" ".tar.gz" (eq .Release.os "darwin") }}"                
+                  extract: true
+                  extract_inner_path: etcd-v3.5.15-{{ .Release.os }}-{{ .Release.arch }}/etcdctl
+              - name: pd-ctl
+                src:
+                  type: oci
+                  url: "{{ .Release.registry }}/tikv/pd/package:{{ .Git.ref }}_{{ .Release.os }}_{{ .Release.arch }}"
+                  path: "pd-ctl-.*-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+                  extract: true
+                  extract_inner_path: pd-ctl
+              - name: tidb-ctl
+                src:
+                  type: oci
+                  url: "{{ .Release.registry }}/pingcap/tidb-ctl/package:master_{{ .Release.os }}_{{ .Release.arch }}"
+                  path: "tidb-ctl-.*-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+                  extract: true
+                  extract_inner_path: tidb-ctl
+              - name: tikv-ctl
+                src:
+                  type: oci
+                  url: "{{ .Release.registry }}/tikv/tikv/package:{{ .Git.ref }}_{{ .Release.os }}_{{ .Release.arch }}"
+                  path: "tikv-ctl-.*-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+                  extract: true
+                  extract_inner_path: tikv-ctl
+              - name: tidb-lightning-ctl
+                src:
+                  type: oci
+                  url: "{{ .Release.registry }}/pingcap/tidb/package:{{ .Git.ref }}_{{ .Release.os }}_{{ .Release.arch }}"
+                  path: "tidb-lightning-ctl-.*-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+                  extract: true
+                  extract_inner_path: tidb-lightning-ctl
+            tiup:
+              description: TiDB controller suite
+              entrypoint: ctl
+      - description: For range [v6.1.0, v8.4.0)
+        if: {{ semver.CheckConstraint ">= 6.1.0-0, < 8.4.0-0" .Release.version }}
+        os: [linux, darwin]
+        arch: [amd64, arm64]
+        profile: [release]
         artifacts:
           - name: "ctl-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
             files:
               - name: binlogctl
-                if: {{ semver.CheckConstraint "< 8.4.0-0" .Release.version }}
                 src:
                   type: oci
                   url: "{{ .Release.registry }}/pingcap/tidb-binlog/package:{{ .Git.ref }}_{{ .Release.os }}_{{ .Release.arch }}"
@@ -82,10 +139,6 @@ components:
                   type: http
                   {{- if eq .Release.os "darwin" }}
                   # ${FILE_SERVER_URL}/download/pingcap/etcd-${ETCDCTL_VERSION}-${os}-${arch}.tar.gz
-                  # Notice: it only support darwin arm64 platform from [v3.5.5](https://github.com/etcd-io/etcd/releases/tag/v3.5.5) in office artifacts.
-                  # When we upgrade to 3.5.x, we should replace it with the following url:
-                  # "https://github.com/etcd-io/etcd/releases/download/v3.4.30/etcd-v3.4.30-{{ .Release.os }}-{{ .Release.arch }}.zip"
-                  # Also we need support extract from zip archive.
                   url: "http://fileserver.pingcap.net/download/pingcap/etcd-v3.4.30-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
                   {{- else }}
                   url: "https://github.com/etcd-io/etcd/releases/download/v3.4.30/etcd-v3.4.30-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"

diff --git a/packages/scripts/build-package-artifacts.sh.tmpl b/packages/scripts/build-package-artifacts.sh.tmpl
@@ -133,7 +133,13 @@ function archive() {
     # - {{ .name }}
     mkdir -p "$(dirname ${archive_dir}/{{ .name }})"
     {{- if has .src "extract" }}
+      {{- if strings.HasSuffix ".tar.gz" .src.path }}
     tar -zxvf {{ .src.path }} --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path | strings.TrimSuffix "/") | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }}
+      {{- else if strings.HasSuffix ".zip" .src.path }}
+    unzip -j {{ .src.path }} {{ default "" .src.extract_inner_path }} -d $archive_dir
+      {{- else }}
+        {{ fail "Unsupported archive format, only support .tar.gz and .zip" }}
+      {{- end }}
       {{- if and (has .src "extract_inner_path") (ne (path.Base .src.extract_inner_path) .name) }}
     mv $archive_dir/{{ path.Base .src.extract_inner_path }} ${archive_dir}/{{ .name }}
       {{- end }}
@@ -150,7 +156,13 @@ function archive() {
     tarball_file=$(fetch_file_from_oci_artifact {{ .src.url }} "{{ .src.path }}")
     mkdir -p $(dirname ${archive_dir}/{{ .name }})
     {{- if has .src "extract" }}
+      {{- if strings.HasSuffix ".tar.gz" .src.path }}
     tar -zxvf $tarball_file --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path | strings.TrimSuffix "/") | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }}
+      {{- else if strings.HasSuffix ".zip" .src.path }}
+    unzip -j $tarball_file {{ default "" .src.extract_inner_path }} -d $archive_dir
+      {{- else }}
+        {{ fail .src.path }}
+      {{- end }}
       {{- if and (has .src "extract_inner_path") (ne (path.Base .src.extract_inner_path) .name) }}
     mv $archive_dir/{{ path.Base .src.extract_inner_path }} ${archive_dir}/{{ .name }}
       {{- end }}
@@ -167,7 +179,13 @@ function archive() {
     wget --tries=3 -c {{ .src.url }}
     mkdir -p $(dirname ${archive_dir}/{{ .name }})
     {{- if has .src "extract" }}
+      {{- if strings.HasSuffix ".tar.gz" .src.url }}
     tar -zxvf {{ path.Base .src.url }} --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path | strings.TrimSuffix "/") | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }}
+      {{- else if strings.HasSuffix ".zip" .src.url }}
+    unzip -j {{ path.Base .src.url }} {{ default "" .src.extract_inner_path }} -d $archive_dir
+      {{- else }}
+        {{ fail "Unsupported archive format, only support .tar.gz and .zip" }}
+      {{- end }}
       {{- if and (has .src "extract_inner_path") (ne (path.Base .src.extract_inner_path) .name) }}
     mv $archive_dir/{{ path.Base .src.extract_inner_path }} ${archive_dir}/{{ .name }}
       {{- end }}

diff --git a/packages/scripts/build-package-images.sh.tmpl b/packages/scripts/build-package-images.sh.tmpl
@@ -148,7 +148,13 @@ build_and_push_images() {
     # - {{ .name }}
     mkdir -p "$(dirname ${archive_dir}/{{ .name }})"
     {{- if has .src "extract" }}
+      {{- if strings.HasSuffix ".tar.gz" .src.path }}
     tar -zxvf {{ .src.path }} --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path | strings.TrimSuffix "/") | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }}
+      {{- else if strings.HasSuffix ".zip" .src.path }}
+    unzip -j {{ .src.path }} {{ default "" .src.extract_inner_path }} -d $archive_dir
+      {{- else }}
+        {{ fail "Unsupported archive format, only support .tar.gz and .zip" }}
+      {{- end }}
       {{- if and (has .src "extract_inner_path") (ne (path.Base .src.extract_inner_path) .name) }}
     mv $archive_dir/{{ path.Base .src.extract_inner_path }} ${archive_dir}/{{ .name }}
       {{- end }}
@@ -165,7 +171,13 @@ build_and_push_images() {
     wget --tries=3 -c {{ .src.url }}
     mkdir -p $(dirname ${archive_dir}/{{ .name }})
     {{- if has .src "extract" }}
+      {{- if strings.HasSuffix ".tar.gz" .src.url }}
     tar -zxvf {{ path.Base .src.url }} --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path | strings.TrimSuffix "/") | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }}
+      {{- else if strings.HasSuffix ".zip" .src.url }}
+    unzip -j {{ path.Base .src.url }} {{ default "" .src.extract_inner_path }} -d $archive_dir
+      {{- else }}
+        {{ fail "Unsupported archive format, only support .tar.gz and .zip" }}
+      {{- end }}
       {{- if and (has .src "extract_inner_path") (ne (path.Base .src.extract_inner_path) .name) }}
     mv $archive_dir/{{ path.Base .src.extract_inner_path }} ${archive_dir}/{{ .name }}
       {{- end }}

diff --git a/packages/scripts/compose-offline-packages-artifacts.sh.tmpl b/packages/scripts/compose-offline-packages-artifacts.sh.tmpl
@@ -207,7 +207,13 @@ function compose_artifact{{ $i }}() {
     # {{ .name }}
     tarball_file=$(fetch_file_from_oci_artifact {{ .src.url }} "{{ .src.path }}")
     {{- if has .src "extract" }}
+      {{- if strings.HasSuffix ".tar.gz" .src.path }}
     tar -zxvf $tarball_file --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path) | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }}
+      {{- else if strings.HasSuffix ".zip" .src.path }}
+    unzip -j $tarball_file {{ default "" .src.extract_inner_path }} -d $archive_dir
+      {{- else }}
+        {{ fail "Unsupported archive format, only support .tar.gz and .zip" }}
+      {{- end }}
     {{- else if has . "publish" }}
     tiup mirror publish {{ default .name .publish.name }} $version $tarball_file {{ .publish.entrypoint }} ${tiup_publish_options}
     {{- else }}
@@ -220,7 +226,13 @@ function compose_artifact{{ $i }}() {
     # {{ .name }}
     wget --tries=3 -c {{ .src.url }}
     {{- if has .src "extract" }}
+      {{- if strings.HasSuffix ".tar.gz" .src.url }}
     tar -zxvf {{ path.Base .src.url }} --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path) | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }}
+      {{- else if strings.HasSuffix ".zip" .src.url }}
+    unzip -j {{ path.Base .src.url }} {{ default "" .src.extract_inner_path }} -d $archive_dir
+      {{- else }}
+        {{ fail "Unsupported archive format, only support .tar.gz and .zip" }}
+      {{- end }}
     {{- else if has . "publish" }}
     tiup mirror publish {{ default .name .publish.name }} $version {{ path.Base .src.url }} {{ .publish.entrypoint }} ${tiup_publish_options}
     {{- else }}