quickube · GoshaDo · Jun 12, 2024 · Jun 12, 2024 · Jun 12, 2024 · Jun 12, 2024
@@ -3,13 +3,13 @@ name: E2E tests
 on:
   push:
     branches:
-      - "main"
+      - "add-gitlab-provider"
     paths:
-      - '**'
-      - '!docs/**'
+      - "**"
+      - "!docs/**"
   pull_request:
     branches:
-      - "main"
+      - "add-gitlab-provider"
 
 concurrency:
   group: ${{ github.workflow }} # Bottleneck in ngrok tunnel, only one tunnel can exist for specific token -${{ github.ref }}
@@ -19,8 +19,8 @@ permissions:
   contents: read
 
 jobs:
-  e2e-env-init:
-    name: E2E Tests (on development)
+  gitlab-e2e-env:
+    name: Gitlab E2E Tests (on development)
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -48,7 +48,140 @@ jobs:
           chmod +x kind
           sudo mv kind /usr/local/bin/kind
           kind version
-      - name:  Install Kubectl
+      - name: Install Kubectl
+        run: |
+          curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl"
+          chmod +x kubectl
+          sudo mv kubectl /usr/local/bin/kubectl
+          kubectl version --client --output=yaml
+      - name: Kubernetes KinD Cluster
+        run: |
+          make init-kind
+      - name: install nginx
+        run: |
+          make init-nginx
+      - name: install workflows
+        run: |
+          make init-argo-workflows
+      - name: install gitlab
+        run: |
+          make init-gitlab
+      - name: populate gitlab
+        run: |
+          sh ./scripts/setup-gitlab.sh
+      - name: Build Docker Image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: localhost:5001/piper:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Check tunnel existence
+        run: |
+          echo "NGROK_URL=$(cat ~/ngrok.log | grep -o 'url=https://.*' | cut -d '=' -f 2)" >> $GITHUB_ENV
+          cat ~/ngrok.log | grep -o 'url=https://.*' | cut -d '=' -f 2
+      - name: init piper
+        run: |
+          helm upgrade --install piper ./helm-chart \
+          -f ./examples/template.values.dev.yaml \
+          --set piper.gitProvider.token="gitlab" \
+          --set piper.gitProvider.token="${{ secrets.GIT_TOKEN }}" \
+          --set piper.gitProvider.webhook.url="${{ env.NGROK_URL }}/piper/webhook" \
+          --set piper.gitProvider.webhook.repoList={piper-e2e-test} \
+          --set piper.gitProvider.organization.name="Pied Pipers" \
+          --set image.repository=localhost:5001 \
+          --set piper.argoWorkflows.server.address="${{ env.NGROK_URL }}/argo" \
+          --set-string env\[0\].name=GIT_WEBHOOK_AUTO_CLEANUP,env\[0\].value="true" \
+          --set-string rookout.token="${{ secrets.ROOKOUT_TOKEN }}" && \
+          sleep 20 && kubectl logs deployment/piper
+          kubectl wait  \
+          --for=condition=ready pod \
+          --selector=app=piper \
+          --timeout=60s
+      - uses: actions/checkout@v3
+        with:
+          repository: "quickube/piper-e2e-test"
+          path: piper-e2e-test
+          ref: "main"
+      - name: clone-project
+        run: |
+          GITLAB_TOKEN=$(curl --location "http://localhost:8080/api/v4/users/${user_id}/personal_access_tokens" --header "PRIVATE-TOKEN: ${GITLAB_ROOT_TOKEN}" --header "Content-Type: application/json" \
+            --data '{"name":"clone-token", "scopes": ["read_repository"]}' | jq -r ".token")
+          cd ./gitlab
+          git clone http://oauth2:${GITLAB_TOKEN}@localhost:8080/piper-user/piper-e2e-test.git
+          cp ../piper-e2e-test ./gitlab/piper-e2e-test
+      - name: inject some changes to piper-e2e-test repo
+        run: |
+          echo "" >> .workflows/triggers.yaml
+          git config user.name 'piper-user'
+          git config user.email 'piper@example.com'
+          git commit -am "trigger e2e test"
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GIT_TOKEN }}
+          path: piper-e2e-test
+          branch: ${{ github.ref_name }}-test
+          title: ${{ github.ref_name }}-test
+          delete-branch: true
+      - name: Wait for workflow creation
+        run: |
+          sleep 10
+      - name: Close Pull Request
+        uses: peter-evans/close-pull@v3
+        with:
+          token: ${{ secrets.GIT_TOKEN }}
+          pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
+          repository: "quickube/piper-e2e-test"
+          comment: Auto-closing pull request
+          delete-branch: true
+      - name: Check Result
+        run: |
+          kubectl logs deployment/piper
+          kubectl get workflows.argoproj.io -n workflows
+          BRANCH_VALID_STRING=$(echo ${{ github.ref_name }}-test | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd 'a-z0-9.\-')
+
+          ## check if created
+          RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers | grep piper-e2e-test)
+          [ ! -z "$RESULT" ] && echo "CRD created $RESULT" || { echo "Workflow not exists, existing..."; exit 1; }
+
+          ## check if status phase not Failed, if yes, show message
+          RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING  --no-headers -o custom-columns="Status:status.phase")
+          MESSAGE=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.message")
+          [ ! "$RESULT" == "Failed"  ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; }
+
+  github-e2e-env:
+    name: Github E2E Tests (on development)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: network=host
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+          cache: true
+      - name: Install Ngrok Tunnel
+        run: |
+          curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && \
+          echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | sudo tee /etc/apt/sources.list.d/ngrok.list && \
+          sudo apt update && \
+          sudo apt install ngrok
+          touch ~/ngrok.log
+          (timeout 30m ngrok http 80 --authtoken ${{ secrets.NGROK_AUTHTOKEN }} --log ~/ngrok.log) &
+          echo $?
+      - name: Install kind
+        run: |
+          curl -sSLo kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.19.0/kind-linux-amd64"
+          chmod +x kind
+          sudo mv kind /usr/local/bin/kind
+          kind version
+      - name: Install Kubectl
         run: |
           curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl"
           chmod +x kubectl
@@ -94,9 +227,9 @@ jobs:
           --timeout=60s
       - uses: actions/checkout@v3
         with:
-          repository: 'quickube/piper-e2e-test'
+          repository: "quickube/piper-e2e-test"
           path: piper-e2e-test
-          ref: 'main'
+          ref: "main"
       - name: inject some changes to piper-e2e-test repo
         run: |
           cd ./piper-e2e-test
@@ -116,26 +249,25 @@ jobs:
       - name: Wait for workflow creation
         run: |
           sleep 10
-
       - name: Close Pull Request
         uses: peter-evans/close-pull@v3
         with:
           token: ${{ secrets.GIT_TOKEN }}
           pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
-          repository: 'quickube/piper-e2e-test'
+          repository: "quickube/piper-e2e-test"
           comment: Auto-closing pull request
           delete-branch: true
       - name: Check Result
         run: |
           kubectl logs deployment/piper
           kubectl get workflows.argoproj.io -n workflows
           BRANCH_VALID_STRING=$(echo ${{ github.ref_name }}-test | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd 'a-z0-9.\-')
-          
+
           ## check if created
           RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers | grep piper-e2e-test)
           [ ! -z "$RESULT" ] && echo "CRD created $RESULT" || { echo "Workflow not exists, existing..."; exit 1; }
 
           ## check if status phase not Failed, if yes, show message
           RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING  --no-headers -o custom-columns="Status:status.phase")
           MESSAGE=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.message")
-          [ ! "$RESULT" == "Failed"  ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; }
+          [ ! "$RESULT" == "Failed"  ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; }
@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine3.16 as builder
+FROM golang:1.20-alpine3.16 AS builder
 
 WORKDIR /piper
 
@@ -25,7 +25,7 @@ RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache
 RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build  go build -gcflags='all=-N -l' -tags=alpine -buildvcs=false  -trimpath ./cmd/piper
 
 
-FROM alpine:3.16 as piper-release
+FROM alpine:3.16 AS piper-release
 
 ENV GIN_MODE=release
 

@@ -28,7 +28,6 @@ func main() {
 			log.Printf("failed to start Rookout, error: %v\n", err)
 		}
 	}
-
 	err = cfg.WorkflowsConfig.WorkflowsSpecLoad("/piper-config/..data")
 	if err != nil {
 		log.Panicf("Failed to load workflow spec configuration, error: %v", err)
@@ -53,4 +52,4 @@ func main() {
 	defer stop()
 	event_handler.Start(ctx, stop, cfg, globalClients)
 	server.Start(ctx, stop, cfg, globalClients)
-}
+}
@@ -5,19 +5,22 @@ The helm chart populates them using [values.yaml](https://github.com/quickube/pi
 
 ### Git
 
-* GIT_PROVIDER
-  The git provider that Piper will use, possible variables: GitHub . We plan to support Bitbucket and GitLab, as well.
+- GIT_PROVIDER
+  The git provider that Piper will use, possible variables: gitHub, gitlab or bitbucket
 
-* GIT_TOKEN
-  The git token that will be used to connect to the git provider.
+- GIT_TOKEN
+  The git token that will be used.
 
-* GIT_ORG_NAME
+- GIT_URL
+  the git url that will be used, only relevant when running gitlab self hosted
+
+- GIT_ORG_NAME
   The organization name.
 
 * GIT_ORG_LEVEL_WEBHOOK
   Boolean variable, whether to config webhook at the organization level. Defaults to `false`.
 
-* GIT_WEBHOOK_REPO_LIST
+- GIT_WEBHOOK_REPO_LIST
   List of repositories to configure webhooks to.
 
 * GIT_WEBHOOK_URL
@@ -39,16 +42,16 @@ The helm chart populates them using [values.yaml](https://github.com/quickube/pi
 * ARGO_WORKFLOWS_TOKEN
   This token is used to authenticate with the Argo Workflows server.
 
-* ARGO_WORKFLOWS_ADDRESS
+- ARGO_WORKFLOWS_ADDRESS
   The address of Argo Workflows Server.
 
 * ARGO_WORKFLOWS_CREATE_CRD
   Boolean variable that deterines whether to directly send Workflows instructions or create a CRD in the Cluster.
 
-* ARGO_WORKFLOWS_NAMESPACE
+- ARGO_WORKFLOWS_NAMESPACE
   The namespace of Workflows creation for Argo Workflows.
 
-* KUBE_CONFIG
+- KUBE_CONFIG
   Used to configure Argo Workflows client with local kube configurations.
 
 ### Rookout

@@ -34,9 +34,9 @@ Refer to [values.yaml](https://github.com/quickube/piper/tree/main/helm-chart/va
 
 Piper will use git to fetch the `.workflows` folder and receive events using webhooks.
 
-To pick which git provider you are using provide `gitProvider.name` configuration in helm chart (Currently we only support GitHub and Bitbucket).
+To pick which git provider you are using provide `gitProvider.name` configuration in helm chart (supports GitHub, Bitbucket and Gitlab).
 
-You must also configure your organization (GitHub) or workspace (Bitbucket) name using `gitProvider.organization.name` in the helm chart.
+Also configure you organization (Github), workspace (Bitbucket) or group (Gitlab) name using `gitProvider.organization.name` in helm chart.
 
 #### Git Token Permissions
 

@@ -1,6 +1,6 @@
 gitlab:
   toolbox:
-    enabled: false
+    enabled: true
   gitlab-shell:
     enabled: false
   gitlab-pages:

@@ -45,6 +45,8 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -67,6 +69,7 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/xanzy/go-gitlab v0.105.0 // indirect
 	github.com/yhirose/go-peg v0.0.0-20210804202551-de25d6753cf1 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect

@@ -879,6 +879,11 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
+github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
@@ -1032,6 +1037,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/xanzy/go-gitlab v0.105.0 h1:3nyLq0ESez0crcaM19o5S//SvezOQguuIHZ3wgX64hM=
+github.com/xanzy/go-gitlab v0.105.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI=
 github.com/yhirose/go-peg v0.0.0-20210804202551-de25d6753cf1 h1:7iTmQ0lZwTtfm4XMgP5ezzWMDCjo7GTS0ZgCj6jpVzM=
 github.com/yhirose/go-peg v0.0.0-20210804202551-de25d6753cf1/go.mod h1:q2QWLflHsZxT6ixYcXveTYicEvxGh5Uv6CnI7f7BfjQ=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

@@ -100,6 +100,8 @@ spec:
                 key: token
           - name: GIT_ORG_NAME
             value: {{ .Values.piper.gitProvider.organization.name | quote }}
+          - name: GIT_URL
+            value: {{ .Values.piper.gitProvider.url | quote }}
           - name: GIT_WEBHOOK_URL
             value: {{ .Values.piper.gitProvider.webhook.url | quote }}
           - name: GIT_WEBHOOK_SECRET