feat: Gitlab provider (#27)

* fix: gitlab get scopes added * fix: finished newgitlabClient * fix: ListFiles complete * fix: get File and getFiles * fix: SetStatus + PinkHook still need to do tests on them, not sure if working * fix: set and del webhook done * fix: small change setWebhook * fix: done handlePaylod will now start tests * fix: pr review changes and fixed scopes * fix: first test done * fix: some pr changes resolving * docs: added provider and url to chart * fix: done utils tests * fix: started gitlab tests * fix: added some more tests and fixing bugs * fix: finished tests now runnning qa * fix: added some docs * fix: added some validations and docs * fix: argo ingress * fix: gitlab init ruby script * ci: add gitlab local support * ci: add gitlab local support * ci: fix e2e service account permissions * fix: gitlab setup script * ci: change rules to working branch * ci: start work on actions * fix: cleaning up * fix: context and git provider factory * fix: add context * fix: changes to make it work * ci: add local gitlab at localhost:8080 (#24) * fix: changes to make it work * fix: pipe works, small fixes left * fix: pipe works, small fixes left * test: fixed unit tests * test: running e2e * test: running e2e * test: gitlab e2e check * fix: gitlab unsetting webhook * fix: pr changes * fix: gitlab ruby init script finished * fix: gitlab init change * docs: updated docs for gitlab * fix: gitlab script cleanup * fix: some space * ci: changed running branch for testing * ci: changed order of jobs in e2e * ci: e2e to run on branch * test: gitlab e2e test * test: gitlab e2e test * test: gitlab e2e test * test: gitlab e2e test * test: gitlab e2e test * test: fix gitlab test * fix: gitlab rails script * ci: e2e test revert to main * ci: e2e check on main * ci: parallel e2e jobs * fix: gitlab e2e * fix: e2e tests * docs: align with main * fix: changed e2e * fix: gitlab license as env * fix: gitlab script add sleep * fix: lock gitlab helm version * fix: gitlab script * fix: gitlab ruby script * fix: gitlab ruby script --------- Co-authored-by: goshado <goshatoo@gmail.com> Co-authored-by: GoshaDo <86723475+GoshaDo@users.noreply.github.com>
quickube · Dec 24, 2024 · 8974887 · 8974887
1 parent fa7656a
commit 8974887
Show file tree

Hide file tree

Showing 27 changed files with 1,391 additions and 60 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -5,8 +5,8 @@ on:
     branches:
       - "main"
     paths:
-      - '**'
-      - '!docs/**'
+      - "**"
+      - "!docs/**"
   pull_request:
     branches:
       - "main"
@@ -19,8 +19,125 @@ permissions:
   contents: read
 
 jobs:
-  e2e-env-init:
-    name: E2E Tests (on development)
+  gitlab-e2e-env:
+    env:
+      GITLAB_LICENSE: ${{ secrets.GITLAB_LICENSE }}
+    name: Gitlab E2E Tests (on development)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: network=host
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+          cache: true
+      - name: Install kind
+        run: |
+          curl -sSLo kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.19.0/kind-linux-amd64"
+          chmod +x kind
+          sudo mv kind /usr/local/bin/kind
+          kind version
+      - name: Install Kubectl
+        run: |
+          curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl"
+          chmod +x kubectl
+          sudo mv kubectl /usr/local/bin/kubectl
+          kubectl version --client --output=yaml
+      - name: Kubernetes KinD Cluster
+        run: |
+          make init-kind
+      - name: install workflows
+        run: |
+          make init-argo-workflows
+      - name: install gitlab
+        run: |
+          tokens=$(make init-gitlab | tail -n1)
+          GROUP_TOKEN=$(echo "$tokens" | grep -oP "(?<=GROUP_TOKEN )\S+")
+          echo "GITLAB_TOKEN=$GROUP_TOKEN" >> $GITHUB_ENV
+      - name: Build Docker Image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: localhost:5001/piper:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Check tunnel existence
+        run: |
+          echo "NGROK_URL=$(cat ~/ngrok.log | grep -o 'url=https://.*' | cut -d '=' -f 2)" >> $GITHUB_ENV
+          cat ~/ngrok.log | grep -o 'url=https://.*' | cut -d '=' -f 2
+      - name: init piper
+        run: |
+          helm upgrade --install piper ./helm-chart \
+          -f ./examples/template.values.dev.yaml \
+          --set piper.gitProvider.name="gitlab" \
+          --set piper.gitProvider.token="${{ env.GITLAB_TOKEN }}" \
+          --set piper.gitProvider.url="http://gitlab-webservice-default.gitlab:8080" \
+          --set piper.gitProvider.webhook.url="http://piper.default/webhook" \
+          --set piper.gitProvider.webhook.repoList="piper-e2e-test" \
+          --set piper.gitProvider.organization.name="pied-pipers" \
+          --set image.repository=localhost:5001 \
+          --set piper.argoWorkflows.server.address="${{ env.NGROK_URL }}/argo" \
+          --set-string env\[0\].name=GIT_WEBHOOK_AUTO_CLEANUP,env\[0\].value="true" && \
+          sleep 20 && kubectl logs deployment/piper 
+          kubectl wait \
+          --for=condition=ready pod \
+          --selector=app=piper \
+          --timeout=60s
+      - uses: actions/checkout@v3
+        with:
+          repository: "quickube/piper-e2e-test"
+          path: piper-e2e-test
+          ref: "main"
+      - name: inject some changes to piper-e2e-test repo
+        run: |
+          mkdir ./gitlab
+          cd ./gitlab
+          git clone http://oauth2:${{ env.GITLAB_TOKEN }}@localhost:8080/pied-pipers/piper-e2e-test.git
+          cp -r ../piper-e2e-test/.workflows ./piper-e2e-test/
+          cd ./piper-e2e-test
+          git config user.name 'piper-user'
+          git config user.email 'piper@example.com'
+          git add -A
+          git commit -m "add stuff"
+          git push
+          git checkout -b ${{ github.ref_name }}-test      
+          rm ./.workflows/triggers.yaml
+          cat <<EOF > ./.workflows/triggers.yaml
+          - events:
+              - merge_request
+              - merge_request.open
+            branches: ["*"]
+            onStart: ["main.yaml"]
+            onExit: ["exit.yaml"]
+            templates: ["templates.yaml"]
+          EOF
+          git add -A
+          git commit -m "${{ github.ref_name }}-test"
+          git push --set-upstream origin ${{ github.ref_name }}-test -o merge_request.create
+      - name: Wait for workflow creation
+        run: |
+          sleep 10
+      - name: Check Result
+        run: |
+          kubectl logs deployment/piper
+          kubectl get workflows.argoproj.io -n workflows
+          BRANCH_VALID_STRING=$(echo ${{ github.ref_name }}-test | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd 'a-z0-9.\-')
+
+          ## check if created
+          RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers | grep piper-e2e-test)
+          [ ! -z "$RESULT" ] && echo "CRD created $RESULT" || { echo "Workflow not exists, existing..."; exit 1; }
+
+          ## check if status phase not Failed, if yes, show message
+          RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING  --no-headers -o custom-columns="Status:status.phase")
+          MESSAGE=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.message")
+          [ ! "$RESULT" == "Failed"  ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; }
+  github-e2e-env:
+    name: Github E2E Tests (on development)
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -48,7 +165,7 @@ jobs:
           chmod +x kind
           sudo mv kind /usr/local/bin/kind
           kind version
-      - name:  Install Kubectl
+      - name: Install Kubectl
         run: |
           curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl"
           chmod +x kubectl
@@ -79,24 +196,24 @@ jobs:
         run: |
           helm upgrade --install piper ./helm-chart \
           -f ./examples/template.values.dev.yaml \
+          --set piper.gitProvider.name="github" \
           --set piper.gitProvider.token="${{ secrets.GIT_TOKEN }}" \
           --set piper.gitProvider.webhook.url="${{ env.NGROK_URL }}/piper/webhook" \
           --set piper.gitProvider.webhook.repoList={piper-e2e-test} \
           --set piper.gitProvider.organization.name="quickube" \
           --set image.repository=localhost:5001 \
           --set piper.argoWorkflows.server.address="${{ env.NGROK_URL }}/argo" \
-          --set-string env\[0\].name=GIT_WEBHOOK_AUTO_CLEANUP,env\[0\].value="true" \
-          --set-string rookout.token="${{ secrets.ROOKOUT_TOKEN }}" && \
+          --set-string env\[0\].name=GIT_WEBHOOK_AUTO_CLEANUP,env\[0\].value="true" && \
           sleep 20 && kubectl logs deployment/piper
-          kubectl wait  \
+          kubectl wait \
           --for=condition=ready pod \
           --selector=app=piper \
           --timeout=60s
       - uses: actions/checkout@v3
         with:
-          repository: 'quickube/piper-e2e-test'
+          repository: "quickube/piper-e2e-test"
           path: piper-e2e-test
-          ref: 'main'
+          ref: "main"
       - name: inject some changes to piper-e2e-test repo
         run: |
           cd ./piper-e2e-test
@@ -116,26 +233,25 @@ jobs:
       - name: Wait for workflow creation
         run: |
           sleep 10
-
       - name: Close Pull Request
         uses: peter-evans/close-pull@v3
         with:
           token: ${{ secrets.GIT_TOKEN }}
           pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
-          repository: 'quickube/piper-e2e-test'
+          repository: "quickube/piper-e2e-test"
           comment: Auto-closing pull request
           delete-branch: true
       - name: Check Result
         run: |
           kubectl logs deployment/piper
           kubectl get workflows.argoproj.io -n workflows
           BRANCH_VALID_STRING=$(echo ${{ github.ref_name }}-test | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd 'a-z0-9.\-')
-          
+
           ## check if created
           RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers | grep piper-e2e-test)
           [ ! -z "$RESULT" ] && echo "CRD created $RESULT" || { echo "Workflow not exists, existing..."; exit 1; }
 
           ## check if status phase not Failed, if yes, show message
           RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING  --no-headers -o custom-columns="Status:status.phase")
           MESSAGE=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.message")
-          [ ! "$RESULT" == "Failed"  ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; }
+          [ ! "$RESULT" == "Failed"  ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; }
diff --git a/.gitignore b/.gitignore
@@ -104,6 +104,9 @@ venv.bak/
 # mkdocs documentation
 /site
 
+#mirrord config
+.mirrord/
+
 # mypy
 .mypy_cache/
 *.iml

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine3.16 as builder
+FROM golang:1.20-alpine3.16 AS builder
 
 WORKDIR /piper
 
@@ -25,7 +25,7 @@ RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache
 RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build  go build -gcflags='all=-N -l' -tags=alpine -buildvcs=false  -trimpath ./cmd/piper
 
 
-FROM alpine:3.16 as piper-release
+FROM alpine:3.16 AS piper-release
 
 ENV GIN_MODE=release
 

diff --git a/cmd/piper/piper.go b/cmd/piper/piper.go
@@ -53,4 +53,4 @@ func main() {
 	defer stop()
 	event_handler.Start(ctx, stop, cfg, globalClients)
 	server.Start(ctx, stop, cfg, globalClients)
-}
+}
diff --git a/docs/configuration/environment_variables.md b/docs/configuration/environment_variables.md
@@ -5,13 +5,16 @@ The helm chart populates them using [values.yaml](https://github.com/quickube/pi
 
 ### Git
 
-* GIT_PROVIDER
-  The git provider that Piper will use, possible variables: GitHub . We plan to support Bitbucket and GitLab, as well.
+- GIT_PROVIDER
+  The git provider that Piper will use, possible variables: GitHub | Gitlab | Bitbucket
 
 * GIT_TOKEN
   The git token that will be used to connect to the git provider.
 
-* GIT_ORG_NAME
+- GIT_URL
+  the git url that will be used, only relevant when running gitlab self hosted
+
+- GIT_ORG_NAME
   The organization name.
 
 * GIT_ORG_LEVEL_WEBHOOK

diff --git a/docs/configuration/health_check.md b/docs/configuration/health_check.md
@@ -1,5 +1,7 @@
 ## Health Check
 
+currently not supported for gitlab / bitbucket
+
 The following examples shows a health check being executed every 1 minute as configured in the helm chart under `livenessProbe`, and triggered by `/healthz` endpoint:
 
 ```yaml

diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md
@@ -36,13 +36,14 @@ Piper will use git to fetch the `.workflows` folder and receive events using web
 
 To pick which git provider you are using provide `gitProvider.name` configuration in helm chart (Currently we only support GitHub and Bitbucket).
 
-You must also configure your organization (GitHub) or workspace (Bitbucket) name using `gitProvider.organization.name` in the helm chart.
+Also configure you organization (Github), workspace (Bitbucket) or group (Gitlab) name using `gitProvider.organization.name` in helm chart.
 
 #### Git Token Permissions
 
-The token should have access for creating webhooks and read repositories content.
-For GitHub, configure `admin:org` and `write:org` permissions in Classic Token.
-For Bitbucket, configure `Repositories:read`, `Webhooks:read and write` and `Pull requests:read` permissions (for multiple repos use workspace token).
+The token should have access for creating webhooks and read repositories content.</br>
+<b>For GitHub</b>, configure `admin:org` and `write:org` permissions in Classic Token. </br>
+<b>For Bitbucket</b>, configure `Repositories:read`, `Webhooks:read and write` and `Pull requests:read` permissions (for multiple repos use workspace token). </br>
+<b>For Gitlab</b>, configure `read_api`, `write_repository` and `api` (for multiple repos use group token with owner role). </br>
 
 #### Token
 

diff --git a/examples/template.values.dev.yaml b/examples/template.values.dev.yaml
@@ -1,6 +1,6 @@
 piper:
   gitProvider:
-    name: github
+    name: ""  # github/bitbucket/gitlab | env: GIT_PROVIDER
     token: "GIT_TOKEN"
     organization:
       name: "ORG_NAME"

diff --git a/gitlab.values.yaml b/gitlab.values.yaml
@@ -1,19 +1,33 @@
 gitlab:
   toolbox:
-    enabled: false
+    enabled: true
+    extraVolumes: |-
+      - name: piper-config
+        configMap:
+          name: piper-setup
+    extraVolumeMounts: |-
+      - mountPath: /tmp/scripts/piper-setup.rb
+        name: piper-config
+        subPath: piper-setup.rb
+        readOnly: true
   gitlab-shell:
-    enabled: false
+    enabled: true
   gitlab-pages:
     enabled: false
   gitlab-exporter:
     enabled: false
   kas:
     minReplicas: 1
   webservice:
+    enabled: true
     minReplicas: 1
     ingress:
       requireBasePath: false
 global:
+  gitlab:
+    license:
+      key: license_key
+      secret: gitlab-license
   hosts:
     domain: localhost
     https: false
@@ -38,7 +52,6 @@ prometheus:
 certmanager:
   installCRDs: false
   install: false
-
 nginx-ingress:
   controller:
     ingressClassResource:

diff --git a/go.mod b/go.mod
@@ -13,6 +13,7 @@ require (
 	github.com/ktrysmt/go-bitbucket v0.9.66
 	github.com/stretchr/testify v1.8.4
 	github.com/tidwall/gjson v1.16.0
+	github.com/xanzy/go-gitlab v0.113.0
 	golang.org/x/net v0.17.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apimachinery v0.24.3
@@ -45,6 +46,8 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -53,7 +56,7 @@ require (
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -71,7 +74,7 @@ require (
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect