From 8974887966ef429ab7d5ecb92db97bd6060e4d46 Mon Sep 17 00:00:00 2001 From: omri assa <89311475+omri2001@users.noreply.github.com> Date: Tue, 24 Dec 2024 15:19:04 +0200 Subject: [PATCH] feat: Gitlab provider (#27) * fix: gitlab get scopes added * fix: finished newgitlabClient * fix: ListFiles complete * fix: get File and getFiles * fix: SetStatus + PinkHook still need to do tests on them, not sure if working * fix: set and del webhook done * fix: small change setWebhook * fix: done handlePaylod will now start tests * fix: pr review changes and fixed scopes * fix: first test done * fix: some pr changes resolving * docs: added provider and url to chart * fix: done utils tests * fix: started gitlab tests * fix: added some more tests and fixing bugs * fix: finished tests now runnning qa * fix: added some docs * fix: added some validations and docs * fix: argo ingress * fix: gitlab init ruby script * ci: add gitlab local support * ci: add gitlab local support * ci: fix e2e service account permissions * fix: gitlab setup script * ci: change rules to working branch * ci: start work on actions * fix: cleaning up * fix: context and git provider factory * fix: add context * fix: changes to make it work * ci: add local gitlab at localhost:8080 (#24) * fix: changes to make it work * fix: pipe works, small fixes left * fix: pipe works, small fixes left * test: fixed unit tests * test: running e2e * test: running e2e * test: gitlab e2e check * fix: gitlab unsetting webhook * fix: pr changes * fix: gitlab ruby init script finished * fix: gitlab init change * docs: updated docs for gitlab * fix: gitlab script cleanup * fix: some space * ci: changed running branch for testing * ci: changed order of jobs in e2e * ci: e2e to run on branch * test: gitlab e2e test * test: gitlab e2e test * test: gitlab e2e test * test: gitlab e2e test * test: gitlab e2e test * test: fix gitlab test * fix: gitlab rails script * ci: e2e test revert to main * ci: e2e check on main * ci: parallel e2e jobs * fix: gitlab e2e * fix: e2e tests * docs: align with main * fix: changed e2e * fix: gitlab license as env * fix: gitlab script add sleep * fix: lock gitlab helm version * fix: gitlab script * fix: gitlab ruby script * fix: gitlab ruby script --------- Co-authored-by: goshado Co-authored-by: GoshaDo <86723475+GoshaDo@users.noreply.github.com> --- .github/workflows/e2e.yaml | 144 +++++++- .gitignore | 3 + Dockerfile | 4 +- cmd/piper/piper.go | 2 +- docs/configuration/environment_variables.md | 9 +- docs/configuration/health_check.md | 2 + docs/getting_started/installation.md | 9 +- examples/template.values.dev.yaml | 2 +- gitlab.values.yaml | 19 +- go.mod | 7 +- go.sum | 17 +- helm-chart/templates/deployment.yaml | 2 + helm-chart/values.yaml | 32 +- makefile | 2 +- pkg/conf/git_provider.go | 7 +- pkg/event_handler/github_event_notifier.go | 19 +- pkg/git_provider/github.go | 1 + pkg/git_provider/gitlab.go | 374 ++++++++++++++++++++ pkg/git_provider/gitlab_test.go | 374 ++++++++++++++++++++ pkg/git_provider/gitlab_utils.go | 150 ++++++++ pkg/git_provider/gitlab_utils_test.go | 146 ++++++++ pkg/git_provider/main.go | 6 + pkg/git_provider/test_utils.go | 31 +- pkg/server/routes/webhook.go | 1 - pkg/webhook_creator/main.go | 7 +- scripts/gitlab-setup.yaml | 54 +++ scripts/init-gitlab.sh | 27 +- 27 files changed, 1391 insertions(+), 60 deletions(-) create mode 100644 pkg/git_provider/gitlab.go create mode 100644 pkg/git_provider/gitlab_test.go create mode 100644 pkg/git_provider/gitlab_utils.go create mode 100644 pkg/git_provider/gitlab_utils_test.go create mode 100644 scripts/gitlab-setup.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index fe87b11..bf79f92 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -5,8 +5,8 @@ on: branches: - "main" paths: - - '**' - - '!docs/**' + - "**" + - "!docs/**" pull_request: branches: - "main" @@ -19,8 +19,125 @@ permissions: contents: read jobs: - e2e-env-init: - name: E2E Tests (on development) + gitlab-e2e-env: + env: + GITLAB_LICENSE: ${{ secrets.GITLAB_LICENSE }} + name: Gitlab E2E Tests (on development) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v3 + - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-buildx-action@v2 + with: + driver-opts: network=host + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + cache: true + - name: Install kind + run: | + curl -sSLo kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.19.0/kind-linux-amd64" + chmod +x kind + sudo mv kind /usr/local/bin/kind + kind version + - name: Install Kubectl + run: | + curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/kubectl + kubectl version --client --output=yaml + - name: Kubernetes KinD Cluster + run: | + make init-kind + - name: install workflows + run: | + make init-argo-workflows + - name: install gitlab + run: | + tokens=$(make init-gitlab | tail -n1) + GROUP_TOKEN=$(echo "$tokens" | grep -oP "(?<=GROUP_TOKEN )\S+") + echo "GITLAB_TOKEN=$GROUP_TOKEN" >> $GITHUB_ENV + - name: Build Docker Image + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: localhost:5001/piper:latest + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Check tunnel existence + run: | + echo "NGROK_URL=$(cat ~/ngrok.log | grep -o 'url=https://.*' | cut -d '=' -f 2)" >> $GITHUB_ENV + cat ~/ngrok.log | grep -o 'url=https://.*' | cut -d '=' -f 2 + - name: init piper + run: | + helm upgrade --install piper ./helm-chart \ + -f ./examples/template.values.dev.yaml \ + --set piper.gitProvider.name="gitlab" \ + --set piper.gitProvider.token="${{ env.GITLAB_TOKEN }}" \ + --set piper.gitProvider.url="http://gitlab-webservice-default.gitlab:8080" \ + --set piper.gitProvider.webhook.url="http://piper.default/webhook" \ + --set piper.gitProvider.webhook.repoList="piper-e2e-test" \ + --set piper.gitProvider.organization.name="pied-pipers" \ + --set image.repository=localhost:5001 \ + --set piper.argoWorkflows.server.address="${{ env.NGROK_URL }}/argo" \ + --set-string env\[0\].name=GIT_WEBHOOK_AUTO_CLEANUP,env\[0\].value="true" && \ + sleep 20 && kubectl logs deployment/piper + kubectl wait \ + --for=condition=ready pod \ + --selector=app=piper \ + --timeout=60s + - uses: actions/checkout@v3 + with: + repository: "quickube/piper-e2e-test" + path: piper-e2e-test + ref: "main" + - name: inject some changes to piper-e2e-test repo + run: | + mkdir ./gitlab + cd ./gitlab + git clone http://oauth2:${{ env.GITLAB_TOKEN }}@localhost:8080/pied-pipers/piper-e2e-test.git + cp -r ../piper-e2e-test/.workflows ./piper-e2e-test/ + cd ./piper-e2e-test + git config user.name 'piper-user' + git config user.email 'piper@example.com' + git add -A + git commit -m "add stuff" + git push + git checkout -b ${{ github.ref_name }}-test + rm ./.workflows/triggers.yaml + cat < ./.workflows/triggers.yaml + - events: + - merge_request + - merge_request.open + branches: ["*"] + onStart: ["main.yaml"] + onExit: ["exit.yaml"] + templates: ["templates.yaml"] + EOF + git add -A + git commit -m "${{ github.ref_name }}-test" + git push --set-upstream origin ${{ github.ref_name }}-test -o merge_request.create + - name: Wait for workflow creation + run: | + sleep 10 + - name: Check Result + run: | + kubectl logs deployment/piper + kubectl get workflows.argoproj.io -n workflows + BRANCH_VALID_STRING=$(echo ${{ github.ref_name }}-test | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd 'a-z0-9.\-') + + ## check if created + RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers | grep piper-e2e-test) + [ ! -z "$RESULT" ] && echo "CRD created $RESULT" || { echo "Workflow not exists, existing..."; exit 1; } + + ## check if status phase not Failed, if yes, show message + RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.phase") + MESSAGE=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.message") + [ ! "$RESULT" == "Failed" ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; } + github-e2e-env: + name: Github E2E Tests (on development) runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -48,7 +165,7 @@ jobs: chmod +x kind sudo mv kind /usr/local/bin/kind kind version - - name: Install Kubectl + - name: Install Kubectl run: | curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl" chmod +x kubectl @@ -79,24 +196,24 @@ jobs: run: | helm upgrade --install piper ./helm-chart \ -f ./examples/template.values.dev.yaml \ + --set piper.gitProvider.name="github" \ --set piper.gitProvider.token="${{ secrets.GIT_TOKEN }}" \ --set piper.gitProvider.webhook.url="${{ env.NGROK_URL }}/piper/webhook" \ --set piper.gitProvider.webhook.repoList={piper-e2e-test} \ --set piper.gitProvider.organization.name="quickube" \ --set image.repository=localhost:5001 \ --set piper.argoWorkflows.server.address="${{ env.NGROK_URL }}/argo" \ - --set-string env\[0\].name=GIT_WEBHOOK_AUTO_CLEANUP,env\[0\].value="true" \ - --set-string rookout.token="${{ secrets.ROOKOUT_TOKEN }}" && \ + --set-string env\[0\].name=GIT_WEBHOOK_AUTO_CLEANUP,env\[0\].value="true" && \ sleep 20 && kubectl logs deployment/piper - kubectl wait \ + kubectl wait \ --for=condition=ready pod \ --selector=app=piper \ --timeout=60s - uses: actions/checkout@v3 with: - repository: 'quickube/piper-e2e-test' + repository: "quickube/piper-e2e-test" path: piper-e2e-test - ref: 'main' + ref: "main" - name: inject some changes to piper-e2e-test repo run: | cd ./piper-e2e-test @@ -116,13 +233,12 @@ jobs: - name: Wait for workflow creation run: | sleep 10 - - name: Close Pull Request uses: peter-evans/close-pull@v3 with: token: ${{ secrets.GIT_TOKEN }} pull-request-number: ${{ steps.cpr.outputs.pull-request-number }} - repository: 'quickube/piper-e2e-test' + repository: "quickube/piper-e2e-test" comment: Auto-closing pull request delete-branch: true - name: Check Result @@ -130,7 +246,7 @@ jobs: kubectl logs deployment/piper kubectl get workflows.argoproj.io -n workflows BRANCH_VALID_STRING=$(echo ${{ github.ref_name }}-test | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd 'a-z0-9.\-') - + ## check if created RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers | grep piper-e2e-test) [ ! -z "$RESULT" ] && echo "CRD created $RESULT" || { echo "Workflow not exists, existing..."; exit 1; } @@ -138,4 +254,4 @@ jobs: ## check if status phase not Failed, if yes, show message RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.phase") MESSAGE=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.message") - [ ! "$RESULT" == "Failed" ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; } \ No newline at end of file + [ ! "$RESULT" == "Failed" ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; } diff --git a/.gitignore b/.gitignore index 087275e..1f1ab50 100644 --- a/.gitignore +++ b/.gitignore @@ -104,6 +104,9 @@ venv.bak/ # mkdocs documentation /site +#mirrord config +.mirrord/ + # mypy .mypy_cache/ *.iml diff --git a/Dockerfile b/Dockerfile index 0c27447..1b103fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-alpine3.16 as builder +FROM golang:1.20-alpine3.16 AS builder WORKDIR /piper @@ -25,7 +25,7 @@ RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go build -gcflags='all=-N -l' -tags=alpine -buildvcs=false -trimpath ./cmd/piper -FROM alpine:3.16 as piper-release +FROM alpine:3.16 AS piper-release ENV GIN_MODE=release diff --git a/cmd/piper/piper.go b/cmd/piper/piper.go index a0ec7ac..30412a0 100644 --- a/cmd/piper/piper.go +++ b/cmd/piper/piper.go @@ -53,4 +53,4 @@ func main() { defer stop() event_handler.Start(ctx, stop, cfg, globalClients) server.Start(ctx, stop, cfg, globalClients) -} \ No newline at end of file +} diff --git a/docs/configuration/environment_variables.md b/docs/configuration/environment_variables.md index 483a62a..ee17d2d 100644 --- a/docs/configuration/environment_variables.md +++ b/docs/configuration/environment_variables.md @@ -5,13 +5,16 @@ The helm chart populates them using [values.yaml](https://github.com/quickube/pi ### Git -* GIT_PROVIDER - The git provider that Piper will use, possible variables: GitHub . We plan to support Bitbucket and GitLab, as well. +- GIT_PROVIDER + The git provider that Piper will use, possible variables: GitHub | Gitlab | Bitbucket * GIT_TOKEN The git token that will be used to connect to the git provider. -* GIT_ORG_NAME +- GIT_URL + the git url that will be used, only relevant when running gitlab self hosted + +- GIT_ORG_NAME The organization name. * GIT_ORG_LEVEL_WEBHOOK diff --git a/docs/configuration/health_check.md b/docs/configuration/health_check.md index d91dfec..2dafd89 100644 --- a/docs/configuration/health_check.md +++ b/docs/configuration/health_check.md @@ -1,5 +1,7 @@ ## Health Check +currently not supported for gitlab / bitbucket + The following examples shows a health check being executed every 1 minute as configured in the helm chart under `livenessProbe`, and triggered by `/healthz` endpoint: ```yaml diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index cb12a3a..682a8d1 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -36,13 +36,14 @@ Piper will use git to fetch the `.workflows` folder and receive events using web To pick which git provider you are using provide `gitProvider.name` configuration in helm chart (Currently we only support GitHub and Bitbucket). -You must also configure your organization (GitHub) or workspace (Bitbucket) name using `gitProvider.organization.name` in the helm chart. +Also configure you organization (Github), workspace (Bitbucket) or group (Gitlab) name using `gitProvider.organization.name` in helm chart. #### Git Token Permissions -The token should have access for creating webhooks and read repositories content. -For GitHub, configure `admin:org` and `write:org` permissions in Classic Token. -For Bitbucket, configure `Repositories:read`, `Webhooks:read and write` and `Pull requests:read` permissions (for multiple repos use workspace token). +The token should have access for creating webhooks and read repositories content.
+For GitHub, configure `admin:org` and `write:org` permissions in Classic Token.
+For Bitbucket, configure `Repositories:read`, `Webhooks:read and write` and `Pull requests:read` permissions (for multiple repos use workspace token).
+For Gitlab, configure `read_api`, `write_repository` and `api` (for multiple repos use group token with owner role).
#### Token diff --git a/examples/template.values.dev.yaml b/examples/template.values.dev.yaml index b9420f0..49d98c1 100644 --- a/examples/template.values.dev.yaml +++ b/examples/template.values.dev.yaml @@ -1,6 +1,6 @@ piper: gitProvider: - name: github + name: "" # github/bitbucket/gitlab | env: GIT_PROVIDER token: "GIT_TOKEN" organization: name: "ORG_NAME" diff --git a/gitlab.values.yaml b/gitlab.values.yaml index 1cad99a..e63ed28 100644 --- a/gitlab.values.yaml +++ b/gitlab.values.yaml @@ -1,8 +1,17 @@ gitlab: toolbox: - enabled: false + enabled: true + extraVolumes: |- + - name: piper-config + configMap: + name: piper-setup + extraVolumeMounts: |- + - mountPath: /tmp/scripts/piper-setup.rb + name: piper-config + subPath: piper-setup.rb + readOnly: true gitlab-shell: - enabled: false + enabled: true gitlab-pages: enabled: false gitlab-exporter: @@ -10,10 +19,15 @@ gitlab: kas: minReplicas: 1 webservice: + enabled: true minReplicas: 1 ingress: requireBasePath: false global: + gitlab: + license: + key: license_key + secret: gitlab-license hosts: domain: localhost https: false @@ -38,7 +52,6 @@ prometheus: certmanager: installCRDs: false install: false - nginx-ingress: controller: ingressClassResource: diff --git a/go.mod b/go.mod index 264a837..4773bcd 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/ktrysmt/go-bitbucket v0.9.66 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.16.0 + github.com/xanzy/go-gitlab v0.113.0 golang.org/x/net v0.17.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.24.3 @@ -45,6 +46,8 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -53,7 +56,7 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -71,7 +74,7 @@ require ( golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index c788bdd..276c1d8 100644 --- a/go.sum +++ b/go.sum @@ -696,6 +696,7 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fallais/logrus-lumberjack-hook v0.0.0-20210917073259-3227e1ab93b0 h1:6pt47P8Q9rWTQrS7LbP91HI8hjMN4zqupFn+IkxKFvI= github.com/fallais/logrus-lumberjack-hook v0.0.0-20210917073259-3227e1ab93b0/go.mod h1:m7ERym9P7Ic5dCEl43v3vWPC1Zn2thLbxW+o72yvlco= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -879,6 +880,11 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -937,10 +943,11 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= @@ -1032,6 +1039,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/xanzy/go-gitlab v0.113.0 h1:v5O4R+YZbJGxKqa9iIZxjMyeKkMKBN8P6sZsNl+YckM= +github.com/xanzy/go-gitlab v0.113.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY= github.com/yhirose/go-peg v0.0.0-20210804202551-de25d6753cf1 h1:7iTmQ0lZwTtfm4XMgP5ezzWMDCjo7GTS0ZgCj6jpVzM= github.com/yhirose/go-peg v0.0.0-20210804202551-de25d6753cf1/go.mod h1:q2QWLflHsZxT6ixYcXveTYicEvxGh5Uv6CnI7f7BfjQ= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1337,8 +1346,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/helm-chart/templates/deployment.yaml b/helm-chart/templates/deployment.yaml index e26b84e..10959fa 100644 --- a/helm-chart/templates/deployment.yaml +++ b/helm-chart/templates/deployment.yaml @@ -100,6 +100,8 @@ spec: key: token - name: GIT_ORG_NAME value: {{ .Values.piper.gitProvider.organization.name | quote }} + - name: GIT_URL + value: {{ .Values.piper.gitProvider.url | quote }} - name: GIT_WEBHOOK_URL value: {{ .Values.piper.gitProvider.webhook.url | quote }} - name: GIT_WEBHOOK_SECRET diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index acf5809..7d1e21b 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -4,7 +4,7 @@ # Map of Piper configurations. piper: gitProvider: - # -- Name of your git provider (github/bitbucket). + # -- Name of your git provider (github/bitbucket/gitlab). name: github # -- The token for authentication with the Git provider. # -- This will create a secret named -git-token and with the key 'token' @@ -13,9 +13,12 @@ piper: # -- Reference to existing token with 'token' key. # -- can be created with `kubectl create secret generic piper-git-token --from-literal=token=YOUR_TOKEN` existingSecret: #piper-git-token + # -- git provider url + # -- relevant when using gitlab self hosted + url: "" # Map of organization configurations. organization: - # -- Name of your Git Organization (GitHub) or Workspace (Bitbucket) + # -- Name of your Git Organization (GitHub) / Workspace (Bitbucket) or Group (Gitlab) name: "" # Map of webhook configurations. webhook: @@ -31,7 +34,7 @@ piper: url: "" #https://piper.example.local/webhook # -- Whether config webhook on org level (GitHub) or at workspace level (Bitbucket - not supported yet) orgLevel: false - # -- (Github) Used of orgLevel=false, to configure webhook for each of the repos provided. + # -- (Github/Gitlab) Used of orgLevel=false, to configure webhook for each of the repos provided. repoList: [] # Map of Argo Workflows configurations. @@ -51,8 +54,9 @@ piper: existingSecret: #piper-argo-token # -- Whether create Workflow CRD or send direct commands to Argo Workflows server. crdCreation: true - - workflowsConfig: {} + + workflowsConfig: + {} # default: | # spec: # volumes: @@ -126,22 +130,21 @@ podAnnotations: {} # -- Security Context to set on the pod level podSecurityContext: - fsGroup: 1001 - runAsUser: 1001 - runAsGroup: 1001 + fsGroup: 1001 + runAsUser: 1001 + runAsGroup: 1001 # -- Security Context to set on the container level securityContext: runAsUser: 1001 capabilities: drop: - - ALL + - ALL readOnlyRootFilesystem: true runAsNonRoot: true - service: - # -- Sets the type of the Service + # -- Sets the type of the Service type: ClusterIP # -- Service port # For TLS mode change the port to 443 @@ -158,10 +161,11 @@ ingress: # -- Piper ingress class name className: "" # -- Piper ingress annotations - annotations: {} + annotations: + {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - + # -- Piper ingress hosts ## Hostnames must be provided if Ingress is enabled. hosts: @@ -215,4 +219,4 @@ volumes: [] volumeMounts: [] # -- Specify postStart and preStop lifecycle hooks for Piper container -lifecycle: {} \ No newline at end of file +lifecycle: {} diff --git a/makefile b/makefile index bca3f7a..11c4e9f 100644 --- a/makefile +++ b/makefile @@ -36,7 +36,7 @@ init-piper: init-kind local-build .PHONY: init-gitlab init-gitlab: init-kind - sh ./scripts/init-gitlab.sh + @sh ./scripts/init-gitlab.sh $(GITLAB_LICENSE) .PHONY: deploy deploy: init-kind init-nginx init-argo-workflows local-build local-push init-piper diff --git a/pkg/conf/git_provider.go b/pkg/conf/git_provider.go index 9d6a0ad..5bf3d9a 100644 --- a/pkg/conf/git_provider.go +++ b/pkg/conf/git_provider.go @@ -1,7 +1,7 @@ package conf import ( - "fmt" + "fmt" "github.com/kelseyhightower/envconfig" ) @@ -9,13 +9,14 @@ import ( type GitProviderConfig struct { Provider string `envconfig:"GIT_PROVIDER" required:"true"` Token string `envconfig:"GIT_TOKEN" required:"true"` - OrgName string `envconfig:"GIT_ORG_NAME" required:"true"` + Url string `envconfig:"GIT_URL" required:"false"` + OrgName string `envconfig:"GIT_ORG_NAME" required:"true"` OrgLevelWebhook bool `envconfig:"GIT_ORG_LEVEL_WEBHOOK" default:"false" required:"false"` RepoList string `envconfig:"GIT_WEBHOOK_REPO_LIST" required:"false"` WebhookURL string `envconfig:"GIT_WEBHOOK_URL" required:"false"` WebhookSecret string `envconfig:"GIT_WEBHOOK_SECRET" required:"false"` WebhookAutoCleanup bool `envconfig:"GIT_WEBHOOK_AUTO_CLEANUP" default:"false" required:"false"` - EnforceOrgBelonging bool `envconfig:"GIT_ENFORCE_ORG_BELONGING" default:"false" required:"false"` + EnforceOrgBelonging bool `envconfig:"GIT_ENFORCE_ORG_BELONGING" default:"false" required:"false"` OrgID int64 FullHealthCheck bool `envconfig:"GIT_FULL_HEALTH_CHECK" default:"false" required:"false"` } diff --git a/pkg/event_handler/github_event_notifier.go b/pkg/event_handler/github_event_notifier.go index e5c3609..b878829 100644 --- a/pkg/event_handler/github_event_notifier.go +++ b/pkg/event_handler/github_event_notifier.go @@ -27,6 +27,15 @@ var workflowTranslationToBitbucketMap = map[string]string{ "Error": "STOPPED", } +var workflowTranslationToGitlabMap = map[string]string{ + "": "pending", + "Pending": "pending", + "Running": "running", + "Succeeded": "success", + "Failed": "failed", + "Error": "failed", +} + type githubNotifier struct { cfg *conf.GlobalConfig clients *clients.Clients @@ -72,13 +81,19 @@ func (gn *githubNotifier) translateWorkflowStatus(status string, workflowName st case "github": result, ok := workflowTranslationToGithubMap[status] if !ok { - return "", fmt.Errorf("failed to translate workflow status to github stasuts for %s status: %s", workflowName, status) + return "", fmt.Errorf("failed to translate workflow status to github status for %s status: %s", workflowName, status) } return result, nil case "bitbucket": result, ok := workflowTranslationToBitbucketMap[status] if !ok { - return "", fmt.Errorf("failed to translate workflow status to bitbucket stasuts for %s status: %s", workflowName, status) + return "", fmt.Errorf("failed to translate workflow status to bitbucket status for %s status: %s", workflowName, status) + } + return result, nil + case "gitlab": + result, ok := workflowTranslationToGitlabMap[status] + if !ok { + return "", fmt.Errorf("failed to translate workflow status to gitlab status for %s status: %s", workflowName, status) } return result, nil } diff --git a/pkg/git_provider/github.go b/pkg/git_provider/github.go index ec9e290..72467de 100644 --- a/pkg/git_provider/github.go +++ b/pkg/git_provider/github.go @@ -22,6 +22,7 @@ func NewGithubClient(cfg *conf.GlobalConfig) (Client, error) { ctx := context.Background() client := github.NewTokenClient(ctx, cfg.GitProviderConfig.Token) + err := ValidatePermissions(ctx, client, cfg) if err != nil { return nil, fmt.Errorf("failed to validate permissions: %v", err) diff --git a/pkg/git_provider/gitlab.go b/pkg/git_provider/gitlab.go new file mode 100644 index 0000000..2ddad80 --- /dev/null +++ b/pkg/git_provider/gitlab.go @@ -0,0 +1,374 @@ +package git_provider + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "strings" + + "github.com/quickube/piper/pkg/conf" + "github.com/quickube/piper/pkg/utils" + + "github.com/xanzy/go-gitlab" +) + +type GitlabClientImpl struct { + client *gitlab.Client + cfg *conf.GlobalConfig +} + +func NewGitlabClient(cfg *conf.GlobalConfig) (Client, error) { + var options []gitlab.ClientOptionFunc + ctx := context.Background() + + if cfg.GitProviderConfig.Url != "" { + options = append(options, gitlab.WithBaseURL(cfg.GitProviderConfig.Url)) + } + client, err := gitlab.NewClient(cfg.GitProviderConfig.Token, options...) + if err != nil { + return nil, fmt.Errorf("failed to authenticate user: %v", err) + } + + err = ValidateGitlabPermissions(ctx, client, cfg) + if err != nil { + return nil, fmt.Errorf("failed to validate permissions: %v", err) + } + + group, resp, err := client.Groups.GetGroup(cfg.GitProviderConfig.OrgName, nil) + if err != nil { + return nil, fmt.Errorf("failed to get organization: %v", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get organization data %s", resp.Status) + } + + cfg.GitProviderConfig.OrgID = int64(group.ID) + + log.Printf("Group ID is: %d\n", cfg.OrgID) + + return &GitlabClientImpl{ + client: client, + cfg: cfg, + }, err +} + +func (c *GitlabClientImpl) ListFiles(ctx *context.Context, repo string, branch string, path string) ([]string, error) { + log.Printf("Listing files for repo: %s", repo) + var files []string + opt := &gitlab.ListTreeOptions{ + Ref: &branch, + Path: &path} + + projectId, err := GetProjectId(ctx, c, &repo) + if err != nil { + return nil, err + } + dirFiles, resp, err := c.client.Repositories.ListTree(*projectId, opt, gitlab.WithContext(*ctx)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("gitlab provider returned %d: failed to get contents of %s/%s%s", resp.StatusCode, repo, branch, path) + } + for _, file := range dirFiles { + files = append(files, file.Name) + } + return files, nil +} + +func (c *GitlabClientImpl) GetFile(ctx *context.Context, repo string, branch string, path string) (*CommitFile, error) { + log.Printf("Getting file: %s", path) + commitFile := &CommitFile{} + projectId, err := GetProjectId(ctx, c, &repo) + if err != nil { + return nil, err + } + fmt.Println("got project id: ", *projectId) + fileContent, resp, err := c.client.RepositoryFiles.GetFile(*projectId, path, &gitlab.GetFileOptions{Ref: &branch}, gitlab.WithContext(*ctx)) + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + return nil, err + } + + decodedText, err := DecodeBase64ToStringPtr(fileContent.Content) + if err != nil { + return nil, err + } + + commitFile.Path = &fileContent.FilePath + commitFile.Content = decodedText + + return commitFile, nil +} + +func (c *GitlabClientImpl) GetFiles(ctx *context.Context, repo string, branch string, paths []string) ([]*CommitFile, error) { + log.Println("Getting multiple files") + var commitFiles []*CommitFile + for _, path := range paths { + file, err := c.GetFile(ctx, repo, branch, path) + if err != nil { + return nil, err + } + if file == nil { + log.Printf("file %s not found in repo %s branch %s", path, repo, branch) + continue + } + commitFiles = append(commitFiles, file) + } + log.Println("commit file", commitFiles) + return commitFiles, nil +} + +func (c *GitlabClientImpl) SetWebhook(ctx *context.Context, repo *string) (*HookWithStatus, error) { + var gitlabHookId *int + if *repo == "" { + log.Println("starting with group level hooks") + respHook, ok := IsGroupWebhookEnabled(ctx, c) + if !ok { + groupHookOptions := gitlab.AddGroupHookOptions{ + URL: &c.cfg.GitProviderConfig.WebhookURL, + Token: &c.cfg.GitProviderConfig.WebhookSecret, + MergeRequestsEvents: gitlab.Ptr(true), + PushEvents: gitlab.Ptr(true), + ReleasesEvents: gitlab.Ptr(true), + } + + gitlabHook, resp, err := c.client.Groups.AddGroupHook(c.cfg.GitProviderConfig.OrgName, &groupHookOptions, gitlab.WithContext(*ctx)) + if resp != nil { + if resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("for org level webhook, group token must be Owner level") + } else if resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("failed to create group level webhhok, API returned %d", resp.StatusCode) + } + } + if err != nil { + return nil, err + } + gitlabHookId = &gitlabHook.ID + log.Printf("added webhook: %d for %s name: %s\n", gitlabHook.ID, c.cfg.GitProviderConfig.OrgName, gitlabHook.URL) + } else { + editedGroupHookOpt := gitlab.EditGroupHookOptions{ + URL: gitlab.Ptr(c.cfg.GitProviderConfig.WebhookURL), + Token: gitlab.Ptr(c.cfg.GitProviderConfig.WebhookSecret), + MergeRequestsEvents: gitlab.Ptr(true), + PushEvents: gitlab.Ptr(true), + ReleasesEvents: gitlab.Ptr(true), + } + gitlabHook, resp, err := c.client.Groups.EditGroupHook(c.cfg.GitProviderConfig.OrgName, respHook.ID, &editedGroupHookOpt, gitlab.WithContext(*ctx)) + if resp != nil { + if resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("for org level webhook, group token must be Owner level") + } else if resp.StatusCode != http.StatusOK { + fmt.Println(resp.Request.URL, err) + return nil, fmt.Errorf( + "failed to update group level webhook for %s, API returned %d", + c.cfg.GitProviderConfig.OrgName, + resp.StatusCode, + ) + } + } + if err != nil { + return nil, err + } + gitlabHookId = &gitlabHook.ID + log.Printf("edited webhook for %s: %s\n", c.cfg.GitProviderConfig.OrgName, gitlabHook.URL) + } + } else { + projectId, err := GetProjectId(ctx, c, repo) + if err != nil { + return nil, err + } + log.Printf("project id is: %d\n", *projectId) + respHook, ok := IsProjectWebhookEnabled(ctx, c, *projectId) + + if !ok { + addProjectHookOpts := gitlab.AddProjectHookOptions{ + URL: &c.cfg.GitProviderConfig.WebhookURL, + Token: &c.cfg.GitProviderConfig.WebhookSecret, + MergeRequestsEvents: gitlab.Ptr(true), + PushEvents: gitlab.Ptr(true), + ReleasesEvents: gitlab.Ptr(true), + } + gitlabHook, resp, err := c.client.Projects.AddProjectHook(*projectId, &addProjectHookOpts, gitlab.WithContext(*ctx)) + if resp != nil { + if resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("for projects specific webhook, group token must be Maintainer level or above") + } else if resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("failed to create repo level webhhok for %s, API returned %d", *repo, resp.StatusCode) + } + } + if err != nil { + log.Println("url", *addProjectHookOpts.URL, *projectId, *addProjectHookOpts.Token) + return nil, fmt.Errorf("failed to add project hook ,%d", err) + } + gitlabHookId = &gitlabHook.ID + log.Printf("created webhook: %d for %s: %s\n", gitlabHook.ID, *repo, gitlabHook.URL) + } else { + editProjectHookOpts := gitlab.EditProjectHookOptions{ + URL: gitlab.Ptr(c.cfg.GitProviderConfig.WebhookURL), + Token: gitlab.Ptr(c.cfg.GitProviderConfig.WebhookSecret), + MergeRequestsEvents: gitlab.Ptr(true), + PushEvents: gitlab.Ptr(true), + ReleasesEvents: gitlab.Ptr(true), + } + gitlabHook, resp, err := c.client.Projects.EditProjectHook(*projectId, respHook.ID, &editProjectHookOpts, gitlab.WithContext(*ctx)) + if resp != nil { + if resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("for projects specific webhook, group token must be Maintainer level or above") + } else if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to update repo level webhhok for %s, API returned %d", *repo, resp.StatusCode) + } + } + if err != nil { + return nil, err + } + gitlabHookId = &gitlabHook.ID + log.Printf("edited webhook: %d for %s: %s\n", *gitlabHookId, *repo, gitlabHook.URL) + } + } + + hookID := int64(*gitlabHookId) + return &HookWithStatus{HookID: hookID, HealthStatus: true, RepoName: repo}, nil +} + +func (c *GitlabClientImpl) UnsetWebhook(ctx *context.Context, hook *HookWithStatus) error { + log.Println("unsetting webhook", *hook.RepoName) + if *hook.RepoName == "" { + resp, err := c.client.Groups.DeleteGroupHook(c.cfg.GitProviderConfig.OrgName, int(hook.HookID), gitlab.WithContext(*ctx)) + if resp != nil { + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("failed to delete group level webhhok, API call returned %d", resp.StatusCode) + } + } + if err != nil { + return err + } + log.Printf("removed group webhook, hookID :%d\n", hook.HookID) + } else { + projectId, err := GetProjectId(ctx, c, hook.RepoName) + if err != nil { + return err + } + resp, err := c.client.Projects.DeleteProjectHook(*projectId, int(hook.HookID), gitlab.WithContext(*ctx)) + if resp != nil { + if resp.StatusCode != http.StatusNoContent { + log.Printf("failed to delete project level webhhok for %s, API call returned %d", *hook.RepoName, resp.StatusCode) + return fmt.Errorf("failed to delete project level webhhok for %s, API call returned %d", *hook.RepoName, resp.StatusCode) + } + } + if err != nil { + log.Printf("failed to delete project level webhhok for %s, API call returned %s", *hook.RepoName, err) + return err + } + log.Printf("removed project webhook, project:%s hookID :%d\n", *hook.RepoName, hook.HookID) // INFO + } + + return nil +} + +func (c *GitlabClientImpl) HandlePayload(ctx *context.Context, request *http.Request, secret []byte) (*WebhookPayload, error) { + log.Printf("starting with payload") + var webhookPayload WebhookPayload + payload, err := io.ReadAll(request.Body) + if err != nil { + return nil, fmt.Errorf("error reading request body: %v", err) + } + event, err := gitlab.ParseWebhook(gitlab.WebhookEventType(request), payload) + if err != nil { + return nil, err + } + switch e := event.(type) { + case *gitlab.PushEvent: + webhookPayload = WebhookPayload{ + Event: "push", + Repo: e.Project.Name, + Branch: strings.TrimPrefix(e.Ref, "refs/heads/"), + Commit: e.CheckoutSHA, + User: e.UserName, + UserEmail: e.UserEmail, + OwnerID: int64(e.UserID), + } + case *gitlab.MergeEvent: + webhookPayload = WebhookPayload{ + Event: "merge_request", + Action: e.ObjectAttributes.Action, //open, close, reopen, update, approved, unapproved, approval, unapproval, merge + Repo: e.Repository.Name, + Branch: e.ObjectAttributes.SourceBranch, + Commit: e.ObjectAttributes.LastCommit.ID, + User: e.User.Name, + UserEmail: e.User.Email, + PullRequestTitle: e.ObjectAttributes.Title, + PullRequestURL: e.ObjectAttributes.URL, + DestBranch: e.ObjectAttributes.TargetBranch, + Labels: ExtractLabelsId(e.Labels), + OwnerID: int64(e.User.ID), + } + case *gitlab.ReleaseEvent: + webhookPayload = WebhookPayload{ + Event: "release", + Action: e.Action, // "create" | "update" | "delete" + Repo: e.Project.Name, + Branch: e.Tag, + Commit: e.Commit.ID, + User: e.Commit.Author.Name, + UserEmail: e.Commit.Author.Email, + } + } + log.Printf("sending payload: %s, %s", webhookPayload.Repo, webhookPayload.User) + return &webhookPayload, nil +} + +func (c *GitlabClientImpl) SetStatus(ctx *context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error { + if !utils.ValidateHTTPFormat(*linkURL) { + log.Println("invalid link URL", *linkURL) + return fmt.Errorf("invalid linkURL") + } + projectId, err := GetProjectId(ctx, c, repo) + if err != nil { + return err + } + + currCommit, resp, err := c.client.Commits.GetCommitStatuses(*projectId, *commit, nil, gitlab.WithContext(*ctx)) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to get commit status on repo:%s, commit:%s, API call returned %d", *repo, *commit, resp.StatusCode) + } + + if len(currCommit) != 0 { + if currCommit[0].Status == *status { + // https://forum.gitlab.com/t/cannot-transition-status-via-run-from-running-reason-s-status-cannot-transition-via-run/42588/6 + log.Printf("cannot change commit description without status also, status stays: %s", *status) + return nil + } + } + + repoStatus := gitlab.SetCommitStatusOptions{ + State: gitlab.BuildStateValue(*status), // pending, success, error, or failure. + TargetURL: linkURL, + Description: gitlab.Ptr(fmt.Sprintf("Workflow %s %s", *status, *message)), + Context: gitlab.Ptr("Piper/ArgoWorkflows"), + } + _, resp, err = c.client.Commits.SetCommitStatus(*projectId, *commit, &repoStatus, gitlab.WithContext(*ctx)) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusCreated { + return fmt.Errorf("failed to set status on repo:%s, commit:%s, API call returned %d", *repo, *commit, resp.StatusCode) + } + + log.Printf("successfully set status on repo:%s commit: %s to status: %s\n", *repo, *commit, *status) + return nil +} + +func (c *GitlabClientImpl) PingHook(ctx *context.Context, hook *HookWithStatus) error { + //TODO implement me + panic("implement me") +} diff --git a/pkg/git_provider/gitlab_test.go b/pkg/git_provider/gitlab_test.go new file mode 100644 index 0000000..beadfa1 --- /dev/null +++ b/pkg/git_provider/gitlab_test.go @@ -0,0 +1,374 @@ +package git_provider + +import ( + "encoding/base64" + "errors" + "fmt" + "github.com/quickube/piper/pkg/conf" + "github.com/quickube/piper/pkg/utils" + assertion "github.com/stretchr/testify/assert" + "github.com/xanzy/go-gitlab" + "golang.org/x/net/context" + "net/http" + "testing" +) + +func TestGitlabListFiles(t *testing.T) { + // Prepare + mux, client := setupGitlab(t) + + repoContent := gitlab.TreeNode{ + Type: "file", + Name: "exit.yaml", + Path: ".workflows/exit.yaml", + } + + repoContent2 := gitlab.TreeNode{ + Type: "file", + Name: "main.yaml", + Path: ".workflows/main.yaml", + } + + treeNodes := []gitlab.TreeNode{repoContent, repoContent2} + expectedRef := "branch1" + project := "project1" + + c := GitlabClientImpl{ + client: client, + cfg: &conf.GlobalConfig{ + GitProviderConfig: conf.GitProviderConfig{ + OrgLevelWebhook: true, + OrgName: "group1", + RepoList: project, + }, + }, + } + mux.HandleFunc("/api/v4/projects/1/repository/tree", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + ref := r.URL.Query().Get("ref") + + // Check if the ref value matches the expected value + if ref != expectedRef { + http.Error(w, "Invalid ref value", http.StatusBadRequest) + return + } + mockHTTPResponse(t, w, treeNodes) + }) + url := fmt.Sprintf("/api/v4/projects/%s/%s", c.cfg.GitProviderConfig.OrgName, project) + mockProject := gitlab.Project{ID: 1} + mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + + mockHTTPResponse(t, w, mockProject) + }) + + ctx := context.Background() + + // Execute + actualContent, err := c.ListFiles(&ctx, project, expectedRef, ".workflows") + + var expectedFilesNames []string + for _, file := range treeNodes { + expectedFilesNames = append(expectedFilesNames, file.Name) + } + + // Assert + assert := assertion.New(t) + assert.NotNil(t, err) + assert.Equal(expectedFilesNames, actualContent) +} + +func TestGitlabGetFile(t *testing.T) { + // Prepare + mux, client := setupGitlab(t) + project := "project1" + fileName := "file.yaml" + filePath := fmt.Sprintf(".workflows/%s", fileName) + c := GitlabClientImpl{ + client: client, + cfg: &conf.GlobalConfig{ + GitProviderConfig: conf.GitProviderConfig{ + OrgLevelWebhook: true, + OrgName: "group1", + RepoList: project, + }, + }, + } + branch := "branch1" + projectUrl := fmt.Sprintf("/api/v4/projects/%s/%s", c.cfg.GitProviderConfig.OrgName, project) + mockProject := &gitlab.Project{ID: 1} + mux.HandleFunc(projectUrl, func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + mockHTTPResponse(t, w, mockProject) + }) + decodedString := "file data" + encoded := base64.StdEncoding.EncodeToString([]byte(decodedString)) + + expectedFile := gitlab.File{ + Content: encoded, + FileName: fileName, + CommitID: "1", + FilePath: filePath, + } + url := fmt.Sprintf("/api/v4/projects/%d/repository/files/%s", mockProject.ID, filePath) + mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("ref") != branch { + t.Errorf("Unexpected request: %s", r.URL.String()) + } + testMethod(t, r, "GET") + mockHTTPResponse(t, w, expectedFile) + }) + + ctx := context.Background() + + // Execute + actualFile, err := c.GetFile(&ctx, project, branch, filePath) + // Assert + assert := assertion.New(t) + assert.NotNil(t, err) + assert.Equal(*actualFile.Path, expectedFile.FilePath) + assert.Equal(*actualFile.Content, decodedString) +} + +func TestGitlabSetStatus(t *testing.T) { + // Prepare + ctx := context.Background() + assert := assertion.New(t) + mux, client := setupGitlab(t) + + project := "test-repo1" + commit := "test-commit" + c := GitlabClientImpl{ + client: client, + cfg: &conf.GlobalConfig{ + GitProviderConfig: conf.GitProviderConfig{ + OrgLevelWebhook: false, + OrgName: "test", + RepoList: project, + }, + }, + } + projectUrl := fmt.Sprintf("/api/v4/projects/%s/%s", c.cfg.GitProviderConfig.OrgName, project) + mockProject := &gitlab.Project{ID: 1} + mux.HandleFunc(projectUrl, func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + mockHTTPResponse(t, w, mockProject) + }) + currCommitUrl := fmt.Sprintf("/api/v4/projects/%d/repository/commits/%s/statuses", mockProject.ID, commit) + + mux.HandleFunc(currCommitUrl, func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + mockHTTPResponse(t, w, []gitlab.CommitStatus{}) + }) + mux.HandleFunc("/api/v4/projects/1/statuses/test-commit", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "POST") + + w.WriteHeader(http.StatusCreated) + jsonBytes := []byte(`{"status": "ok"}`) + _, _ = fmt.Fprint(w, string(jsonBytes)) + }) + + // Define test cases + tests := []struct { + name string + repo *string + commit *string + linkURL *string + status *string + message *string + wantedError error + }{ + { + name: "Notify success", + repo: utils.SPtr(project), + commit: utils.SPtr(commit), + linkURL: utils.SPtr("https://argo"), + status: utils.SPtr("success"), + message: utils.SPtr(""), + wantedError: nil, + }, + { + name: "Notify pending", + repo: utils.SPtr(project), + commit: utils.SPtr(commit), + linkURL: utils.SPtr("https://argo"), + status: utils.SPtr("pending"), + message: utils.SPtr(""), + wantedError: nil, + }, + { + name: "Notify error", + repo: utils.SPtr(project), + commit: utils.SPtr(commit), + linkURL: utils.SPtr("https://argo"), + status: utils.SPtr("error"), + message: utils.SPtr("some message"), + wantedError: nil, + }, + { + name: "Notify failure", + repo: utils.SPtr(project), + commit: utils.SPtr(commit), + linkURL: utils.SPtr("https://argo"), + status: utils.SPtr("failure"), + message: utils.SPtr(""), + wantedError: nil, + }, + { + name: "Non managed repo", + repo: utils.SPtr("non-existing-repo"), + commit: utils.SPtr(commit), + linkURL: utils.SPtr("https://argo"), + status: utils.SPtr("error"), + message: utils.SPtr(""), + wantedError: errors.New("404 Not Found"), + }, + { + name: "Non existing commit", + repo: utils.SPtr(project), + commit: utils.SPtr("not-exists"), + linkURL: utils.SPtr("https://argo"), + status: utils.SPtr("error"), + message: utils.SPtr(""), + wantedError: errors.New("404 Not Found"), + }, + { + name: "Wrong URL", + repo: utils.SPtr(project), + commit: utils.SPtr(commit), + linkURL: utils.SPtr("argo"), + status: utils.SPtr("error"), + message: utils.SPtr(""), + wantedError: errors.New("invalid linkURL"), + }, + } + // Run test cases + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + // Call the function being tested + err := c.SetStatus(&ctx, test.repo, test.commit, test.linkURL, test.status, test.message) + + if test.wantedError != nil { + assert.NotNil(err) + assert.Equal(test.wantedError.Error(), err.Error()) + } else { + assert.Nil(err) + } + }) + } +} + +func TestGitlabSetWebhook(t *testing.T) { + // Prepare + ctx := context.Background() + assert := assertion.New(t) + mux, client := setupGitlab(t) + + hookUrl := "https://url" + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" && r.URL.String() == "/api/v4/groups/groupA/hooks" { + // new group webhook check for existing + mockHTTPResponse(t, w, []*gitlab.GroupHook{}) + } else if r.Method == "POST" && r.URL.String() == "/api/v4/groups/groupA/hooks" { + // new group webhook creation of new webhook + w.WriteHeader(http.StatusCreated) + mockHTTPResponse(t, w, gitlab.GroupHook{ID: 123, URL: hookUrl}) + } else if r.Method == "GET" && r.URL.String() == "/api/v4/groups/groupB/hooks" { + // existing group Webhook check for existing + mockHTTPResponse(t, w, []*gitlab.GroupHook{{ID: 123, URL: hookUrl}}) + } else if r.Method == "PUT" && r.URL.String() == "/api/v4/groups/groupB/hooks/123" { + // existing group Webhook editing the existing one + w.WriteHeader(http.StatusOK) + mockHTTPResponse(t, w, gitlab.GroupHook{ID: 123, URL: hookUrl}) + } else if r.Method == "GET" && r.URL.String() == "/api/v4/projects/test%2Ftest-repo1" { + // new project Webhook get project id + mockHTTPResponse(t, w, &gitlab.Project{ID: 1}) + } else if r.Method == "GET" && r.URL.String() == "/api/v4/projects/test%2Ftest-repo2" { + // new project Webhook get project id + mockHTTPResponse(t, w, &gitlab.Project{ID: 2}) + } else if r.Method == "GET" && r.URL.String() == "/api/v4/projects/1/hooks" { + // new project Webhook check for existing + mockHTTPResponse(t, w, []*gitlab.ProjectHook{{}}) + } else if r.Method == "POST" && r.URL.String() == "/api/v4/projects/1/hooks" { + // new project Webhook create new webhook + w.WriteHeader(http.StatusCreated) + mockHTTPResponse(t, w, gitlab.ProjectHook{ID: 123, URL: hookUrl}) + } else if r.Method == "GET" && r.URL.String() == "/api/v4/projects/2/hooks" { + // new project Webhook check for existing + mockHTTPResponse(t, w, []*gitlab.ProjectHook{{ID: 123, URL: hookUrl}}) + } else if r.Method == "PUT" && r.URL.String() == "/api/v4/projects/2/hooks/123" { + // new project Webhook edit existing webhook + w.WriteHeader(http.StatusOK) + mockHTTPResponse(t, w, gitlab.ProjectHook{ID: 123, URL: hookUrl}) + } else { + fmt.Println("unhandled ", r.Method, " route: ", r.URL.String()) + } + }) + + c := GitlabClientImpl{ + client: client, + cfg: &conf.GlobalConfig{ + GitProviderConfig: conf.GitProviderConfig{}, + }, + } + + // Define test cases + tests := []struct { + name string + repo *string + config conf.GitProviderConfig + }{ + { + name: "New group webhook", + repo: nil, + config: conf.GitProviderConfig{ + OrgLevelWebhook: true, + OrgName: "groupA", + RepoList: "", + WebhookURL: hookUrl, + }, + }, + { + name: "Existing group webhook", + repo: nil, + config: conf.GitProviderConfig{ + OrgLevelWebhook: true, + OrgName: "groupB", + RepoList: "", + WebhookURL: hookUrl, + }, + }, + { + name: "New project webhook", + repo: utils.SPtr("test-repo1"), + config: conf.GitProviderConfig{ + OrgLevelWebhook: false, + OrgName: "test", + RepoList: "test-repo1", + WebhookURL: hookUrl, + }, + }, + { + name: "Existing project webhook", + repo: utils.SPtr("test-repo2"), + config: conf.GitProviderConfig{ + OrgLevelWebhook: false, + OrgName: "test", + RepoList: "test-repo2", + WebhookURL: hookUrl, + }, + }, + } + // Run test cases + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c.cfg.GitProviderConfig = test.config + _, err := c.SetWebhook(&ctx, &c.cfg.GitProviderConfig.RepoList) + + // Use assert to check the equality of the error + assert.Nil(err) + }) + } +} diff --git a/pkg/git_provider/gitlab_utils.go b/pkg/git_provider/gitlab_utils.go new file mode 100644 index 0000000..726c38f --- /dev/null +++ b/pkg/git_provider/gitlab_utils.go @@ -0,0 +1,150 @@ +package git_provider + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "strings" + + "github.com/quickube/piper/pkg/conf" + "github.com/quickube/piper/pkg/utils" + "github.com/xanzy/go-gitlab" + "golang.org/x/net/context" +) + +func ValidateGitlabPermissions(ctx context.Context, client *gitlab.Client, cfg *conf.GlobalConfig) error { + + repoAdminScopes := []string{"api"} + repoGranularScopes := []string{"write_repository", "read_api"} + + token, _, err := client.PersonalAccessTokens.GetSinglePersonalAccessToken() + if err != nil { + return fmt.Errorf("failed to get scopes: %v", err) + } + scopes := token.Scopes + + if len(scopes) == 0 { + return fmt.Errorf("permissions error: no scopes found for the gitlab client") + } + + if utils.ListContains(repoAdminScopes, scopes) { + return nil + } + if utils.ListContains(repoGranularScopes, scopes) { + return nil + } + + return fmt.Errorf("permissions error: %v is not a valid scope for the project level permissions", scopes) +} + +func IsGroupWebhookEnabled(ctx *context.Context, c *GitlabClientImpl) (*gitlab.GroupHook, bool) { + emptyHook := gitlab.GroupHook{} + hooks, resp, err := c.client.Groups.ListGroupHooks(c.cfg.GitProviderConfig.OrgName, nil, gitlab.WithContext(*ctx)) + + if err != nil { + return &emptyHook, false + } + if resp.StatusCode != 200 { + return &emptyHook, false + } + if len(hooks) != 0 { + for _, hook := range hooks { + if hook.URL == c.cfg.GitProviderConfig.WebhookURL { + return hook, true + } + } + } + return &emptyHook, false +} + +func IsProjectWebhookEnabled(ctx *context.Context, c *GitlabClientImpl, projectId int) (*gitlab.ProjectHook, bool) { + emptyHook := gitlab.ProjectHook{} + + hooks, resp, err := c.client.Projects.ListProjectHooks(projectId, nil, gitlab.WithContext(*ctx)) + if err != nil { + return &emptyHook, false + } + if resp.StatusCode != 200 { + return &emptyHook, false + } + if len(hooks) == 0 { + return &emptyHook, false + } + + for _, hook := range hooks { + if hook.URL == c.cfg.GitProviderConfig.WebhookURL { + return hook, true + } + } + + return &emptyHook, false +} + +func ExtractLabelsId(labels []*gitlab.EventLabel) []string { + var returnLabelsList []string + for _, label := range labels { + returnLabelsList = append(returnLabelsList, fmt.Sprint(label.ID)) + } + return returnLabelsList +} + +func GetProjectId(ctx *context.Context, c *GitlabClientImpl, repo *string) (*int, error) { + projectFullName := fmt.Sprintf("%s/%s", c.cfg.GitProviderConfig.OrgName, *repo) + IProject, _, err := c.client.Projects.GetProject(projectFullName, nil, gitlab.WithContext(*ctx)) + if err != nil { + log.Printf("Failed to get project (%s): %v", *repo, err) + return nil, err + } + return &IProject.ID, nil +} + +func ValidatePayload(r *http.Request, secret []byte) ([]byte, error) { + payload, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("error reading request body: %v", err) + } + + // Get GitLab signature from headers + gitlabSignature := r.Header.Get("X-Gitlab-Token") + if gitlabSignature == "" { + return nil, fmt.Errorf("no GitLab signature found in headers") + } + + h := hmac.New(sha256.New, secret) + _, err = h.Write(payload) + if err != nil { + return nil, fmt.Errorf("error computing HMAC: %v", err) + } + expectedMAC := hex.EncodeToString(h.Sum(nil)) + + isEqual := hmac.Equal([]byte(gitlabSignature), []byte(expectedMAC)) + if !isEqual { + return nil, fmt.Errorf("secret not correct") + } + return payload, nil +} + +func FixRepoNames(c *GitlabClientImpl) error { + var formattedRepos []string + for _, repo := range strings.Split(c.cfg.GitProviderConfig.RepoList, ",") { + userRepo := fmt.Sprintf("%s/%s", c.cfg.GitProviderConfig.OrgName, repo) + formattedRepos = append(formattedRepos, userRepo) + } + c.cfg.GitProviderConfig.RepoList = strings.Join(formattedRepos, ",") + return nil +} + +func DecodeBase64ToStringPtr(encoded string) (*string, error) { + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, err + } + + result := string(decoded) + return &result, nil +} diff --git a/pkg/git_provider/gitlab_utils_test.go b/pkg/git_provider/gitlab_utils_test.go new file mode 100644 index 0000000..3ceee0f --- /dev/null +++ b/pkg/git_provider/gitlab_utils_test.go @@ -0,0 +1,146 @@ +package git_provider + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + + "github.com/quickube/piper/pkg/conf" + assertion "github.com/stretchr/testify/assert" + "github.com/xanzy/go-gitlab" + "golang.org/x/net/context" +) + +func mockHTTPResponse(t *testing.T, w io.Writer, response interface{}) { + err := json.NewEncoder(w).Encode(response) + if err != nil { + fmt.Printf("error %s", err) + } +} + +func TestValidateGitlabPermissions(t *testing.T) { + // + // Prepare + // + type testData = struct { + name string + scopes []string + raiseErr bool + } + var CurrentTest testData + mux, client := setupGitlab(t) + c := GitlabClientImpl{ + client: client, + cfg: &conf.GlobalConfig{ + GitProviderConfig: conf.GitProviderConfig{ + OrgLevelWebhook: false, + OrgName: "test", + RepoList: "test-repo1", + }, + }, + } + ctx := context.Background() + + mux.HandleFunc("/api/v4/personal_access_tokens/self", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + mockHTTPResponse(t, w, gitlab.PersonalAccessToken{Scopes: CurrentTest.scopes}) + }) + // + // Execute + // + tests := []testData{ + {name: "validScope", scopes: []string{"api"}, raiseErr: false}, + {name: "invalidScope", scopes: []string{"invalid"}, raiseErr: true}, + } + for _, test := range tests { + CurrentTest = test + t.Run(test.name, func(t *testing.T) { + err := ValidateGitlabPermissions(ctx, c.client, c.cfg) + // + // Assert + // + assert := assertion.New(t) + if test.raiseErr { + assert.NotNil(err) + } else { + assert.Nil(err) + } + }) + } +} + +func TestIsGroupWebhookEnabled(t *testing.T) { + // Prepare + ctx := context.Background() + mux, client := setupGitlab(t) + c := GitlabClientImpl{ + client: client, + cfg: &conf.GlobalConfig{ + GitProviderConfig: conf.GitProviderConfig{ + OrgLevelWebhook: true, + OrgName: "group1", + WebhookURL: "testing-url", + }, + }, + } + + hook := []gitlab.GroupHook{{ + ID: 1234, + URL: c.cfg.GitProviderConfig.WebhookURL, + }} + url := fmt.Sprintf("/api/v4/groups/%s/hooks", c.cfg.GitProviderConfig.OrgName) + mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + mockHTTPResponse(t, w, hook) + }) + // Execute + groupHook, isEnabled := IsGroupWebhookEnabled(&ctx, &c) + // Assert + assert := assertion.New(t) + assert.Equal(isEnabled, true) + assert.Equal(groupHook.URL, c.cfg.GitProviderConfig.WebhookURL) +} + +func TestIsProjectWebhookEnabled(t *testing.T) { + // + // Prepare + // + ctx := context.Background() + mux, client := setupGitlab(t) + project := "test-repo1" + projectId := 1 + c := GitlabClientImpl{ + client: client, + cfg: &conf.GlobalConfig{ + GitProviderConfig: conf.GitProviderConfig{ + OrgLevelWebhook: false, + OrgName: "group1", + WebhookURL: "testing-url", + RepoList: project, + }, + }, + } + + hook := []gitlab.ProjectHook{{ + ID: 1234, + URL: c.cfg.GitProviderConfig.WebhookURL, + }} + + url := fmt.Sprintf("/api/v4/projects/%d/hooks", projectId) + mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + mockHTTPResponse(t, w, hook) + }) + // + // Execute + // + projectHook, isEnabled := IsProjectWebhookEnabled(&ctx, &c, projectId) + // + // Assert + // + assert := assertion.New(t) + assert.Equal(isEnabled, true) + assert.Equal(projectHook.URL, c.cfg.GitProviderConfig.WebhookURL) +} diff --git a/pkg/git_provider/main.go b/pkg/git_provider/main.go index 9ebc5fa..6143b9a 100644 --- a/pkg/git_provider/main.go +++ b/pkg/git_provider/main.go @@ -20,6 +20,12 @@ func NewGitProviderClient(cfg *conf.GlobalConfig) (Client, error) { return nil, err } return gitClient, nil + case "gitlab": + gitClient, err := NewGitlabClient(cfg) + if err != nil { + return nil, err + } + return gitClient, nil } return nil, fmt.Errorf("didn't find matching git provider %s", cfg.GitProviderConfig.Provider) diff --git a/pkg/git_provider/test_utils.go b/pkg/git_provider/test_utils.go index 20c13b5..909032f 100644 --- a/pkg/git_provider/test_utils.go +++ b/pkg/git_provider/test_utils.go @@ -2,14 +2,17 @@ package git_provider import ( "fmt" - "github.com/google/go-cmp/cmp" - "github.com/google/go-github/v52/github" - "github.com/ktrysmt/go-bitbucket" "net/http" "net/http/httptest" "net/url" "os" "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-github/v52/github" + "github.com/ktrysmt/go-bitbucket" + "github.com/xanzy/go-gitlab" ) const ( @@ -88,3 +91,25 @@ func setupBitbucket() (client *bitbucket.Client, mux *http.ServeMux, serverURL s return client, mux, server.URL, server.Close } +func setupGitlab(t *testing.T) (*http.ServeMux, *gitlab.Client) { + // mux is the HTTP request multiplexer used with the test server. + mux := http.NewServeMux() + + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + + // client is the Gitlab client being tested. + client, err := gitlab.NewClient("", + gitlab.WithBaseURL(server.URL), + // Disable backoff to speed up tests that expect errors. + gitlab.WithCustomBackoff(func(_, _ time.Duration, _ int, _ *http.Response) time.Duration { + return 0 + }), + ) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + + return mux, client +} \ No newline at end of file diff --git a/pkg/server/routes/webhook.go b/pkg/server/routes/webhook.go index 1d2a4ec..7e035a3 100644 --- a/pkg/server/routes/webhook.go +++ b/pkg/server/routes/webhook.go @@ -22,7 +22,6 @@ func AddWebhookRoutes(cfg *conf.GlobalConfig, clients *clients.Clients, rg *gin. c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - if webhookPayload.Event == "ping" { if cfg.GitProviderConfig.FullHealthCheck { err = wc.SetWebhookHealth(webhookPayload.HookID, true) diff --git a/pkg/webhook_creator/main.go b/pkg/webhook_creator/main.go index 6e1b390..044b530 100644 --- a/pkg/webhook_creator/main.go +++ b/pkg/webhook_creator/main.go @@ -31,6 +31,10 @@ func NewWebhookCreator(cfg *conf.GlobalConfig, clients *clients.Clients) *Webhoo return wr } +func (wc *WebhookCreatorImpl) GetHooks() *map[int64]*git_provider.HookWithStatus { + return &wc.hooks +} + func (wc *WebhookCreatorImpl) Start() { err := wc.initWebhooks() @@ -86,6 +90,8 @@ func (wc *WebhookCreatorImpl) initWebhooks() error { ctx := context.Background() if wc.cfg.GitProviderConfig.OrgLevelWebhook && len(wc.cfg.GitProviderConfig.RepoList) != 0 { return fmt.Errorf("org level webhook wanted but provided repositories list") + } else if !wc.cfg.GitProviderConfig.OrgLevelWebhook && len(wc.cfg.GitProviderConfig.RepoList) == 0 { + return fmt.Errorf("either org level webhook or repos list must be provided") } for _, repo := range strings.Split(wc.cfg.GitProviderConfig.RepoList, ",") { if wc.cfg.GitProviderConfig.Provider == "bitbucket" { @@ -111,7 +117,6 @@ func (wc *WebhookCreatorImpl) Stop(ctx *context.Context) { } func (wc *WebhookCreatorImpl) deleteWebhooks(ctx *context.Context) error { - for hookID, hook := range wc.hooks { err := wc.clients.GitProvider.UnsetWebhook(ctx, hook) if err != nil { diff --git a/scripts/gitlab-setup.yaml b/scripts/gitlab-setup.yaml new file mode 100644 index 0000000..3c3d21a --- /dev/null +++ b/scripts/gitlab-setup.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: piper-setup +data: + piper-setup.rb: | + #enable the Admin->Network->Outbound requests -> allow for webhooks + ApplicationSetting.current.update!(allow_local_requests_from_web_hooks_and_services: true) + + #create new group + Group.create!(name: "Pied Pipers", path: "pied-pipers") + g = Group.find_by(name: "Pied Pipers") + n = Namespace.find_by(name: "Pied Pipers") + + #create new user + + u = User.new(username: 'piper-user', email: 'piper@example.com', name: 'piper-user', password: 'Aa123456', password_confirmation: 'Aa123456') + u.assign_personal_namespace(g.organization) + u.skip_confirmation! # Use only if you want the user to be automatically confirmed. If you do not use this, the user receives a confirmation email. + u.save! + + # create user token + token = u.personal_access_tokens.create(scopes: [:read_api, :write_repository, :api], name: 'p-token', expires_at: 365.days.from_now) + utoken = token.token + token.save! + + #add user to group + g.add_member(u, :owner) + g.save! + u.save! + + #create new project + project = g.projects.create(name: "piper-e2e-test", path: "piper-e2e-test", creator:u, organization:g.organization, namespace: n, visibility_level: 20) + project.create_repository + project.add_member(u, :owner) + project.save! + g.save! + + #GROUP ACCESS TOKEN: + # Create the group bot user. For further group access tokens, the username should be `group_{group_id}_bot_{random_string}` and email address `group_{group_id}_bot_{random_string}@noreply.{Gitlab.config.gitlab.host}`. + admin = User.find(1) + random_string = SecureRandom.hex(16) + bot = Users::CreateService.new(admin, {name: 'g_token', username: "group_#{g.id}_bot_#{random_string}", email: "group_#{g.id}_bot_#{random_string}@noreply.gitlab.local", user_type: :project_bot }).execute + bot.confirm + + # Add the bot to the group with the required role. + g.add_member(bot, :owner) + token = bot.personal_access_tokens.create(scopes:[:read_api, :write_repository, :api], name: 'g-token', expires_at: 365.days.from_now) + + # Get the token value. + gtoken = token.token + + puts "GROUP_TOKEN #{gtoken} USER_TOKEN #{utoken}" + diff --git a/scripts/init-gitlab.sh b/scripts/init-gitlab.sh index 8662bf5..a2832a1 100644 --- a/scripts/init-gitlab.sh +++ b/scripts/init-gitlab.sh @@ -1,10 +1,35 @@ #!/bin/sh set -o errexit + +LICENSE="${1:-$GITLAB_LICENSE}" + +if [ -z "$LICENSE" ]; then + echo "no gitlab license was entered for init-gitlab.sh as argument or env: GITLAB_LICENSE" + exit 2 +fi + if [ -z "$(helm list -n gitlab | grep gitlab)" ]; then + #start gitlab namespace + kubectl create namespace gitlab + # add gitlab secret + kubectl create secret generic gitlab-license -n gitlab --from-literal=license_key=$1 + kubectl apply -f ./scripts/gitlab-setup.yaml -n gitlab # 8. Install gitlab helm repo add gitlab https://charts.gitlab.io/ - helm upgrade --install gitlab --create-namespace -n gitlab gitlab/gitlab -f gitlab.values.yaml + helm upgrade --install gitlab -n gitlab gitlab/gitlab --version 8.6.1 -f gitlab.values.yaml + + echo "waiting for gitlab toolbox pod to ready" + kubectl wait --namespace gitlab --for=condition=ready pod -l app=toolbox --timeout=600s + echo "waiting for gitlab webservice pod to ready" + kubectl wait --namespace gitlab --for=condition=ready pod -l app=webservice --timeout=600s + + echo "setup gitlab configs" + GITLAB_TOOLBOX_POD=$(kubectl get pods --namespace gitlab -l app=toolbox -o name) + + sleep 30 + TOKENS_OUTPUT=$(kubectl exec -it -c toolbox ${GITLAB_TOOLBOX_POD} -n gitlab -- gitlab-rails runner /tmp/scripts/piper-setup.rb) + echo $TOKENS_OUTPUT else echo "Gitlab release exists, skipping installation" fi \ No newline at end of file