feat(pre-commit): Pre commit parametrisation - cont (#14)

Co-authored-by: Łukasz Pawlęga <lpawlega@paloaltonetworks.com>
Co-authored-by: Łukasz Pawlęga <42772730+FoSix@users.noreply.github.com>

.github/workflows/_pre_commit.yml

@@ -7,9 +7,18 @@ defaults:
   run:
     shell: bash
 
-on: 
+on:
   workflow_call:
-
+    inputs:
+      pre-commit-hooks:
+        description: "Pre-commit hook list . Possible values are a combination of any of the following: terraform_fmt, terraform_docs, terraform_tflint, checkov (space spearated) "
+        type: string
+        required: true
+      pre-commit-files:
+        description: "Files for pre-commit to scan - passed dynamically via PR CI variable. "
+        type: string
+        default: "all-files"
+        required: false
 env:
   REQUIREMENTS: >
     appdirs==1.4.4
@@ -32,8 +41,26 @@ env:
     zipp==3.4.1
 
 jobs:
+
+  prerequisites:
+    name: prep data for matrix strategy
+    runs-on: ubuntu-latest
+    outputs:
+      pre-commit-hooks: ${{ steps.preqs.outputs.pre-commit-hooks }}
+      pre-commit-files: ${{ steps.preqs.outputs.pre-commit-files}}
+    steps:
+      - name: set outputs
+        id: preqs
+        run: |
+          echo "pre-commit-hooks=$(echo "${{ inputs.pre-commit-hooks }} " | jq -R -s -c 'split(" ")[:-1]')" >> $GITHUB_OUTPUT
+          echo "pre-commit-files=$(echo "${{ inputs.pre-commit-files }}" | tr '\n' ' ')" >> $GITHUB_OUTPUT
+
   pre_commit_framework:
+    needs: [prerequisites]
     name: Pre-Commit
+    strategy:
+      matrix:
+        pre_commit_hook: ${{ fromJson(needs.prerequisites.outputs.pre-commit-hooks) }}
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
@@ -48,22 +75,16 @@ jobs:
         run: mkdir tmp
 
       - name: install TF Docs
+        if: ${{ matrix.pre_commit_hook == 'terraform_docs' }}
         working-directory: tmp
         run: |
           curl -sL https://github.com/terraform-docs/terraform-docs/releases/download/v0.16.0/terraform-docs-v0.16.0-linux-amd64.tar.gz > terraform-docs.tar.gz 
           tar zxf terraform-docs.tar.gz
           mv terraform-docs /usr/local/bin/
           terraform-docs --version
 
-      - name: install TF Sec
-        working-directory: tmp
-        run: |
-          curl -sL https://github.com/tfsec/tfsec/releases/download/v0.34.0/tfsec-linux-amd64 > tfsec
-          chmod +x tfsec
-          mv tfsec /usr/local/bin/
-          tfsec --version
-
       - name: install TF Lint
+        if: ${{ matrix.pre_commit_hook == 'terraform_tflint' }}
         working-directory: tmp
         run: |
           curl -sL https://github.com/terraform-linters/tflint/releases/download/v0.29.0/tflint_linux_amd64.zip > tflint.zip
@@ -74,9 +95,48 @@ jobs:
       - name: install python requirements
         run: python3 -m pip install $REQUIREMENTS
 
-      - name: run pre-commit
+      - name: run pre-commit terraform_fmt
+        shell: bash
+        if: ${{ matrix.pre_commit_hook == 'terraform_fmt' }}
+        env:
+          INPUT_FILES: ${{ inputs.pre-commit-files }}
+        run: |
+          rm -rf tmp
+          if grep -q "all-files" <<< "$INPUT_FILES"; then
+            pre-commit run ${{ matrix.pre_commit_hook }} -a
+          else
+            pre-commit run ${{ matrix.pre_commit_hook }} --files ${{ needs.prerequisites.outputs.pre-commit-files }}
+          fi
+
+      - name: run pre-commit terraform_tflint
+        shell: bash
+        if: ${{ matrix.pre_commit_hook == 'terraform_tflint' }}
+        env:
+          INPUT_FILES: ${{ inputs.pre-commit-files }}
+        run: |
+          rm -rf tmp
+          if grep -q "all-files" <<< "$INPUT_FILES"; then
+            pre-commit run ${{ matrix.pre_commit_hook }} -a
+          else
+            pre-commit run ${{ matrix.pre_commit_hook }} --files ${{ needs.prerequisites.outputs.pre-commit-files }}
+          fi
+
+      - name: run pre-commit terraform_docs
+        shell: bash
+        if: ${{ matrix.pre_commit_hook == 'terraform_docs' }}
+        env:
+          INPUT_FILES: ${{ inputs.pre-commit-files }}
+        run: |
+          rm -rf tmp
+          if grep -q "all-files" <<< "$INPUT_FILES"; then
+            pre-commit run ${{ matrix.pre_commit_hook }} -a
+          else
+            pre-commit run ${{ matrix.pre_commit_hook }} --files ${{ needs.prerequisites.outputs.pre-commit-files }}
+          fi
+
+      - name: run pre-commit for checkov
+        shell: bash
+        if: ${{ contains(matrix.pre_commit_hook, 'checkov') }}
         run: |
           rm -rf tmp
-          pre-commit run --all-files terraform_fmt
-          pre-commit run --all-files terraform_tflint
-          pre-commit run --all-files terraform_docs
+          pre-commit run checkov -a

.github/workflows/_tf_plan_apply.yml

@@ -46,8 +46,8 @@ jobs:
       - name: set outputs
         id: preqs
         run: |
-          echo "::set-output name=paths::$(echo -n "${{ inputs.paths }}" | jq -R -s -c 'split(",")')"
-          echo "::set-output name=tf_versions::$(echo "${{ inputs.tf_version }} " | jq -R -s -c 'split(" ")[:-1]')"
+          echo "paths=$(echo -n "${{ inputs.paths }}" | jq -R -s -c 'split(",")')" >> $GITHUB_OUTPUT
+          echo "tf_versions=$(echo "${{ inputs.tf_version }} " | jq -R -s -c 'split(" ")[:-1]')" >> $GITHUB_OUTPUT
         
   terraform:
     needs: [prerequisites]

.github/workflows/_tf_validate.yml

@@ -30,8 +30,8 @@ jobs:
       - name: set outputs
         id: preqs
         run: |
-          echo "::set-output name=paths::$(echo -n "${{ inputs.paths }}" | jq -R -s -c 'split(",")')"
-          echo "::set-output name=tf_versions::$(echo "${{ inputs.tf_version }} " | jq -R -s -c 'split(" ")[:-1]')"
+          echo "paths=$(echo -n "${{ inputs.paths }}" | jq -R -s -c 'split(",")')" >> $GITHUB_OUTPUT
+          echo "tf_versions=$(echo "${{ inputs.tf_version }} " | jq -R -s -c 'split(" ")[:-1]')" >> $GITHUB_OUTPUT
         
   validate:
     needs: [prerequisites]

.github/workflows/actions_pr_ci.yml

@@ -0,0 +1,21 @@
+name: Actions PR CI
+run-name: Actions PR CI
+
+permissions:
+  contents: write
+  issues: read
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+
+  pre_commit:
+    name: Pre-Commit
+    uses: ./.github/workflows/_pre_commit.yml
+    with:
+      pre-commit-hooks: checkov

.github/workflows/actions_release_ci.yml

@@ -10,6 +10,13 @@ on:
     branches: [main]
 
 jobs:
+
+  pre_commit:
+    name: Pre-Commit
+    uses: ./.github/workflows/_pre_commit.yml
+    with:
+      pre-commit-hooks: checkov
+
   release:
     name: Semantic release
     runs-on: ubuntu-latest

.github/workflows/lint_pr_title.yml

@@ -21,6 +21,6 @@ jobs:
     name: Validate PR title matches conventional commits
     runs-on: ubuntu-latest
     steps:
-      - uses: amannn/action-semantic-pull-request@v4
+      - uses: amannn/action-semantic-pull-request@v5
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pr_ci.yml

@@ -26,24 +26,20 @@ on:
         description: "Decide against which public cloud the code will be run. Possible values: azure, aws, gcp"
         type: string
         required: true
+      pre-commit-hooks:
+        description: "Pre-commit hook list. Possible values are a combination of any of the following: terraform_fmt, terraform_docs, terraform_tflint, checkov (space spearated)."
+        type: string
+        default: terraform_fmt terraform_docs terraform_tflint checkov
 
 jobs:
-  pre_commit:
-    name: Pre-Commit
-    uses: ./.github/workflows/_pre_commit.yml
-
-
-  checkov:
-    name: Checkov
-    uses: ./.github/workflows/_checkov.yml
-
 
   tf_prereqs:
     name: terraform modules discovery
     runs-on: ubuntu-latest
     outputs:
       validate_paths: ${{ steps.format.outputs.dir_diff }}
       plan_paths: ${{ steps.deps.outputs.examples }}
+      changed_files: ${{ steps.format.outputs.files_diff }}
     steps:
       - name: checkout code
         uses: actions/checkout@v3
@@ -54,7 +50,7 @@ jobs:
         id: diff
         uses: tj-actions/changed-files@v32
         with:
-          separator: "\n"
+          separator: "@"
           files: |
             modules/**/*.tf
             examples/**/*.tf
@@ -63,8 +59,10 @@ jobs:
         id: format
         env:
           DIFFS: ${{ steps.diff.outputs.all_changed_files }}
+        shell: bash
         run: |
-          echo "::set-output name=dir_diff::$(echo -n "$DIFFS" | sed -E "s/^(modules|examples)\/(.+)\/.*$/\1\/\2/"  | tr '\n' ',')"
+          echo "dir_diff=$(echo -n "$DIFFS" | tr -d '\n' | tr '@' '\n' | sed -E "s/^(modules|examples)\/(.+)\/.*$/\1\/\2/" | sort -u | tr '\n' ',' | rev | cut -c2- | rev)" >> $GITHUB_OUTPUT
+          echo "files_diff=$(echo "$DIFFS" | tr '@' ' ')" >> $GITHUB_OUTPUT
 
       - name: discover module->example dependencies
         id: deps
@@ -79,10 +77,18 @@ jobs:
 
             if [ "$(echo "$(echo $EXAMPLES_DISCOVERY | tr ' ' ','),$DIFFS" | grep examples)" ]; then
               EXAMPLES_COMBINED=$(echo "$(echo $EXAMPLES_DISCOVERY | tr ' ' ','),$DIFFS" | tr ',' '\n' | awk NF | grep examples | sort -u)
-              echo "::set-output name=examples::$(echo -n $EXAMPLES_COMBINED | tr ' ' ',')"
+              echo "examples=$(echo -n $EXAMPLES_COMBINED | tr ' ' ',')" >> $GITHUB_OUTPUT
             fi
           fi
 
+  pre_commit:
+    name: Pre-Commit
+    needs: [tf_prereqs]
+    uses: ./.github/workflows/_pre_commit.yml
+    with:
+      pre-commit-hooks: ${{ inputs.pre-commit-hooks }}
+      pre-commit-files: ${{ needs.tf_prereqs.outputs.changed_files }}
+
 
   validate:
     name: validate all changed modules
@@ -99,7 +105,6 @@ jobs:
     needs:
       - validate
       - pre_commit
-      - checkov
     if: always()
     permissions:
       actions: read

.github/workflows/release_ci.yml

@@ -64,14 +64,9 @@ jobs:
     uses: ./.github/workflows/_pre_commit.yml
     needs: [release-prereqs]
     if: needs.release-prereqs.outputs.rc  == 'true'
-
-
-  checkov:
-    name: Checkov
-    uses: ./.github/workflows/_checkov.yml
-    needs: [release-prereqs]
-    if: needs.release-prereqs.outputs.rc  == 'true'
-
+    with:
+      pre-commit-hooks: terraform_fmt terraform_docs terraform_tflint checkov
+      pre-commit-files: all-files
 
   tf_prereqs:
     name: validate prerequisites
@@ -88,8 +83,8 @@ jobs:
         id: paths
         shell: bash
         run: |
-          echo "::set-output name=modules::$(echo $(ls -d1 examples/* modules/*) | tr ' ' ',')"
-          echo "::set-output name=examples::$(echo $(ls -d1 examples/*) | tr ' ' ',')"
+          echo "modules=$(echo $(ls -d1 examples/* modules/*) | tr ' ' ',')" >> $GITHUB_OUTPUT
+          echo "examples=$(echo $(ls -d1 examples/*) | tr ' ' ',')" >> $GITHUB_OUTPUT
 
 
   validate:
@@ -106,7 +101,6 @@ jobs:
     name: release sem version
     needs: 
       - validate
-      - checkov
       - pre_commit
     runs-on: ubuntu-latest
     permissions:

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+  - repo: https://github.com/bridgecrewio/checkov.git
+    rev: '2.2.125'
+    hooks:
+    - id: checkov
+      verbose: true
+      files: '^\.github/workflows/.*\.yml$'
+      args: [
+        --compact,
+        --quiet,
+      ]

README.md

@@ -71,11 +71,7 @@ flowchart
     subgraph P [Pre-Commit Sub-Workflow]
         direction TB
         p_start(Init point of a sub-workflow)-->p_prereq(Install pre-commit prerequisites)
-        p_prereq-->p_pre_commit(Run Pre-Commit)
-    end
-    subgraph C [Checkov Sub-Workflow]
-        direction TB
-        c_start(Init point of a sub-workflow)-->c_checkov(Run Checkov)
+        p_prereq-->p_pre_commit(Execute pre-commit hooks based on input parametrization)
     end
   end
   t_call-->g_start