-
Notifications
You must be signed in to change notification settings - Fork 11
256 lines (219 loc) · 8.33 KB
/
test_and_publish.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
name: Test & Publish
on:
push:
branches:
- master
- develop
paths-ignore:
- "docs/**"
pull_request:
branches:
- master
- develop
paths-ignore:
- "docs/**"
jobs:
unit_tests:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10']
steps:
- uses: actions/checkout@v4
- name: Setup python ${{ matrix.python-version }}
uses: actions/setup-python@v4.7.1
with:
python-version: ${{ matrix.python-version }}
- name: Setup poetry
run: |
python -V
python -m pip install poetry
- name: Check pre-commit status
run: |
poetry install -v
poetry run pre-commit run --all-files
- name: Test with tox
run: |
pip install tox-pip-version tox-gh-actions "tox<4.0.0"
tox -v
- name: Store coverage reports
uses: actions/upload-artifact@v3
with:
name: coverage-${{ matrix.python-version }}
path: coverage.xml
if-no-files-found: error
sonarcloud:
runs-on: ubuntu-latest
needs: unit_tests
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/download-artifact@v3
with:
name: coverage-3.9
path: .
- name: SonarCloud Scan
uses: sonarsource/sonarcloud-github-action@master
with:
projectBaseDir: ${{ github.workspace }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
codeql:
runs-on: ubuntu-latest
needs: unit_tests
permissions:
actions: read
contents: read
security-events: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: python
- name: Autobuild
uses: github/codeql-action/autobuild@v2
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
### PROJECT SPECIFIC CONFIGURATION HERE
e2e_tests:
runs-on: ubuntu-latest
timeout-minutes: 60
if: (github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request == null)
needs: [unit_tests, sonarcloud]
strategy:
matrix:
e2e_case: ["standard", "grouping"]
python-version: ['3.8']
env:
PROJECT_NAME: kedro-vertexai
IMAGE_REGISTRY: gcr.io/gid-ml-ops-sandbox
steps:
- uses: actions/checkout@v4
- name: Setup python
uses: actions/setup-python@v4.7.1
with:
python-version: ${{ matrix.python-version }}
- name: Setup poetry
run: |
python -V
python -m pip install poetry wheel setuptools
- name: Build the package
run: |
poetry build -f sdist
- name: Initialize starter project
run: |
pip install $(find "./dist" -name "*.tar.gz")
kedro new --starter spaceflights --config tests/e2e/${{ matrix.e2e_case }}/starter-config.yml --verbose
- name: Install starter requirements
working-directory: ./spaceflights
run: |
find "../dist" -name "*.tar.gz" | xargs -I@ cp @ $PROJECT_NAME.tar.gz
echo -e "\n./$PROJECT_NAME.tar.gz\n" >> src/requirements.txt
echo -e "kedro-docker\n" >> src/requirements.txt
sed -i '/kedro-telemetry/d' src/requirements.txt
echo $(cat src/requirements.txt)
pip install -r src/requirements.txt
- name: Init and update starter configuration
working-directory: ./spaceflights
run: |
kedro docker init
sed -i "s/\(COPY src\/requirements.txt.*\)$/\1\nCOPY $PROJECT_NAME.tar.gz ./g" Dockerfile
cat Dockerfile
echo "!data/01_raw" >> .dockerignore
kedro vertexai init gid-ml-ops-sandbox europe-west4
cp -r ../tests/e2e/${{ matrix.e2e_case }}/overwrite/* .
- name: Prepare docker env
uses: docker/setup-buildx-action@v3
id: buildx
with:
install: true
- name: Set up GCP Credentials
uses: google-github-actions/auth@v1.1.1
with:
credentials_json: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
create_credentials_file: true
cleanup_credentials: true
- name: Build docker image
working-directory: ./spaceflights
run: |
docker pull $IMAGE_REGISTRY/$PROJECT_NAME-e2e:${{ matrix.e2e_case }} || true
docker build --build-arg BASE_IMAGE=python:${{ matrix.python-version }}-buster --tag $PROJECT_NAME-e2e:${{ matrix.e2e_case }} --load --cache-from=$IMAGE_REGISTRY/$PROJECT_NAME-e2e:${{ matrix.e2e_case }} .
- name: Publish docker image to docker registry
uses: mattes/gce-docker-push-action@v1
with:
creds: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
src: ${{ env.PROJECT_NAME }}-e2e:${{ matrix.e2e_case }}
dst: ${{ env.IMAGE_REGISTRY }}/${{ env.PROJECT_NAME }}-e2e:${{ matrix.e2e_case }}
- name: Run project on vertex pipeline
working-directory: ./spaceflights
run: |
export KEDRO_CONFIG_COMMIT_ID=$GITHUB_SHA
kedro vertexai run-once --wait-for-completion
publish:
# only run on push to master
if: github.event.pull_request == null && github.ref == 'refs/heads/master'
needs: [e2e_tests, codeql]
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.8']
env:
PYTHON_PACKAGE: kedro_vertexai
steps:
- name: Checkout the repo
uses: actions/checkout@v4
with:
fetch-depth: 0 # necessary to enable merging, all the history is needed
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4.7.1
with:
python-version: ${{ matrix.python-version }}
- name: Build package dist from source # A better way will be : https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ but pep 517 is still marked as experimental
run: |
pip install poetry
poetry build
- name: Merge back to develop # we have to set the config first on a fresh machine
run: |
git config user.name github-actions
git config user.email github-actions@github.com
git checkout -B develop --track origin/develop
git merge master
git push
- name: Set dynamically package version as output variable # see https://github.com/actions/create-release/issues/39
# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
id: set_package_version
run: |
echo "::set-output name=PACKAGE_VERSION::$(cat $PYTHON_PACKAGE/__init__.py | grep -Po '\d+\.\d+\.\d+')"
- name: Create temporary file with the body content for the release
run: |
grep -Poz "## \[${{steps.set_package_version.outputs.PACKAGE_VERSION}}] - \d{4}-\d{2}-\d{2}[\S\s]+?(?=## \[\d+\.\d+\.\d+\]|\[.+\]:)" CHANGELOG.md > release_body.md
- name: Create Release # https://github.com/actions/create-release
id: create_release
uses: actions/create-release@v1.1.4
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
with:
tag_name: ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
release_name: Release ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
body_path: ./release_body.md
draft: false
prerelease: false
- name: Rollback Release in case of run failure
if: failure() && steps.create_release.outputs.id != ''
uses: author/action-rollback@stable
with:
# Using a known release ID
release_id: ${{ steps.create_release.outputs.id }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Publish distribution to PyPI # official action from python maintainers
uses: pypa/gh-action-pypi-publish@master
with:
user: __token__
password: ${{ secrets.PYPI_PASSWORD }}
verbose: true # trace if the upload fails