From 1a3a716aba9e23e8360e7a6f09c954f17ef63e2d Mon Sep 17 00:00:00 2001 From: Anton Smorodskyi Date: Fri, 2 Feb 2024 15:40:42 +0100 Subject: [PATCH] Create dedicated folder for containers and add base image --- .dockerignore | 4 ++ .github/workflows/container.yml | 16 ++--- .github/workflows/container_base.yml | 58 +++++++++++++++++++ .github/workflows/lint.yml | 14 +++-- Makefile | 26 +++++---- README.md | 42 +++++++------- Dockerfile => containers/Dockerfile | 9 +-- containers/Dockerfile_base | 6 ++ containers/Dockerfile_base_k8s | 11 ++++ Dockerfile_dev => containers/Dockerfile_dev | 0 Dockerfile_k8s => containers/Dockerfile_k8s | 0 .../Dockerfile_k8s_dev | 0 .../container-startup | 0 tests/test_webui.py | 2 +- 14 files changed, 137 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/container_base.yml rename Dockerfile => containers/Dockerfile (88%) create mode 100644 containers/Dockerfile_base create mode 100644 containers/Dockerfile_base_k8s rename Dockerfile_dev => containers/Dockerfile_dev (100%) rename Dockerfile_k8s => containers/Dockerfile_k8s (100%) rename Dockerfile_k8s_dev => containers/Dockerfile_k8s_dev (100%) rename container-startup => containers/container-startup (100%) diff --git a/.dockerignore b/.dockerignore index f2f3a83c..c6cdb049 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,11 @@ # Ignore git, data and cache files .git +.github .cache +.pytest_cache + __pycache__ /db +/tests # Also ignore templates /templates diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 57d26252..f00a9258 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -48,19 +48,19 @@ jobs: uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 with: context: . - file: Dockerfile + file: containers/Dockerfile push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - name: Build Docker image (K8S_GCE) + - name: Build Docker image (K8S) if: ${{ matrix.suffix == 'k8s' }} uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 with: context: . - file: Dockerfile_${{ matrix.suffix }} + file: containers/Dockerfile_${{ matrix.suffix }} push: false tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + labels: ${{ steps.meta.outputs.labels }} build-and-push: if: ${{ github.event_name == 'release' }} @@ -94,16 +94,16 @@ jobs: uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 with: context: . - file: Dockerfile + file: containers/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - name: Build and push Docker image (K8S_GCE) + - name: Build and push Docker image (K8S) if: ${{ matrix.suffix == 'k8s' }} uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 with: context: . - file: Dockerfile_${{ matrix.suffix }} + file: containers/Dockerfile_${{ matrix.suffix }} push: true tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/container_base.yml b/.github/workflows/container_base.yml new file mode 100644 index 00000000..fa9dd67a --- /dev/null +++ b/.github/workflows/container_base.yml @@ -0,0 +1,58 @@ +--- +# See https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages + +name: Container workflow + +on: + schedule: + - cron: 0 8 * * * + +env: + REGISTRY: ghcr.io + PCW_IMAGE_PREFIX: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + strategy: + matrix: + suffix: [base_main, base_k8s] + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c + with: + images: ${{ env.REGISTRY }}/${{ env.PCW_IMAGE_PREFIX }}_${{ matrix.suffix }} + + - name: Build and push Docker base image (PCW) + if: ${{ matrix.suffix == 'base_main' }} + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 + with: + context: . + file: containers/Dockerfile_base + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - name: Build and push Docker base image (K8S) + if: ${{ matrix.suffix == 'base_k8s' }} + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 + with: + context: . + file: containers/Dockerfile_${{ matrix.suffix }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9dafc912..02eb713f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,13 +41,19 @@ jobs: - uses: actions/checkout@v4 - uses: hadolint/hadolint-action@v3.1.0 with: - dockerfile: 'Dockerfile' + dockerfile: 'containers/Dockerfile' - uses: hadolint/hadolint-action@v3.1.0 with: - dockerfile: 'Dockerfile_dev' + dockerfile: 'containers/Dockerfile_dev' - uses: hadolint/hadolint-action@v3.1.0 with: - dockerfile: 'Dockerfile_k8s' + dockerfile: 'containers/Dockerfile_k8s' - uses: hadolint/hadolint-action@v3.1.0 with: - dockerfile: 'Dockerfile_k8s_dev' + dockerfile: 'containers/Dockerfile_k8s_dev' + - uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: 'containers/Dockerfile_base' + - uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: 'containers/Dockerfile_base_k8s' diff --git a/Makefile b/Makefile index 1adb23d0..ded2a18a 100644 --- a/Makefile +++ b/Makefile @@ -27,18 +27,20 @@ codecov: pytest -v --cov --cov-report=html && xdg-open htmlcov/index.html # Build containers -docker-container: - docker build . -t ${CONT_TAG} -podman-container: - podman build . -t ${CONT_TAG} -podman-container-devel: - podman build -f Dockerfile_dev -t pcw-devel -podman-container-k8s: - podman build -f Dockerfile_k8s -t pcw-k8s-cleaner -podman-container-k8s-devel: - podman build -f Dockerfile_k8s_dev -t pcw-k8s-cleaner-devel +container: + podman build . -t ${CONT_TAG} -f containers/Dockerfile +container-base: + podman build . -t ${CONT_TAG}-base -f containers/Dockerfile_base +container-base-k8s: + podman build . -t ${CONT_TAG}-base-k8s -f containers/Dockerfile_base_k8s +container-devel: + podman build . -t ${CONT_TAG}-devel -f containers/Dockerfile_dev +container-k8s: + podman build . -t ${CONT_TAG}-k8s-cleaner -f containers/Dockerfile_k8s +container-k8s-devel: + podman build . -t ${CONT_TAG}-k8s-cleaner-devel -f containers/Dockerfile_k8s_dev # Container linting .PHONY: container-lint -container-lint: Dockerfile* - hadolint Dockerfile* +container-lint: containers/Dockerfile* + hadolint containers/Dockerfile* diff --git a/README.md b/README.md index ff355abf..64aebf5a 100644 --- a/README.md +++ b/README.md @@ -9,18 +9,14 @@ PublicCloud-Watcher (PCW) is a web app which monitors, displays and deletes resources on various Cloud Service Providers (CSPs). PCW has two main flows : -1. **Update run ( implemented in [ocw/lib/db.py](ocw/lib/db.py) )** Executed every 5 minutes. Concentrates on deleting VMs (in case of Azure Resource Groups). - - Each update run scans accounts defined in configuration file and - writes the obtained results into a local sqlite database. Newly discovered entities get assigned an obligatory time-to-life value (TTL). - TTL may be taken from tag `openqa_ttl` if entity is tagged with such tag if not PCW will check `pcw.ini` for `updaterun/default_ttl` setting - and if setting is not defined than PCW will use hard-coded value from [webui/settings.py](webui/settings.py). Database has a web UI where - you can manually trigger certain entity deletion. +1. **Update run ( implemented in [ocw/lib/db.py](ocw/lib/db.py) )** Executed every 45 minutes. Concentrates on deleting VMs (in case of Azure Resource Groups). + - Each update scans accounts defined in configuration file and writes the obtained results into a local sqlite database. Newly discovered entities get assigned an obligatory time-to-life value (TTL). TTL may be taken from tag `openqa_ttl` if entity is tagged with such tag if not PCW will check `pcw.ini` for `updaterun/default_ttl` setting and if setting is not defined than PCW will use hard-coded value from [webui/settings.py](webui/settings.py). Database has a web UI where you can manually trigger certain entity deletion. - After persisting results into db PCW deciding which entities needs to be deleted. There are two ways to survive for entity: a. Having tag `pcw_ignore` ( with any value) b. Age of entity is lower than TTL defined. Age is calculated as delta of last_seen and first_seen - For entities that survive cleanup PCW will sent notification email to the list defined in config. -2. **Cleanup ( implemented in [ocw/lib/cleanup.py](ocw/lib/cleanup.py) )** Executed every hour. Concentrates on everything except VM deletion. This vary a lot per CSP so let's clarify that on per provider level. +2. **Cleanup ( implemented in [ocw/lib/cleanup.py](ocw/lib/cleanup.py) )** Execution via django command. Concentrates on everything except VM deletion. This vary a lot per CSP so let's clarify that on per provider level. - For Azure such entities monitored (check details in [ocw/lib/azure.py](ocw/lib/azure.py)): a. bootdiagnostics b. Blobs in `sle-images` container @@ -39,7 +35,11 @@ The fastest way to run PCW is via the provided containers, as described in the [ ## Install -See the [requirements.txt](requirements.txt). It's recommended to setup `pcw` in a virtual environment to avoid package collisions: +PCW has 3 sets of virtual env requirements files : + - [requirements.txt](requirements.txt) common usage for everything except K8S related cleanups + - [requirements_k8s.txt](requirements_k8s.txt) due to high volume of dependencies needed only in single use case (k8s cleanups) they excluded in independent category + - [requirements_test.txt](requirements_test.txt) contains dependencies allowing to run pcw's unit tests +It's recommended to setup `pcw` in a virtual environment to avoid package collisions: ```bash virtualenv venv @@ -79,16 +79,15 @@ python manage.py runserver By default, PCW runs on http://127.0.0.1:8000/ -## Building a container +## Building PCW containers -To build a docker/podman container with the default `suse/qac/pcw` tag, run +In [containers](containers/) folder you main find several Dockerfiles to build several different images: -```bash -make docker-container -make podman-container -``` - -This repository contains the skeleton `Dockerfile` for building a PCW docker/podman container. + - [Dockerfile_base](containers/Dockerfile_base) basic image based on [bci-python3.11](https://registry.suse.com/categories/bci-devel/repositories/bci-python311) image which contains all dependencies needed for PCW execution except k8s cleanup. Image used as base for final image with actual PCW code + - [Dockerfile_base_k8s](containers/Dockerfile_base_k8s) basic image based on [bci-python3.11](https://registry.suse.com/categories/bci-devel/repositories/bci-python311) image which contains all dependencies needed for k8s cleanup PCW execution. Image used as base for final image with actual PCW code + - [Dockerfile](containers/Dockerfile) image based on [Dockerfile_base](containers/Dockerfile_base) and can be used to run all PCW functionality except k8s cleanup + - [Dockerfile_k8s](containers/Dockerfile_k8s) image based on [Dockerfile_base_k8s](containers/Dockerfile_base_k8s) and can be used to run k8s cleanup + - [Dockerfile_k8s_dev](containers/Dockerfile_k8s_dev) and [Dockerfile_dev](containers/Dockerfile_dev) images which contains same set of dependencies as [Dockerfile](containers/Dockerfile) and [Dockerfile_k8s](containers/Dockerfile_k8s) and expecting PCW source code to be mounted as volumes. Very usefull for development experiments ## Running a container @@ -96,6 +95,7 @@ You can use the already build containers within [this repository](https://github ```bash podman pull ghcr.io/suse/pcw:latest +podman pull ghcr.io/suse/pcw_k8s:latest ``` The PCW container supports two volumes to be mounted: @@ -110,9 +110,7 @@ podman create --hostname pcw --name pcw -v /srv/pcw/pcw.ini:/etc/pcw.ini -v /srv podman start pcw ``` -For usage in docker simply replace `podman` by `docker` in the above command. - -The `pcw` container runs by default the `/pcw/container-startup` startup helper script. You can interact with it by running +The `pcw` container runs by default the [/pcw/container-startup](containers/container-startup) startup helper script. You can interact with it by running ```bash podman exec pcw /pcw/container-startup help @@ -128,7 +126,7 @@ podman exec pcw /pcw/container-startup createuser admin USE_A_STRONG_PASSWORD ## Devel version of container -There is [devel version](Dockerfile_dev) of container file. Main difference is that source files are not copied into image but expected to be mounted via volume. This ease development in environment close as much as possible to production run. +There is [devel version](containers/Dockerfile_dev) of container file. Main difference is that source files are not copied into image but expected to be mounted via volume. This ease development in environment close as much as possible to production run. Expected use would be : @@ -158,6 +156,10 @@ To simplify problem investigation pcw has two [django commands](https://docs.dja [updaterun](ocw/management/commands/updaterun.py) +[dumpstate](ocw/management/commands/dumpstate.py) + +[rmclusters](ocw/management/commands/rmclusters.py) + those allows triggering core functionality without web UI. It is highly recommended to use `dry_run = True` in `pcw.ini` in such cases. diff --git a/Dockerfile b/containers/Dockerfile similarity index 88% rename from Dockerfile rename to containers/Dockerfile index e7dda81f..3f3638e0 100644 --- a/Dockerfile +++ b/containers/Dockerfile @@ -1,10 +1,5 @@ FROM registry.suse.com/bci/python:3.11 -ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 UWSGI_WSGI_FILE=/pcw/webui/wsgi.py UWSGI_MASTER=1 -ENV UWSGI_HTTP_AUTO_CHUNKED=1 UWSGI_HTTP_KEEPALIVE=1 UWSGI_LAZY_APPS=1 UWSGI_WSGI_ENV_BEHAVIOR=holy - -## System preparation steps ################################################# ## - # We do the whole installation and configuration in one layer: COPY requirements.txt /pcw/ # * Install system requirements @@ -17,7 +12,7 @@ RUN source /etc/os-release && zypper addrepo -G -cf "https://download.opensuse.o # Copy program files only COPY ocw /pcw/ocw/ COPY webui /pcw/webui/ -COPY container-startup manage.py LICENSE README.md setup.cfg pyproject.toml /pcw/ +COPY containers/container-startup manage.py LICENSE /pcw/ WORKDIR /pcw @@ -32,6 +27,8 @@ EXPOSE 8000/tcp # Required to use system certs in python-requests ENV REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 UWSGI_WSGI_FILE=/pcw/webui/wsgi.py UWSGI_MASTER=1 +ENV UWSGI_HTTP_AUTO_CHUNKED=1 UWSGI_HTTP_KEEPALIVE=1 UWSGI_LAZY_APPS=1 UWSGI_WSGI_ENV_BEHAVIOR=holy # Once we are certain that this runs nicely, replace this with ENTRYPOINT. ENTRYPOINT ["/pcw/container-startup", "run"] diff --git a/containers/Dockerfile_base b/containers/Dockerfile_base new file mode 100644 index 00000000..3cacfe6c --- /dev/null +++ b/containers/Dockerfile_base @@ -0,0 +1,6 @@ +FROM registry.suse.com/bci/python:3.11 + +COPY requirements.txt /pcw/ +RUN source /etc/os-release && zypper addrepo -G -cf "https://download.opensuse.org/repositories/SUSE:/CA/$VERSION_ID/SUSE:CA.repo" && \ + zypper -n in ca-certificates-suse gcc libffi-devel && \ + pip install --no-cache-dir wheel && pip install --no-cache-dir -r /pcw/requirements.txt && zypper clean && rm -rf /var/cache diff --git a/containers/Dockerfile_base_k8s b/containers/Dockerfile_base_k8s new file mode 100644 index 00000000..459e8a9d --- /dev/null +++ b/containers/Dockerfile_base_k8s @@ -0,0 +1,11 @@ +FROM registry.suse.com/bci/python:3.11 + +RUN zypper -n in gcc tar gzip kubernetes1.24-client aws-cli && zypper clean && rm -rf /var/cache + +# Google cli installation +RUN curl -sf https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-415.0.0-linux-x86_64.tar.gz | tar -zxf - -C /opt \ +&& /opt/google-cloud-sdk/bin/gcloud components install gke-gcloud-auth-plugin + +# Install python dependences +COPY requirements_k8s.txt /pcw/ +RUN pip install --no-cache-dir wheel && pip install --no-cache-dir -r /pcw/requirements_k8s.txt diff --git a/Dockerfile_dev b/containers/Dockerfile_dev similarity index 100% rename from Dockerfile_dev rename to containers/Dockerfile_dev diff --git a/Dockerfile_k8s b/containers/Dockerfile_k8s similarity index 100% rename from Dockerfile_k8s rename to containers/Dockerfile_k8s diff --git a/Dockerfile_k8s_dev b/containers/Dockerfile_k8s_dev similarity index 100% rename from Dockerfile_k8s_dev rename to containers/Dockerfile_k8s_dev diff --git a/container-startup b/containers/container-startup similarity index 100% rename from container-startup rename to containers/container-startup diff --git a/tests/test_webui.py b/tests/test_webui.py index c80b5cef..aaa784a7 100644 --- a/tests/test_webui.py +++ b/tests/test_webui.py @@ -59,7 +59,7 @@ def image(random_port, client): try: client.images.build( path=".", - dockerfile="Dockerfile", + dockerfile="containers/Dockerfile", tag=image_name, ) except APIError as exc: