From 4ea4347e8ad80ed8d33222a2f3517036b6674318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Tue, 25 Jun 2024 11:55:43 +0200 Subject: [PATCH] Build CI (#31) * FIX: fix bash call, use jammy (22.04), use erad2024 docker endpoint * FIX: try to build image on pullrequest * WIP: refer to own workflows * WIP: add baltrad build deps * WIP: use 2024.03.0 repo2docker * WIP: hardcode python3.11 for baltrad install scripts * WIP: use openradar fork of baltrad hlhdf * WIP: do not build baltrad for now * WIP: do not build baltrad for now * WIP: use correct repo2docker version * WIP: add environment.ipynb, remove nbstripout.sh call * WIP: add environment.ipynb, remove nbstripout.sh call * WIP: fix workflows * WIP: use auto instead of cache/binder --- .github/workflows/build-book-pullrequest.yaml | 81 +++++++++++++++++++ .github/workflows/build-book.yaml | 11 +-- .github/workflows/trigger-book-build.yaml | 8 +- _config.yml | 2 +- _toc.yml | 3 + binder/Dockerfile | 2 +- binder/appendix.txt | 2 - binder/environment.yml | 12 +++ install/baltrad/install_baltrad_hlhdf.sh | 6 +- install/baltrad/install_baltrad_rave.sh | 4 +- notebooks/environment.ipynb | 63 +++++++++++++++ 11 files changed, 178 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/build-book-pullrequest.yaml create mode 100644 notebooks/environment.ipynb diff --git a/.github/workflows/build-book-pullrequest.yaml b/.github/workflows/build-book-pullrequest.yaml new file mode 100644 index 000000000..ec33272d5 --- /dev/null +++ b/.github/workflows/build-book-pullrequest.yaml @@ -0,0 +1,81 @@ +name: build-book-pullrequest + +on: + workflow_call: + inputs: + environment_name: + description: 'Name of conda environment to activate' + required: false + default: 'cookbook-dev' + type: string + environment_file: + description: 'Name of conda environment file' + required: false + default: 'environment.yml' + type: string + path_to_notebooks: + description: 'Location of the JupyterBook source relative to repo root' + required: false + default: './' + type: string + use_cached_environment: + description: 'Flag for whether we should attempt to retrieve a previously cached environment.' + required: false + default: 'true' + type: string # had a lot of trouble with boolean types, see https://github.com/actions/runner/issues/1483 + +jobs: + build-book: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + - name: checkout files in repo + uses: actions/checkout@v3 + - name: remove Dockerfile + run: | + rm binder/Dockerfile + - name: update jupyter dependencies with repo2docker + uses: jupyterhub/repo2docker-action@master + with: + NO_PUSH: true + DOCKER_USERNAME: "openradar" + DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + DOCKER_REGISTRY: "ghcr.io" + PUBLIC_REGISTRY_CHECK: true + APPENDIX_FILE: "binder/appendix.txt" + REPO2DOCKER_EXTRA_ARGS: --Repo2Docker.base_image=docker.io/library/buildpack-deps:jammy + FORCE_REPO2DOCKER_VERSION: jupyter-repo2docker==2024.03.0 + + - name: list docker images + run: | + docker image ls -a + - name: Build the book + uses: addnab/docker-run-action@v3 + with: + image: ghcr.io/openradar/erad2024:latest + options: --user root -v ${{ github.workspace }}:/work + shell: bash -l {0} + run: | + # copy baltrad notebooks to book dir + # re add when baltrad issues are sorted out + # cp -rp /home/jovyan/notebooks/baltrad* /work/notebooks/. + # cp -rp /home/jovyan/notebooks/pyart2baltrad* /work/notebooks/. + # /work/install/nbstripout_notebooks.sh + source /srv/conda/etc/profile.d/conda.sh + conda activate notebook + jupyter-book build /work/${{ inputs.path_to_notebooks }} + - name: Zip the book + run: | + set -x + set -e + if [ -f book.zip ]; then + rm -rf book.zip + fi + zip -r book.zip ${{ inputs.path_to_notebooks }}/_build/html + - name: Upload zipped book artifact + uses: actions/upload-artifact@v3 + with: + name: book-zip-${{github.event.number}} + path: ./book.zip diff --git a/.github/workflows/build-book.yaml b/.github/workflows/build-book.yaml index d9341b110..742e4a975 100644 --- a/.github/workflows/build-book.yaml +++ b/.github/workflows/build-book.yaml @@ -41,8 +41,8 @@ jobs: DOCKER_REGISTRY: "ghcr.io" PUBLIC_REGISTRY_CHECK: true APPENDIX_FILE: "binder/appendix.txt" - REPO2DOCKER_EXTRA_ARGS: --Repo2Docker.base_image=docker.io/library/buildpack-deps:bionic - FORCE_REPO2DOCKER_VERSION: jupyter-repo2docker==2022.02.0 + REPO2DOCKER_EXTRA_ARGS: --Repo2Docker.base_image=docker.io/library/buildpack-deps:jammy + FORCE_REPO2DOCKER_VERSION: jupyter-repo2docker==2024.03.0 build-book: runs-on: ubuntu-latest @@ -58,9 +58,10 @@ jobs: - name: Build the book run: | # copy baltrad notebooks to book dir - cp -rp /home/jovyan/notebooks/baltrad* notebooks/. - cp -rp /home/jovyan/notebooks/pyart2baltrad* notebooks/. - install/nbstripout_notebooks.sh + # re add when baltrad issues are sorted out + # cp -rp /home/jovyan/notebooks/baltrad* notebooks/. + # cp -rp /home/jovyan/notebooks/pyart2baltrad* notebooks/. + # install/nbstripout_notebooks.sh jupyter-book build ${{ inputs.path_to_notebooks }} - name: Zip the book run: | diff --git a/.github/workflows/trigger-book-build.yaml b/.github/workflows/trigger-book-build.yaml index bd014bf8a..46bef0e64 100644 --- a/.github/workflows/trigger-book-build.yaml +++ b/.github/workflows/trigger-book-build.yaml @@ -4,8 +4,12 @@ on: jobs: build: - uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main + # uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main + uses: ./.github/workflows/build-book-pullrequest.yaml with: environment_name: cookbook-dev - artifact_name: book-zip-${{ github.event.number }} + environment_file: environment.yml + path_to_notebooks: ./ + use_cached_environment: 'false' + # artifact_name: book-zip-${{ github.event.number }} # Other input options are possible, see ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml diff --git a/_config.yml b/_config.yml index bf28a7fc8..05b5414f7 100644 --- a/_config.yml +++ b/_config.yml @@ -8,7 +8,7 @@ copyright: "2024" execute: # To execute notebooks via a Binder instead, replace 'cache' with 'binder' - execute_notebooks: binder + execute_notebooks: auto timeout: 600 allow_errors: False # cells with expected failures must set the `raises-exception` cell tag diff --git a/_toc.yml b/_toc.yml index 995f86b61..882179e71 100644 --- a/_toc.yml +++ b/_toc.yml @@ -7,3 +7,6 @@ parts: - caption: Introduction chapters: - file: notebooks/notebook-template + - caption: Notebook Environment + chapters: + - file: notebooks/environment diff --git a/binder/Dockerfile b/binder/Dockerfile index e93da14b7..c582bcd4b 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,3 +1,3 @@ -FROM ghcr.io/openradar/erad2022:latest +FROM ghcr.io/openradar/erad2024:latest RUN mamba env update -f binder/environment.yml \ No newline at end of file diff --git a/binder/appendix.txt b/binder/appendix.txt index 819c590e2..e47d9846c 100644 --- a/binder/appendix.txt +++ b/binder/appendix.txt @@ -16,6 +16,4 @@ RUN CONDA_DIR=/srv/conda && \ echo "export RADARENV=$RADARENV" >> ~/.profile && \ echo "export CONDA_PREFIX=$CONDA_PREFIX" >> ~/.profile && \ echo "export PROJ_NETWORK=$PROJ_NETWORK" >> ~/.profile && \ - ${CONDA_PREFIX}/bin -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad.sh && \ - ${CONDA_PREFIX}/bin -l $BALTRAD_INSTALL_ROOT/install/nbstripout_notebooks.sh && \ cp binder/kernel.json $CONDA_PREFIX/share/jupyter/kernels/python3/. diff --git a/binder/environment.yml b/binder/environment.yml index 37cd0b353..e9a885498 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -16,3 +16,15 @@ dependencies: - xradar - numpy<2.0 - python<=3.11 + # this is needed for baltrad compilation + - compilers + - make + - gnuconfig + - bash + - libaec + - coreutils + - autoconf + - automake + - tar + - libblas=*=*netlib + - liblapacke=*=*netlib diff --git a/install/baltrad/install_baltrad_hlhdf.sh b/install/baltrad/install_baltrad_hlhdf.sh index 6a8426737..b2b9c52c9 100755 --- a/install/baltrad/install_baltrad_hlhdf.sh +++ b/install/baltrad/install_baltrad_hlhdf.sh @@ -7,7 +7,7 @@ set +e # download cd $BALTRAD_INSTALL_ROOT/tmp -git clone --depth=1 https://github.com/baltrad/hlhdf.git +git clone --depth=1 https://github.com/openradar/hlhdf.git cd hlhdf/ # build, test and install @@ -15,11 +15,11 @@ cd hlhdf/ --with-hdf5=$CONDA_PREFIX/include,$CONDA_PREFIX/lib \ --enable-py3support \ --with-py3bin=$CONDA_PREFIX/bin/python3 \ - --with-numpy=$CONDA_PREFIX/lib/python3.9/site-packages/numpy/core/include/numpy/ + --with-numpy=$CONDA_PREFIX/lib/python3.11/site-packages/numpy/core/include/numpy/ make make test make install -mv $CONDA_PREFIX/hlhdf/hlhdf.pth $CONDA_PREFIX/lib/python3.9/site-packages/. +mv $CONDA_PREFIX/hlhdf/hlhdf.pth $CONDA_PREFIX/lib/python3.11/site-packages/. # activation script grep -l hlhdf ${CONDA_PREFIX}/etc/conda/activate.d/baltrad.sh diff --git a/install/baltrad/install_baltrad_rave.sh b/install/baltrad/install_baltrad_rave.sh index d32e22905..5d26700c6 100755 --- a/install/baltrad/install_baltrad_rave.sh +++ b/install/baltrad/install_baltrad_rave.sh @@ -25,9 +25,9 @@ sed -i -e 's/from keyczar import keyczar/#from keyczar import keyczar/g' Lib/Bal --with-hlhdf=$CONDA_PREFIX/hlhdf \ --with-proj=$CONDA_PREFIX/include,$CONDA_PREFIX/lib \ --with-expat=$CONDA_PREFIX/include,$CONDA_PREFIX/lib \ - --with-numpy=$CONDA_PREFIX/lib/python3.9/site-packages/numpy/core/include/numpy/ \ + --with-numpy=$CONDA_PREFIX/lib/python3.11/site-packages/numpy/core/include/numpy/ \ --with-netcdf=$CONDA_PREFIX/include,$CONDA_PREFIX/lib \ - --with-python-makefile=$CONDA_PREFIX/lib/python3.9/config-3.9-x86_64-linux-gnu/Makefile + --with-python-makefile=$CONDA_PREFIX/lib/python3.11/config-3.11-x86_64-linux-gnu/Makefile make make test make install diff --git a/notebooks/environment.ipynb b/notebooks/environment.ipynb new file mode 100644 index 000000000..026df6198 --- /dev/null +++ b/notebooks/environment.ipynb @@ -0,0 +1,63 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Environment overview" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!env" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!conda list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import wradlib as wrl" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wrl.show_versions()" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file