Skip to content

Commit

Permalink
Build CI (#31)
Browse files Browse the repository at this point in the history
* FIX: fix bash call, use jammy (22.04), use erad2024 docker endpoint

* FIX: try to build image on pullrequest

* WIP: refer to own workflows

* WIP: add baltrad build deps

* WIP: use 2024.03.0 repo2docker

* WIP: hardcode python3.11 for baltrad install scripts

* WIP: use openradar fork of baltrad hlhdf

* WIP: do not build baltrad for now

* WIP: do not build baltrad for now

* WIP: use correct repo2docker version

* WIP: add environment.ipynb, remove nbstripout.sh call

* WIP: add environment.ipynb, remove nbstripout.sh call

* WIP: fix workflows

* WIP: use auto instead of cache/binder
  • Loading branch information
kmuehlbauer authored Jun 25, 2024
1 parent 5136fb7 commit 4ea4347
Show file tree
Hide file tree
Showing 11 changed files with 178 additions and 16 deletions.
81 changes: 81 additions & 0 deletions .github/workflows/build-book-pullrequest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
name: build-book-pullrequest

on:
workflow_call:
inputs:
environment_name:
description: 'Name of conda environment to activate'
required: false
default: 'cookbook-dev'
type: string
environment_file:
description: 'Name of conda environment file'
required: false
default: 'environment.yml'
type: string
path_to_notebooks:
description: 'Location of the JupyterBook source relative to repo root'
required: false
default: './'
type: string
use_cached_environment:
description: 'Flag for whether we should attempt to retrieve a previously cached environment.'
required: false
default: 'true'
type: string # had a lot of trouble with boolean types, see https://github.com/actions/runner/issues/1483

jobs:
build-book:
runs-on: ubuntu-latest
defaults:
run:
shell: bash -l {0}
steps:
- name: checkout files in repo
uses: actions/checkout@v3
- name: remove Dockerfile
run: |
rm binder/Dockerfile
- name: update jupyter dependencies with repo2docker
uses: jupyterhub/repo2docker-action@master
with:
NO_PUSH: true
DOCKER_USERNAME: "openradar"
DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
DOCKER_REGISTRY: "ghcr.io"
PUBLIC_REGISTRY_CHECK: true
APPENDIX_FILE: "binder/appendix.txt"
REPO2DOCKER_EXTRA_ARGS: --Repo2Docker.base_image=docker.io/library/buildpack-deps:jammy
FORCE_REPO2DOCKER_VERSION: jupyter-repo2docker==2024.03.0

- name: list docker images
run: |
docker image ls -a
- name: Build the book
uses: addnab/docker-run-action@v3
with:
image: ghcr.io/openradar/erad2024:latest
options: --user root -v ${{ github.workspace }}:/work
shell: bash -l {0}
run: |
# copy baltrad notebooks to book dir
# re add when baltrad issues are sorted out
# cp -rp /home/jovyan/notebooks/baltrad* /work/notebooks/.
# cp -rp /home/jovyan/notebooks/pyart2baltrad* /work/notebooks/.
# /work/install/nbstripout_notebooks.sh
source /srv/conda/etc/profile.d/conda.sh
conda activate notebook
jupyter-book build /work/${{ inputs.path_to_notebooks }}
- name: Zip the book
run: |
set -x
set -e
if [ -f book.zip ]; then
rm -rf book.zip
fi
zip -r book.zip ${{ inputs.path_to_notebooks }}/_build/html
- name: Upload zipped book artifact
uses: actions/upload-artifact@v3
with:
name: book-zip-${{github.event.number}}
path: ./book.zip
11 changes: 6 additions & 5 deletions .github/workflows/build-book.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ jobs:
DOCKER_REGISTRY: "ghcr.io"
PUBLIC_REGISTRY_CHECK: true
APPENDIX_FILE: "binder/appendix.txt"
REPO2DOCKER_EXTRA_ARGS: --Repo2Docker.base_image=docker.io/library/buildpack-deps:bionic
FORCE_REPO2DOCKER_VERSION: jupyter-repo2docker==2022.02.0
REPO2DOCKER_EXTRA_ARGS: --Repo2Docker.base_image=docker.io/library/buildpack-deps:jammy
FORCE_REPO2DOCKER_VERSION: jupyter-repo2docker==2024.03.0

build-book:
runs-on: ubuntu-latest
Expand All @@ -58,9 +58,10 @@ jobs:
- name: Build the book
run: |
# copy baltrad notebooks to book dir
cp -rp /home/jovyan/notebooks/baltrad* notebooks/.
cp -rp /home/jovyan/notebooks/pyart2baltrad* notebooks/.
install/nbstripout_notebooks.sh
# re add when baltrad issues are sorted out
# cp -rp /home/jovyan/notebooks/baltrad* notebooks/.
# cp -rp /home/jovyan/notebooks/pyart2baltrad* notebooks/.
# install/nbstripout_notebooks.sh
jupyter-book build ${{ inputs.path_to_notebooks }}
- name: Zip the book
run: |
Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/trigger-book-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,12 @@ on:

jobs:
build:
uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main
# uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main
uses: ./.github/workflows/build-book-pullrequest.yaml
with:
environment_name: cookbook-dev
artifact_name: book-zip-${{ github.event.number }}
environment_file: environment.yml
path_to_notebooks: ./
use_cached_environment: 'false'
# artifact_name: book-zip-${{ github.event.number }}
# Other input options are possible, see ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml
2 changes: 1 addition & 1 deletion _config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ copyright: "2024"

execute:
# To execute notebooks via a Binder instead, replace 'cache' with 'binder'
execute_notebooks: binder
execute_notebooks: auto
timeout: 600
allow_errors: False # cells with expected failures must set the `raises-exception` cell tag

Expand Down
3 changes: 3 additions & 0 deletions _toc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,6 @@ parts:
- caption: Introduction
chapters:
- file: notebooks/notebook-template
- caption: Notebook Environment
chapters:
- file: notebooks/environment
2 changes: 1 addition & 1 deletion binder/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
FROM ghcr.io/openradar/erad2022:latest
FROM ghcr.io/openradar/erad2024:latest

RUN mamba env update -f binder/environment.yml
2 changes: 0 additions & 2 deletions binder/appendix.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,4 @@ RUN CONDA_DIR=/srv/conda && \
echo "export RADARENV=$RADARENV" >> ~/.profile && \
echo "export CONDA_PREFIX=$CONDA_PREFIX" >> ~/.profile && \
echo "export PROJ_NETWORK=$PROJ_NETWORK" >> ~/.profile && \
${CONDA_PREFIX}/bin -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad.sh && \
${CONDA_PREFIX}/bin -l $BALTRAD_INSTALL_ROOT/install/nbstripout_notebooks.sh && \
cp binder/kernel.json $CONDA_PREFIX/share/jupyter/kernels/python3/.
12 changes: 12 additions & 0 deletions binder/environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,15 @@ dependencies:
- xradar
- numpy<2.0
- python<=3.11
# this is needed for baltrad compilation
- compilers
- make
- gnuconfig
- bash
- libaec
- coreutils
- autoconf
- automake
- tar
- libblas=*=*netlib
- liblapacke=*=*netlib
6 changes: 3 additions & 3 deletions install/baltrad/install_baltrad_hlhdf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,19 @@ set +e

# download
cd $BALTRAD_INSTALL_ROOT/tmp
git clone --depth=1 https://github.com/baltrad/hlhdf.git
git clone --depth=1 https://github.com/openradar/hlhdf.git
cd hlhdf/

# build, test and install
./configure --prefix=$CONDA_PREFIX/hlhdf \
--with-hdf5=$CONDA_PREFIX/include,$CONDA_PREFIX/lib \
--enable-py3support \
--with-py3bin=$CONDA_PREFIX/bin/python3 \
--with-numpy=$CONDA_PREFIX/lib/python3.9/site-packages/numpy/core/include/numpy/
--with-numpy=$CONDA_PREFIX/lib/python3.11/site-packages/numpy/core/include/numpy/
make
make test
make install
mv $CONDA_PREFIX/hlhdf/hlhdf.pth $CONDA_PREFIX/lib/python3.9/site-packages/.
mv $CONDA_PREFIX/hlhdf/hlhdf.pth $CONDA_PREFIX/lib/python3.11/site-packages/.

# activation script
grep -l hlhdf ${CONDA_PREFIX}/etc/conda/activate.d/baltrad.sh
Expand Down
4 changes: 2 additions & 2 deletions install/baltrad/install_baltrad_rave.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ sed -i -e 's/from keyczar import keyczar/#from keyczar import keyczar/g' Lib/Bal
--with-hlhdf=$CONDA_PREFIX/hlhdf \
--with-proj=$CONDA_PREFIX/include,$CONDA_PREFIX/lib \
--with-expat=$CONDA_PREFIX/include,$CONDA_PREFIX/lib \
--with-numpy=$CONDA_PREFIX/lib/python3.9/site-packages/numpy/core/include/numpy/ \
--with-numpy=$CONDA_PREFIX/lib/python3.11/site-packages/numpy/core/include/numpy/ \
--with-netcdf=$CONDA_PREFIX/include,$CONDA_PREFIX/lib \
--with-python-makefile=$CONDA_PREFIX/lib/python3.9/config-3.9-x86_64-linux-gnu/Makefile
--with-python-makefile=$CONDA_PREFIX/lib/python3.11/config-3.11-x86_64-linux-gnu/Makefile
make
make test
make install
Expand Down
63 changes: 63 additions & 0 deletions notebooks/environment.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Environment overview"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!env"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!conda list"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import wradlib as wrl"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"wrl.show_versions()"
]
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

0 comments on commit 4ea4347

Please sign in to comment.