From daceed563bf9bf5bb1c48d110941c6334502402d Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 15 Feb 2024 16:48:13 -0500 Subject: [PATCH 01/15] ci: fix main python path --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7778468..d87b16d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -17,7 +17,7 @@ jobs: - name: Dry-run pipeline run: | docker run -v $PWD:/opt2 -w /opt2 snakemake/snakemake:v5.24.2 \ - python ./renee.py run \ + python src/renee/__main__.py run \ --input .tests/KO_S3.R1.fastq.gz .tests/KO_S3.R2.fastq.gz .tests/KO_S4.R1.fastq.gz .tests/KO_S4.R2.fastq.gz .tests/WT_S1.R1.fastq.gz .tests/WT_S1.R2.fastq.gz .tests/WT_S2.R1.fastq.gz .tests/WT_S2.R2.fastq.gz \ --output output \ --genome config/genomes/biowulf/hg38_30.json \ From 75a5945f4138c8b2472fda6f35c8c7a24291eecf Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Fri, 16 Feb 2024 10:27:57 -0500 Subject: [PATCH 02/15] chore: remove print statement --- src/renee/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/renee/__main__.py b/src/renee/__main__.py index 7996560..4f342d2 100755 --- a/src/renee/__main__.py +++ b/src/renee/__main__.py @@ -812,7 +812,6 @@ def setup(sub_args, ifiles, repo_path, output_path): ), end="", ) - # print(json.dumps(config, indent = 4, sort_keys=True)) with open(os.path.join(output_path, "config.json"), "w") as fh: json.dump(config, fh, indent=4, sort_keys=True) print("Done!") From 5bb0a30ef2255420f65282211352c734e7fb5137 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Fri, 16 Feb 2024 10:28:48 -0500 Subject: [PATCH 03/15] test: expect error when output arg missing --- tests/test_cli.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 71f5c96..2cd1324 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,12 @@ import pytest import subprocess -from src.renee.__main__ import exists + +renee_run = ( + "src/renee/__main__.py run " + "--mode local --runmode init --dry-run " + "--input .tests/*.fastq.gz " + "--genome hg38_30 " +) def test_help(): @@ -15,3 +21,12 @@ def test_version(): "src/renee/__main__.py --version", capture_output=True, shell=True, text=True ).stdout assert "renee v" in output + + +def test_run_error(): + assert ( + "the following arguments are required: --output" + in subprocess.run( + f"{renee_run}", capture_output=True, shell=True, text=True + ).stderr + ) From 60ac60703ad13fcb5d0d63ab5a6fc985b29ebec5 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Fri, 16 Feb 2024 10:29:11 -0500 Subject: [PATCH 04/15] test: image caching behavior with & without --sif-cache --- tests/data/sifs/ccbr_arriba_2.0.0_v0.0.1.sif | 0 tests/test_cache.py | 45 ++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/data/sifs/ccbr_arriba_2.0.0_v0.0.1.sif create mode 100644 tests/test_cache.py diff --git a/tests/data/sifs/ccbr_arriba_2.0.0_v0.0.1.sif b/tests/data/sifs/ccbr_arriba_2.0.0_v0.0.1.sif new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..00c7fb3 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,45 @@ +import tempfile +import json +import os.path +import subprocess + +renee_run = ( + "src/renee/__main__.py run " + "--mode local --runmode init --dry-run " + "--input .tests/*.fastq.gz " + "--genome hg38_30 " +) + + +def run_in_temp(command_str): + with tempfile.TemporaryDirectory() as tmp_dir: + outdir = os.path.join(tmp_dir, "testout") + output = subprocess.run( + f"{command_str} --output {outdir}", + capture_output=True, + shell=True, + text=True, + ) + with open(os.path.join(outdir, "config.json"), "r") as infile: + config = json.load(infile) + return output, config + + +def test_cache_sif(): + output, config = run_in_temp(f"{renee_run} --sif-cache tests/data/sifs/") + assertions = [ + config["images"]["arriba"].endswith( + "tests/data/sifs/ccbr_arriba_2.0.0_v0.0.1.sif" + ), + "does not exist in singularity cache" in output.stderr, + ] + assert all(assertions) + + +def test_cache_nosif(): + output, config = run_in_temp(f"{renee_run}") + assertions = [ + config["images"]["arriba"] == "docker://nciccbr/ccbr_arriba_2.0.0:v0.0.1", + "The singularity command has to be available" in output.stderr, + ] + assert all(assertions) From e47a2fc264bbf88af346924f2f1d636b50b9b1ee Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 19 Feb 2024 19:07:33 -0500 Subject: [PATCH 05/15] ci: set TMPDIR=runner.temp for pytest step should fix the problem where /tmp can't be found --- .github/workflows/main.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d87b16d..771245f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -52,6 +52,8 @@ jobs: - name: Test run: | python -m pytest + env: + TMPDIR: ${{ runner.temp }} build-status: # https://github.com/orgs/community/discussions/4324#discussioncomment-3477871 runs-on: ubuntu-latest From 06e9e0c24baa119a9f533a6c0036229487c935c7 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 19 Feb 2024 19:25:15 -0500 Subject: [PATCH 06/15] ci: need snakemake to test renee dryrun --- .github/workflows/main.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 771245f..c577a73 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -37,23 +37,31 @@ jobs: python-version: ["3.11"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + - uses: mamba-org/setup-micromamba@v1 with: - python-version: ${{ matrix.python-version }} - cache: "pip" + micromamba-version: "1.3.1-0" + cache-environment: true + create-args: | + python=${{ matrix.python-version }} + snakemake + setuptools + pip + pytest - name: pip run: | - python -m pip install --upgrade pip setuptools pytest + python -m pip install + shell: micromamba-shell {0} - name: check CLI basics run: | ./bin/renee --help ./bin/renee --version + shell: micromamba-shell {0} - name: Test run: | python -m pytest env: TMPDIR: ${{ runner.temp }} + shell: micromamba-shell {0} build-status: # https://github.com/orgs/community/discussions/4324#discussioncomment-3477871 runs-on: ubuntu-latest From 6349904d0325e77a72a026988ec921392010270b Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 19 Feb 2024 19:26:44 -0500 Subject: [PATCH 07/15] ci: specify snakemake version --- .github/workflows/main.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c577a73..dbb595e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -35,6 +35,7 @@ jobs: strategy: matrix: python-version: ["3.11"] + snakemake-version: ["7.32.3"] steps: - uses: actions/checkout@v4 - uses: mamba-org/setup-micromamba@v1 @@ -43,7 +44,7 @@ jobs: cache-environment: true create-args: | python=${{ matrix.python-version }} - snakemake + snakemake=${{ matrix.snakemake-version }} setuptools pip pytest From bfa9d48970e17546f4354e040bcc7bca4bdcb64b Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 19 Feb 2024 19:30:02 -0500 Subject: [PATCH 08/15] ci: remove cache env for mamba --- .github/workflows/main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dbb595e..2aadeae 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -41,7 +41,6 @@ jobs: - uses: mamba-org/setup-micromamba@v1 with: micromamba-version: "1.3.1-0" - cache-environment: true create-args: | python=${{ matrix.python-version }} snakemake=${{ matrix.snakemake-version }} From 63582b55d21a2590babc81dba989c9108792f9ec Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 19 Feb 2024 19:58:40 -0500 Subject: [PATCH 09/15] ci: env name required for mamba --- .github/workflows/main.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2aadeae..2d282b6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -40,7 +40,9 @@ jobs: - uses: actions/checkout@v4 - uses: mamba-org/setup-micromamba@v1 with: + environment-name: test micromamba-version: "1.3.1-0" + cache-environment: true create-args: | python=${{ matrix.python-version }} snakemake=${{ matrix.snakemake-version }} From fe889917be6a081e266c741bcd53460a3d6aeb0d Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 19 Feb 2024 22:08:52 -0500 Subject: [PATCH 10/15] ci: remove mamba version --- .github/workflows/main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2d282b6..a2bde79 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -41,7 +41,6 @@ jobs: - uses: mamba-org/setup-micromamba@v1 with: environment-name: test - micromamba-version: "1.3.1-0" cache-environment: true create-args: | python=${{ matrix.python-version }} From 23441c593d4fd751d5ad0604ae5d2ae04c07800d Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 19 Feb 2024 22:16:26 -0500 Subject: [PATCH 11/15] ci: remove py version --- .github/workflows/main.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a2bde79..1783b6b 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -34,7 +34,6 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11"] snakemake-version: ["7.32.3"] steps: - uses: actions/checkout@v4 @@ -43,7 +42,6 @@ jobs: environment-name: test cache-environment: true create-args: | - python=${{ matrix.python-version }} snakemake=${{ matrix.snakemake-version }} setuptools pip From 9269021db04d71f8ca26848fc4e663d52c3d8749 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Tue, 20 Feb 2024 09:29:09 -0500 Subject: [PATCH 12/15] ci: fix create-args multiline delim --- .github/workflows/main.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1783b6b..37022c6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -34,6 +34,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + python-version: ["3.11"] snakemake-version: ["7.32.3"] steps: - uses: actions/checkout@v4 @@ -41,7 +42,8 @@ jobs: with: environment-name: test cache-environment: true - create-args: | + create-args: >- + python=${{ matrix.python-version }} snakemake=${{ matrix.snakemake-version }} setuptools pip From 5df6d745bc7e5e19cb7b11105bbc48717bcb9455 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Tue, 20 Feb 2024 09:32:44 -0500 Subject: [PATCH 13/15] ci: need bioconda channel for snakemake --- .github/workflows/main.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 37022c6..43cdfbd 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -42,6 +42,10 @@ jobs: with: environment-name: test cache-environment: true + condarc: | + channels: + - conda-forge + - bioconda create-args: >- python=${{ matrix.python-version }} snakemake=${{ matrix.snakemake-version }} From 4b0682e5761f4578b63b7d02cd360a430a47d19a Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Tue, 20 Feb 2024 09:46:58 -0500 Subject: [PATCH 14/15] ci: pip is not needed since renee isn't a package --- .github/workflows/main.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 43cdfbd..5d4e4b2 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -52,10 +52,6 @@ jobs: setuptools pip pytest - - name: pip - run: | - python -m pip install - shell: micromamba-shell {0} - name: check CLI basics run: | ./bin/renee --help From fc3c93624080526bb3638f284e389b01d8370f96 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Tue, 20 Feb 2024 14:00:33 -0500 Subject: [PATCH 15/15] test: specify path to genome config --- tests/test_cache.py | 9 ++++++--- tests/test_cli.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_cache.py b/tests/test_cache.py index 00c7fb3..384c9c4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -7,7 +7,7 @@ "src/renee/__main__.py run " "--mode local --runmode init --dry-run " "--input .tests/*.fastq.gz " - "--genome hg38_30 " + "--genome config/genomes/biowulf/hg38_30.json " ) @@ -20,8 +20,11 @@ def run_in_temp(command_str): shell=True, text=True, ) - with open(os.path.join(outdir, "config.json"), "r") as infile: - config = json.load(infile) + if os.path.exists(os.path.join(outdir, "config.json")): + with open(os.path.join(outdir, "config.json"), "r") as infile: + config = json.load(infile) + else: + config = None return output, config diff --git a/tests/test_cli.py b/tests/test_cli.py index 2cd1324..ff6c2a9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,7 +5,7 @@ "src/renee/__main__.py run " "--mode local --runmode init --dry-run " "--input .tests/*.fastq.gz " - "--genome hg38_30 " + "--genome config/genomes/biowulf/hg38_30.json " )