From 0f703b90cc10f43deceda336de1cb3ae2b263d52 Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:48:15 +0100
Subject: [PATCH 01/11] add pip_install.sh

---
 .github/workflows/nbdev_tests.yml        |  2 +-
 .github/workflows/quick_tests.yml        |  4 ++--
 .github/workflows/quick_tests_ubuntu.yml |  2 +-
 .github/workflows/ratio_tests.yml        |  2 +-
 .github/workflows/unused/all_tests.yml   |  2 +-
 misc/loose_pip_install.sh                |  5 -----
 misc/pip_install.sh                      | 25 ++++++++++++++++++++++++
 misc/stable_pip_install.sh               |  5 -----
 8 files changed, 31 insertions(+), 16 deletions(-)
 delete mode 100644 misc/loose_pip_install.sh
 create mode 100644 misc/pip_install.sh
 delete mode 100644 misc/stable_pip_install.sh

diff --git a/.github/workflows/nbdev_tests.yml b/.github/workflows/nbdev_tests.yml
index 73c2e5e..626ede9 100644
--- a/.github/workflows/nbdev_tests.yml
+++ b/.github/workflows/nbdev_tests.yml
@@ -27,7 +27,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          . ./loose_pip_install.sh
+          ./pip_install.sh loose,tests
 
       - name: Unittests
         shell: bash -le {0}
diff --git a/.github/workflows/quick_tests.yml b/.github/workflows/quick_tests.yml
index aab6ff6..4ab1240 100644
--- a/.github/workflows/quick_tests.yml
+++ b/.github/workflows/quick_tests.yml
@@ -32,7 +32,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          . ./stable_pip_install.sh
+          ./pip_install.sh stable,tests
       - name: Run pipeline
         shell: bash -le {0}
         run: |
@@ -63,7 +63,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          . ./loose_pip_install.sh
+          ./pip_install.sh loose,tests
       - name: Run pipeline
         shell: bash -le {0}
         run: |
diff --git a/.github/workflows/quick_tests_ubuntu.yml b/.github/workflows/quick_tests_ubuntu.yml
index 9b046bf..d1b18f9 100644
--- a/.github/workflows/quick_tests_ubuntu.yml
+++ b/.github/workflows/quick_tests_ubuntu.yml
@@ -27,7 +27,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          . ./stable_pip_install.sh
+          ./pip_install.sh stable,tests
 
       - name: Unittests
         shell: bash -le {0}
diff --git a/.github/workflows/ratio_tests.yml b/.github/workflows/ratio_tests.yml
index 9f44bc4..fbf1db4 100644
--- a/.github/workflows/ratio_tests.yml
+++ b/.github/workflows/ratio_tests.yml
@@ -25,7 +25,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          . ./stable_pip_install.sh
+          ./pip_install.sh stable,tests
 
       - name: Ratio tests
         shell: bash -le {0}
diff --git a/.github/workflows/unused/all_tests.yml b/.github/workflows/unused/all_tests.yml
index 0fce3f9..71db82b 100644
--- a/.github/workflows/unused/all_tests.yml
+++ b/.github/workflows/unused/all_tests.yml
@@ -28,7 +28,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          . ./loose_pip_install.sh
+          ./pip_install.sh loose,tests
       - name: Unittests
         shell: bash -le {0}
         run: |
diff --git a/misc/loose_pip_install.sh b/misc/loose_pip_install.sh
deleted file mode 100644
index 64e5fcf..0000000
--- a/misc/loose_pip_install.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-conda create -n directlfq python=3.9 -y
-conda activate directlfq
-pip install -e '../.[development, gui]'
-directlfq
-conda deactivate
diff --git a/misc/pip_install.sh b/misc/pip_install.sh
new file mode 100644
index 0000000..38dddb4
--- /dev/null
+++ b/misc/pip_install.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+### Install the package with a given type in a defined conda environment with a define python version,
+### and call it to check if it works
+### example usage:
+### ./pip_install.sh stable my_env 3.9
+set -e -u
+
+INSTALL_TYPE=$1 # stable, loose, etc..
+ENV_NAME=${2:-directlfq}
+PYTHON_VERSION=${3:-3.9}
+
+conda create -n $ENV_NAME python=$PYTHON_VERSION -y
+
+if [ "$INSTALL_TYPE" = "loose" ]; then
+  INSTALL_STRING=""
+else
+  INSTALL_STRING="[${INSTALL_TYPE}]"
+fi
+
+# print pip environment for reproducibility
+conda run -n $ENV_NAME --no-capture-output pip freeze
+
+# conda 'run' vs. 'activate', cf. https://stackoverflow.com/a/72395091
+conda run -n $ENV_NAME --no-capture-output pip install -e "../.$INSTALL_STRING"
+conda run -n $ENV_NAME --no-capture-output directlfq
diff --git a/misc/stable_pip_install.sh b/misc/stable_pip_install.sh
deleted file mode 100644
index 69dedb2..0000000
--- a/misc/stable_pip_install.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-conda create -n directlfq python=3.9 -y
-conda activate directlfq
-pip install -e '../.[stable, gui-stable, development]'
-directlfq
-conda deactivate

From 40f336ed7b522c92fd8b65a6e9a01cf98a635574 Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:51:10 +0100
Subject: [PATCH 02/11] split requirements

---
 pyproject.toml                              |  3 ++-
 release/windows/build_installer_windows.ps1 |  1 -
 requirements/requirements.txt               |  1 -
 requirements/requirements_development.txt   | 15 ++-------------
 requirements/requirements_loose.txt         |  1 -
 requirements/requirements_tests.txt         |  9 +++++++++
 6 files changed, 13 insertions(+), 17 deletions(-)
 create mode 100644 requirements/requirements_tests.txt

diff --git a/pyproject.toml b/pyproject.toml
index 0e51b32..98a19ca 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -51,7 +51,8 @@ dependencies = {file = ["requirements/requirements_loose.txt"]}
 optional-dependencies = { stable = { file = ["requirements/requirements.txt",
 ] }, gui = { file = [ "requirements/requirements_gui_loose.txt",
 ] }, gui-stable = { file = [ "requirements/requirements_gui.txt",
-] }, development = { file = ["requirements/requirements_development.txt"
+] }, tests = { file = [ "requirements/requirements_tests.txt",
+] }, development = { file = ["requirements/requirements_development.txt", "requirements/requirements_tests.txt",
 ] }}
 
 version = {attr = "directlfq.__version__"}
diff --git a/release/windows/build_installer_windows.ps1 b/release/windows/build_installer_windows.ps1
index f552c1c..44e91fb 100644
--- a/release/windows/build_installer_windows.ps1
+++ b/release/windows/build_installer_windows.ps1
@@ -18,4 +18,3 @@ pip install "dist/$WHL_NAME[stable,gui-stable]"
 # Creating the stand-alone pyinstaller folder
 pip install pyinstaller
 pyinstaller release/pyinstaller/directlfq.spec --distpath dist_pyinstaller --workpath build_pyinstaller -y
-# pip install jinja2==3.0
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 0df71fa..5b626de 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -1,7 +1,6 @@
 # Dependencies required for running the "stable" version of directlfq.
 # Only usage of fixed versions is allowed, and all dependencies listed here must also be
 # included in `requirements_loose.txt` (enforced by a test).
-Jinja2==3.1.2
 numpy==1.23.5
 pandas>=1.5.3  # test: tolerate_version
 dask>=2023.1.0  # test: tolerate_version
diff --git a/requirements/requirements_development.txt b/requirements/requirements_development.txt
index f9e2ca3..2e00ead 100644
--- a/requirements/requirements_development.txt
+++ b/requirements/requirements_development.txt
@@ -1,20 +1,9 @@
+# Dependencies helping for development
 jupyter
 jupyter_contrib_nbextensions
-pyinstaller
 autodocsumm
 sphinx-rtd-theme
-twine
 bump-my-version
 pipdeptree
 ipykernel
-tqdm
-psutil
-matplotlib_venn
-wget
-seaborn
-matplotlib
-nbdev>=2.3.9
-notebook
-alphabase>=1.4.0
-progressbar
-pytest
\ No newline at end of file
+notebook
\ No newline at end of file
diff --git a/requirements/requirements_loose.txt b/requirements/requirements_loose.txt
index 3ea836c..363f4e7 100644
--- a/requirements/requirements_loose.txt
+++ b/requirements/requirements_loose.txt
@@ -1,6 +1,5 @@
 # Dependencies required for running the "loose" version of direclfq.
 # All dependencies that are also included in `requirements.txt` must be added also here (enforced by a test).
-Jinja2
 numpy
 pandas
 dask
diff --git a/requirements/requirements_tests.txt b/requirements/requirements_tests.txt
new file mode 100644
index 0000000..adc79cd
--- /dev/null
+++ b/requirements/requirements_tests.txt
@@ -0,0 +1,9 @@
+# Dependencies required for running tests
+matplotlib_venn
+matplotlib
+wget
+seaborn
+nbdev>=2.3.9
+alphabase>=1.4.0
+progressbar
+pytest
\ No newline at end of file

From cc100ea58b7da3dbcf09c7bb5d9df67be06d696b Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:51:43 +0100
Subject: [PATCH 03/11] deactivate a workflow

---
 .github/workflows/quick_tests_ubuntu.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/quick_tests_ubuntu.yml b/.github/workflows/quick_tests_ubuntu.yml
index d1b18f9..6d82259 100644
--- a/.github/workflows/quick_tests_ubuntu.yml
+++ b/.github/workflows/quick_tests_ubuntu.yml
@@ -1,6 +1,7 @@
 on:
   push:
-    branches: [ main, development ]
+  #  branches: [ main, development ]
+  # TODO: this is covered already by quick_tests.yml
   pull_request:
   workflow_dispatch:
 

From 75648bb77b8ea160137b9ffd310a1f3d53b399f4 Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:58:48 +0100
Subject: [PATCH 04/11] fix permissions

---
 release/pyinstaller/directlfq.spec | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/release/pyinstaller/directlfq.spec b/release/pyinstaller/directlfq.spec
index 65ee566..fcb253d 100644
--- a/release/pyinstaller/directlfq.spec
+++ b/release/pyinstaller/directlfq.spec
@@ -49,10 +49,10 @@ a = Analysis(
 	binaries=binaries,
 	datas=datas,
 	hiddenimports=hidden_imports,
-	hookspath=[],
+	hookspath=['./release/pyinstaller/hookdir'],
 	runtime_hooks=[],
-	excludes=[h for h in hidden_imports if "datashader" in h],
-	win_no_prefer_redirects=False,
+	excludes=[],
+    win_no_prefer_redirects=False,
 	win_private_assemblies=False,
 	cipher=block_cipher,
 	noarchive=False

From 245deeb249ed7eae8b4a3704d9da49d275aa87f5 Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 13:01:17 +0100
Subject: [PATCH 05/11] fix permissions

---
 misc/pip_install.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 misc/pip_install.sh

diff --git a/misc/pip_install.sh b/misc/pip_install.sh
old mode 100644
new mode 100755

From 47fbbea62215d7d4dc39194b67e281816b82206b Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 14:03:50 +0100
Subject: [PATCH 06/11] fix windows tests

---
 .github/workflows/nbdev_tests.yml        | 2 +-
 .github/workflows/quick_tests.yml        | 4 ++--
 .github/workflows/quick_tests_ubuntu.yml | 2 +-
 .github/workflows/ratio_tests.yml        | 2 +-
 .github/workflows/unused/all_tests.yml   | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/nbdev_tests.yml b/.github/workflows/nbdev_tests.yml
index 626ede9..40b7fef 100644
--- a/.github/workflows/nbdev_tests.yml
+++ b/.github/workflows/nbdev_tests.yml
@@ -27,7 +27,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          ./pip_install.sh loose,tests
+          . ./pip_install.sh loose,tests
 
       - name: Unittests
         shell: bash -le {0}
diff --git a/.github/workflows/quick_tests.yml b/.github/workflows/quick_tests.yml
index 4ab1240..8f4d5de 100644
--- a/.github/workflows/quick_tests.yml
+++ b/.github/workflows/quick_tests.yml
@@ -32,7 +32,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          ./pip_install.sh stable,tests
+          . ./pip_install.sh stable,tests
       - name: Run pipeline
         shell: bash -le {0}
         run: |
@@ -63,7 +63,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          ./pip_install.sh loose,tests
+          . ./pip_install.sh loose,tests
       - name: Run pipeline
         shell: bash -le {0}
         run: |
diff --git a/.github/workflows/quick_tests_ubuntu.yml b/.github/workflows/quick_tests_ubuntu.yml
index 6d82259..2b32b7d 100644
--- a/.github/workflows/quick_tests_ubuntu.yml
+++ b/.github/workflows/quick_tests_ubuntu.yml
@@ -28,7 +28,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          ./pip_install.sh stable,tests
+          . ./pip_install.sh stable,tests
 
       - name: Unittests
         shell: bash -le {0}
diff --git a/.github/workflows/ratio_tests.yml b/.github/workflows/ratio_tests.yml
index fbf1db4..8989355 100644
--- a/.github/workflows/ratio_tests.yml
+++ b/.github/workflows/ratio_tests.yml
@@ -25,7 +25,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          ./pip_install.sh stable,tests
+          . ./pip_install.sh stable,tests
 
       - name: Ratio tests
         shell: bash -le {0}
diff --git a/.github/workflows/unused/all_tests.yml b/.github/workflows/unused/all_tests.yml
index 71db82b..2095954 100644
--- a/.github/workflows/unused/all_tests.yml
+++ b/.github/workflows/unused/all_tests.yml
@@ -28,7 +28,7 @@ jobs:
         shell: bash -le {0}
         run: |
           cd misc
-          ./pip_install.sh loose,tests
+          . ./pip_install.sh loose,tests
       - name: Unittests
         shell: bash -le {0}
         run: |

From 1f4939f043c813e5617ea2c2cb3fd2515104aac6 Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 14:03:56 +0100
Subject: [PATCH 07/11] revert accidentl commit

---
 release/pyinstaller/directlfq.spec | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/release/pyinstaller/directlfq.spec b/release/pyinstaller/directlfq.spec
index fcb253d..65ee566 100644
--- a/release/pyinstaller/directlfq.spec
+++ b/release/pyinstaller/directlfq.spec
@@ -49,10 +49,10 @@ a = Analysis(
 	binaries=binaries,
 	datas=datas,
 	hiddenimports=hidden_imports,
-	hookspath=['./release/pyinstaller/hookdir'],
+	hookspath=[],
 	runtime_hooks=[],
-	excludes=[],
-    win_no_prefer_redirects=False,
+	excludes=[h for h in hidden_imports if "datashader" in h],
+	win_no_prefer_redirects=False,
 	win_private_assemblies=False,
 	cipher=block_cipher,
 	noarchive=False

From fe7bf7df244e05742abc2a8f409c16d159d949f2 Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 14:12:52 +0100
Subject: [PATCH 08/11] clean reqs

---
 requirements/requirements.txt       | 1 -
 requirements/requirements_loose.txt | 1 -
 2 files changed, 2 deletions(-)

diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 5b626de..b3f6959 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -6,6 +6,5 @@ pandas>=1.5.3  # test: tolerate_version
 dask>=2023.1.0  # test: tolerate_version
 numba==0.56.4
 multiprocess==0.70.14
-wget==3.2
 PyYAML==6.0.2
 pyarrow==17.0.0
\ No newline at end of file
diff --git a/requirements/requirements_loose.txt b/requirements/requirements_loose.txt
index 363f4e7..e4a1483 100644
--- a/requirements/requirements_loose.txt
+++ b/requirements/requirements_loose.txt
@@ -5,6 +5,5 @@ pandas
 dask
 numba
 multiprocess
-wget
 PyYAML
 pyarrow
\ No newline at end of file

From e9e06509b0e43ea7c7407e99b5d48e14212162c0 Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 14:32:18 +0100
Subject: [PATCH 09/11] fix stable versions

---
 requirements/requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index b3f6959..5319986 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -2,8 +2,8 @@
 # Only usage of fixed versions is allowed, and all dependencies listed here must also be
 # included in `requirements_loose.txt` (enforced by a test).
 numpy==1.23.5
-pandas>=1.5.3  # test: tolerate_version
-dask>=2023.1.0  # test: tolerate_version
+pandas==2.2.3
+dask==2024.8.0
 numba==0.56.4
 multiprocess==0.70.14
 PyYAML==6.0.2

From f4eaa20889c8dddecc418533970b21917684181f Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 14:32:23 +0100
Subject: [PATCH 10/11] remove PyYAML

---
 requirements/requirements.txt       | 1 -
 requirements/requirements_loose.txt | 1 -
 2 files changed, 2 deletions(-)

diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 5319986..fac980d 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -6,5 +6,4 @@ pandas==2.2.3
 dask==2024.8.0
 numba==0.56.4
 multiprocess==0.70.14
-PyYAML==6.0.2
 pyarrow==17.0.0
\ No newline at end of file
diff --git a/requirements/requirements_loose.txt b/requirements/requirements_loose.txt
index e4a1483..5b2d326 100644
--- a/requirements/requirements_loose.txt
+++ b/requirements/requirements_loose.txt
@@ -5,5 +5,4 @@ pandas
 dask
 numba
 multiprocess
-PyYAML
 pyarrow
\ No newline at end of file

From f722a84cd938399dac4eeeb8b5a0b9e73f03302c Mon Sep 17 00:00:00 2001
From: mschwoerer <82171591+mschwoer@users.noreply.github.com>
Date: Thu, 16 Jan 2025 16:17:23 +0100
Subject: [PATCH 11/11] move filterdict.yaml

---
 README.md                                  | 2 +-
 {release => misc}/examples/filterdict.yaml | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename {release => misc}/examples/filterdict.yaml (100%)

diff --git a/README.md b/README.md
index 9c623a6..5af3705 100644
--- a/README.md
+++ b/README.md
@@ -240,7 +240,7 @@ directLFQ is started internally via the directlfq.lfq_manager.run_lfq() command.
 -  **num_cores**: The number of cores to use (default is to use multiprocessing).
 -  **filename_suffix**: Suffix to append to the output files.
 -  **deactivate_normalization**: Set to true, if no between-sample normalization should be performed before processing.
--  **filter_dict**: In case you want to define specific filters in addition to the standard filters, you can add a yaml file where the filters are defined (see example [here](release/examples/filterdict.yaml)). In the Python API you can also directly put in the dictionary instead of the .yaml file.
+-  **filter_dict**: In case you want to define specific filters in addition to the standard filters, you can add a yaml file where the filters are defined (see example [here](misc/examples/filterdict.yaml)). In the Python API you can also directly put in the dictionary instead of the .yaml file.
 
 ---
 
diff --git a/release/examples/filterdict.yaml b/misc/examples/filterdict.yaml
similarity index 100%
rename from release/examples/filterdict.yaml
rename to misc/examples/filterdict.yaml