Merge branch 'master' into SPD-417

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 199

.github/workflows/python-app.yml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
 
 
 

README.md

@@ -8,26 +8,39 @@ Python client using CouchDB as a token pool server.
 Installation
 ============
 
-To install, first clone the repository and then use pip to install:
+Development & Testing
+---------------------
+
+To install `picas` source code for development, first clone the repository and then use [`poetry`](https://python-poetry.org/docs/) to install. `poetry` is a tool for dependency managing and packaging in Python. If you don't have `poetry`, install it first with `pipx install poetry`.
 ```
 git clone git@github.com:sara-nl/picasclient.git
 cd picasclient
-pip install -U .
+poetry install --with test
 ```
+Note that poetry will create a virtual environment if it is not running within an activated virtual environment already. In that case, you will need to run `poetry run` before your commands to execute them within the poetry virtual environment.
 
-Testing
-=======
-
-First, install the test dependencies with 
+If you prefer not to use `poetry`, then you can install with (in a virtual environment):
 ```
-pip install ".[test]"
+pip install -U .
+pip install flake8 pytest
 ```
+
 To test, run
 ```
 flake8 picas tests
 pytest tests
 ```
 
+Installing package
+------------------
+The latest release of `picas` can be installed as a package from PyPI with:
+```
+pip install picas
+```
+You can then write your custom Python program to use `picas` based on the examples below. 
+
+
+
 Examples
 ========
 
@@ -159,13 +172,13 @@ cc src/fractals.c -o bin/fractals -lm
 And finally, the `process_task.sh` code needs to call a different command. Replace
 
 ```
-eval $INPUT
+bash -c "$INPUT"
 ```
 
 with:
 
 ```
-./fractals -o $OUTPUT $INPUT
+bin/fractals -o $OUTPUT $INPUT
 ```
 
 to ensure the fractal code is called.

examples/process_task.sh

@@ -28,9 +28,6 @@ if [[ "$?" != "0" ]]; then
     exit 1
 fi
 
-#Copy output to the grid storage
-#globus-url-copy file:///${PWD}/${OUTPUT} gsiftp://gridftp.grid.sara.nl:2811/pnfs/grid.sara.nl/data/lsgrid/homer/${OUTPUT}
-
 echo `date`
 
 exit 0

examples/slurm-example.sh

@@ -14,8 +14,8 @@
 #    Attach the logs to the token
 
 
-cd $PWD
-# You need to load your environment here
-# mamba activate MAMBA-ENV
-# source /PATH/TO/VENV/bin/activate
-python local_example.py
+
+# You may set environmental variables needed in the SLURM job
+# For example, when using the LUMI container wrapper:
+# export PATH="/path/to/install_dir/bin:$PATH"
+python local-example.py

picas/actors.py

@@ -60,24 +60,28 @@ def _run(self, task, timeout):
             with ThreadingTimeout(timeout, swallow_exc=False) as context_manager:
                 self.process_task(task)
         except Exception as ex:
-            msg = ("Exception {0} occurred during processing: {1}"
-                   .format(type(ex), ex))
+            msg = f"Exception {type(ex)} occurred during processing: {ex}"
             task.error(msg, exception=ex)
             log.info(msg)
 
         if context_manager.state == context_manager.TIMED_OUT:
-            msg = ("Token execution exceeded timeout limit of {0} seconds".format(timeout))
+            msg = f"Token execution exceeded timeout limit of {timeout} seconds"
             log.info(msg)
 
-        while True:
-            try:
-                self.db.save(task)
-                break
-            except ResourceConflict:
-                # simply overwrite changes - model results are more
-                # important
-                new_task = self.db.get(task.id)
-                task['_rev'] = new_task.rev
+        try:
+            self.db.save(task)
+        except ResourceConflict as ex:
+            # simply overwrite changes - model results are more important
+            msg = f"Warning: {type(ex)} occurred while saving task to database: " + \
+                "Document exists with different revision or was deleted"
+            log.info(msg)
+            new_task = self.db.get(task.id)
+            task['_rev'] = new_task.rev
+        except Exception as ex:
+            # re-raise Exception
+            msg = f"Error: {type(ex)} occurred while saving task to database: {ex}"
+            log.info(msg)
+            raise
 
         self.cleanup_run()
         self.tasks_processed += 1

picas/jobid.py

@@ -14,14 +14,15 @@ def add_job_id(doc):
     A glite wms system makes underneath use of a cream system which makes use
     of PBS. In such a case only the glite wms id instead of all of them.
     """
+
     dirac_jobid = environ.get("DIRACJOBID")
-    slurm_jobid = environ.get("SLURM_JOB_ID")
     wms_jobid = environ.get("GLITE_WMS_JOBID")
     cream_jobid = environ.get("CREAM_JOBID")
     pbs_jobid = environ.get("PBS_JOBID")
+    slurm_jobid = environ.get("SLURM_JOB_ID")
+    slurm_array_jobid = environ.get("SLURM_ARRAY_JOB_ID")
+    slurm_array_taskid = environ.get("SLURM_ARRAY_TASK_ID")
 
-    if slurm_jobid is not None:
-        doc["slurm_job_id"] = slurm_jobid
     if dirac_jobid is not None:
         doc["dirac_job_id"] = dirac_jobid
     elif wms_jobid is not None:
@@ -32,6 +33,10 @@ def add_job_id(doc):
         doc["cream_job_id"] = cream_jobid
     elif pbs_jobid is not None:
         doc["pbs_job_id"] = pbs_jobid
+    elif slurm_array_jobid is not None:
+        doc["slurm_job_id"] = slurm_array_jobid+"_"+slurm_array_taskid
+    elif slurm_jobid is not None:
+        doc["slurm_job_id"] = slurm_jobid
 
 
 def remove_job_id(doc):

pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "picas"
+version = "1.0.0"
+description = "Python client using CouchDB as a token pool server."
+authors = ["Jan Bot, Joris Borgdorff, Lodewijk Nauta, Haili Hu"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+couchdb = "^1.2"
+stopit = "^1.1.2"
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.3.3"
+flake8 = "^7.1.1"
+
+[tool.poetry.group.test]
+optional = true
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"