Merge pull request #11 from juaml/fix_exceptions_handling

Fix_exceptions_handling

changelog.d/11.fixed.md

@@ -0,0 +1 @@
+Fix a bug in which exceptions raised during delayed computation broke the backend without reporting the error and finishing.

joblib_htcondor/backend.py

@@ -1012,8 +1012,7 @@ def _poll_jobs(self) -> tuple[int, bool]:  # noqa: C901
                             level=9,
                             msg=f"Job {job_meta} raised an exception.",
                         )
-                        typ, exc, tb = result
-                        job_meta.tracking_future.set_exception(exc)
+                        job_meta.tracking_future.set_exception(result)
                     else:
                         logger.log(
                             level=9,

joblib_htcondor/tests/test_runtime.py

@@ -0,0 +1,71 @@
+"""Provide runtime tests."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+# License: AGPL
+
+import logging
+import socket
+
+import pytest
+from joblib import Parallel, delayed, parallel_config
+
+from joblib_htcondor import register_htcondor
+
+
+# Check if the test is running on juseless
+if socket.gethostname() != "juseless":
+    pytest.skip("These tests are only for juseless", allow_module_level=True)
+
+
+register_htcondor()
+
+
+def test_normal() -> None:
+    """Test normal execution."""
+    from operator import neg
+
+    with parallel_config(
+        backend="htcondor", n_jobs=-1, request_cpus=1, request_memory="1Gb"
+    ):
+        out = Parallel()(delayed(neg)(i + 1) for i in range(5))
+
+    assert out == [-1, -2, -3, -4, -5]
+
+
+def test_exception() -> None:
+    """Test running with an exception."""
+
+    def neg_with_exception(a: int) -> int:
+        """Negate if a!=2, raise an exception otherwise.
+
+        Parameters
+        ----------
+        a : int
+            The number to negate.
+
+        Returns
+        -------
+        int
+            The negated number.
+
+        Raises
+        ------
+        ValueError
+            If `a` is 2.
+
+        """
+        if a == 2:
+            raise ValueError("This is an exception")
+        else:
+            return -a
+
+    with parallel_config(
+        backend="htcondor",
+        n_jobs=-1,
+        request_cpus=1,
+        request_memory="1Gb",
+        verbose=1000,
+        worker_log_level=logging.INFO,
+    ):
+        with pytest.raises(ValueError, match="This is an exception"):
+            Parallel()(delayed(neg_with_exception)(i + 1) for i in range(5))

joblib_htcondor/ui/treeparser.py

@@ -6,6 +6,7 @@
 import json
 from copy import copy
 from datetime import datetime
+from json.decoder import JSONDecodeError
 from pathlib import Path
 
 from ..backend import (
@@ -150,6 +151,9 @@ def update(self) -> None:
             except OSError as e:
                 logger.error(f"Error loading {f}: {e}")
                 continue
+            except JSONDecodeError as e:
+                logger.error(f"Error decoding {f}: {e}")
+                continue
         all_meta = sorted(all_meta, key=lambda x: x.meta.start_timestamp)
         # Update the tree from this list of files
         logger.log(