Singularity images are now reported with their right processor archit…

…ecture.

This one is related to issue #34

wfexs_backend/common.py

@@ -707,6 +707,7 @@ class Container(NamedTuple):
     origTaggedName: "str"
     taggedName: "URIType"
     type: "ContainerType"
+    architecture: "Optional[ProcessorArchitecture]" = None
     localPath: "Optional[AbsPath]" = None
     signature: "Optional[Fingerprint]" = None
     fingerprint: "Optional[Fingerprint]" = None

wfexs_backend/ro_crate.py

@@ -962,8 +962,12 @@ def add_containers_to_workflow(
             software_container["softwareVersion"] = container.fingerprint
             if containerEngineOs is not None:
                 software_container["operatingSystem"] = containerEngineOs
-            if arch is not None:
-                software_container["processorRequirements"] = arch
+            # Getting the processor architecture of the container
+            container_arch = container.architecture
+            if container_arch is None:
+                container_arch = arch
+            if container_arch is not None:
+                software_container["processorRequirements"] = container_arch
             software_container["softwareRequirements"] = crate_cont_type
 
             crate_cont = crate.add(software_container)

wfexs_backend/singularity_container.py

@@ -19,6 +19,7 @@
 import json
 import os
 import os.path
+import re
 import shutil
 import subprocess
 import tempfile
@@ -59,6 +60,7 @@
         ContainerLocalConfig,
         ContainerTaggedName,
         Fingerprint,
+        ProcessorArchitecture,
         RelPath,
         URIType,
     )
@@ -163,6 +165,119 @@ def __init__(
     def ContainerType(cls) -> "ContainerType":
         return ContainerType.Singularity
 
+    def _getContainerArchitecture(
+        self, container_filename: "AnyPath", matEnv: "Mapping[str, str]" = {}
+    ) -> "Optional[ProcessorArchitecture]":
+        with tempfile.NamedTemporaryFile() as s_out, tempfile.NamedTemporaryFile() as s_err:
+            self.logger.debug(f"Describing container {container_filename}")
+            # Singularity command line borrowed from
+            # https://github.com/nextflow-io/nextflow/blob/539a22b68c114c94eaf4a88ea8d26b7bfe2d0c39/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy#L221
+            s_retval = subprocess.Popen(
+                [self.runtime_cmd, "sif", "list", container_filename],
+                env=matEnv,
+                stdout=s_out,
+                stderr=s_err,
+            ).wait()
+
+            self.logger.debug(f"singularity sif list retval: {s_retval}")
+
+            with open(s_out.name, "r") as c_stF:
+                s_out_v = c_stF.read()
+            with open(s_err.name, "r") as c_stF:
+                s_err_v = c_stF.read()
+
+            self.logger.debug(f"singularity sif list stdout: {s_out_v}")
+
+            self.logger.debug(f"singularity sif list stderr: {s_err_v}")
+
+            if s_retval != 0:
+                errstr = """Could not describe singularity image {}. Retval {}
+======
+STDOUT
+======
+{}
+
+======
+STDERR
+======
+{}""".format(
+                    container_filename, s_retval, s_out_v, s_err_v
+                )
+                raise ContainerEngineException(errstr)
+
+            # The default for images translated from docker are usually these
+            data_bundle_id = "4"
+            type_column_id = 3
+            column_id = 0
+            parse_header = True
+            with open(s_out.name, mode="r") as c_stF:
+                for line in c_stF:
+                    if line.startswith("-"):
+                        continue
+
+                    cells = re.split(r"\s*\|\s*", line.strip())
+                    if parse_header:
+                        for i_cell, cell_name in enumerate(cells):
+                            if cell_name.startswith("TYPE"):
+                                type_column_id = i_cell
+                            elif cell_name.startswith("ID"):
+                                column_id = i_cell
+                        parse_header = False
+                    elif cells[type_column_id].startswith("FS"):
+                        data_bundle_id = cells[column_id]
+                        break
+
+        # Now, the details
+        architecture = None
+        with tempfile.NamedTemporaryFile() as s_out, tempfile.NamedTemporaryFile() as s_err:
+            self.logger.debug(
+                f"Learning container architecture from {container_filename}"
+            )
+            # Singularity command line borrowed from
+            # https://github.com/nextflow-io/nextflow/blob/539a22b68c114c94eaf4a88ea8d26b7bfe2d0c39/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy#L221
+            s_retval = subprocess.Popen(
+                [self.runtime_cmd, "sif", "info", data_bundle_id, container_filename],
+                env=matEnv,
+                stdout=s_out,
+                stderr=s_err,
+            ).wait()
+
+            self.logger.debug(f"singularity sif info retval: {s_retval}")
+
+            with open(s_out.name, "r") as c_stF:
+                s_out_v = c_stF.read()
+            with open(s_err.name, "r") as c_stF:
+                s_err_v = c_stF.read()
+
+            self.logger.debug(f"singularity sif info stdout: {s_out_v}")
+
+            self.logger.debug(f"singularity sif info stderr: {s_err_v}")
+
+            if s_retval != 0:
+                errstr = """Could not describe bundle {}  from singularity image {}. Retval {}
+======
+STDOUT
+======
+{}
+
+======
+STDERR
+======
+{}""".format(
+                    data_bundle_id, container_filename, s_retval, s_out_v, s_err_v
+                )
+                raise ContainerEngineException(errstr)
+
+            # Learning the architecture
+            with open(s_out.name, mode="r") as c_stF:
+                for line in c_stF:
+                    key, value = re.split(r":\s*", line.strip(), maxsplit=1)
+                    if key == "Architecture":
+                        architecture = value
+                        break
+
+        return cast("ProcessorArchitecture", architecture)
+
     def _materializeSingleContainer(
         self,
         tag: "ContainerTaggedName",
@@ -501,6 +616,7 @@ def _materializeSingleContainer(
             taggedName=cast("URIType", singTag),
             signature=imageSignature,
             fingerprint=fingerprint,
+            architecture=self._getContainerArchitecture(containerPath, matEnv),
             type=self.containerType,
             localPath=containerPath,
         )