feat(nipa-update): Add to pipeline new kind of executor for NIPA

To handle NIPA update, after test completion we need to execute
nipa-update script.

Signed-off-by: Denys Fedoryshchenko <denys.f@collabora.com>

config/pipeline.yaml

@@ -103,6 +103,11 @@ _anchors:
     result: pass
     kind: kbuild
 
+  job-event: &job-event
+    channel: node
+    state: done
+    kind: job
+
 ### Frequently used rules
 
   build-only-trees-rules: &build-only-trees-rules
@@ -2071,6 +2076,13 @@ jobs:
         - mainline
         - stable-rc
     kcidb_test_suite: kernelci_wifi_basic
+
+  nipa-update:
+    template: nipa-update.jinja2
+    kind: test
+    image: kernelci/{image_prefix}kernelci
+    kcidb_test_suite: kernelci_nipa-update
+
 
 trees:
 
@@ -3682,6 +3694,17 @@ scheduler:
       - rk3399-gru-kevin
       - rk3399-rock-pi-4b
 
+  # Execute nipa-update on the blktests-ddp-x86 node
+  # complete with the blktests-ddp-x86 event
+  - job: nipa-update
+    event:
+      <<: *job-event
+      name: blktests-ddp-x86
+    runtime:
+      type: shell
+
+
+
 # -----------------------------------------------------------------------------
 # Legacy configuration data (still used by trigger service)
 #

config/runtime/nipa-update.jinja2

@@ -0,0 +1,88 @@
+{# -*- mode: Python -*- -#}
+{# SPDX-License-Identifier: LGPL-2.1-or-later -#}
+
+{%- extends 'base/python.jinja2' %}
+
+{%- block python_imports %}
+{{ super() }}
+import json
+import subprocess
+{%- endblock %}
+
+{%- block python_local_imports %}
+{{ super() }}
+import kernelci.api.helper
+import kernelci.runtime
+{%- endblock %}
+
+{%- block python_globals %}
+{{ super() }}
+REVISION = {{ node.data.kernel_revision }}
+NODEID = "{{ node.id }}"
+API_NAME = "{{ api_config.name }}"
+{% endblock %}
+
+{% block python_job_constr -%}
+REVISION, NODEID, API_NAME, {{ super() }}
+{%- endblock %}
+
+{% block python_job -%}
+class Job(BaseJob):
+
+    def __init__(self, revision, nodeid, api_name, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._revision = revision
+        self._nodeid = nodeid
+        self._api_name = api_name
+
+    def _run(self, src_path):
+        print(f"Executing nipa-update for node {self._nodeid}")
+        api_helper = kernelci.api.helper.APIHelper(self._api)
+        # TODO: Implement nipa-update
+        jobnode =self._api.node.get(self._nodeid)
+        # retrieve parent job to feed the nipa-update
+        parent_job = jobnode.get('parent')
+        if not parent_job:
+            raise Exception(f"No parent job found for node {self._nodeid}")
+        # temporary we get kernelci-nipa from git
+        # TODO: Embed it into pipeline as a submodule
+        nipa_path = "/tmp/kernelci-nipa"
+        # if the path does not exist, clone the repository
+        if not os.path.exists(nipa_path):
+            subprocess.run(["git", "clone", "https://github.com/nuclearcat/kernelci-nipa.git", nipa_path])
+            # branch various-improvements
+            subprocess.run(["git", "checkout", "various-improvements"], cwd=nipa_path)            
+        else:
+            subprocess.run(["git", "pull"], cwd=nipa_path)
+        # chdir to the nipa_path
+        os.chdir(nipa_path)
+        # run the nipa-update
+        args = ["/tmp/kernelci-nipa/nipa-results", "--id", parent_job]
+        # if api_name is not production, add --staging
+        if self._api_name != "production":
+            args.append("--staging")
+        r = subprocess.run(args, cwd=nipa_path)
+        if r.returncode != 0:
+            raise Exception(f"Failed to run nipa-update for node {self._nodeid}: {r.stderr}")
+        # Upload results to the storage
+        # Check key env var KCI_SSH_KEY
+        ssh_key_path = os.getenv("KCI_SSH_KEY")
+        if not ssh_key_path:
+            raise Exception("KCI_SSH_KEY is not set")
+        # Upload results to the storage
+        # Check key env var KCI_SSH_KEY
+        ssh_key_path = os.getenv("KCI_SSH_KEY")
+        ssh_username = "nipa"
+        ssh_host = "storage.kernelci.org"
+        ssh_port = "22022"
+        if self._api_name != "production":
+            ssh_username = "nipa-staging"
+            ssh_host = "storage.staging.kernelci.org"
+
+        args = ["scp", "-P", ssh_port, "-r", "-i", ssh_key_path, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", nipa_path + "/netdev-results/*", f"{ssh_username}@{ssh_host}:"]
+        print(args)
+        # scp nipa-results to the storage, recursive
+        subprocess.run(args, cwd=nipa_path)
+
+        return 'pass'
+{% endblock %}