Use dynamic branch triggers

config/pipeline.yaml

@@ -2081,6 +2081,9 @@ trees:
   net-next:
     url: "https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git"
 
+  netdev-testing:
+    url: "https://github.com/linux-netdev/testing.git"
+
   next:
     url: 'https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git'
 
@@ -3803,6 +3806,10 @@ build_configs:
     tree: net-next
     branch: 'main'
 
+  netdev-testing:
+    tree: netdev-testing
+    branch: 'https://netdev.bots.linux.dev/static/nipa/branches.json'
+
   next_master:
     tree: next
     branch: 'master'

docker-compose.yaml

@@ -145,7 +145,7 @@ services:
       - './pipeline/trigger.py'
       - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
       - 'run'
-      - '--trees=kernelci'
+      - '--trees=kernelci,netdev-testing'
       - '--name=trigger'
     extra_hosts:
       - "host.docker.internal:host-gateway"

src/base.py

@@ -7,6 +7,7 @@
 
 import logging
 import os
+import requests
 
 import kernelci
 from kernelci.api.helper import APIHelper
@@ -76,3 +77,24 @@ def run(self, args=None):
             self._stop(context)
 
         return status
+
+
+def validate_url(url):
+    '''
+    Validate URL by:
+    - checking if it's not empty
+    - checking if it starts with HTTP* scheme
+    - requesting HEAD to see if it can be accessed
+    '''
+    if not url:
+        return False
+    if not url.startswith('http'):
+        return False
+    try:
+        r = requests.head(url)
+        if r.status_code != 200:
+            return False
+    except Exception as e:
+        logging.error(f'Error accessing URL: {e}')
+        return False
+    return True

src/lava_callback.py

@@ -28,6 +28,8 @@
 import kernelci.config
 from concurrent.futures import ThreadPoolExecutor
 
+from base import validate_url
+
 
 SETTINGS = toml.load(os.getenv('KCI_SETTINGS', 'config/kernelci.toml'))
 CONFIGS = kernelci.config.load(
@@ -360,7 +362,7 @@ def find_tree(url, branch):
 
     for bconfig in YAMLCFG['build_configs']:
         data = YAMLCFG['build_configs'].get(bconfig)
-        if data.get('tree') == treename and data.get('branch') == branch:
+        if data.get('tree') == treename:
             return treename
 
     return None
@@ -632,24 +634,6 @@ async def checkout(data: ManualCheckout, request: Request,
         return JSONResponse(content=item, status_code=200)
 
 
-def validate_patch_url(patchurl):
-    '''
-    Validate patch URL
-    '''
-    if not patchurl:
-        return False
-    if not patchurl.startswith('http'):
-        return False
-    try:
-        r = requests.get(patchurl)
-        if r.status_code != 200:
-            return False
-    except Exception as e:
-        logging.error(f'Error fetching patch URL: {e}')
-        return False
-    return True
-
-
 @app.post('/api/patchset')
 async def patchset(data: PatchSet, request: Request,
                    Authorization: str = Header(None)):
@@ -696,7 +680,7 @@ async def patchset(data: PatchSet, request: Request,
                 if not isinstance(patchurl, str):
                     item['message'] = 'Invalid patch URL element type'
                     return JSONResponse(content=item, status_code=400)
-                if not validate_patch_url(patchurl):
+                if not validate_url(patchurl):
                     item['message'] = 'Invalid patch URL'
                     return JSONResponse(content=item, status_code=400)
         else:

src/tarball.py

@@ -57,6 +57,9 @@ def _find_build_config(self, node):
         for config in self._build_configs.values():
             if config.tree.name == tree and config.branch == branch:
                 return config
+            if config.tree.name == tree and config.branch.startswith('http'):
+                config._branch = branch
+                return config
 
     def _find_build_commit(self, node):
         revision = node['data'].get('kernel_revision')

src/trigger.py

@@ -21,7 +21,7 @@
 import requests
 import hashlib
 
-from base import Service
+from base import Service, validate_url
 
 
 class Trigger(Service):
@@ -44,6 +44,14 @@ def _run_trigger(self, build_config, force, timeout, trees):
                 return
 
         try:
+            if validate_url(build_config.branch):
+                response = requests.get(build_config.branch)
+                # Following extractor supports only NIPA JSON scheme.
+                # Adding support for other schemas will force moving it to a separate function.
+                branches = response.json()
+                latest = sorted(branches, key=lambda x: x['date'], reverse=True)[0]
+                build_config._branch = latest['branch']
+
             head_commit = kernelci.build.get_branch_head(build_config)
         except Exception as ex:
             self.log.error(f"Failed to get branch head for {build_config.name:32s}")