Merge pull request #100 from cevich/update_reqs

Fix installer failures + update ccia requirements + use exec in wrapper

bin/install_automation.sh

@@ -259,7 +259,7 @@ elif [[ "$_MAGIC_JUJU" == "$_DEFAULT_MAGIC_JUJU" ]]; then
                 INSTALLATION_SOURCE=$INSTALLATION_SOURCE \
                 A_DEBUG=$A_DEBUG \
                 MAGIC_JUJU=$_MAGIC_JUJU \
-                /bin/bash $CHAIN_TO
+                $CHAIN_TO
             msg "##### Installation complete for '$arg' subcomponent"
         else
             msg "Warning: Cannot find installer for $CHAIN_TO"

build-push/.install.sh

@@ -1,8 +1,11 @@
+#!/bin/bash
 
 # Installs 'build-push' script system-wide.  NOT intended to be used directly
 # by humans, should only be used indirectly by running
 # ../bin/install_automation.sh <ver> build-push
 
+set -eo pipefail
+
 source "$AUTOMATION_LIB_PATH/anchors.sh"
 source "$AUTOMATION_LIB_PATH/console_output.sh"
 

build-push/test/testbin-build-push.sh

@@ -8,7 +8,7 @@ source $TEST_SOURCE_DIRPATH/testlib.sh || exit 1
 SUBJ_FILEPATH="$TEST_DIR/$SUBJ_FILENAME"
 TEST_CONTEXT="$TEST_SOURCE_DIRPATH/test_context"
 EMPTY_CONTEXT=$(mktemp -d -p '' .tmp_$(basename ${BASH_SOURCE[0]})_XXXX)
-export NATIVE_GOARCH=$($RUNTIME info --format='{{.host.arch}}')
+export NATIVE_GOARCH=$(buildah info --format='{{.host.arch}}')
 
 test_cmd "Verify error when automation library not found" \
     2 'ERROR: Expecting \$AUTOMATION_LIB_PATH' \

cirrus-ci_artifacts/.install.sh

@@ -1,9 +1,12 @@
+#!/bin/bash
 
 # Installs cirrus-ci_artifacts and a python virtual environment
 # to execute with.  NOT intended to be used directly
 # by humans, should only be used indirectly by running
 # ../bin/install_automation.sh <ver> cirrus-ci_artifacts
 
+set -eo pipefail
+
 source "$AUTOMATION_LIB_PATH/anchors.sh"
 source "$AUTOMATION_LIB_PATH/console_output.sh"
 
@@ -25,11 +28,11 @@ if [[ $UID -eq 0 ]]; then
 fi
 
 cd $(dirname $(realpath "${BASH_SOURCE[0]}"))
-virtualenv --quiet --clear --download --activators bash \
+virtualenv --clear --download \
     $AUTOMATION_LIB_PATH/ccia.venv
 (
     source $AUTOMATION_LIB_PATH/ccia.venv/bin/activate
-    pip3 install --quiet --requirement ./requirements.txt
+    pip3 install --requirement ./requirements.txt
     deactivate
 )
 install -v $INST_PERM_ARG -m '0644' -D -t "$INSTALL_PREFIX/lib/ccia.venv/bin" \

cirrus-ci_artifacts/cirrus-ci_artifacts

@@ -21,4 +21,4 @@ if [[ -z "$AUTOMATION_LIB_PATH" ]]; then
 fi
 
 source $AUTOMATION_LIB_PATH/ccia.venv/bin/activate
-python3 $AUTOMATION_LIB_PATH/ccia.venv/bin/cirrus-ci_artifacts.py "$@"
+exec python3 $AUTOMATION_LIB_PATH/ccia.venv/bin/cirrus-ci_artifacts.py "$@"

cirrus-ci_artifacts/cirrus-ci_artifacts.py

@@ -1,33 +1,34 @@
 #!/usr/bin/env python3
 
 """
-Download all artifacts from a Cirrus-CI Build into a subdirectory named
-<build ID>/<task-name>/<artifact-name>/<file-path>
+Download all artifacts from a Cirrus-CI Build into a subdirectory tree.
+
+Subdirectory naming format: <build ID>/<task-name>/<artifact-name>/<file-path>
 
 Input arguments (in order):
     Build ID - string, the build containing tasks w/ artifacts to download
                e.g. "5790771712360448"
-    Path RX - Optional, regular expression to match against task,
-              artifact, and/or file path.  Input string will be
-              of the form task_name/artifact_name/file_path
+    Path RX - Optional, regular expression to match against subdirectory
+              tree naming format.
 """
 
+import asyncio
+import re
 import sys
-from urllib.parse import quote, unquote
+from argparse import ArgumentParser
 from os import makedirs
 from os.path import split
-import re
-from argparse import ArgumentParser
-import asyncio
+from urllib.parse import quote, unquote
+
 # Ref: https://docs.aiohttp.org/en/stable/http_request_lifecycle.html
 from aiohttp import ClientSession
-
 # Ref: https://gql.readthedocs.io/en/latest/index.html
 # pip3 install --user --requirement ./requirements.txt
 # (and/or in a python virtual environment)
+
+from gql import Client as GQLClient
 from gql import gql
 from gql.transport.requests import RequestsHTTPTransport
-from gql import Client as GQLClient
 
 
 # GraphQL API URL for Cirrus-CI
@@ -41,8 +42,8 @@
 # Set True when --verbose is first argument
 VERBOSE = False
 
-def get_tasks(gqlclient, buildId):
-    """Given a build ID, return a list of task objects"""
+def get_tasks(gqlclient, buildId):  # noqa N803
+    """Given a build ID, return a list of task objects."""
     # Ref: https://cirrus-ci.org/api/
     query = gql('''
         query tasksByBuildId($buildId: ID!) {
@@ -67,33 +68,36 @@ def get_tasks(gqlclient, buildId):
         b = tasks["build"]
         if "tasks" in b and len(b["tasks"]):
             return b["tasks"]
-        raise RuntimeError(f"No tasks found for build with ID {build_id}")
-    raise RuntimeError(f"No Cirrus-CI build found with id {build_id}")
+        raise RuntimeError(f"No tasks found for build with ID {buildId}")
+    raise RuntimeError(f"No Cirrus-CI build found with ID {buildId}")
+
 
 def task_art_url_sfxs(task):
-    """Given a task dict return list CCI_ART_URL suffixes for all artifacts"""
+    """Given a task dict return list CCI_ART_URL suffixes for all artifacts."""
     result = []
     bid = task["buildId"]
     tname = quote(task["name"])  # Make safe for URLs
     for art in task["artifacts"]:
-        aname=quote(art["name"])
+        aname = quote(art["name"])
         for _file in art["files"]:
             fpath = quote(_file["path"])
             result.append(f"{bid}/{tname}/{aname}/{fpath}")
     return result
 
+
 async def download_artifact(session, dest_path, dl_url):
-    """Asynchronous download contents of art_url as a byte-stream"""
+    """Asynchronous download contents of art_url as a byte-stream."""
     # Last path component assumed to be the filename
     makedirs(split(dest_path)[0], exist_ok=True)  # os.path.split
     async with session.get(dl_url) as response:
         with open(dest_path, "wb") as dest_file:
             dest_file.write(await response.read())
 
+
 async def download_artifacts(task, path_rx=None):
-    """Given a task dict, download all artifacts or matches to path_rx"""
-    downloaded=[]
-    skipped=[]
+    """Given a task dict, download all artifacts or matches to path_rx."""
+    downloaded = []
+    skipped = []
     async with ClientSession() as session:
         for art_url_sfx in task_art_url_sfxs(task):
             dest_path = unquote(art_url_sfx)  # Strip off URL encoding
@@ -108,25 +112,28 @@ async def download_artifacts(task, path_rx=None):
                 if VERBOSE:
                     print(f"       Skipping '{dest_path}'")
                 skipped.append(dest_path)
-    return {"downloaded":downloaded, "skipped":skipped}
+    return {"downloaded": downloaded, "skipped": skipped}
+
 
 def get_args(argv):
-    """Return parsed argument namespace object"""
+    """Return parsed argument namespace object."""
     parser = ArgumentParser(prog="cirrus-ci_artifacts",
-        description=('Download Cirrus-CI artifacts by Build ID number, into'
-        ' a subdirectory of the form <Build ID>/<Task Name>/<Artifact Name>'
-        '/<File Path>'))
+                            description=('Download Cirrus-CI artifacts by Build ID'
+                                         ' number, into a subdirectory of the form'
+                                         ' <Build ID>/<Task Name>/<Artifact Name>'
+                                         '/<File Path>'))
     parser.add_argument('-v', '--verbose',
-        dest='verbose', action='store_true', default=False,
-        help='Show "Downloaded" | "Skipped" + relative artifact file-path.')
+                        dest='verbose', action='store_true', default=False,
+                        help='Show "Downloaded" | "Skipped" + relative artifact file-path.')
     parser.add_argument('buildId', nargs=1, metavar='<Build ID>', type=int,
-        help="A Cirrus-CI Build ID number.")
+                        help="A Cirrus-CI Build ID number.")
     parser.add_argument('path_rx', nargs='?', default=None, metavar='[Reg. Exp.]',
-        help="Reg. exp. include only <task>/<artifact>/<file-path> matches.")
+                        help="Reg. exp. include only <task>/<artifact>/<file-path> matches.")
     return parser.parse_args(args=argv[1:])
 
+
 async def download(tasks, path_rx=None):
-    """Main async entrypoint function with nested async download tasks"""
+    """Return results from all async operations."""
     # Python docs say to retain a reference to all tasks so they aren't
     # "garbage-collected" while still active.
     results = []
@@ -136,7 +143,8 @@ async def download(tasks, path_rx=None):
     await asyncio.gather(*results)
     return results
 
-def main(buildId, path_rx=None):
+
+def main(buildId, path_rx=None):  # noqa: N803,D103
     if path_rx is not None:
         path_rx = re.compile(path_rx)
     transport = RequestsHTTPTransport(url=CCI_GQL_URL, verify=True, retries=3)
@@ -146,6 +154,7 @@ def main(buildId, path_rx=None):
     async_results = asyncio.run(download(tasks, path_rx))
     return [r.result() for r in async_results]
 
+
 if __name__ == "__main__":
     args = get_args(sys.argv)
     VERBOSE = args.verbose

cirrus-ci_artifacts/requirements.txt

@@ -1,16 +1,18 @@
+# Producing this list was done using the following process:
+# 1. Create a temporary `req.txt` file containing only the basic
+#    non-distribution provided packages, e.g. `aiohttp[speedups]`,
+#    `PyYAML`, `gql[requests]`, `requests` (see cirrus-ci_artifacts.py,
+#    actual requirements may have changed)
+# 2. From a Fedora:latest container, install python3 & python3-virtualenv
+# 3. Setup & activate a temporary virtual environment
+# 4. Execute `pip3 install --requirements req.txt`
+# 5. Run pip3 freeze
+# 6. Edit `requirements.txt`, add the `~=` specifier to each line along
+#    with the correct two-component version number (from freeze output)
+# 7. In a fresh container, confirm the automation installer
+#    functions with the cirrus-ci_artifacts component (see main README
+#    for installer instructions)
 PyYAML~=6.0
-aiohttp~=3.8
-aiosignal~=1.2
-async-timeout~=4.0
-attrs~=21.4
-certifi~=2021.10
-charset-normalizer~=2.0.0
-frozenlist~=1.3
-gql~=3.0
-graphql-core~=3.2
-idna~=3.3
-multidict~=6.0
-requests-toolbelt~=0.9
-requests~=2.27
-urllib3~=1.26
-yarl~=1.7
+aiohttp[speedups]~=3.8
+gql[requests]~=3.3
+requests~=2.28

cirrus-ci_artifacts/test/run_all_tests.sh

@@ -1,13 +1,29 @@
 #!/bin/bash
 
+set -e
+
 TESTDIR=$(dirname ${BASH_SOURCE[0]})
-cd "$TESTDIR"
 
-set -a
-virtualenv testvenv
-source testvenv/bin/activate
-testvenv/bin/python -m pip install --upgrade pip
-pip3 install --requirement ../requirements.txt
-set +a
+if [[ "$GITHUB_ACTIONS" == "true" ]]; then
+    echo "Lint/Style checking not supported under github actions: Skipping"
+    exit 0
+fi
+
+if [[ -x $(type -P flake8-3) ]]; then
+    cd "$TESTDIR"
+    set -a
+    virtualenv testvenv
+    source testvenv/bin/activate
+    testvenv/bin/python -m pip install --upgrade pip
+    pip3 install --requirement ../requirements.txt
+    set +a
+
+    ./test_cirrus-ci_artifacts.py -v
 
-./test_cirrus-ci_artifacts.py
+    cd ..
+    flake8-3 --max-line-length=100 ./cirrus-ci_artifacts.py
+    flake8-3 --max-line-length=100 --extend-ignore=D101,D102,D103,D105 test/test_cirrus-ci_artifacts.py
+else
+    echo "Can't find flake-8-3 binary, is script executing inside CI container?"
+    exit 1
+fi