diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..ff8fb7f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,4 @@ +# Default owners of the repository +* @metoffice/ssdteam + +# Add component owners as appropriate diff --git a/.github/linters/.isort.cfg b/.github/linters/.isort.cfg new file mode 100644 index 0000000..b4b81dd --- /dev/null +++ b/.github/linters/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile=black +src_paths=. \ No newline at end of file diff --git a/.github/linters/.perlcriticrc b/.github/linters/.perlcriticrc new file mode 100644 index 0000000..56c7874 --- /dev/null +++ b/.github/linters/.perlcriticrc @@ -0,0 +1,5 @@ +# Report slightly less severe violations (severity >= 4) +severity = 4 +# Verbosity format +# filename:line:column [severity policy] message near 'string of source code that caused the violation' +verbose = %f:%l:%c [%s %p] %m near '%r'\n diff --git a/.github/linters/.python-black b/.github/linters/.python-black new file mode 100644 index 0000000..e69de29 diff --git a/.github/linters/.python-lint b/.github/linters/.python-lint new file mode 100644 index 0000000..b49d38e --- /dev/null +++ b/.github/linters/.python-lint @@ -0,0 +1,20 @@ +[MASTER] +# Use multiple processes to speed up Pylint. +jobs=2 + +[MESSAGES CONTROL] +# C0114: Missing module docstring (missing-module-docstring) +# C0116: Missing function or method docstring (missing-function-docstring) +# C0103: Constant name doesn't conform to UPPER_CASE naming style (invalid-name) +# C0209: Formatting a regular string which could be an f-string (consider-using-f-string) +# E0401: Unable to import module (import-error) +# W0611: Unused import sphinx_rtd_theme (unused-import) +# W0622: Redefining built-in 'copyright' (redefined-builtin) +# R0801: Similar lines in 2 files +disable=C0114,C0116,C0103,C0209,E0401,W0611,W0622,R0801, + +[SIMILARITIES] +# Minimum lines number of a similarity. +min-similarity-lines=4 +ignore-comments=yes +ignore-docstrings=yes diff --git a/.github/linters/.ruff.toml b/.github/linters/.ruff.toml new file mode 100644 index 0000000..6fd57be --- /dev/null +++ b/.github/linters/.ruff.toml @@ -0,0 +1,13 @@ +cache-dir = "/tmp/.ruff_cache" +line-length = 80 + + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[format] +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false diff --git a/.github/linters/.shellcheckrc b/.github/linters/.shellcheckrc new file mode 100644 index 0000000..472bfba --- /dev/null +++ b/.github/linters/.shellcheckrc @@ -0,0 +1 @@ +source-path=SCRIPTDIR diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..15fbc1c --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,32 @@ +# Description + +## Summary + +_Briefly describe the feature being introduced._ + +## Changes + +_List the major changes made in this pull request._ + +## Dependency + +_List dependent changes. Can use build-group logic here._ + +## Impact + +_Discuss any potential impacts this feature may have on existing functionalities._ + +## Issues addressed + +Resolves + +_List issue(s) related to this PR._ + +## Coordinated merge + +_Specify any coordinated merges here._ + + +## Checklist + +- [ ] I have performed a self-review of my own changes diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..a20f454 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,58 @@ +--- +# Lint/analyse code with Super-Linter. +name: Checks + +on: + pull_request: + branches: [main] + push: + branches: [main, 'releases/**'] + workflow_dispatch: + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: read-all + +jobs: + check: + name: Lint + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + permissions: + contents: read + packages: read + statuses: write + + steps: + - name: Checkout current + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Super-Linter + uses: super-linter/super-linter/slim@v7.2.1 + env: + FILTER_REGEX_EXCLUDE: (.*[.]conf.py|pull_request_template.md|/linters/) + IGNORE_GITIGNORED_FILES: true + VALIDATE_BASH_EXEC: false + VALIDATE_JAVASCRIPT_STANDARD: false + VALIDATE_JSCPD: false + VALIDATE_MARKDOWN_PRETTIER: false + VALIDATE_PYTHON_FLAKE8: false + VALIDATE_PYTHON_MYPY: false + VALIDATE_PYTHON_PYINK: false + VALIDATE_PYTHON_PYLINT: false + VALIDATE_PYTHON_RUFF: false + VALIDATE_SHELL_SHFMT: false + VALIDATE_YAML_PRETTIER: false + VALIDATE_YAML: false + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +... diff --git a/.gitignore b/.gitignore index 372c13e..79d47bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ +*.sublime-workspace +*.code-workspace +.vscode __pycache__/ - +*.pyc +*~ diff --git a/README.md b/README.md index ca28858..8ffcdb3 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ -# SimSys Scripts +# Simulation Systems Scripts -This repository contains support scripts that are common across the many simulation and modelling codes owned by the Met Office. -Particularily those owned and maintained by the SSD team. +[![Checks](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/lint.yml/badge.svg)](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/lint.yml) +[![CodeQL](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/github-code-scanning/codeql) -Also contains a copy of script_updater.sh which is intended to live in the fcm repositories to pull from -this repo. +This repository contains support scripts that are common across the many +simulation and modelling codes owned by the Met Office. Particularly those +owned and maintained by the Simulation Systems and Deployment (SSD) team. + +Also contains a copy of `script_updater.sh` which is intended to live in the +fcm repositories to pull from this repository. diff --git a/fcm_bdiff.py b/fcm_bdiff.py index 6a9761e..7e20a8e 100644 --- a/fcm_bdiff.py +++ b/fcm_bdiff.py @@ -9,8 +9,9 @@ run tests on based on the branch-difference (to allow checking of only files which a developer has actually modified on their branch) """ -import re + import os +import re import subprocess import time @@ -20,10 +21,11 @@ class FCMError(Exception): """ Exception class for FCM commands """ + def __str__(self): - return ("\nFCM command: \"{0:s}\"" - "\nFailed with error: \"{1:s}\"" - .format(" ".join(self.args[0]), self.args[1].strip())) + return '\nFCM command: "{0:s}"\nFailed with error: "{1:s}"'.format( + " ".join(self.args[0]), self.args[1].strip() + ) # ------------------------------------------------------------------------------ @@ -32,16 +34,20 @@ def is_trunk(url): Given an FCM url, returns True if it appears to be pointing to the UM main trunk """ - search = re.search(r""" + search = re.search( + r""" (svn://fcm\d+/\w+_svn/\w+/trunk| .*/svn/[\w\.]+/\w+/trunk| ..*_svn/\w+/trunk) - """, url, flags=re.VERBOSE) + """, + url, + flags=re.VERBOSE, + ) return search is not None # ------------------------------------------------------------------------------ -def text_decoder(bytes_type_string, codecs=['utf8', 'cp1252']): +def text_decoder(bytes_type_string, codecs=["utf8", "cp1252"]): """ Given a bytes type string variable, attempt to decode it using the codecs listed. @@ -127,18 +133,20 @@ def get_branch_diff_filenames(branch=".", path_override=None): # Strip whitespace, and remove blank lines while turning the output into # a list of strings. bdiff_files = [x.strip() for x in bdiff.split("\n") if x.strip()] - bdiff_files = [bfile.split()[1] for bfile in bdiff_files - if bfile.split()[0].strip() == "M" or - bfile.split()[0].strip() == "A"] + bdiff_files = [ + bfile.split()[1] + for bfile in bdiff_files + if bfile.split()[0].strip() == "M" or bfile.split()[0].strip() == "A" + ] # Convert the file paths to be relative to the current URL; to do this # construct the base path of the trunk URL and compare it to the results # of the bdiff command above repos_root = get_repository_root(info) - relative_paths = [os.path.relpath(bfile, - os.path.join(repos_root, - "main", "trunk")) - for bfile in bdiff_files] + relative_paths = [ + os.path.relpath(bfile, os.path.join(repos_root, "main", "trunk")) + for bfile in bdiff_files + ] # These relative paths can be joined to an appropriate base to complete # the filenames to return @@ -169,9 +177,9 @@ def run_fcm_command(command, max_retries, snooze): """ retries = 0 while True: - output = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + output = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) _ = output.wait() if output.returncode == 0: return text_decoder(output.stdout.read()) @@ -198,8 +206,7 @@ def use_mirror(branch): if mirror_key in os.environ: branch = os.environ[mirror_key] retries = 2 - print("[INFO] Switching branch used for fcm command to : {0:}".format( - branch)) + print(f"[INFO] Switching branch used for fcm command to: {branch}") else: retries = 0 return branch, retries @@ -211,8 +218,9 @@ def get_repository_root(branch_info): Given the raw output from an fcm binfo command - which can be retrieved by calling get_branch_info() - returns the Repository Root field """ - repos_root = re.search(r"^Repository Root:\s*(?P.*)\s*$", - branch_info, flags=re.MULTILINE) + repos_root = re.search( + r"^Repository Root:\s*(?P.*)\s*$", branch_info, flags=re.MULTILINE + ) if repos_root: repos_root = repos_root.group("url") else: @@ -226,8 +234,9 @@ def get_branch_parent(branch_info): Given the raw output from an fcm binfo command - which can be retrieved by calling get_branch_info() - returns the Branch Parent Field """ - parent = re.search(r"^Branch Parent:\s*(?P.*)$", branch_info, - flags=re.MULTILINE) + parent = re.search( + r"^Branch Parent:\s*(?P.*)$", branch_info, flags=re.MULTILINE + ) if parent: parent = parent.group("parent") else: diff --git a/kgo_updates/kgo_update/kgo_update.py b/kgo_updates/kgo_update/kgo_update.py index 4224bf5..1b9435f 100755 --- a/kgo_updates/kgo_update/kgo_update.py +++ b/kgo_updates/kgo_update/kgo_update.py @@ -31,15 +31,14 @@ """ +import argparse import os import re -import sys import sqlite3 -import argparse import subprocess +import sys from collections import defaultdict - # Global criteria for updating - any statuses matched by this list will # be treated as "failed" for the purposes of updating. When running a # "new version" update this list will be updated to include the "OK" @@ -48,8 +47,8 @@ def banner(message): - '''Print a simple banner message''' - return "{0}\n* {1} *\n{0}".format("%"*(len(message)+4), message) + """Print a simple banner message""" + return "{0}\n* {1} *\n{0}".format("%" * (len(message) + 4), message) def confirm(message, skip=False): @@ -91,9 +90,9 @@ def write_update_script(kgo_dirs, new_dirname, script): # Write a sub-header for this KGO directory to help visually split # up the file (for when the user checks it by eye) - script.write("#"*(len(new_kgo_dir)+4)+"\n") + script.write("#" * (len(new_kgo_dir) + 4) + "\n") script.write("# {0} #\n".format(new_kgo_dir)) - script.write("#"*(len(new_kgo_dir)+4)+"\n") + script.write("#" * (len(new_kgo_dir) + 4) + "\n") script.write(f"echo 'Installing {new_kgo_dir}'\n\n") # Build up several different text sections in the upcoming loop @@ -125,8 +124,7 @@ def write_update_script(kgo_dirs, new_dirname, script): # without a source here must have been retired from # the tests continue - keep_description.append( - "# * {0}".format(kgo_file)) + keep_description.append("# * {0}".format(kgo_file)) # Construct the paths old_file = os.path.join(kgo_dir, kgo_file) @@ -134,23 +132,22 @@ def write_update_script(kgo_dirs, new_dirname, script): keep_commands.append( "echo 'ln -s {0} {1}'".format( - os.path.relpath(old_file, - os.path.dirname(new_file)), - new_file)) + os.path.relpath(old_file, os.path.dirname(new_file)), + new_file, + ) + ) keep_commands.append( "ln -s {0} {1}".format( - os.path.relpath(old_file, - os.path.dirname(new_file)), - new_file)) + os.path.relpath(old_file, os.path.dirname(new_file)), + new_file, + ) + ) else: # Files from the suite should be copied from the suite - copy_description.append( - "# * {0}".format(kgo_file)) + copy_description.append("# * {0}".format(kgo_file)) new_file = os.path.join(new_kgo_dir, kgo_file) - copy_commands.append( - "echo 'cp {0} {1}'".format(source, new_file)) - copy_commands.append( - "cp {0} {1}".format(source, new_file)) + copy_commands.append("echo 'cp {0} {1}'".format(source, new_file)) + copy_commands.append("cp {0} {1}".format(source, new_file)) # In this case more disk-space is needed, so update # the running total for reporting later total_filesize += os.path.getsize(source) @@ -181,10 +178,14 @@ def report_space_required(total_filesize, skip=False): units.pop() unit = units[-1] - print(banner("Update will require: {0:.2f} {1} of disk space" - .format(total_filesize, unit))) - if not confirm("Please confirm this much space is available (y/n)? ", - skip=skip): + print( + banner( + "Update will require: {0:.2f} {1} of disk space".format( + total_filesize, unit + ) + ) + ) + if not confirm("Please confirm this much space is available (y/n)? ", skip=skip): sys.exit("Aborting...") @@ -193,8 +194,7 @@ def add_untested_kgo_files(kgo_dirs): for kgo_dir, update_dict in kgo_dirs.items(): for path, _, filenames in os.walk(kgo_dir): for filename in filenames: - kgo_file = ( - os.path.relpath(os.path.join(path, filename), kgo_dir)) + kgo_file = os.path.relpath(os.path.join(path, filename), kgo_dir) if kgo_file not in update_dict: kgo_dirs[kgo_dir][kgo_file] = None return kgo_dirs @@ -217,9 +217,11 @@ def group_comparisons_by_dir(comparisons, skip=False): # If it looks like this KGO file never existed at all, let the user # decide what they want to happen if not os.path.exists(kgo_file) and status.strip() in UPDATE_CRITERIA: - if not confirm("KGO file {0} doesn't appear to exist, should we " - "install it from the suite (y/n)? " - .format(kgo_file), skip=skip): + if not confirm( + "KGO file {0} doesn't appear to exist, should we " + "install it from the suite (y/n)? ".format(kgo_file), + skip=skip, + ): # Note this is negated - i.e. if the user doesn't want to # include this new file, we skip it and move on continue @@ -227,15 +229,18 @@ def group_comparisons_by_dir(comparisons, skip=False): # Extract the base KGO directory by moving backwards through the path # until a directory that matches the KGO directory naming style basedir = os.path.dirname(kgo_file) - while (basedir != "/" and not - re.match(r".*((vn|\d+\.)\d+\.\d+(_t\d+|))$", basedir)): + while basedir != "/" and not re.match( + r".*((vn|\d+\.)\d+\.\d+(_t\d+|))$", basedir + ): basedir = os.path.dirname(basedir) # If the above goes wrong it will eventually hit root; if this happens # we cannot continue as something is seriously not right if basedir == "/": - msg = ("Problem locating KGO directory - " - "is this actually a KGO file?\n {0}") + msg = ( + "Problem locating KGO directory - " + "is this actually a KGO file?\n {0}" + ) sys.exit(msg.format(kgo_file)) # Otherwise add the result to the list - for each entry we store the @@ -247,8 +252,10 @@ def group_comparisons_by_dir(comparisons, skip=False): # and if it has perhaps we can do some additional checking # (for instance, are the changes in answers the same?) relative_kgo_path = os.path.relpath(kgo_file, basedir) - if (relative_kgo_path in kgo_dirs[basedir] and - kgo_dirs[basedir][relative_kgo_path] != suite_file): + if ( + relative_kgo_path in kgo_dirs[basedir] + and kgo_dirs[basedir][relative_kgo_path] != suite_file + ): # Or not... it isn't clear what could be checked here continue @@ -259,8 +266,9 @@ def group_comparisons_by_dir(comparisons, skip=False): def get_all_kgo_comparisons(conn): "Retrieve all comparisons related to KGO tasks" - res = conn.execute("SELECT comp_task, kgo_file, suite_file, " - "status, comparison FROM comparisons") + res = conn.execute( + "SELECT comp_task, kgo_file, suite_file, status, comparison FROM comparisons" + ) return res.fetchall() @@ -269,12 +277,16 @@ def check_for_incomplete_tasks(conn, skip=False): res = conn.execute("SELECT task_name FROM tasks WHERE completed==?", (1,)) failed_tasks = res.fetchall() if len(failed_tasks) > 0: - print("WARNING - Some comparisons are either still running or " - "failed to complete properly:") + print( + "WARNING - Some comparisons are either still running or " + "failed to complete properly:" + ) for task in failed_tasks: - print(" * "+task[0]) - print("Please investigate this manually - no KGO updates will be " - "made for any of the tasks listed above") + print(" * " + task[0]) + print( + "Please investigate this manually - no KGO updates will be " + "made for any of the tasks listed above" + ) if not confirm("Continue anyway (y/n)? ", skip=skip): sys.exit() @@ -304,17 +316,14 @@ def get_site(): Function to return the site this is being run at using rose config """ command = ["rose", "config", "rose-stem", "automatic-options"] - cmd = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = cmd.communicate() retcode = cmd.returncode if retcode == 0: - site = stdout.decode('utf-8').split("=")[1].strip() + site = stdout.decode("utf-8").split("=")[1].strip() print("Found site via rose-config: {0}\n".format(site)) else: - sys.exit( - "No site was detected via rose config - this setting is required" - ) + sys.exit("No site was detected via rose config - this setting is required") return site @@ -332,16 +341,14 @@ def get_variables_file_path(suite_dir, site, platform, variables_extension): if platform is not None: vars_file = f"variables_{platform}{variables_extension}" variables_path = os.path.join(site_path, vars_file) - print("INFO: Looking for a kgo variables file at " - + variables_path) + print("INFO: Looking for a kgo variables file at " + variables_path) if os.path.exists(variables_path): return variables_path # Search for a variables. file if a platform one doesn't exist, eg. VM site vars_file = f"variables{variables_extension}" variables_path = os.path.join(site_path, vars_file) - print("INFO: Looking for a kgo variables file at " - + variables_path) + print("INFO: Looking for a kgo variables file at " + variables_path) if os.path.exists(variables_path): return variables_path @@ -349,42 +356,40 @@ def get_variables_file_path(suite_dir, site, platform, variables_extension): def update_variables_rc( - suite_dir, - kgo_dirs, - new_kgo_dir, - site, - platform, - variables_extension, - skip=False - ): + suite_dir, + kgo_dirs, + new_kgo_dir, + site, + platform, + variables_extension, + skip=False, +): """ Create an updated copy of the variables.rc with the KGO variables for any changed jobs updated """ # Attempt to get a copy of the variables.rc - variables_rc = get_variables_file_path(suite_dir, - site, - platform, - variables_extension) + variables_rc = get_variables_file_path( + suite_dir, site, platform, variables_extension + ) # Create a file to hold the new variables.rc variables_rc_new = os.path.expanduser( f"~/variables{variables_extension}_{new_kgo_dir}" ) if os.path.exists(variables_rc_new): - print("WARNING: New variables.rc file for this update already exists " - "at {0} and will be overwritten".format(variables_rc_new)) - if not confirm("Okay to overwrite variables.rc file (y/n)? ", - skip=skip): + print( + "WARNING: New variables.rc file for this update already exists " + "at {0} and will be overwritten".format(variables_rc_new) + ) + if not confirm("Okay to overwrite variables.rc file (y/n)? ", skip=skip): sys.exit("Aborting...") # Get the KGO variable names that need updating kgo_vars = [] for kgo_dir in kgo_dirs.keys(): - kgo_vars.append( - os.path.basename( - os.path.dirname(kgo_dir)).upper()) + kgo_vars.append(os.path.basename(os.path.dirname(kgo_dir)).upper()) # Get the suffix of the new directory if "_" not in new_kgo_dir: @@ -399,10 +404,13 @@ def update_variables_rc( # named in the database pattern = re.search(r'\s*"(\S+)"\s*:\s*BASE', line) if pattern and pattern.group(1).upper() in kgo_vars: - vrc_new.write(re.sub( - r':\s*BASE.*', - r': BASE~"_{}",'.format(ticket_suffix), - line)) + vrc_new.write( + re.sub( + r":\s*BASE.*", + r': BASE~"_{}",'.format(ticket_suffix), + line, + ) + ) else: # Otherwise just write the line unaltered vrc_new.write(line) @@ -417,9 +425,9 @@ def main(): # adds spaces between the option help text class BlankLinesHelpFormatter(argparse.HelpFormatter): "Formatter which adds blank lines between options" + def _split_lines(self, text, width): - return super( - BlankLinesHelpFormatter, self)._split_lines(text, width) + [''] + return super(BlankLinesHelpFormatter, self)._split_lines(text, width) + [""] parser = argparse.ArgumentParser( usage="%(prog)s [--new-release]", @@ -432,55 +440,69 @@ def _split_lines(self, text, width): is used) """, formatter_class=BlankLinesHelpFormatter, - ) + ) - parser.add_argument('--new-release', - help="if set, indicates that the task being " - "performed is installing a full set of new KGO " - "for a major model release instead of simply " - "updating failed KGO tasks.", - action="store_true", - ) + parser.add_argument( + "--new-release", + help="if set, indicates that the task being " + "performed is installing a full set of new KGO " + "for a major model release instead of simply " + "updating failed KGO tasks.", + action="store_true", + ) - parser.add_argument('--non-interactive', - help="if set, no questions will be asked and " - "the script will proceed without any checking " - "or warnings. Use with caution.", - action="store_true", - ) + parser.add_argument( + "--non-interactive", + help="if set, no questions will be asked and " + "the script will proceed without any checking " + "or warnings. Use with caution.", + action="store_true", + ) - parser.add_argument('-S', "--suite-name", - help="specifies the (cylc-run dir) name of " - "the suite containing the KGO database; for " - "use with non-interactive mode. If not given " - "non-interactive mode will try to take " - "this from the environment ($CYLC_SUITE_NAME)." - ) + parser.add_argument( + "-S", + "--suite-name", + help="specifies the (cylc-run dir) name of " + "the suite containing the KGO database; for " + "use with non-interactive mode. If not given " + "non-interactive mode will try to take " + "this from the environment ($CYLC_SUITE_NAME).", + ) - parser.add_argument('-U', "--suite-user", - help="specifies the (cylc-run dir) username " - "where the KGO database suite can be found; for " - "use with non-interactive mode. If not given " - "non-interactive mode will try to take " - "this from the environment ($USER)." - ) + parser.add_argument( + "-U", + "--suite-user", + help="specifies the (cylc-run dir) username " + "where the KGO database suite can be found; for " + "use with non-interactive mode. If not given " + "non-interactive mode will try to take " + "this from the environment ($USER).", + ) - parser.add_argument('-N', "--new-kgo-dir", - help="specifies the name of the new KGO " - "subdirectory; for use with non-interactive " - "mode. If not given the KGO files will be " - "installed directly to the location currently " - "specified by the suite." - ) + parser.add_argument( + "-N", + "--new-kgo-dir", + help="specifies the name of the new KGO " + "subdirectory; for use with non-interactive " + "mode. If not given the KGO files will be " + "installed directly to the location currently " + "specified by the suite.", + ) - parser.add_argument('-P', "--platform", - help="specifies the platform this script is " - "running on. Defaults as ''. If the site is meto then" - "the platform will be autopopulated.") + parser.add_argument( + "-P", + "--platform", + help="specifies the platform this script is " + "running on. Defaults as ''. If the site is meto then" + "the platform will be autopopulated.", + ) - parser.add_argument('-E', "--extension", default=".rc", - help="The extension of the variables file, either .rc " - "or .cylc") + parser.add_argument( + "-E", + "--extension", + default=".rc", + help="The extension of the variables file, either .rc " "or .cylc", + ) args = parser.parse_args() @@ -498,10 +520,12 @@ def _split_lines(self, text, width): variables_extension = args.extension if suite_name is None or suite_user is None: - message = "'kgo_update.py' will no longer ask for your suite "\ - "information\n.To pass this into the script please use "\ - "the command line arguments.\nThese are documented at "\ - "the top of the file." + message = ( + "'kgo_update.py' will no longer ask for your suite " + "information\n.To pass this into the script please use " + "the command line arguments.\nThese are documented at " + "the top of the file." + ) sys.exit(message) # Prompt the user for the key options @@ -516,17 +540,19 @@ def _split_lines(self, text, width): # Get the site being run at site = get_site() - if site == 'meto' and new_kgo_dir == "install_in_place": - message = "Site meto should not use the 'install_in_place' option. "\ - "Check --new-kgo-dir is specified on the command line." \ - "Do you want to continue with the update?" + if site == "meto" and new_kgo_dir == "install_in_place": + message = ( + "Site meto should not use the 'install_in_place' option. " + "Check --new-kgo-dir is specified on the command line." + "Do you want to continue with the update?" + ) if not confirm(message, False): sys.exit() # Populate platform if not provided and at meto - if platform is None and site == 'meto': + if platform is None and site == "meto": hostname = os.uname()[1] - if hostname == 'uan01': + if hostname == "uan01": platform = "ex1a" elif hostname.startswith("xc"): platform = "xc40" @@ -553,10 +579,11 @@ def _split_lines(self, text, width): # Create a file to hold the update script script_path = os.path.expanduser("~/kgo_update_{0}.sh".format(new_kgo_dir)) if os.path.exists(script_path): - print("WARNING: Script file for this update already exists at {0} " - "and will be overwritten".format(script_path)) - if not confirm("Okay to overwrite script file (y/n)? ", - skip=confirm_skip): + print( + "WARNING: Script file for this update already exists at {0} " + "and will be overwritten".format(script_path) + ) + if not confirm("Okay to overwrite script file (y/n)? ", skip=confirm_skip): sys.exit("Aborting...") # Write the script file @@ -578,65 +605,76 @@ def _split_lines(self, text, width): site, platform, variables_extension, - skip=confirm_skip + skip=confirm_skip, ) # Open the file for viewing in user's editor if interactive: - print(f"\n\nOpening {script_path}\nHit Return to Step through, " - "q to print all\n\n") + print( + f"\n\nOpening {script_path}\nHit Return to Step through, " + "q to print all\n\n" + ) with open(script_path, "r") as f: l = 0 print_all = False for line in f: - print(line, end='') + print(line, end="") l += 1 if l == 10 and not print_all: l = 0 key = input() - if key.lower() == 'q': + if key.lower() == "q": print_all = True - print("\n\nPlease carefully check the commands in the above file " - "before continuing.\nIf changes need to be made then open a " - "new terminal and edit the file there.") + print( + "\n\nPlease carefully check the commands in the above file " + "before continuing.\nIf changes need to be made then open a " + "new terminal and edit the file there." + ) # Final confirmation before running print("WARNING: Once launched the script will make permanent changes") - if not confirm("Are you sure you want to continue (y/n)? ", - skip=confirm_skip): + if not confirm("Are you sure you want to continue (y/n)? ", skip=confirm_skip): sys.exit("Aborting...") - print(banner("Running KGO Update commands from {0}".format(script_path)), - flush=True) + print( + banner("Running KGO Update commands from {0}".format(script_path)), + flush=True, + ) - process = subprocess.Popen(["bash", script_path], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + process = subprocess.Popen( + ["bash", script_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) while True: line = process.stdout.readline() if not line and process.poll() is not None: break - print(line.decode(), end='', flush=True) + print(line.decode(), end="", flush=True) retcode = process.returncode if retcode != 0: - sys.exit(f"Script did not run successfully, update failed\n" - f"{process.stderr.read().decode()}\n") + sys.exit( + f"Script did not run successfully, update failed\n" + f"{process.stderr.read().decode()}\n" + ) # Print a final message print(banner("Generated KGO Script has run successfully")) - if site != 'meto': - print("\nThe script has created a copy of the variables.rc in your " + if site != "meto": + print( + "\nThe script has created a copy of the variables.rc in your " "$HOME directory; you should merge this with the one in your " - "working copy using xxdiff to update the KGO variables") + "working copy using xxdiff to update the KGO variables" + ) else: - print(f"\nThe kgo update for {platform} is complete.\n\nIf this was " - "run from 'meto_update_kgo.sh' in the UM, the generated files " - "will be moved to UMDIR on spice and the process will continue " - "for other platforms requested.\n\nOtherwise, eg. for " - "lfric_inputs, you will need to merge the generated variables " - "file with the one in your working copy and run the script " - "again on the next platform.") + print( + f"\nThe kgo update for {platform} is complete.\n\nIf this was " + "run from 'meto_update_kgo.sh' in the UM, the generated files " + "will be moved to UMDIR on spice and the process will continue " + "for other platforms requested.\n\nOtherwise, eg. for " + "lfric_inputs, you will need to merge the generated variables " + "file with the one in your working copy and run the script " + "again on the next platform." + ) if __name__ == "__main__": diff --git a/kgo_updates/kgo_update/meto_run_kgo_script.sh b/kgo_updates/kgo_update/meto_run_kgo_script.sh index bfce8b5..0e16dd5 100755 --- a/kgo_updates/kgo_update/meto_run_kgo_script.sh +++ b/kgo_updates/kgo_update/meto_run_kgo_script.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # *****************************COPYRIGHT******************************* # (C) Crown copyright Met Office. All rights reserved. # For further details please refer to the file COPYRIGHT.txt @@ -14,6 +14,8 @@ # rsyncs with the xcs. # This script is NOT intended to be run independently of '../meto_update_kgo.sh' +# shellcheck disable=SC2059 + # Set colour codes RED='\033[0;31m' GREEN='\033[0;32m' @@ -31,12 +33,13 @@ do P) platforms=${OPTARG};; F) variables_extension=${OPTARG};; V) version_number=${OPTARG};; + *) echo "Invalid option: -$flag" >&2; exit 1;; esac done # Create command to run python script kgo_command="./kgo_update.py" -if [ $new_release -eq 1 ]; then +if [[ $new_release -eq 1 ]]; then kgo_command="${kgo_command} --new-release" fi kgo_command="${kgo_command} -S $suite_name -N $new_kgo_dir -E $variables_extension" @@ -47,25 +50,25 @@ kgo_command_ex1a="$kgo_command -U $suite_user_ex1a -P ex1a" # Make a directory to store the script and variables file file in variables_dir=~/kgo_update_files/vn$version_number/$new_kgo_dir -mkdir -p $variables_dir +mkdir -p "$variables_dir" # If spice has kgo updates if [[ $platforms == *"spice"* ]] && [[ $platforms != *"azspice"* ]]; then printf "${GREEN}\n\nRunning KGO Update Script on spice.\n${NC}" # Run the Update Script - $kgo_command_spice - - if [[ $? -ne 0 ]]; then - printf "${RED}\nThe installation script has failed on spice.\n${NC}" - else + if $kgo_command_spice; then # Move the updated variables file and script into the ticket folder printf "${GREEN}\n\nSuccessfully installed on spice.\n${NC}" printf "${GREEN}Moving the generated files into spice ${variables_dir}.${NC}\n" - if [ $new_release -ne 1 ]; then - mv ~/variables${variables_extension}_${new_kgo_dir} ${variables_dir}/spice_updated_variables${variables_extension} + if [[ $new_release -ne 1 ]]; then + mv ~/"variables${variables_extension}_${new_kgo_dir}" \ + "${variables_dir}/spice_updated_variables${variables_extension}" fi - mv ~/kgo_update_${new_kgo_dir}.sh ${variables_dir}/spice_update_script.sh + mv ~/"kgo_update_${new_kgo_dir}.sh" \ + "${variables_dir}/spice_update_script.sh" + else + printf "${RED}\nThe installation script has failed on spice.\n${NC}" fi fi @@ -74,18 +77,18 @@ if [[ $platforms == *"azspice"* ]]; then printf "${GREEN}\n\nRunning KGO Update Script on azspice.\n${NC}" # Run the Update Script - $kgo_command_azspice - - if [[ $? -ne 0 ]]; then - printf "${RED}\nThe installation script has failed on azspice.\n${NC}" - else + if $kgo_command_azspice; then # Move the updated variables file and script into the ticket folder printf "${GREEN}\n\nSuccessfully installed on azspice.\n${NC}" printf "${GREEN}Moving the generated files into azspice ${variables_dir}.${NC}\n" - if [ $new_release -ne 1 ]; then - mv ~/variables${variables_extension}_${new_kgo_dir} ${variables_dir}/azspice_updated_variables${variables_extension} + if [[ $new_release -ne 1 ]]; then + mv ~/"variables${variables_extension}_${new_kgo_dir}" \ + "${variables_dir}/azspice_updated_variables${variables_extension}" fi - mv ~/kgo_update_${new_kgo_dir}.sh ${variables_dir}/azspice_update_script.sh + mv ~/"kgo_update_${new_kgo_dir}.sh" \ + "${variables_dir}/azspice_update_script.sh" + else + printf "${RED}\nThe installation script has failed on azspice.\n${NC}" fi fi @@ -96,34 +99,32 @@ if [[ $platforms == *"xc40"* ]]; then host_xc40=$(rose host-select xc) # SCP the python script to the xc40 - scp -q kgo_update.py frum@$host_xc40:~ + scp -q kgo_update.py "frum@${host_xc40}":~ # Define the commands to run on xc40 command=". /etc/profile ; module load scitools ; ${kgo_command_xc40}" # SSH to the xc40 with the run command - ssh -Y $host_xc40 $command - - if [[ $? -ne 0 ]]; then - printf "${RED}\nThe installation script has failed on xc40.\n${NC}" - else + if ssh -Y "$host_xc40" "$command"; then # rsync the generated variables file and script back to frum on linux # This cleans up the original files printf "${GREEN}\n\nSuccessfully installed on xc40.\n${NC}" printf "${GREEN}Rsyncing the generated files into spice ${variables_dir}.\n${NC}" - if [ $new_release -ne 1 ]; then + if [[ $new_release -ne 1 ]]; then rsync --remove-source-files -avz \ - frum@$host_xc40:~/variables${variables_extension}_${new_kgo_dir} \ - ${variables_dir}/xc40_updated_variables${variables_extension} + "frum@$host_xc40:~/variables${variables_extension}_${new_kgo_dir}" \ + "${variables_dir}/xc40_updated_variables${variables_extension}" fi rsync --remove-source-files -avz \ - frum@$host_xc40:~/kgo_update_${new_kgo_dir}.sh \ - ${variables_dir}/xc40_update_script.sh + "frum@${host_xc40}:~/kgo_update_${new_kgo_dir}.sh" \ + "${variables_dir}/xc40_update_script.sh" + else + printf "${RED}\nThe installation script has failed on xc40.\n${NC}" fi # Clean up kgo_update.py on xc40 - ssh -Yq $host_xc40 "rm kgo_update.py" + ssh -Yq "$host_xc40" "rm kgo_update.py" fi # If ex1a has kgo updates @@ -133,32 +134,30 @@ if [[ $platforms == *"ex1a"* ]]; then host_ex=$(rose host-select exab) # SCP the python script to the ex1a - scp -q kgo_update.py umadmin@$host_ex:~ + scp -q kgo_update.py "umadmin@$host_ex":~ # Define the commands to run on ex1a command="module load scitools ; ${kgo_command_ex1a}" # SSH to the ex1a with the run command - ssh -Y $host_ex $command - - if [[ $? -ne 0 ]]; then - printf "${RED}\nThe installation script has failed on ex1a.\n${NC}" - else + if ssh -Y "$host_ex" "$command"; then # rsync the generated variables file and script back to frum on linux # This cleans up the original files printf "${GREEN}\n\nSuccessfully installed on ex1a.\n${NC}" printf "${GREEN}Rsyncing the generated files into azspice ${variables_dir}.\n${NC}" - if [ $new_release -ne 1 ]; then + if [[ $new_release -ne 1 ]]; then rsync --remove-source-files -avz \ - umadmin@$host_ex:~/variables${variables_extension}_${new_kgo_dir} \ - ${variables_dir}/ex1a_updated_variables${variables_extension} + "umadmin@$host_ex:~/variables${variables_extension}_${new_kgo_dir}" \ + "${variables_dir}/ex1a_updated_variables${variables_extension}" fi rsync --remove-source-files -avz \ - umadmin@$host_ex:~/kgo_update_${new_kgo_dir}.sh \ - ${variables_dir}/ex1a_update_script.sh + "umadmin@$host_ex:~/kgo_update_${new_kgo_dir}.sh" \ + "${variables_dir}/ex1a_update_script.sh" + else + printf "${RED}\nThe installation script has failed on ex1a.\n${NC}" fi # Clean up kgo_update.py on ex1a - ssh -Yq $host_ex "rm kgo_update.py" + ssh -Yq "$host_ex" "rm kgo_update.py" fi diff --git a/kgo_updates/meto_update_kgo.sh b/kgo_updates/meto_update_kgo.sh index 1c9e3e1..bc1e633 100755 --- a/kgo_updates/meto_update_kgo.sh +++ b/kgo_updates/meto_update_kgo.sh @@ -11,6 +11,8 @@ # The kgo_update.py script can be run with the --new-release option by providing # 'new-release' as a command line option to this script +# shellcheck disable=SC2059 + # Set colour codes RED='\033[0;31m' GREEN='\033[0;32m' @@ -23,11 +25,10 @@ script_loc="$(dirname "$0")" if [[ $HOSTNAME == "caz"* ]]; then launch_platform=azspice # Check you can sudo in as umadmin - sudo -iu umadmin bash -c "echo ''" - if [[ $? -ne 0 ]]; then + sudo -iu umadmin bash -c "echo ''" || { printf "${RED} You were unable to run commands as umadmin - this is required to run this script" printf "This may be because of a password typo or similar" - fi + } else launch_platform=spice fi @@ -41,7 +42,7 @@ if [ $# -ne 0 ]; then else printf "${RED}'%s' is not a recognised command line argument.\n" "${1}" printf "The only command line option available is --new-release.\n" - read -p "Would you like to run in new-release mode (default n)? " answer + read -rp "Would you like to run in new-release mode (default n)? " answer answer=${answer:-"n"} if [[ $answer == "y" ]]; then new_release=1 @@ -54,14 +55,14 @@ fi # Prompt user for Update Details echo "Enter the platforms requiring a kgo update" echo "Enter platforms lowercase and space separated, eg. spice xc40 ex1a azspice" -read platforms +read -r platforms if [[ $platforms == *"spice"* ]] && [[ $platforms != *"azspice"* ]] || [[ $platforms == *"xc40"* ]]; then # Check we're not trying to install to spice while on azspice if [[ $launch_platform == "azspice" ]] && [[ $platforms == *"spice"* ]] && [[ $platforms != *"azspice"* ]]; then printf "${RED}Attempting to install spice kgo from azspice - this isn't possible" exit 1 fi - read -p "spice/xc40 Suite Username: " suite_user + read -rp "spice/xc40 Suite Username: " suite_user else suite_user=None fi @@ -71,26 +72,26 @@ if [[ $platforms == *"ex1a"* ]] || [[ $platforms == *"azspice"* ]]; then printf "${RED}Attempting to install azspice kgo from spice - this isn't possible" exit 1 fi - read -p "ex1a/azspice Suite Username: " suite_user_ex1a + read -rp "ex1a/azspice Suite Username: " suite_user_ex1a else suite_user_ex1a=None fi -read -p "Suite Name: " suite_name -read -p "Enter the path to the merged trunk WC (top directory): " wc_path +read -rp "Suite Name: " suite_name +read -rp "Enter the path to the merged trunk WC (top directory): " wc_path # Trim any trailing / from the end of the path wc_path=${wc_path%/} if [ ! -d "${wc_path}" ]; then printf "${RED}${wc_path} is not a valid path${NC}\n" exit 1 fi -read -p "Version Number (VV.V): " version_number -read -p "Ticket Number (TTTT): " ticket_number -read -p "How should the new kgo directory be named (default vn${version_number}_t${ticket_number}): " new_kgo_dir +read -rp "Version Number (VV.V): " version_number +read -rp "Ticket Number (TTTT): " ticket_number +read -rp "How should the new kgo directory be named (default vn${version_number}_t${ticket_number}): " new_kgo_dir new_kgo_dir=${new_kgo_dir:-"vn${version_number}_t${ticket_number}"} # Check in the working_copy rose-stem for .rc or .cylc files # Need this for the variables file extension -# Can't use Cylc version as .rc can be used in compatability mode +# Can't use Cylc version as .rc can be used in compatibility mode if [ -f "${wc_path}/rose-stem/suite.rc" ]; then variables_extension=".rc" elif [ -f "${wc_path}/rose-stem/suite.cylc" ] || [ -f "${wc_path}/rose-stem/flow.cylc" ]; then @@ -117,17 +118,17 @@ echo "New KGO Dir: ${new_kgo_dir}" if [ $new_release -eq 1 ]; then printf "${RED}WARNING: Running with --new-release enabled${NC}\n" fi -read -p "Run with the above settings y/n (default n): " run_script +read -rp "Run with the above settings y/n (default n): " run_script run_script=${run_script:-n} -if [ $run_script != "y" ]; then +if [[ $run_script != "y" ]]; then exit 0 fi # Move the kgo_update directory to frum on linux if [[ $launch_platform == "spice" ]]; then - scp -rq $script_loc/kgo_update frum@localhost:~ + scp -rq "${script_loc}/kgo_update" frum@localhost:~ else - sudo -iu umadmin bash -c "cp -r $script_loc/kgo_update /home/users/umadmin" + sudo -iu umadmin bash -c "cp -r ${script_loc}/kgo_update /home/users/umadmin" fi # Define command to run as frum @@ -144,13 +145,13 @@ command=". /etc/profile ; module load scitools ; cd kgo_update ; # Run the command as frum if [[ $launch_platform == "spice" ]]; then - ssh -Y frum@localhost $command + ssh -Y frum@localhost "$command" else sudo -iu umadmin bash -c "cd $UMDIR ; $command" fi # Error Checking and rsyncing -variables_dir=kgo_update_files/vn${version_number}/${new_kgo_dir} +variables_dir="kgo_update_files/vn${version_number}/${new_kgo_dir}" succeeded_spice=0 succeeded_azspice=0 succeeded_xc40=0 @@ -162,13 +163,12 @@ if [[ $platforms == *"spice"* ]] && [[ $platforms != *"azspice"* ]]; then succeeded_spice=1 if [[ $new_release -ne 1 ]]; then printf "${GREEN}\n\nCopying the spice variables file into this working copy.\n${NC}" - scp -q frum@localhost:~/${variables_dir}/spice_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_spice${variables_extension} - if [[ $? -ne 0 ]]; then + scp -q frum@localhost:~/"${variables_dir}/spice_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_spice${variables_extension}" || { printf "${RED}The copy of the spice variables file into this working copy has failed.\n${NC}" succeeded_spice=0 succeeded_all=0 - fi + } fi else succeeded_all=0 @@ -180,13 +180,12 @@ if [[ $platforms == *"azspice"* ]]; then succeeded_azspice=1 if [[ $new_release -ne 1 ]]; then printf "${GREEN}\n\nCopying the azspice variables file into this working copy.\n${NC}" - cp /home/users/umadmin/${variables_dir}/azspice_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_azspice${variables_extension} - if [[ $? -ne 0 ]]; then + cp "/home/users/umadmin/${variables_dir}/azspice_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_azspice${variables_extension}" || { printf "${RED}The copy of the azspice variables file into this working copy has failed.\n${NC}" succeeded_azspice=0 succeeded_all=0 - fi + } fi else succeeded_all=0 @@ -199,17 +198,20 @@ if [[ $platforms == *"xc40"* ]]; then if [[ $new_release -ne 1 ]]; then printf "${GREEN}\n\nCopying the xc40 variables file into this working copy.\n${NC}" if [[ $launch_platform == "spice" ]]; then - scp -q frum@localhost:~/${variables_dir}/xc40_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_xc40${variables_extension} + scp -q "frum@localhost:~/${variables_dir}/xc40_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_xc40${variables_extension}" + rc=$? else - cp /home/users/umadmin/${variables_dir}/xc40_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_xc40${variables_extension} + cp "/home/users/umadmin/${variables_dir}/xc40_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_xc40${variables_extension}" + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The copy of the xc40 variables file into this working copy has failed.\n${NC}" succeeded_xc40=0 succeeded_all=0 fi + rc= fi else succeeded_all=0 @@ -222,17 +224,20 @@ if [[ $platforms == *"ex1a"* ]]; then if [[ $new_release -ne 1 ]]; then printf "${GREEN}\n\nCopying the ex1a variables file into this working copy.\n${NC}" if [[ $launch_platform == "spice" ]]; then - scp -q frum@localhost:~/${variables_dir}/ex1a_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_ex1a${variables_extension} + scp -q "frum@localhost:~/${variables_dir}/ex1a_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_ex1a${variables_extension}" + rc=$? else - cp /home/users/umadmin/${variables_dir}/ex1a_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_ex1a${variables_extension} + cp "/home/users/umadmin/${variables_dir}/ex1a_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_ex1a${variables_extension}" + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The copy of the ex1a variables file into this working copy has failed.\n${NC}" succeeded_ex1a=0 succeeded_all=0 fi + rc= fi else succeeded_all=0 @@ -246,7 +251,7 @@ else fi if [[ $platforms == *"xc40"* ]] || [[ $platforms == *"ex1a"* ]]; then - read -p "Enter 1 to rsync UM KGO, 2 to rsync lfricinputs KGO (default 1): " rsync_type + read -rp "Enter 1 to rsync UM KGO, 2 to rsync lfricinputs KGO (default 1): " rsync_type if [[ $rsync_type == "2" ]]; then rsync_dir="lfricinputs/kgo/" else @@ -260,15 +265,18 @@ if [[ $succeeded_xc40 -eq 1 ]]; then host_rsync=$(rose host-select xc) rsync_com="ssh -Y ${host_rsync} 'rsync -av /projects/um1/standard_jobs/${rsync_dir} xcslr0:/common/um1/standard_jobs/${rsync_dir}'" if [[ $launch_platform == "spice" ]]; then - ssh -Y frum@localhost $rsync_com + ssh -Y frum@localhost "$rsync_com" + rc=$? else sudo -iu umadmin bash -c '$rsync_com' + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The rsync to the xcs has failed.\n${NC}" else - printf "${Green}The rsync to the xcs has succeeded.\n${NC}" + printf "${GREEN}The rsync to the xcs has succeeded.\n${NC}" fi + rc= elif [[ $platforms == *"xc40"* ]]; then printf "${RED}\n\nSkipping the rsync to the xcs as the xc40 install failed.\n${NC}" fi @@ -277,21 +285,24 @@ fi # This process will need modifying as we go forward # Currently hardcoded to UM kgo as lfricinputs not on ex machines if [[ $succeeded_ex1a -eq 1 ]]; then - printf "${GREEN}\n\nrsyncing the kgo to exz.\n${NC}" + printf "${GREEN}\n\nrsyncing the kgo to exz + excd.\n${NC}" printf "Warning: Always rsyncing UM KGO (not lfricinputs) on ex1a" rsync_dir="kgo/" host_rsync=$(rose host-select exab) rsync_com="ssh -Y ${host_rsync} 'rsync -av /common/umdir/standard_jobs/${rsync_dir} login.exz:/common/internal/umdir/standard_jobs/${rsync_dir}'" if [[ $launch_platform == "spice" ]]; then - ssh -Y frum@localhost $rsync_com + ssh -Y frum@localhost "$rsync_com" + rc=$? else sudo -iu umadmin bash -c '$rsync_com' + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The rsync to the exz has failed.\n${NC}" else - printf "${Green}The rsync to the exz has succeeded.\n${NC}" + printf "${GREEN}The rsync to the exz has succeeded.\n${NC}" fi + rc= elif [[ $platforms == *"ex1a"* ]]; then printf "${RED}\n\nSkipping the rsync to the exa as the exz install failed.\n${NC}" fi @@ -324,4 +335,4 @@ if [[ $platforms == *"ex1a"* ]]; then else printf "${RED}Installation on ex1a unsuccessful. Review output for error.\n${NC}" fi -fi \ No newline at end of file +fi diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index b17760b..93bc6e5 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -10,13 +10,13 @@ Warning: Should only be run on a Test branch or by CR on commit to trunk """ +import argparse +import ast import os import re -import ast import shutil -import argparse -import tempfile import subprocess +import tempfile BLACK_COMMAND = "black --line-length=80" CLASS_NAME_REGEX = r"vn\d+(_t\d+\w*)?" @@ -132,9 +132,7 @@ def split_macros(parsed_versions): in_macro = False in_comment = False for line in parsed_versions: - if line.startswith("class vn") and not line.startswith( - "class vnXX_txxx" - ): + if line.startswith("class vn") and not line.startswith("class vnXX_txxx"): # If the macro string is set, then append to the list. If it's # empty then this is the first macro we're looking at, so nothing to # append @@ -174,9 +172,7 @@ def match_python_import(line): Inputs: - line, str to match """ - if re.match(r"import \w+", line) or re.match( - r"from [\.\w]+ import [\.\w]+", line - ): + if re.match(r"import \w+", line) or re.match(r"from [\.\w]+ import [\.\w]+", line): return True return False @@ -510,13 +506,10 @@ def find_last_macro(self, macros, meta_dir): regexp = re.compile(rf"BEFORE_TAG\s*=\s*[\"']{after_tag}[\"']") if regexp.search(macro): try: - after_tag = re.search( - rf"AFTER_TAG{TAG_REGEX}", macro - ).group(1) + after_tag = re.search(rf"AFTER_TAG{TAG_REGEX}", macro).group(1) except AttributeError as exc: raise Exception( - "Couldn't find an after tag in the macro:\n" - f"{macro}" + "Couldn't find an after tag in the macro:\n" f"{macro}" ) from exc found_macro = macro macros.remove(found_macro) @@ -546,9 +539,7 @@ def find_macro(self, meta_dir, macros): # Find the macro we're interested in for macro in macros: try: - macro_name = re.search( - rf"class ({CLASS_NAME_REGEX})\(", macro - ).group(1) + macro_name = re.search(rf"class ({CLASS_NAME_REGEX})\(", macro).group(1) except AttributeError as exc: raise Exception( "Unable to determine macro class name in " @@ -571,9 +562,7 @@ def get_full_import_path(self, imp): """ core_imp = os.path.join(self.core_source, imp) - if os.path.exists(core_imp) or os.path.exists( - os.path.dirname(core_imp) - ): + if os.path.exists(core_imp) or os.path.exists(os.path.dirname(core_imp)): return core_imp # Reinstate when using Jules Shared from Jules @@ -584,9 +573,7 @@ def get_full_import_path(self, imp): # return jules_imp apps_imp = os.path.join(self.root_path, imp) - if os.path.exists(apps_imp) or os.path.exists( - os.path.dirname(apps_imp) - ): + if os.path.exists(apps_imp) or os.path.exists(os.path.dirname(apps_imp)): return apps_imp raise Exception( @@ -719,9 +706,7 @@ def combine_macros(self, import_order): f"{self.parse_application_section(meta_import)}\n" ) if self.parsed_macros[meta_import]["commands"].strip("\n"): - full_command += ( - self.parsed_macros[meta_import]["commands"] + "\n" - ) + full_command += self.parsed_macros[meta_import]["commands"] + "\n" else: full_command += " # Blank Upgrade Macro\n" return full_command @@ -800,9 +785,7 @@ def preprocess_macros(self): # Read through rose-meta files for import statements # of other metadata - self.parsed_macros[meta_dir]["imports"] = self.read_meta_imports( - meta_dir - ) + self.parsed_macros[meta_dir]["imports"] = self.read_meta_imports(meta_dir) # Read through the versions.py file for python import statements self.python_imports.update( @@ -859,12 +842,8 @@ def apps_to_upgrade(self): upgradeable_apps = [] app_dir_apps = os.path.join(self.root_path, "rose-stem", "app") app_dir_core = os.path.join(self.core_source, "rose-stem", "app") - apps_list = [ - os.path.join(app_dir_apps, f) for f in os.listdir(app_dir_apps) - ] - apps_list += [ - os.path.join(app_dir_core, f) for f in os.listdir(app_dir_core) - ] + apps_list = [os.path.join(app_dir_apps, f) for f in os.listdir(app_dir_apps)] + apps_list += [os.path.join(app_dir_core, f) for f in os.listdir(app_dir_core)] for app_path in apps_list: # Ignore lfric_coupled_rivers as this is based on Jules-standalone # metadata which is not currently available @@ -985,9 +964,7 @@ def parse_args(): Read command line args """ - parser = argparse.ArgumentParser( - "Pre-process and apply LFRic Apps upgrade macros." - ) + parser = argparse.ArgumentParser("Pre-process and apply LFRic Apps upgrade macros.") parser.add_argument( "tag", type=check_tag, @@ -1035,9 +1012,7 @@ def parse_args(): return parser.parse_args() -def apply_macros_main( - tag, cname=None, version=None, apps=".", core=None, jules=None -): +def apply_macros_main(tag, cname=None, version=None, apps=".", core=None, jules=None): """ Main function for this program """ diff --git a/lfric_macros/files/template_versions.py b/lfric_macros/files/template_versions.py index 098890e..fca0043 100644 --- a/lfric_macros/files/template_versions.py +++ b/lfric_macros/files/template_versions.py @@ -2,6 +2,7 @@ from metomi.rose.upgrade import MacroUpgrade + class UpgradeError(Exception): """Exception created when an upgrade fails.""" diff --git a/lfric_macros/release_lfric.py b/lfric_macros/release_lfric.py index f9d667b..c4c1683 100755 --- a/lfric_macros/release_lfric.py +++ b/lfric_macros/release_lfric.py @@ -26,9 +26,9 @@ from apply_macros import ( ApplyMacros, apply_macros_main, + apply_styling, get_root_path, read_versions_file, - apply_styling, split_macros, version_number, ) @@ -72,9 +72,7 @@ def raise_exception(result, command): Raise an exception if a subprocess command has failed """ - raise Exception( - f"[FAIL] Error running command: '{command}'\n" f"{result.stderr}" - ) + raise Exception(f"[FAIL] Error running command: '{command}'\n" f"{result.stderr}") def set_dependency_path(args): @@ -115,11 +113,7 @@ def find_meta_dirs(paths): for path in paths: for dirpath, dirnames, filenames in os.walk(path): exclude_dirs = [".svn", "rose-stem", "integration-test"] - dirnames[:] = [ - d - for d in dirnames - if d not in exclude_dirs - ] + dirnames[:] = [d for d in dirnames if d not in exclude_dirs] if "rose-meta.conf" in filenames: dirs.add(os.path.dirname(dirpath)) return dirs @@ -326,9 +320,7 @@ def parse_args(): Read command line args """ - parser = argparse.ArgumentParser( - "Move and edit files for lfric_apps release" - ) + parser = argparse.ArgumentParser("Move and edit files for lfric_apps release") parser.add_argument( "-o", "--old_version", diff --git a/lfric_macros/tests/test_apply_macros.py b/lfric_macros/tests/test_apply_macros.py index 42ed52d..2585533 100644 --- a/lfric_macros/tests/test_apply_macros.py +++ b/lfric_macros/tests/test_apply_macros.py @@ -1,6 +1,8 @@ -import pytest -import subprocess import shutil +import subprocess + +import pytest + from ..apply_macros import * # A macro that we want to find for these tests @@ -44,10 +46,7 @@ def __repr__(self): test_versions_file = [f"{x}\n" for x in test_versions_file.split("\n")] # The expected result from split_macros for the versions -expected_split_macros = [ - desired_macro, - existing_macro -] +expected_split_macros = [desired_macro, existing_macro] # ApplyMacros below requires an LFRic Apps working copy to work - check out the # head of the lfric_apps trunk for this purpose. The actual contents of the @@ -58,25 +57,19 @@ def __repr__(self): check=False, capture_output=True, text=True, - timeout=120 + timeout=120, ) if result.returncode: raise RuntimeError( "Failed to checkout required LFRic Apps Working Copy with error ", - result.stderr + result.stderr, ) # Create an instance of the apply_macros class # Pass a known directory in as the Jules and Core sources as these are not # required for testing -am = ApplyMacros( - "vn0.0_t001", - None, - None, - appsdir, - "/tmp", - "/tmp" -) +am = ApplyMacros("vn0.0_t001", None, None, appsdir, "/tmp", "/tmp") + def test_split_macros(): m = split_macros(test_versions_file) @@ -103,10 +96,10 @@ def test_parse_macro(): expected_dict = { "before_tag": "vn0.0_t000", "commands": ( - ' self.add_setting(\n' - ' config, ["namelist:namelist1", "opt1"], "value1"\n' - ' )\n' - ) + " self.add_setting(\n" + ' config, ["namelist:namelist1", "opt1"], "value1"\n' + " )\n" + ), } assert am.parsed_macros["meta_dir"] == expected_dict assert am.ticket_number == "#001" @@ -117,14 +110,19 @@ def test_parse_macro(): def test_read_meta_imports(): am.parsed_macros["tests/test_meta_dir"] = {} - am.parsed_macros["tests/test_meta_dir"]["imports"] = am.read_meta_imports("tests/test_meta_dir") + am.parsed_macros["tests/test_meta_dir"]["imports"] = am.read_meta_imports( + "tests/test_meta_dir" + ) expected_imports = [ os.path.join(am.root_path, "science", "gungho"), - os.path.join(am.root_path, "applications", "lfric_atm") + os.path.join(am.root_path, "applications", "lfric_atm"), ] assert am.parsed_macros["tests/test_meta_dir"]["imports"] == expected_imports expected_meta = [os.path.join(am.root_path, "applications", "lfric_atm")] - assert am.read_meta_imports("tests/test_meta_dir/rose-app.conf", "meta") == expected_meta + assert ( + am.read_meta_imports("tests/test_meta_dir/rose-app.conf", "meta") + == expected_meta + ) def test_determine_import_order(): @@ -136,22 +134,13 @@ def test_determine_import_order(): am.parsed_macros["import3"]["imports"] = ["import4"] am.parsed_macros["import4"]["imports"] = [] am.parsed_macros["import2"]["imports"] = [] - expected_order = [ - "import2", - "import4", - "import3", - "import1" - ] + expected_order = ["import2", "import4", "import3", "import1"] assert am.determine_import_order("import1") == expected_order def test_combine_macros(): - am.parsed_macros["importA"] = { - "commands": " importA command" - } - am.parsed_macros["importB"] = { - "commands": " importB command" - } + am.parsed_macros["importA"] = {"commands": " importA command"} + am.parsed_macros["importB"] = {"commands": " importB command"} expected_combined = ( " # Commands From: importA\n importA command\n" " # Commands From: importB\n importB command\n" @@ -167,9 +156,18 @@ def test_parse_application_section(): def test_deduplicate_list(): - assert deduplicate_list([1,2,3]) == [1,2,3] - assert deduplicate_list([1,2,2,3,3,3,]) == [1,2,3] - assert deduplicate_list([1,2,1,3,2]) == [1,2,3] + assert deduplicate_list([1, 2, 3]) == [1, 2, 3] + assert deduplicate_list( + [ + 1, + 2, + 2, + 3, + 3, + 3, + ] + ) == [1, 2, 3] + assert deduplicate_list([1, 2, 1, 3, 2]) == [1, 2, 3] def test_match_python_imports(): @@ -179,8 +177,9 @@ def test_match_python_imports(): assert match_python_import("import m as n") == True assert match_python_import("false") == False + # Remove appsdir -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def remove_tempdir(): yield shutil.rmtree(appsdir) diff --git a/nightly_testing/generate_test_suite_cron.py b/nightly_testing/generate_test_suite_cron.py index 392fa60..d0efc2f 100755 --- a/nightly_testing/generate_test_suite_cron.py +++ b/nightly_testing/generate_test_suite_cron.py @@ -32,12 +32,13 @@ works. """ +import argparse import os -import sys import re -import yaml -import argparse import subprocess +import sys + +import yaml DEFAULT_CYLC_VERSION = "8" DEPENDENCIES = { @@ -130,9 +131,7 @@ def lfric_heads_sed(wc_path): dep_path = os.path.join(wc_path_new, "dependencies.sh") rstr = f"cp -rf {wc_path} {wc_path_new} ; " - rstr += ( - f"sed -i -e 's/^\\(export .*_revision=@\\).*/\\1HEAD/' {dep_path} ; " - ) + rstr += f"sed -i -e 's/^\\(export .*_revision=@\\).*/\\1HEAD/' {dep_path} ; " rstr += f"sed -i -e 's/^\\(export .*_rev=\\).*/\\1HEAD/' {dep_path} ; " return rstr @@ -414,9 +413,7 @@ def parse_cl_args(): main_crontab += f"# {repo.upper()} SUITES\n" main_crontab += 80 * "#" + 2 * "\n" last_repo = repo - main_crontab += generate_cron_job( - suite_name, suites[suite_name], args.cron_log - ) + main_crontab += generate_cron_job(suite_name, suites[suite_name], args.cron_log) main_crontab += 3 * "\n" with open(args.cron_file, "w") as outfile: diff --git a/nightly_testing/retrigger_nightlies.py b/nightly_testing/retrigger_nightlies.py index 1c63809..cb9d1c9 100755 --- a/nightly_testing/retrigger_nightlies.py +++ b/nightly_testing/retrigger_nightlies.py @@ -17,12 +17,12 @@ """ import os -import sys import re -import subprocess import sqlite3 -from time import sleep +import subprocess +import sys from datetime import datetime, timedelta +from time import sleep def run_command(command): @@ -68,8 +68,7 @@ def check_for_failed_tasks(conn): "SELECT name, status FROM task_states WHERE status LIKE '%failed%'" ).fetchall() res_subfail = conn.execute( - "SELECT name, status FROM task_states " - "WHERE status LIKE '%submit-failed%'" + "SELECT name, status FROM task_states WHERE status LIKE '%submit-failed%'" ).fetchall() return res_failed + res_subfail @@ -82,7 +81,7 @@ def ask_yn(message): rval = "" while rval.lower().strip() not in ["y", "n"]: rval = input(f"{message}? (y/n) ") - return rval.lower().strip()=="y" + return rval.lower().strip() == "y" def restart_suite(suite): diff --git a/nightly_testing/tests/test_generate_test_suite_cron.py b/nightly_testing/tests/test_generate_test_suite_cron.py index fa0019a..8d7cc1a 100644 --- a/nightly_testing/tests/test_generate_test_suite_cron.py +++ b/nightly_testing/tests/test_generate_test_suite_cron.py @@ -8,17 +8,19 @@ ( ["um"], "scratch/dir/", - "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; " + "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; ", ), ( ["um", "lfric"], "scratch/dir", - "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; fcm co -q --force fcm:lfric.xm_tr@HEAD scratch/dir/wc_lfric ; " - ) + "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; fcm co -q --force fcm:lfric.xm_tr@HEAD scratch/dir/wc_lfric ; ", + ), ] + + @pytest.mark.parametrize( ("inlist", "scratch", "expected"), - [test_data for test_data in data_join_checkout_commands] + [test_data for test_data in data_join_checkout_commands], ) def test_join_checkout_commands(inlist, scratch, expected): assert join_checkout_commands(inlist, scratch) == expected @@ -31,9 +33,10 @@ def test_join_checkout_commands(inlist, scratch, expected): "cp -rf path/to/wc path/to/wc_heads ; sed -i -e 's/^\\(export .*_revision=@\\).*/\\1HEAD/' path/to/wc_heads/dependencies.sh ; sed -i -e 's/^\\(export .*_rev=\\).*/\\1HEAD/' path/to/wc_heads/dependencies.sh ; ", ) ] + + @pytest.mark.parametrize( - ("wc_path", "expected"), - [test_data for test_data in data_lfric_heads_sed] + ("wc_path", "expected"), [test_data for test_data in data_lfric_heads_sed] ) def test_lfric_heads_sed(wc_path, expected): assert lfric_heads_sed(wc_path) == (expected) @@ -41,55 +44,37 @@ def test_lfric_heads_sed(wc_path, expected): # Test generate_cron_timing_str data_generate_cron_timing_str = [ - ( - {"period": "weekly", "cron_launch": "30 00"}, - "main", - "30 00 * * 1 " - ), - ( - {"period": "nightly", "cron_launch": "30 00"}, - "main", - "30 00 * * 2-5 " - ), + ({"period": "weekly", "cron_launch": "30 00"}, "main", "30 00 * * 1 "), + ({"period": "nightly", "cron_launch": "30 00"}, "main", "30 00 * * 2-5 "), ( {"period": "nightly_all", "cron_launch": "30 00"}, "main", - "30 00 * * 1-5 " - ), - ( - {"period": "weekly", "cron_clean": "30 00"}, - "clean", - "30 00 * * 7 " - ), - ( - {"period": "nightly", "cron_clean": "30 00"}, - "clean", - "30 00 * * 3-6 " + "30 00 * * 1-5 ", ), + ({"period": "weekly", "cron_clean": "30 00"}, "clean", "30 00 * * 7 "), + ({"period": "nightly", "cron_clean": "30 00"}, "clean", "30 00 * * 3-6 "), ( {"period": "nightly_all", "cron_clean": "30 00"}, "clean", - "30 00 * * 2-6 " - ), - ( - {"period": "weekly", "cron_clean": "30 00"}, - "monitoring", - "00 06 * * 1 " + "30 00 * * 2-6 ", ), + ({"period": "weekly", "cron_clean": "30 00"}, "monitoring", "00 06 * * 1 "), ( {"period": "nightly", "cron_clean": "30 00"}, "monitoring", - "00 06 * * 2-5 " + "00 06 * * 2-5 ", ), ( {"period": "nightly_all", "cron_clean": "30 00"}, "monitoring", - "00 06 * * 1-5 " + "00 06 * * 1-5 ", ), ] + + @pytest.mark.parametrize( ("suite", "mode", "expected"), - [test_data for test_data in data_generate_cron_timing_str] + [test_data for test_data in data_generate_cron_timing_str], ) def test_generate_cron_timing_str(suite, mode, expected): assert generate_cron_timing_str(suite, mode) == expected @@ -101,24 +86,26 @@ def test_generate_cron_timing_str(suite, mode, expected): "7", "suite_name", "cron_log", - f"{PROFILE} ; export CYLC_VERSION=7 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; rose suite-clean -y -q suite_name >> cron_log 2>&1\n" + f"{PROFILE} ; export CYLC_VERSION=7 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; rose suite-clean -y -q suite_name >> cron_log 2>&1\n", ), ( "8", "suite_name", "cron_log", - f"{PROFILE} ; export CYLC_VERSION=8 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n" + f"{PROFILE} ; export CYLC_VERSION=8 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n", ), ( "8-next", "suite_name", "cron_log", - f"{PROFILE} ; export CYLC_VERSION=8-next ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n" - ) + f"{PROFILE} ; export CYLC_VERSION=8-next ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n", + ), ] + + @pytest.mark.parametrize( ("cylc_version", "name", "log_file", "expected"), - [test_data for test_data in data_generate_clean_commands] + [test_data for test_data in data_generate_clean_commands], ) def test_generate_clean_commands(cylc_version, name, log_file, expected): assert generate_clean_commands(cylc_version, name, log_file) == expected @@ -131,26 +118,28 @@ def test_generate_clean_commands(cylc_version, name, log_file, expected): "path/to/wc", "7", "suite_name", - "export CYLC_VERSION=7 ; rose stem --group=all --name=suite_name --source=path/to/wc " + "export CYLC_VERSION=7 ; rose stem --group=all --name=suite_name --source=path/to/wc ", ), ( {"groups": "nightly"}, "path/to/wc", "8", "suite_name", - "export CYLC_VERSION=8 ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc " + "export CYLC_VERSION=8 ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc ", ), ( {"groups": "nightly"}, "path/to/wc", "8-next", "suite_name", - "export CYLC_VERSION=8-next ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc " - ) + "export CYLC_VERSION=8-next ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc ", + ), ] + + @pytest.mark.parametrize( ("suite", "wc_path", "cylc_version", "name", "expected"), - [test_data for test_data in data_generate_rose_stem_command] + [test_data for test_data in data_generate_rose_stem_command], ) def test_generate_rose_stem_command(suite, wc_path, cylc_version, name, expected): assert generate_rose_stem_command(suite, wc_path, cylc_version, name) == expected @@ -163,39 +152,35 @@ def test_generate_rose_stem_command(suite, wc_path, cylc_version, name, expected "repo": "um", "revisions": "heads", }, - "--source=fcm:casim.xm_tr@HEAD --source=fcm:jules.xm_tr@HEAD --source=fcm:mule.xm_tr@HEAD --source=fcm:shumlib.xm_tr@HEAD --source=fcm:socrates.xm_tr@HEAD --source=fcm:ukca.xm_tr@HEAD " + "--source=fcm:casim.xm_tr@HEAD --source=fcm:jules.xm_tr@HEAD --source=fcm:mule.xm_tr@HEAD --source=fcm:shumlib.xm_tr@HEAD --source=fcm:socrates.xm_tr@HEAD --source=fcm:ukca.xm_tr@HEAD ", ), ( { "repo": "um", "revisions": "set", }, - "" + "", ), ( { "repo": "um", }, - "" + "", ), ( { "repo": "jules", "revisions": "heads", }, - "" + "", ), - ( - { - "repo": "lfric_apps", - "revisions": "heads" - }, - "" - ) + ({"repo": "lfric_apps", "revisions": "heads"}, ""), ] + + @pytest.mark.parametrize( ("suite", "expected"), - [test_data for test_data in data_populate_heads_sources] + [test_data for test_data in data_populate_heads_sources], ) def test_populate_heads_sources(suite, expected): assert populate_heads_sources(suite) == expected @@ -203,20 +188,14 @@ def test_populate_heads_sources(suite, expected): # Test populate_cl_variables data_populate_cl_variables = [ - ( - { - "vars": ["var1", "var2", "var3"] - }, - "-S var1 -S var2 -S var3 " - ), - ( - {}, - "" - ) + ({"vars": ["var1", "var2", "var3"]}, "-S var1 -S var2 -S var3 "), + ({}, ""), ] + + @pytest.mark.parametrize( ("suite", "expected"), - [test_data for test_data in data_populate_cl_variables] + [test_data for test_data in data_populate_cl_variables], ) def test_populate_cl_variables(suite, expected): assert populate_cl_variables(suite) == expected @@ -224,22 +203,15 @@ def test_populate_cl_variables(suite, expected): # Test major_cylc_version data_major_cylc_version = [ - ( - "7", - "7" - ), - ( - "8", - "8" - ), - ( - "8-next", - "8" - ), + ("7", "7"), + ("8", "8"), + ("8-next", "8"), ] + + @pytest.mark.parametrize( ("version", "expected"), - [test_data for test_data in data_major_cylc_version] + [test_data for test_data in data_major_cylc_version], ) def test_major_cylc_version(version, expected): assert major_cylc_version(version) == expected diff --git a/script_copyright_checker/bin/copyright_checker.py b/script_copyright_checker/bin/copyright_checker.py index 46517d3..54e986c 100755 --- a/script_copyright_checker/bin/copyright_checker.py +++ b/script_copyright_checker/bin/copyright_checker.py @@ -8,21 +8,18 @@ Script which tests code files within the UM repository to ensure they contain a recognised copyright notice. """ -import re +import argparse import os -import sys -import subprocess +import re +from textwrap import wrap from fcm_bdiff import ( get_branch_diff_filenames, - text_decoder, - is_trunk, - use_mirror, get_branch_info, get_url, + is_trunk, + use_mirror, ) -import argparse -from textwrap import wrap # Desired maximum column width for output - we make an exception # for filenames, which are always printed on a single line to aid @@ -56,13 +53,9 @@ def load_templates(filter_pattern): template_path = "." if "CYLC_TASK_WORK_PATH" in os.environ: - template_path = os.path.join( - os.environ["CYLC_TASK_WORK_PATH"], "file", "" - ) + template_path = os.path.join(os.environ["CYLC_TASK_WORK_PATH"], "file", "") - template_files = files_to_process( - template_path, [], filter_pattern=filter_pattern - ) + template_files = files_to_process(template_path, [], filter_pattern=filter_pattern) for filename in template_files: with open(filename) as file: @@ -143,9 +136,7 @@ def main(inputs, ignore_list): regex_templates_raw.extend(load_templates(filter_pattern=filter_tmp)) for filename, template_lines in regex_templates_raw: - regex_templates.append( - (filename, re.compile(r"\n".join(template_lines))) - ) + regex_templates.append((filename, re.compile(r"\n".join(template_lines)))) files_to_check = [] for file_input in inputs: @@ -212,9 +203,7 @@ def parse_options(): action="store", dest="ignore", default=None, - help=( - "ignore filename/s containing (comma separated list of patterns)" - ), + help=("ignore filename/s containing (comma separated list of patterns)"), ) parser.add_argument( "--base_path", @@ -228,8 +217,7 @@ def parse_options(): action="store_true", default=False, help=( - "run on use the full file list when trunk, " - "else run on fcm branch-diff" + "run on use the full file list when trunk, " "else run on fcm branch-diff" ), ) excl_group.add_argument( diff --git a/script_umdp3_checker/bin/umdp3_check.pl b/script_umdp3_checker/bin/umdp3_check.pl index 0a41347..70d6514 100755 --- a/script_umdp3_checker/bin/umdp3_check.pl +++ b/script_umdp3_checker/bin/umdp3_check.pl @@ -1010,13 +1010,13 @@ sub run_checks { } # read in data from file to $data, then - my $io_array = new IO::ScalarArray \@file_lines; + my $io_array = IO::ScalarArray->new(\@file_lines); my $mimetype = mimetype($io_array); # if we can't detect a mime type, try some tricks to aid detection if ( $mimetype =~ /text\/plain/sxm ) { my @mime_file_lines = grep !/^\s*\#/sxm, @file_lines; - $io_array = new IO::ScalarArray \@mime_file_lines; + $io_array = IO::ScalarArray->new(\@mime_file_lines); $mimetype = mimetype($io_array); } @@ -1133,12 +1133,12 @@ sub run_checks { $filename .= "."; } $filename = $log_cylc . "." . $filename . "report"; - my $fileres = open( FH, '>', $filename ); + my $fileres = open( my $fh, '>', $filename ); if ( !defined $fileres ) { die "ERR: $filename\n"; } - print FH $failure_text; - close(FH); + print $fh $failure_text; + close($fh); } } $message = ''; diff --git a/script_updater.sh b/script_updater.sh index 6c4de48..3889a05 100755 --- a/script_updater.sh +++ b/script_updater.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # *****************************COPYRIGHT******************************* # (C) Crown copyright Met Office. All rights reserved. @@ -6,40 +6,20 @@ # which you should have received as part of this distribution. # *****************************COPYRIGHT******************************* -#set -eu +# set -eu +# Note (yp): if this is a standalone script, we should use set -e +# and avoid the || { exit $?; } construct below. git_repos="SimSys_Scripts.git" github_url="git@github.com:MetOffice/$git_repos" -git_branch="add_umdp3checker_script" -#git_branch="main" +git_branch="main" clone_destination="${CYLC_SUITE_SHARE_DIR}/imported_github_scripts" +echo "Remove ${clone_destination} if it exists" +rm -rf "$clone_destination" 1>/dev/null || { exit $?; } -echo "Git-scripts updater has started running" - -rm -rf "${CYLC_SUITE_SHARE_DIR}/imported_github_scripts" -if [[ $? != 0 ]]; then - echo "Couldn't remove specified folder. Try checking permissions" - echo " \"$clone_destination\"" - exit 1 - else - echo "Successfully removed old ${git_repos%.git} git directory" - echo " \"$clone_destination\"" -fi - -# Mildly circular dependency here... The name of the branch needs changing - -# once ths branch has been committed.. See commented out 'branch' name above. -git clone -b $git_branch --single-branch $github_url "$clone_destination" 2>&1 -if [[ $? != 0 ]]; then - echo -e "\nUnable to clone remote git repo into specified location." - echo " Check git branch, git url, git access as well as" - echo -e " destination path, and permissions\n" - echo "GitHub url = \"$github_url\"" - echo "git branch = \"$git_branch\"" - echo "destination path = \"$clone_destination\"" - exit 1 - else - echo -e "\nGit repo successfully cloned\n" -fi - -echo -e "\nGithub scripts updated\n" +echo "Update ${git_repos%.git} in ${clone_destination}" +set -x +git clone -q -b $git_branch --single-branch $github_url "$clone_destination" \ + || { exit $?; } +{ set +x; } 2>/dev/null diff --git a/styling.py b/styling.py index 071ae3b..6aff3a3 100755 --- a/styling.py +++ b/styling.py @@ -4,7 +4,7 @@ # For further details please refer to the file LICENSE # which you should have received as part of this distribution. # *****************************COPYRIGHT******************************* -''' +""" ## NOTE ## This module is one of several for which the Master copy is in the UM repository. When making changes, please ensure the changes are made in the UM @@ -13,71 +13,76 @@ This module contains various functions for applying UMDP3 styling to Fortran source code -''' +""" import re import sys + from fstring_parse import * CODE_REPLACEMENTS = [ # Replace Fortran 77 style conditional keywords - (r'\.eq\.', ' == '), - (r'\.ne\.', ' /= '), - (r'\.gt\.', ' > '), - (r'\.lt\.', ' < '), - (r'\.ge\.', ' >= '), - (r'\.le\.', ' <= '), + (r"\.eq\.", " == "), + (r"\.ne\.", " /= "), + (r"\.gt\.", " > "), + (r"\.lt\.", " < "), + (r"\.ge\.", " >= "), + (r"\.le\.", " <= "), # protect 'operator' definitions e.g. "OPERATOR(/)", from the array # initiialisation enforcement by enforcing spaces around the operators. # Make all operators follow the same style but only (/) and (/=) had issues. - (r'\(\s*\*\s*\)', '( * )'), - (r'\(\s*\+\s*\)', '( + )'), - (r'\(\s*-\s*\)', '( - )'), - (r'\(\s*\/\s*\)', '( / )'), - (r'\(\s*==\s*\)', '( == )'), - (r'\(\s*\/=\s*\)', '( /= )'), - (r'\(\s*<\s*\)', '( < )'), - (r'\(\s*<=\s*\)', '( <= )'), - (r'\(\s*>\s*\)', '( > )'), - (r'\(\s*>=\s*\)', '( >= )'), + (r"\(\s*\*\s*\)", "( * )"), + (r"\(\s*\+\s*\)", "( + )"), + (r"\(\s*-\s*\)", "( - )"), + (r"\(\s*\/\s*\)", "( / )"), + (r"\(\s*==\s*\)", "( == )"), + (r"\(\s*\/=\s*\)", "( /= )"), + (r"\(\s*<\s*\)", "( < )"), + (r"\(\s*<=\s*\)", "( <= )"), + (r"\(\s*>\s*\)", "( > )"), + (r"\(\s*>=\s*\)", "( >= )"), # Replace array initialisations - (r'\(\/', '['), - (r'\/\)', ']'), + (r"\(\/", "["), + (r"\/\)", "]"), # Ensure remaining comparitive logicals have spaces either side - (r'([^\s])(? .not.'), - (r'\.not\.([^\s])', r'.not. \g<1>'), - (r'([^\s])\.and\.', r'\g<1> .and.'), - (r'\.and\.([^\s])', r'.and. \g<1>'), - (r'([^\s])\.or\.', r'\g<1> .or.'), - (r'\.or\.([^\s])', r'.or. \g<1>'), - (r'([^\s])\.eqv\.', r'\g<1> .eqv.'), - (r'\.eqv\.([^\s])', r'.eqv. \g<1>'), - (r'([^\s])\.neqv\.', r'\g<1> .neqv.'), - (r'\.neqv\.([^\s])', r'.neqv. \g<1>'), + (r"([^\s])(? .not."), + (r"\.not\.([^\s])", r".not. \g<1>"), + (r"([^\s])\.and\.", r"\g<1> .and."), + (r"\.and\.([^\s])", r".and. \g<1>"), + (r"([^\s])\.or\.", r"\g<1> .or."), + (r"\.or\.([^\s])", r".or. \g<1>"), + (r"([^\s])\.eqv\.", r"\g<1> .eqv."), + (r"\.eqv\.([^\s])", r".eqv. \g<1>"), + (r"([^\s])\.neqv\.", r"\g<1> .neqv."), + (r"\.neqv\.([^\s])", r".neqv. \g<1>"), # Ensure hard-coded real numbers have a zero after the decimal point - (r'([0-9])\.([^0-9]|$)', r'\g<1>.0\g<2>'), + (r"([0-9])\.([^0-9]|$)", r"\g<1>.0\g<2>"), # Remove start of line ampersands, without changing the spacing of # the line they appear on - (r'^(\s*)&(.*\w.*)$', r'\g<1> \g<2>'), + (r"^(\s*)&(.*\w.*)$", r"\g<1> \g<2>"), # Make constructs which include brackets have exactly one space # between the construct and the bracket character - (r'(^\s*)(\w+\s*:\s*|[0-9]+\s*|)((else|)\s*if)(|\s\s+)\(', - r'\g<1>\g<2>\g<3> ('), - (r'(^\s*)(\w+\s*:\s*|[0-9]+\s*|)where(|\s\s+)\(', r'\g<1>\g<2>where ('), - (r'(^\s*)case(|\s\s+)\(', r'\g<1>case ('), - (r'\)(|\s\s+)then(\W|$)', r') then\g<1>'), + ( + r"(^\s*)(\w+\s*:\s*|[0-9]+\s*|)((else|)\s*if)(|\s\s+)\(", + r"\g<1>\g<2>\g<3> (", + ), + (r"(^\s*)(\w+\s*:\s*|[0-9]+\s*|)where(|\s\s+)\(", r"\g<1>\g<2>where ("), + (r"(^\s*)case(|\s\s+)\(", r"\g<1>case ("), + (r"\)(|\s\s+)then(\W|$)", r") then\g<1>"), # Make intent statements contain no extra spacing inside the brackets - (r'(.*intent\s*)\(\s*in\s*\)(.*)', r'\g<1>(in)\g<2>'), - (r'(.*intent\s*)\(\s*out\s*\)(.*)', r'\g<1>(out)\g<2>'), - (r'(.*intent\s*)\(\s*in out\s*\)(.*)', r'\g<1>(in out)\g<2>'), + (r"(.*intent\s*)\(\s*in\s*\)(.*)", r"\g<1>(in)\g<2>"), + (r"(.*intent\s*)\(\s*out\s*\)(.*)", r"\g<1>(out)\g<2>"), + (r"(.*intent\s*)\(\s*in out\s*\)(.*)", r"\g<1>(in out)\g<2>"), # Make module USE, ONLY statments have exactly no space between the ONLY # and the colon character after it - (r'^(\s*)use(\s*,\s*\w+\s*::|)(\s+\w+\s*,\s*)only\s*:(.*)$', - r'\g<1>use\g<2>\g<3>only:\g<4>'), + ( + r"^(\s*)use(\s*,\s*\w+\s*::|)(\s+\w+\s*,\s*)only\s*:(.*)$", + r"\g<1>use\g<2>\g<3>only:\g<4>", + ), ] COMMENT_REPLACEMENTS = [ # DEPENDS ON fcm constructions - (r'^(\s*!)\s*depends\s*on\s*:\s*', r'\g<1> DEPENDS ON: '), + (r"^(\s*!)\s*depends\s*on\s*:\s*", r"\g<1> DEPENDS ON: "), ] FORTRAN_TYPES = [ @@ -92,690 +97,692 @@ "TYPE", ] -KEYWORDS = set([ - "abort", - "abs", - "abstract", - "access", - "achar", - "acos", - "acosd", - "acosh", - "action", - "adjustl", - "adjustr", - "advance", - "aimag", - "aint", - "alarm", - "algama", - "all", - "allocatable", - "allocate", - "allocated", - "alog", - "alog10", - "amax0", - "amax1", - "amin0", - "amin1", - "amod", - "and", - "anint", - "any", - "asin", - "asind", - "asinh", - "assign", - "assignment", - "associate", - "associated", - "asynchronous", - "atan", - "atan2", - "atan2d", - "atand", - "atanh", - "atomic", - "atomic_add", - "atomic_and", - "atomic_cas", - "atomic_define", - "atomic_fetch_add", - "atomic_fetch_and", - "atomic_fetch_or", - "atomic_fetch_xor", - "atomic_int_kind", - "atomic_logical_kind", - "atomic_or", - "atomic_ref", - "atomic_xor", - "backspace", - "backtrace", - "barrier", - "besj0", - "besj1", - "besjn", - "bessel_j0", - "bessel_j1", - "bessel_jn", - "bessel_y0", - "bessel_y1", - "bessel_yn", - "besy0", - "besy1", - "besyn", - "bge", - "bgt", - "bind", - "bit_size", - "blank", - "ble", - "block", - "blt", - "btest", - "c_alert", - "c_associated", - "c_backspace", - "c_bool", - "c_carriage_return", - "c_char", - "c_double", - "c_double_complex", - "c_f_pointer", - "c_f_procpointer", - "c_float", - "c_float128", - "c_float128_complex", - "c_float_complex", - "c_form_feed", - "c_funloc", - "c_funptr", - "c_horizontal_tab", - "c_int", - "c_int128_t", - "c_int16_t", - "c_int32_t", - "c_int64_t", - "c_int8_t", - "c_int_fast128_t", - "c_int_fast16_t", - "c_int_fast32_t", - "c_int_fast64_t", - "c_int_fast8_t", - "c_int_least128_t", - "c_int_least16_t", - "c_int_least32_t", - "c_int_least64_t", - "c_int_least8_t", - "c_intmax_t", - "c_intptr_t", - "c_loc", - "c_long", - "c_long_double", - "c_long_double_complex", - "c_long_long", - "c_new_line", - "c_null_char", - "c_null_funptr", - "c_null_ptr", - "c_ptr", - "c_ptrdiff_t", - "c_short", - "c_signed_char", - "c_size_t", - "c_sizeof", - "c_vertical_tab", - "cabs", - "call", - "case", - "ccos", - "cdabs", - "cdcos", - "cdexp", - "cdlog", - "cdsin", - "cdsqrt", - "ceiling", - "cexp", - "char", - "character", - "character_kinds", - "character_storage_size", - "chdir", - "chmod", - "class", - "clog", - "close", - "cmplx", - "co_broadcast", - "co_max", - "co_min", - "co_reduce", - "co_sum", - "codimension", - "command_argument_count", - "common", - "compiler_options", - "compiler_version", - "complex", - "concurrent", - "conjg", - "contains", - "contiguous", - "continue", - "convert", - "copyin", - "copyprivate", - "cos", - "cosd", - "cosh", - "cotan", - "cotand", - "count", - "cpp", - "cpu_time", - "cqabs", - "cqcos", - "cqexp", - "cqlog", - "cqsin", - "cqsqrt", - "critical", - "cshift", - "csin", - "csqrt", - "ctime", - "cycle", - "dabs", - "dacos", - "dacosh", - "dasin", - "dasinh", - "data", - "datan", - "datan2", - "datanh", - "date_and_time", - "dbesj0", - "dbesj1", - "dbesjn", - "dbesy0", - "dbesy1", - "dbesyn", - "dble", - "dcmplx", - "dconjg", - "dcos", - "dcosh", - "ddim", - "deallocate", - "decode", - "default", - "deferred", - "delim", - "derf", - "derfc", - "dexp", - "dfloat", - "dgamma", - "digits", - "dim", - "dimag", - "dimension", - "dint", - "direct", - "dlgama", - "dlog", - "dlog10", - "dmax1", - "dmin1", - "dmod", - "dnint", - "do", - "dot_product", - "double", - "dprod", - "dreal", - "dshiftl", - "dshiftr", - "dsign", - "dsin", - "dsinh", - "dsqrt", - "dtan", - "dtanh", - "dtime", - "elemental", - "else", - "elsewhere", - "encode", - "end", - "endfile", - "entry", - "enum", - "enumerator", - "eor", - "eoshift", - "epsilon", - "equivalence", - "eqv", - "erf", - "erfc", - "erfc_scaled", - "errmsg", - "error", - "error_unit", - "etime", - "event_query", - "execute_command_line", - "exist", - "exit", - "exp", - "exponent", - "extends", - "extends_type_of", - "external", - "false", - "fdate", - "fget", - "fgetc", - "file", - "file_storage_size", - "final", - "firstprivate", - "float", - "floor", - "flush", - "fmt", - "fnum", - "forall", - "form", - "format", - "formatted", - "fpp", - "fput", - "fputc", - "fraction", - "free", - "fseek", - "fstat", - "ftell", - "function", - "gamma", - "generic", - "gerror", - "get_command", - "get_command_argument", - "get_environment_variable", - "getarg", - "getcwd", - "getenv", - "getgid", - "getlog", - "getpid", - "getuid", - "gmtime", - "go", - "hostnm", - "huge", - "hypot", - "iabs", - "iachar", - "iall", - "iand", - "iany", - "iargc", - "ibclr", - "ibits", - "ibset", - "ichar", - "idate", - "idim", - "idint", - "idnint", - "ieee_class", - "ieee_class_type", - "ieee_copy_sign", - "ieee_is_finite", - "ieee_is_nan", - "ieee_is_negative", - "ieee_is_normal", - "ieee_logb", - "ieee_negative_denormal", - "ieee_negative_inf", - "ieee_negative_normal", - "ieee_negative_zero", - "ieee_next_after", - "ieee_positive_denormal", - "ieee_positive_inf", - "ieee_positive_normal", - "ieee_positive_zero", - "ieee_quiet_nan", - "ieee_rem", - "ieee_rint", - "ieee_scalb", - "ieee_selected_real_kind", - "ieee_signaling_nan", - "ieee_support_datatype", - "ieee_support_denormal", - "ieee_support_divide", - "ieee_support_inf", - "ieee_support_nan", - "ieee_support_sqrt", - "ieee_support_standard", - "ieee_unordered", - "ieee_value", - "ieor", - "ierrno", - "if", - "ifix", - "imag", - "image_index", - "images", - "imagpart", - "implicit", - "import", - "in", - "include", - "index", - "inout", - "input_unit", - "inquire", - "int", - "int16", - "int2", - "int32", - "int64", - "int8", - "integer", - "integer_kinds", - "intent", - "interface", - "intrinsic", - "iomsg", - "ior", - "iostat", - "iostat_end", - "iostat_eor", - "iostat_inquire_internal_unit", - "iparity", - "iqint", - "irand", - "is", - "is_iostat_end", - "is_iostat_eor", - "isatty", - "ishft", - "ishftc", - "isign", - "isnan", - "iso_c_binding", - "iso_fortran_env", - "itime", - "kill", - "kind", - "lastprivate", - "lbound", - "lcobound", - "leadz", - "len", - "len_trim", - "lgamma", - "lge", - "lgt", - "link", - "lle", - "llt", - "lnblnk", - "loc", - "lock", - "lock_type", - "log", - "log10", - "log_gamma", - "logical", - "logical_kinds", - "long", - "lshift", - "lstat", - "ltime", - "malloc", - "maskl", - "maskr", - "master", - "matmul", - "max", - "max0", - "max1", - "maxexponent", - "maxloc", - "maxval", - "mclock", - "mclock8", - "memory", - "merge", - "merge_bits", - "min", - "min0", - "min1", - "minexponent", - "minloc", - "minval", - "mod", - "module", - "modulo", - "move_alloc", - "mvbits", - "name", - "named", - "namelist", - "nearest", - "neqv", - "new_line", - "nextrec", - "nint", - "nml", - "non_intrinsic", - "non_overridable", - "none", - "nopass", - "norm2", - "not", - "null", - "nullify", - "num_images", - "number", - "numeric_storage_size", - "only", - "open", - "opened", - "operator", - "optional", - "or", - "ordered", - "out", - "output_unit", - "pack", - "pad", - "parallel", - "parameter", - "parity", - "pass", - "perror", - "pointer", - "popcnt", - "poppar", - "position", - "precision", - "present", - "print", - "private", - "procedure", - "product", - "program", - "protected", - "public", - "pure", - "qabs", - "qacos", - "qasin", - "qatan", - "qatan2", - "qcmplx", - "qconjg", - "qcos", - "qcosh", - "qdim", - "qerf", - "qerfc", - "qexp", - "qgamma", - "qimag", - "qlgama", - "qlog", - "qlog10", - "qmax1", - "qmin1", - "qmod", - "qnint", - "qsign", - "qsin", - "qsinh", - "qsqrt", - "qtan", - "qtanh", - "radix", - "ran", - "rand", - "random_number", - "random_seed", - "range", - "rank", - "read", - "readwrite", - "real", - "real128", - "real32", - "real64", - "real_kinds", - "realpart", - "rec", - "recl", - "record", - "recursive", - "reduction", - "rename", - "repeat", - "reshape", - "result", - "return", - "rewind", - "rewrite", - "rrspacing", - "rshift", - "same_type_as", - "save", - "scale", - "scan", - "secnds", - "second", - "sections", - "select", - "selected_char_kind", - "selected_int_kind", - "selected_real_kind", - "sequence", - "sequential", - "set_exponent", - "shape", - "shared", - "shifta", - "shiftl", - "shiftr", - "short", - "sign", - "signal", - "sin", - "sind", - "sinh", - "size", - "sizeof", - "sleep", - "sngl", - "source", - "spacing", - "spread", - "sqrt", - "srand", - "stat", - "stat_failed_image", - "stat_locked", - "stat_locked_other_image", - "stat_stopped_image", - "stat_unlocked", - "status", - "stop", - "storage_size", - "structure", - "submodule", - "subroutine", - "sum", - "symlnk", - "sync", - "system", - "system_clock", - "tan", - "tand", - "tanh", - "target", - "task", - "taskwait", - "then", - "this_image", - "threadprivate", - "time", - "time8", - "tiny", - "to", - "trailz", - "transfer", - "transpose", - "trim", - "true", - "ttynam", - "type", - "ubound", - "ucobound", - "umask", - "unformatted", - "unit", - "unlink", - "unlock", - "unpack", - "use", - "value", - "verif", - "verify", - "volatile", - "wait", - "where", - "while", - "workshare", - "write", - "xor", - "zabs", - "zcos", - "zexp", - "zlog", - "zsin", - "zsqrt", - ]) +KEYWORDS = set( + [ + "abort", + "abs", + "abstract", + "access", + "achar", + "acos", + "acosd", + "acosh", + "action", + "adjustl", + "adjustr", + "advance", + "aimag", + "aint", + "alarm", + "algama", + "all", + "allocatable", + "allocate", + "allocated", + "alog", + "alog10", + "amax0", + "amax1", + "amin0", + "amin1", + "amod", + "and", + "anint", + "any", + "asin", + "asind", + "asinh", + "assign", + "assignment", + "associate", + "associated", + "asynchronous", + "atan", + "atan2", + "atan2d", + "atand", + "atanh", + "atomic", + "atomic_add", + "atomic_and", + "atomic_cas", + "atomic_define", + "atomic_fetch_add", + "atomic_fetch_and", + "atomic_fetch_or", + "atomic_fetch_xor", + "atomic_int_kind", + "atomic_logical_kind", + "atomic_or", + "atomic_ref", + "atomic_xor", + "backspace", + "backtrace", + "barrier", + "besj0", + "besj1", + "besjn", + "bessel_j0", + "bessel_j1", + "bessel_jn", + "bessel_y0", + "bessel_y1", + "bessel_yn", + "besy0", + "besy1", + "besyn", + "bge", + "bgt", + "bind", + "bit_size", + "blank", + "ble", + "block", + "blt", + "btest", + "c_alert", + "c_associated", + "c_backspace", + "c_bool", + "c_carriage_return", + "c_char", + "c_double", + "c_double_complex", + "c_f_pointer", + "c_f_procpointer", + "c_float", + "c_float128", + "c_float128_complex", + "c_float_complex", + "c_form_feed", + "c_funloc", + "c_funptr", + "c_horizontal_tab", + "c_int", + "c_int128_t", + "c_int16_t", + "c_int32_t", + "c_int64_t", + "c_int8_t", + "c_int_fast128_t", + "c_int_fast16_t", + "c_int_fast32_t", + "c_int_fast64_t", + "c_int_fast8_t", + "c_int_least128_t", + "c_int_least16_t", + "c_int_least32_t", + "c_int_least64_t", + "c_int_least8_t", + "c_intmax_t", + "c_intptr_t", + "c_loc", + "c_long", + "c_long_double", + "c_long_double_complex", + "c_long_long", + "c_new_line", + "c_null_char", + "c_null_funptr", + "c_null_ptr", + "c_ptr", + "c_ptrdiff_t", + "c_short", + "c_signed_char", + "c_size_t", + "c_sizeof", + "c_vertical_tab", + "cabs", + "call", + "case", + "ccos", + "cdabs", + "cdcos", + "cdexp", + "cdlog", + "cdsin", + "cdsqrt", + "ceiling", + "cexp", + "char", + "character", + "character_kinds", + "character_storage_size", + "chdir", + "chmod", + "class", + "clog", + "close", + "cmplx", + "co_broadcast", + "co_max", + "co_min", + "co_reduce", + "co_sum", + "codimension", + "command_argument_count", + "common", + "compiler_options", + "compiler_version", + "complex", + "concurrent", + "conjg", + "contains", + "contiguous", + "continue", + "convert", + "copyin", + "copyprivate", + "cos", + "cosd", + "cosh", + "cotan", + "cotand", + "count", + "cpp", + "cpu_time", + "cqabs", + "cqcos", + "cqexp", + "cqlog", + "cqsin", + "cqsqrt", + "critical", + "cshift", + "csin", + "csqrt", + "ctime", + "cycle", + "dabs", + "dacos", + "dacosh", + "dasin", + "dasinh", + "data", + "datan", + "datan2", + "datanh", + "date_and_time", + "dbesj0", + "dbesj1", + "dbesjn", + "dbesy0", + "dbesy1", + "dbesyn", + "dble", + "dcmplx", + "dconjg", + "dcos", + "dcosh", + "ddim", + "deallocate", + "decode", + "default", + "deferred", + "delim", + "derf", + "derfc", + "dexp", + "dfloat", + "dgamma", + "digits", + "dim", + "dimag", + "dimension", + "dint", + "direct", + "dlgama", + "dlog", + "dlog10", + "dmax1", + "dmin1", + "dmod", + "dnint", + "do", + "dot_product", + "double", + "dprod", + "dreal", + "dshiftl", + "dshiftr", + "dsign", + "dsin", + "dsinh", + "dsqrt", + "dtan", + "dtanh", + "dtime", + "elemental", + "else", + "elsewhere", + "encode", + "end", + "endfile", + "entry", + "enum", + "enumerator", + "eor", + "eoshift", + "epsilon", + "equivalence", + "eqv", + "erf", + "erfc", + "erfc_scaled", + "errmsg", + "error", + "error_unit", + "etime", + "event_query", + "execute_command_line", + "exist", + "exit", + "exp", + "exponent", + "extends", + "extends_type_of", + "external", + "false", + "fdate", + "fget", + "fgetc", + "file", + "file_storage_size", + "final", + "firstprivate", + "float", + "floor", + "flush", + "fmt", + "fnum", + "forall", + "form", + "format", + "formatted", + "fpp", + "fput", + "fputc", + "fraction", + "free", + "fseek", + "fstat", + "ftell", + "function", + "gamma", + "generic", + "gerror", + "get_command", + "get_command_argument", + "get_environment_variable", + "getarg", + "getcwd", + "getenv", + "getgid", + "getlog", + "getpid", + "getuid", + "gmtime", + "go", + "hostnm", + "huge", + "hypot", + "iabs", + "iachar", + "iall", + "iand", + "iany", + "iargc", + "ibclr", + "ibits", + "ibset", + "ichar", + "idate", + "idim", + "idint", + "idnint", + "ieee_class", + "ieee_class_type", + "ieee_copy_sign", + "ieee_is_finite", + "ieee_is_nan", + "ieee_is_negative", + "ieee_is_normal", + "ieee_logb", + "ieee_negative_denormal", + "ieee_negative_inf", + "ieee_negative_normal", + "ieee_negative_zero", + "ieee_next_after", + "ieee_positive_denormal", + "ieee_positive_inf", + "ieee_positive_normal", + "ieee_positive_zero", + "ieee_quiet_nan", + "ieee_rem", + "ieee_rint", + "ieee_scalb", + "ieee_selected_real_kind", + "ieee_signaling_nan", + "ieee_support_datatype", + "ieee_support_denormal", + "ieee_support_divide", + "ieee_support_inf", + "ieee_support_nan", + "ieee_support_sqrt", + "ieee_support_standard", + "ieee_unordered", + "ieee_value", + "ieor", + "ierrno", + "if", + "ifix", + "imag", + "image_index", + "images", + "imagpart", + "implicit", + "import", + "in", + "include", + "index", + "inout", + "input_unit", + "inquire", + "int", + "int16", + "int2", + "int32", + "int64", + "int8", + "integer", + "integer_kinds", + "intent", + "interface", + "intrinsic", + "iomsg", + "ior", + "iostat", + "iostat_end", + "iostat_eor", + "iostat_inquire_internal_unit", + "iparity", + "iqint", + "irand", + "is", + "is_iostat_end", + "is_iostat_eor", + "isatty", + "ishft", + "ishftc", + "isign", + "isnan", + "iso_c_binding", + "iso_fortran_env", + "itime", + "kill", + "kind", + "lastprivate", + "lbound", + "lcobound", + "leadz", + "len", + "len_trim", + "lgamma", + "lge", + "lgt", + "link", + "lle", + "llt", + "lnblnk", + "loc", + "lock", + "lock_type", + "log", + "log10", + "log_gamma", + "logical", + "logical_kinds", + "long", + "lshift", + "lstat", + "ltime", + "malloc", + "maskl", + "maskr", + "master", + "matmul", + "max", + "max0", + "max1", + "maxexponent", + "maxloc", + "maxval", + "mclock", + "mclock8", + "memory", + "merge", + "merge_bits", + "min", + "min0", + "min1", + "minexponent", + "minloc", + "minval", + "mod", + "module", + "modulo", + "move_alloc", + "mvbits", + "name", + "named", + "namelist", + "nearest", + "neqv", + "new_line", + "nextrec", + "nint", + "nml", + "non_intrinsic", + "non_overridable", + "none", + "nopass", + "norm2", + "not", + "null", + "nullify", + "num_images", + "number", + "numeric_storage_size", + "only", + "open", + "opened", + "operator", + "optional", + "or", + "ordered", + "out", + "output_unit", + "pack", + "pad", + "parallel", + "parameter", + "parity", + "pass", + "perror", + "pointer", + "popcnt", + "poppar", + "position", + "precision", + "present", + "print", + "private", + "procedure", + "product", + "program", + "protected", + "public", + "pure", + "qabs", + "qacos", + "qasin", + "qatan", + "qatan2", + "qcmplx", + "qconjg", + "qcos", + "qcosh", + "qdim", + "qerf", + "qerfc", + "qexp", + "qgamma", + "qimag", + "qlgama", + "qlog", + "qlog10", + "qmax1", + "qmin1", + "qmod", + "qnint", + "qsign", + "qsin", + "qsinh", + "qsqrt", + "qtan", + "qtanh", + "radix", + "ran", + "rand", + "random_number", + "random_seed", + "range", + "rank", + "read", + "readwrite", + "real", + "real128", + "real32", + "real64", + "real_kinds", + "realpart", + "rec", + "recl", + "record", + "recursive", + "reduction", + "rename", + "repeat", + "reshape", + "result", + "return", + "rewind", + "rewrite", + "rrspacing", + "rshift", + "same_type_as", + "save", + "scale", + "scan", + "secnds", + "second", + "sections", + "select", + "selected_char_kind", + "selected_int_kind", + "selected_real_kind", + "sequence", + "sequential", + "set_exponent", + "shape", + "shared", + "shifta", + "shiftl", + "shiftr", + "short", + "sign", + "signal", + "sin", + "sind", + "sinh", + "size", + "sizeof", + "sleep", + "sngl", + "source", + "spacing", + "spread", + "sqrt", + "srand", + "stat", + "stat_failed_image", + "stat_locked", + "stat_locked_other_image", + "stat_stopped_image", + "stat_unlocked", + "status", + "stop", + "storage_size", + "structure", + "submodule", + "subroutine", + "sum", + "symlnk", + "sync", + "system", + "system_clock", + "tan", + "tand", + "tanh", + "target", + "task", + "taskwait", + "then", + "this_image", + "threadprivate", + "time", + "time8", + "tiny", + "to", + "trailz", + "transfer", + "transpose", + "trim", + "true", + "ttynam", + "type", + "ubound", + "ucobound", + "umask", + "unformatted", + "unit", + "unlink", + "unlock", + "unpack", + "use", + "value", + "verif", + "verify", + "volatile", + "wait", + "where", + "while", + "workshare", + "write", + "xor", + "zabs", + "zcos", + "zexp", + "zlog", + "zsin", + "zsqrt", + ] +) def replace_patterns(line, str_continuation): - '''Replace various patterns according to the styling guidelines on - the provided line, returning the result''' + """Replace various patterns according to the styling guidelines on + the provided line, returning the result""" if len(line.strip()) == 0: return line @@ -785,11 +792,11 @@ def replace_patterns(line, str_continuation): stripline = workline.strip() # Pre-processor lines start with #. Ignore them completely. - pre_proc = (stripline[0] == "#") + pre_proc = stripline[0] == "#" # Lines that are completely commented start with a bang and are also # ignored completely. - all_comment = (stripline[0] == "!") + all_comment = stripline[0] == "!" if pre_proc or all_comment: return line @@ -815,14 +822,14 @@ def replace_patterns(line, str_continuation): for pattern, replacement in CODE_REPLACEMENTS: m = re.search(pattern, match_line, flags=re.IGNORECASE) if m: - for n in range(len(m.groups())+1): - replacement = re.sub(r'(\\{0:s}|\\g<{0:s}>)'.format(str(n)), - line[m.start(n):m.end(n)], - replacement) + for n in range(len(m.groups()) + 1): + replacement = re.sub( + r"(\\{0:s}|\\g<{0:s}>)".format(str(n)), + line[m.start(n) : m.end(n)], + replacement, + ) - line = "".join([line[:m.start(0)], - replacement, - line[m.end(0):]]) + line = "".join([line[: m.start(0)], replacement, line[m.end(0) :]]) workline = clean_str_continuation(line, str_continuation) @@ -836,8 +843,7 @@ def replace_patterns(line, str_continuation): blank_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:" \ - "".format(e.msg)) + print("{0:s} Line simplification has failed for:" "".format(e.msg)) print(line) exit(1) @@ -848,8 +854,8 @@ def replace_patterns(line, str_continuation): def replace_comment_patterns(line, str_continuation): - '''Replace various patterns according to the styling guidelines on - the provided line, returning the result''' + """Replace various patterns according to the styling guidelines on + the provided line, returning the result""" if len(line.strip()) == 0: return line @@ -859,7 +865,7 @@ def replace_comment_patterns(line, str_continuation): stripline = workline.strip() # Pre-processor lines start with #. Ignore them completely. - pre_proc = (stripline[0] == "#") + pre_proc = stripline[0] == "#" if pre_proc: return line @@ -882,14 +888,14 @@ def replace_comment_patterns(line, str_continuation): for pattern, replacement in COMMENT_REPLACEMENTS: m = re.search(pattern, match_line, flags=re.IGNORECASE) if m: - for n in range(len(m.groups())+1): - replacement = re.sub(r'(\\{0:s}|\\g<{0:s}>)'.format(str(n)), - line[m.start(n):m.end(n)], - replacement) + for n in range(len(m.groups()) + 1): + replacement = re.sub( + r"(\\{0:s}|\\g<{0:s}>)".format(str(n)), + line[m.start(n) : m.end(n)], + replacement, + ) - line = "".join([line[:m.start(0)], - replacement, - line[m.end(0):]]) + line = "".join([line[: m.start(0)], replacement, line[m.end(0) :]]) workline = clean_str_continuation(line, str_continuation) @@ -903,8 +909,7 @@ def replace_comment_patterns(line, str_continuation): match_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:" \ - "".format(e.msg)) + print("{0:s} Line simplification has failed for:" "".format(e.msg)) print(line) exit(1) @@ -912,8 +917,8 @@ def replace_comment_patterns(line, str_continuation): def upcase_keywords(line, str_continuation): - '''Upper-case any Fortran keywords on the given line, and down-case any - all capital words which aren't keywords, returning the result''' + """Upper-case any Fortran keywords on the given line, and down-case any + all capital words which aren't keywords, returning the result""" workline = clean_str_continuation(line, str_continuation) @@ -945,20 +950,20 @@ def upcase_keywords(line, str_continuation): for word in line_words: # Exclude special "__FILE__" or "__LINE__" directives - if (word.isupper() and - not re.match(r"__\w+__", word)): - recomp = re.compile(r'(^|\b){0:s}(\b|$)'.format(word)) - simple_line = recomp.sub(r'\g<1>{0:s}' - r'\g<2>'.format(word.lower()), - simple_line) + if word.isupper() and not re.match(r"__\w+__", word): + recomp = re.compile(r"(^|\b){0:s}(\b|$)".format(word)) + simple_line = recomp.sub( + r"\g<1>{0:s}" r"\g<2>".format(word.lower()), simple_line + ) line_words = set([word.lower() for word in line_words]) words_to_upcase = list(line_words.intersection(KEYWORDS)) line = list(line) for keyword in words_to_upcase: - recomp = re.compile(r'(?i)(^|\b){0:s}(\b|$)'.format(keyword)) - simple_line = recomp.sub(r'\g<1>{0:s}\g<2>'.format(keyword.upper()), - simple_line) + recomp = re.compile(r"(?i)(^|\b){0:s}(\b|$)".format(keyword)) + simple_line = recomp.sub( + r"\g<1>{0:s}\g<2>".format(keyword.upper()), simple_line + ) # Now add back any comments/strings simple_line = list(simple_line) @@ -977,10 +982,10 @@ def upcase_keywords(line, str_continuation): def declaration_double_colon(iline, lines, pp_line_previous, line_previous): - ''' + """ Attempt to add the double colon to definition lines which do not already have it - ''' + """ line = lines[iline] @@ -990,8 +995,9 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): found_dec_type = None for declare_type in FORTRAN_TYPES: - if re.search(r"^\s*{0:s}\W".format(declare_type), - workline, flags=re.IGNORECASE): + if re.search( + r"^\s*{0:s}\W".format(declare_type), workline, flags=re.IGNORECASE + ): found_dec_type = declare_type break @@ -1002,8 +1008,7 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): # Pre-process the line to pull in any continuation lines simple_line = simplify_line(xlines) - if not re.search(r"\s+FUNCTION(,|\s|\()", - simple_line, flags=re.IGNORECASE): + if not re.search(r"\s+FUNCTION(,|\s|\()", simple_line, flags=re.IGNORECASE): # The presence of declaration attributes (ALLOCATABLE, # PUBLIC, POINTER, etc) are only valid when used with # the double-colon. Therefore after allowing for the @@ -1011,8 +1016,10 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): # older "*INT" declaration the first character should # not be a comma search = re.search( - r"^(\s*{0:s}\s*?(\(.*?\)|\*\s*[0-9]+|))\s+(\w+)".format( - found_dec_type), simple_line, flags=re.IGNORECASE) + r"^(\s*{0:s}\s*?(\(.*?\)|\*\s*[0-9]+|))\s+(\w+)".format(found_dec_type), + simple_line, + flags=re.IGNORECASE, + ) if search: # avoid CLASS IS, TYPE IS and CLASS DEFAULT statements classtype = search.group(3).strip().upper() @@ -1029,24 +1036,28 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): # Attempt to fit the double-colon into an existing space to # preserve indentation, otherwise just add it to the line - line = re.sub(r"^{0:s}" - r"\s\s\s\s".format(re.escape(statement)), - r"{0:s} :: ".format(statement), - line, count=1) - line = re.sub(r"^{0:s}\s*" - r"((?".format( - statement), line, count=1) + line = re.sub( + r"^{0:s}" r"\s\s\s\s".format(re.escape(statement)), + r"{0:s} :: ".format(statement), + line, + count=1, + ) + line = re.sub( + r"^{0:s}\s*" + r"((?".format(statement), + line, + count=1, + ) return line def apply_styling(lines): - ''' + """ For a lit of lines apply UMDP3 styling to each line and return the result - ''' + """ output_lines = [] continuation = False @@ -1058,8 +1069,7 @@ def apply_styling(lines): line_previous = "" for iline, line in enumerate(lines): - line = declaration_double_colon(iline, lines, pp_line_previous, - line_previous) + line = declaration_double_colon(iline, lines, pp_line_previous, line_previous) if pp_continuation: if not pseudo_comment: @@ -1098,22 +1108,22 @@ def apply_styling(lines): # if we are a (pp) continuation, save the partial line if pp_continuation: - pp_line_previous = ''.join([re.sub(r"\\\s*$", "", - pp_line_previous), - re.sub(r"&\s*$", "", line_previous), - line]) + pp_line_previous = "".join( + [ + re.sub(r"\\\s*$", "", pp_line_previous), + re.sub(r"&\s*$", "", line_previous), + line, + ] + ) line_previous = "" pseudo_line = re.sub(r"\\\s*$", "&", pp_line_previous) - pseudo_str_continuation = is_str_continuation(pseudo_line, - str_continuation) + pseudo_str_continuation = is_str_continuation(pseudo_line, str_continuation) if not pseudo_comment: - pseudo_line = partial_blank_fstring(pseudo_line, - str_continuation) + pseudo_line = partial_blank_fstring(pseudo_line, str_continuation) if pseudo_line.strip()[0] == "#": pseudo_comment = True if pseudo_line.find("!") != -1: - pseudo_line = blank_fcomments(pseudo_line, - str_continuation) + pseudo_line = blank_fcomments(pseudo_line, str_continuation) if pseudo_line.find("!") == -1: pseudo_comment = True elif continuation: @@ -1126,10 +1136,10 @@ def apply_styling(lines): def main(): - '''Main toplevel function for testing''' + """Main toplevel function for testing""" input_file = sys.argv[-1] with open(input_file, "r+") as file_in: - print("Styling "+input_file) + print("Styling " + input_file) lines_in = file_in.read().split("\n") new_lines = apply_styling(lines_in) file_in.seek(0) @@ -1137,5 +1147,5 @@ def main(): file_in.truncate() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/suite_report.py b/suite_report.py index eab28b1..f5d179f 100755 --- a/suite_report.py +++ b/suite_report.py @@ -26,18 +26,18 @@ from __future__ import print_function import glob +import json import os import re import sqlite3 +import subprocess import sys -import traceback import time -import subprocess -import json -from optparse import OptionParser, OptionGroup +import traceback from collections import defaultdict -from fcm_bdiff import get_branch_diff_filenames +from optparse import OptionGroup, OptionParser +from fcm_bdiff import get_branch_diff_filenames CYLC_SUITE_ENV_FILE = "cylc-suite-env" PROCESSED_SUITE_RC = "suite.rc.processed" @@ -161,6 +161,7 @@ "Unknown": [], } + def _read_file(filename): """Takes filename (str) Return contents of a file, as list of strings.""" @@ -169,9 +170,7 @@ def _read_file(filename): lines = filehandle.readlines() else: print('[ERROR] Unable to find file :\n "{0:s}"'.format(filename)) - raise IOError( - '_read_file got invalid filename : "{0:s}"'.format(filename) - ) + raise IOError('_read_file got invalid filename : "{0:s}"'.format(filename)) return lines @@ -192,9 +191,7 @@ def _run_command(command, ignore_fail=False): Runs the command with subprocess.Popen. Returns the exit code, standard out and standard error as list. """ - pobj = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) + pobj = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pobj.wait() retcode, stdout, stderr = ( pobj.returncode, @@ -348,9 +345,7 @@ def __init__( status when generating the task table in the report. """ self.suite_path = os.path.abspath(suite_path) - self.is_cylc8 = os.path.isdir( - os.path.join(self.suite_path, "log", "config") - ) + self.is_cylc8 = os.path.isdir(os.path.join(self.suite_path, "log", "config")) self.log_path = log_path self.sort_by_name = sort_by_name @@ -377,8 +372,7 @@ def __init__( suite_dir, self.suitename = suitename.split("cylc-run/") # Default to userID from suite path unless CYLC_SUITE_OWNER is present self.suite_owner = os.environ.get( - "CYLC_SUITE_OWNER", - os.path.basename(suite_dir.rstrip("/")) + "CYLC_SUITE_OWNER", os.path.basename(suite_dir.rstrip("/")) ) self.parse_rose_suite_run() @@ -406,9 +400,7 @@ def __init__( invalid = [] for project in self.job_sources: proj_dict = self.job_sources[project] - proj_dict["tested source"] = _remove_quotes( - proj_dict["tested source"] - ) + proj_dict["tested source"] = _remove_quotes(proj_dict["tested source"]) if "repo loc" in proj_dict: proj_dict["repo loc"] = self.convert_to_srs( proj_dict["repo loc"], self.projects @@ -443,13 +435,9 @@ def __init__( if ":" in url and "@" not in url: revision = _get_current_head_revision(mirror_url, fcm_exec) proj_dict[location + " loc"] = url + "@" + revision - proj_dict[location + " mirror"] = ( - mirror_url + "@" + revision - ) + proj_dict[location + " mirror"] = mirror_url + "@" + revision proj_dict["repo link"] = self.generate_link(proj_dict["repo loc"]) - proj_dict["parent link"] = self.generate_link( - proj_dict["parent loc"] - ) + proj_dict["parent link"] = self.generate_link(proj_dict["parent loc"]) # If those attempts to generate links didn't work, try the hope # and guess approach. if proj_dict["repo link"] is None: @@ -495,10 +483,7 @@ def __init__( self.only_common_groups = True else: self.only_common_groups = all( - [ - group.strip() in COMMON_GROUPS[self.site] - for group in self.groups - ] + [group.strip() in COMMON_GROUPS[self.site] for group in self.groups] ) # Finally, remove any projects which were deemed invalid. @@ -525,14 +510,10 @@ def debug_print_obj(self): elif key == "verbosity": text = "Verbosity level is set to : " if value >= 4: - print( - text - + "Hide Housekeeping, Gatekeeping and Successful tasks" - ) + print(text + "Hide Housekeeping, Gatekeeping and Successful tasks") elif value >= 3: print( - text - + "Hide Housekeeping, Gatekeeping and if all groups run" + text + "Hide Housekeeping, Gatekeeping and if all groups run" 'were "common" groups also hide Successful tasks' ) elif value >= 2: @@ -545,9 +526,7 @@ def debug_print_obj(self): self.print_job_sources(value) else: print('{0:s} is :"{1:}"'.format(key, value)) - print( - "\n" + "-" * 80 + "\nEnd of SuiteReport object\n" + "-" * 80 + "\n" - ) + print("\n" + "-" * 80 + "\nEnd of SuiteReport object\n" + "-" * 80 + "\n") @staticmethod def print_job_sources(job_srcs_dict): @@ -589,8 +568,7 @@ def parse_processed_config_file(self): break else: sys.exit( - "Error: Couldn't find a *-rose-suite.conf file in " - + f"{srp_file}" + "Error: Couldn't find a *-rose-suite.conf file in " + f"{srp_file}" ) else: srp_file = os.path.join(suite_dir, PROCESSED_SUITE_RC) @@ -622,13 +600,11 @@ def parse_processed_config_file(self): sources[result.group(1)] = {} if " " in result.group(3): multiple_branches[(result.group(1))] = result.group(3) - sources[result.group(1)][ - "tested source" - ] = result.group(3).split()[0] + sources[result.group(1)]["tested source"] = result.group( + 3 + ).split()[0] else: - sources[result.group(1)][ - "tested source" - ] = result.group(3) + sources[result.group(1)]["tested source"] = result.group(3) self.rose_orig_host = rose_orig_host self.job_sources = sources @@ -706,11 +682,8 @@ def initialise_projects(self): # Check for keywords conforming to the meto prescribed pattern # of ending in '.x' for the external repo and '.xm' for the # local mirror. - if ( - find_x_keyword.search(project) and find_srs_url.match(url) - ) or ( - find_xm_keyword.search(project) - and find_mirror_url.match(url) + if (find_x_keyword.search(project) and find_srs_url.match(url)) or ( + find_xm_keyword.search(project) and find_mirror_url.match(url) ): projects[project] = url self.projects = projects @@ -931,9 +904,7 @@ def generate_owner_dictionary(self, mode): file_path = self.export_file("fcm:um.xm_tr", fname, exported_file) if file_path is None: # Couldn't check out file - use working copy Owners file instead - wc_path = get_working_copy_path( - self.job_sources["UM"]["tested source"] - ) + wc_path = get_working_copy_path(self.job_sources["UM"]["tested source"]) if not wc_path: wc_path = "" file_path = os.path.join(wc_path, fname) @@ -964,9 +935,7 @@ def generate_owner_dictionary(self, mode): others = "" except IndexError: others = "" - owners_dict.update( - {section.lower(): [owners, others]} - ) + owners_dict.update({section.lower(): [owners, others]}) except EnvironmentError: print("Can't find working copy for Owners File") return None @@ -997,9 +966,7 @@ def create_approval_table(self, needed_approvals, mode): if needed_approvals is None: table += [ - " |||||| No UM " - + mode.capitalize() - + " Owner Approvals Required || " + " |||||| No UM " + mode.capitalize() + " Owner Approvals Required || " ] else: for owner in needed_approvals.keys(): @@ -1152,7 +1119,10 @@ def get_code_owners(self, code_owners): # Couldn't check out working copy file - deleted? Use trunk instead file_path = self.export_file("fcm:um.xm_tr", fpath) if file_path is None: - print('[WARN] Unable to establish code section for file: ', fle) + print( + "[WARN] Unable to establish code section for file: ", + fle, + ) file_path = "" try: @@ -1314,9 +1284,7 @@ def check_lfric_extract_list(self): if extract_list_path: try: - extract_list_dict = self.parse_lfric_extract_list( - extract_list_path - ) + extract_list_dict = self.parse_lfric_extract_list(extract_list_path) except (EnvironmentError, TypeError, AttributeError): # Potential error here changed type between python2 and 3 extract_list_path = None @@ -1390,9 +1358,7 @@ def forced_status_sort(item_tuple): find_monitor = re.compile(r"monitor") find_gatekeeper = re.compile(r"gatekeeper") failed_configs = [] - for task, state in sorted( - list(data.items()), key=key_by_name_or_status - ): + for task, state in sorted(list(data.items()), key=key_by_name_or_status): # Count the number of times task have any given status. status_counts[state] += 1 if (verbosity >= 1) and find_housekeep.match(task): @@ -1419,9 +1385,7 @@ def forced_status_sort(item_tuple): # Check if task requires extra care for extra_care_string in HIGHLIGHT_ROSE_ANA_FAILS: if extra_care_string in task: - highlight_start = ( - "'''[[span(style=color: #FF00FF, *****" - ) + highlight_start = "'''[[span(style=color: #FF00FF, *****" highlight_end = "***** )]]'''" status_counts[PINK_FAIL_TEXT] += 1 status_counts[state] -= 1 @@ -1437,8 +1401,7 @@ def forced_status_sort(item_tuple): ) if len(lines) == 1: lines.append( - " |||| This table is deliberately empty as all tasks " - "are hidden || " + " |||| This table is deliberately empty as all tasks " "are hidden || " ) status_summary = ["'''Suite Output'''"] @@ -1446,18 +1409,12 @@ def forced_status_sort(item_tuple): status_summary += [""] status_summary.append(" |||| '''All Tasks''' || ") status_summary.append(" || '''Status''' || '''No. of Tasks''' || ") - for status, count in sorted( - status_counts.items(), key=forced_status_sort - ): - status_summary.append( - " || {0:s} || {1:6d} || ".format(status, count) - ) + for status, count in sorted(status_counts.items(), key=forced_status_sort): + status_summary.append(" || {0:s} || {1:6d} || ".format(status, count)) status_summary.append("") if len(hidden_counts) > 0: status_summary.append(" |||| '''Hidden Tasks''' || ") - status_summary.append( - " || '''Type''' || '''No. of Tasks Hidden''' || " - ) + status_summary.append(" || '''Type''' || '''No. of Tasks Hidden''' || ") for task_type, count in hidden_counts.items(): status_summary.append( " || {0:s} || {1:6d} || ".format(task_type, count) @@ -1477,9 +1434,7 @@ def forced_status_sort(item_tuple): co_approval_table = self.required_co_approvals() if co_approval_table: return_list += co_approval_table - config_approval_table = self.required_config_approvals( - failed_configs - ) + config_approval_table = self.required_config_approvals(failed_configs) if config_approval_table: return_list += config_approval_table return lfric_testing_message + return_list + status_summary + lines @@ -1503,9 +1458,7 @@ def convert_to_mirror(url, projects_dict): if new_proj in projects_dict: old_proj_url = proj_url new_proj_url = projects_dict[new_proj] - mirror_url = re.sub( - old_proj_url, new_proj_url, url, count=1 - ) + mirror_url = re.sub(old_proj_url, new_proj_url, url, count=1) break # checking given url against keywords in the projects_dict elif proj in url: @@ -1677,13 +1630,9 @@ def gen_table_element(text_list, link, bold=False): text = _select_preferred(text_list) highlight = "'''" if bold else "" if text is not None and link is not None: - element = " {2:s}[{0:s} {1:s}]{2:s} || ".format( - link, text, highlight - ) + element = " {2:s}[{0:s} {1:s}]{2:s} || ".format(link, text, highlight) elif text is not None: - element = " {1:s}{0:s}{1:s} || ".format( - _escape_svn(text), highlight - ) + element = " {1:s}{0:s}{1:s} || ".format(_escape_svn(text), highlight) else: element = " || " return element @@ -1748,9 +1697,7 @@ def gen_resources_table(self): ] found_nothing = False lines.append( - " || {0:s} || {1:} || {2:} || ".format( - job, wallclock, memory - ) + " || {0:s} || {1:} || {2:} || ".format(job, wallclock, memory) ) lines.append("") return lines @@ -1761,15 +1708,11 @@ def get_wallclock_and_memory(filename): and memory.""" wallclock = "Unavailable" memory = "Unavailable" - find_wallclock = re.compile( - r"PE\s*0\s*Elapsed Wallclock Time:\s*(\d+(\.\d+|))" - ) + find_wallclock = re.compile(r"PE\s*0\s*Elapsed Wallclock Time:\s*(\d+(\.\d+|))") find_total_mem = re.compile(r"Total Mem\s*(\d+)") find_um_atmos_exe = re.compile(r"um-atmos.exe") check_for_percentage = re.compile("[0-9]+[%]") - find_mem_n_units = re.compile( - r"(?P[0-9]*\.[0-9]*)(?P[A-Za-z])" - ) + find_mem_n_units = re.compile(r"(?P[0-9]*\.[0-9]*)(?P[A-Za-z])") try: for line in _read_file(filename): result = find_wallclock.search(line) @@ -1794,9 +1737,7 @@ def get_wallclock_and_memory(filename): memory = "Failure processing EOJ" stacktr = traceback.format_exc() print( - "[ERROR] Processing wallclock and memory use :\n{0:s}".format( - stacktr - ) + "[ERROR] Processing wallclock and memory use :\n{0:s}".format(stacktr) ) print("Error type : {0:s}".format(type(err))) print(err) @@ -1845,17 +1786,12 @@ def get_altered_files_list(mirror_loc): for attempt in range(5): try: # Get a list of altered files from the fcm mirror url - bdiff_files = get_branch_diff_filenames( - mirror_loc, path_override="" - ) + bdiff_files = get_branch_diff_filenames(mirror_loc, path_override="") break except Exception as err: print(err) if attempt == 4: - print( - "Cant get list of alterered files - returning " - "empty list." - ) + print("Cant get list of alterered files - returning " "empty list.") bdiff_files = [] break else: @@ -1918,33 +1854,21 @@ def print_report(self): trac_log.append(" = Testing Results - rose-stem output = ") trac_log.append("") - trac_log.append( - " || Suite Name: || {0:s} || ".format(self.suitename) - ) + trac_log.append(" || Suite Name: || {0:s} || ".format(self.suitename)) - trac_log.append( - " || Suite Owner: || {0:s} || ".format(self.suite_owner) - ) + trac_log.append(" || Suite Owner: || {0:s} || ".format(self.suite_owner)) if self.trustzone: - trac_log.append( - " || Trustzone: || {0:s} || ".format(self.trustzone) - ) + trac_log.append(" || Trustzone: || {0:s} || ".format(self.trustzone)) if self.fcm: - trac_log.append( - " || FCM version: || {0:s} || ".format(self.fcm) - ) + trac_log.append(" || FCM version: || {0:s} || ".format(self.fcm)) if self.rose: - trac_log.append( - " || Rose version: || {0:s} || ".format(self.rose) - ) + trac_log.append(" || Rose version: || {0:s} || ".format(self.rose)) if self.cylc: - trac_log.append( - " || Cylc version: || {0:s} || ".format(self.cylc) - ) + trac_log.append(" || Cylc version: || {0:s} || ".format(self.cylc)) trac_log.append( " || Report Generated: || {0:s} || ".format(self.creation_time) @@ -1961,15 +1885,11 @@ def print_report(self): trac_log.append(" || Site: || {0:s} || ".format(self.site)) trac_log.append( - " || Groups Run: || {0:s} || ".format( - self.generate_groups(self.groups) - ) + " || Groups Run: || {0:s} || ".format(self.generate_groups(self.groups)) ) if self.rose_orig_host is not None: trac_log.append( - " || ''ROSE_ORIG_HOST:'' || {0:s} || ".format( - self.rose_orig_host - ) + " || ''ROSE_ORIG_HOST:'' || {0:s} || ".format(self.rose_orig_host) ) if self.host_xcs: trac_log.append(" || HOST_XCS || True || ") @@ -1992,10 +1912,7 @@ def print_report(self): trac_log.append("-----") trac_log.append("") - if ( - not self.required_comparisons - and "LFRIC_APPS" not in self.job_sources - ): + if not self.required_comparisons and "LFRIC_APPS" not in self.job_sources: trac_log.append("") trac_log.append("-----") trac_log.append(" = WARNING !!! = ") @@ -2033,9 +1950,7 @@ def print_report(self): db_file = "" if self.is_cylc8: - db_file = os.path.join( - self.suite_path, "log", SUITE_DB_FILENAME_CYLC8 - ) + db_file = os.path.join(self.suite_path, "log", SUITE_DB_FILENAME_CYLC8) else: db_file = os.path.join(self.suite_path, SUITE_DB_FILENAME) @@ -2061,8 +1976,7 @@ def print_report(self): suite_dir = "--cylc_suite_dir--" trac_log.extend( [ - "There has been an exception in " - + "SuiteReport.print_report()", + "There has been an exception in " + "SuiteReport.print_report()", "See output for more information", "rose-stem suite output will be in the files :\n", "~/cylc-run/{0:s}/log/suite/log".format(suite_dir), @@ -2094,9 +2008,7 @@ def print_report(self): print("----- Start of {0:s}.log -----".format(TRAC_LOG_FILE)) for line in trac_log: print(line) - print( - "\n----- End of {0:s}.log -----\n\n".format(TRAC_LOG_FILE) - ) + print("\n----- End of {0:s}.log -----\n\n".format(TRAC_LOG_FILE)) raise @@ -2135,8 +2047,7 @@ def parse_options(): dest="increase_verbosity", action="count", default=0, - help="Increases Verbosity level. Default = " - "{0:2d}".format(DEFAULT_VERBOSITY), + help="Increases Verbosity level. Default = " "{0:2d}".format(DEFAULT_VERBOSITY), ) parser.add_option( "--decrease-verbosity", @@ -2207,8 +2118,7 @@ def parse_options(): dest="sort_by_status", default=False, action="store_true", - help="Sort by task status is now default and this option" - + " is deprecated", + help="Sort by task status is now default and this option" + " is deprecated", ) parser.add_option_group(group) # -- End of deprecated options -- block to be removed around Dec 2023 @@ -2231,7 +2141,7 @@ def parse_options(): # Cylc7 environment variable "CYLC_SUITE_RUN_DIR", # Default to Cylc8 environment variable - os.environ.get("CYLC_WORKFLOW_RUN_DIR") + os.environ.get("CYLC_WORKFLOW_RUN_DIR"), ) if opts.suite_path is None: print("Available Environment Variables:") @@ -2252,8 +2162,7 @@ def parse_options(): message = [] message.append("") message.append( - " ---------------------------------------------" - "----------------" + " ---------------------------------------------" "----------------" ) message.append( """ ### WARNING !!! ### @@ -2263,8 +2172,7 @@ def parse_options(): becomes a fatal error""" ) message.append( - " ---------------------------------------------" - "----------------" + " ---------------------------------------------" "----------------" ) message.append("") if opts.sort_by_status: diff --git a/tests/test_fcm_bdiff.py b/tests/test_fcm_bdiff.py index 1808b0b..4d03815 100644 --- a/tests/test_fcm_bdiff.py +++ b/tests/test_fcm_bdiff.py @@ -20,20 +20,23 @@ # Use Case 2 - Test changes made to a working copy, # use url of the branch as the first parameter # svn/um/main variant -use_case_2_1_branch = 'https://code.metoffice.gov.uk/svn/um/main/branches/'\ - + 'dev/Share/r118291_fcm_bdiff_share_testing_branch' +use_case_2_1_branch = ( + "https://code.metoffice.gov.uk/svn/um/main/branches/" + + "dev/Share/r118291_fcm_bdiff_share_testing_branch" +) use_case_2_1_expected = [ - 'https://code.metoffice.gov.uk/svn/um/main/branches/' - + 'dev/Share/r118291_fcm_bdiff_share_testing_branch/src/atmosphere/' - + 'AC_assimilation/ac2.F90' - ] + "https://code.metoffice.gov.uk/svn/um/main/branches/" + + "dev/Share/r118291_fcm_bdiff_share_testing_branch/src/atmosphere/" + + "AC_assimilation/ac2.F90" +] # trac/um/browser variant -use_case_2_2_branch = 'fcm:um.x_br/'\ - + 'dev/Share/r118291_fcm_bdiff_share_testing_branch' +use_case_2_2_branch = ( + "fcm:um.x_br/" + "dev/Share/r118291_fcm_bdiff_share_testing_branch" +) use_case_2_2_expected = [ - 'fcm:um.x_br/dev/Share/r118291_fcm_bdiff_share_testing_branch/src/' - + 'atmosphere/AC_assimilation/ac2.F90' - ] + "fcm:um.x_br/dev/Share/r118291_fcm_bdiff_share_testing_branch/src/" + + "atmosphere/AC_assimilation/ac2.F90" +] # Use Case 3 - Test changes made to a working copy, @@ -51,8 +54,8 @@ ("../../_svn/main/trunk", True), # PASS ("../incorrect_value/trunk", False), # PASS ("..*_svn/main/trunk", True), # PASS - ("..*incorrect_value/trunk", False) # PASS - ] + ("..*incorrect_value/trunk", False), # PASS + ], ) def test_is_trunk(url, expected): assert is_trunk(url) == expected @@ -61,15 +64,9 @@ def test_is_trunk(url, expected): @pytest.mark.parametrize( ("bytes_type_string", "expected"), [ - ( - bytes("my_test_string", encoding='utf-8'), - "my_test_string" - ), # PASS - ( - bytes("my_test_string", encoding='cp1252'), - "my_test_string" - ) # PASS - ] + (bytes("my_test_string", encoding="utf-8"), "my_test_string"), # PASS + (bytes("my_test_string", encoding="cp1252"), "my_test_string"), # PASS + ], ) def test_text_decoder(bytes_type_string, expected): assert text_decoder(bytes_type_string) == expected @@ -80,8 +77,8 @@ def test_text_decoder(bytes_type_string, expected): [ # Use Case 2 (use_case_2_1_branch, use_case_2_1_expected), # PASS - (use_case_2_2_branch, use_case_2_2_expected) # PASS - ] + (use_case_2_2_branch, use_case_2_2_expected), # PASS + ], ) def test_get_branch_diff_filenames(branch, expected): assert get_branch_diff_filenames(branch) == expected