diff --git a/.github/workflows/script_module_list.yml b/.github/workflows/script_module_list.yml new file mode 100644 index 0000000000..08578ab938 --- /dev/null +++ b/.github/workflows/script_module_list.yml @@ -0,0 +1,50 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Module overview script (lint + test) +on: + push: + paths: + - 'scripts/README.mds/**' + - './.github/**' + pull_request: + paths: + - 'scripts/**' + - './.github/**' + +# Declare default permissions as read only. +permissions: read-all +jobs: + + flake8-lint: + runs-on: ubuntu-20.04 + name: Lint + steps: + - name: Check out source repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - name: Set up Python environment + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: "3.6" + - name: Run flake8 + uses: py-actions/flake8@v2 + with: + max-line-length: "120" + path: "scripts/available_software" + + pytest-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - name: Set up Python + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: '3.6' + - name: Install dependencies + run: | + cd scripts/available_software + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements_tests.txt + - name: Test with pytest + run: | + cd scripts/available_software + ./test.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000..eed5fd8938 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,3 @@ +Scripts that can be used to automatically generate markdown files, can be found here. + +* [`available_software`](available_software): script to generate overview of available environment modules; diff --git a/scripts/available_software/README.md b/scripts/available_software/README.md new file mode 100644 index 0000000000..c25d58b620 --- /dev/null +++ b/scripts/available_software/README.md @@ -0,0 +1,85 @@ +# Available software + +`available_software.py` is a script that generates an overview of all software modules that are available on an HPC system. +It also generates a detailed overview per software that includes specific software versions. +To do this, it generates 3 things: +1. `json_data.json`: This JSON file is used to populate the global overview table. +2. `json_data_detail.json`: This JSON file is used to automatically generate all the detailed markdown pages. +3. A lot of MarkDown files: These are the detailed overview pages per software. + +The generated files will be placed in the [available_software](/docs/available_software) directory, +more specifically the [data](/docs/available_software/data) +and [detail](/docs/available_software/detail) subdirectories. + +## Requirements +- Required Python packages are listed in `requirements.txt` and `requirements_tests.txt` +- [Lmod](https://github.com/TACC/Lmod) must be available, and `$LMOD_CMD` must specify the path to the `lmod` binary. + + +### Creating a virtual environment (optional) + +If the required Python packages are not available in your Python setup, +you can easily create a dedicated virtual environment as follows: + +```shell +python -m venv module_overview_venv +source module_overview_venv/bin/activate +pip install -r requirements.txt +pip install -r requirements_tests.txt +# to exit the virtual environment, run 'deactivate' +``` + +## Usage +You can run the script with following command: + +```shell +python available_software.py +``` + +## Testing +You can run the tests by running the `test.sh` script. +```shell +./test.sh +``` + +The tests make use of a mocked `$LMOD_CMD` script, which you can find [here](tests/data/lmod_mock.sh). + +### Write tests +If you want to write additional tests and use the script effectively, follow these guidelines: + + +1. **Setting up mocked Lmod:** + + Before each test, ensure that you set the path to the script that mocks the `lmod` binary. + This can be done within the setup_class function. + ```python + path = os.path.dirname(os.path.realpath(__file__)) + + @classmethod + def setup_class(cls): + os.environ["LMOD_CMD"] = cls.path + "/data/lmod_mock.sh" + ``` + +### Example +An example of a possible `setup_class` function is given below. +```python +import os + +@classmethod +def setup_class(cls): + os.environ["TESTS_PATH"] = cls.path + os.environ["LMOD_CMD"] = cls.path + "/data/lmod_mock.sh" + os.environ["SHELL"] = cls.path + "/data/bash_mock.sh" + os.environ["MOCK_FILE_SWAP"] = cls.path + "/data/data_swap_TARGET.txt" + os.environ["MOCK_FILE_AVAIL_TARGET"] = cls.path + "/data/data_avail_target_simple.txt" + os.environ["MOCK_FILE_AVAIL_TARGET_AMD_INTEL"] = cls.path + "/data/data_avail_target_amd_intel.txt" +``` + +This does multiple things: +1. Set the path of the tests folder in `$TESTS_PATH` +2. Set the path to the `lmod_mock.sh` script in the environment variable `$LMOD_CMD` +3. set the path to the `bash_mock.sh` script in the environment variable `$SHELL` +4. Set the output file for the `module swap` to the `MOCK_FILE_SWAP` variable. + For example, `data/data_swap_generic.txt` could be a possible file. +3. Set the output file for the `module avail` to the `MOCK_FILE_AVAIL_TARGET` variable. + The actual output can be found in the `data/data_avail_target_simple.txt` file or `/data/data_avail_target_amd_intel.txt` file. diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py new file mode 100644 index 0000000000..5bc94f5fe5 --- /dev/null +++ b/scripts/available_software/available_software.py @@ -0,0 +1,614 @@ +# +# Copyright 2023-2023 Ghent University +# +# This file is part of vsc_user_docs, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/hpcugent/vsc_user_docs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# The original program is update and is now a part of the EESSI docs +""" +Python script to generate an overview of available modules across different CPU and GPU targets, +in MarkDown format. + +@author: Michiel Lachaert (Ghent University) +@author: Lara Peeters (Ghent University) +""" +import argparse +import json +import os +import re +import subprocess +import time +from pathlib import Path +from typing import Union, Tuple +import numpy as np +from mdutils.mdutils import MdUtils +from natsort import natsorted + + +# -------------------------------------------------------------------------------------------------------- +# MAIN +# -------------------------------------------------------------------------------------------------------- + +def main(): + os.environ["SHELL"] = "/bin/bash" + current_dir = Path(__file__).resolve() + project_name = 'docs' + root_dir = next( + p for p in current_dir.parents if p.parts[-1] == project_name + ) + path_data_dir = os.path.join(root_dir, "docs/available_software/data") + + # Generate the JSON overviews + modules = modules_eessi() + print(modules) + print("Generate JSON overview... ", end="", flush=True) + generate_json_overview(modules, path_data_dir) + print("Done!") + + # Generate the JSON detail + json_data = generate_json_detailed_data(modules) + json_data = get_extra_info_eessi(json_data) + print("Generate JSON detailed... ", end="", flush=True) + json_path = generate_json_detailed(json_data, path_data_dir) + print("Done!") + + # Generate detail markdown pages + print("Generate detailed pages... ", end="", flush=True) + generate_detail_pages(json_path, os.path.join(root_dir, "docs/available_software/detail")) + print("Done!") + + +# -------------------------------------------------------------------------------------------------------- +# Functions to run bash commands +# -------------------------------------------------------------------------------------------------------- + +def bash_command(cmd: str) -> np.ndarray: + bash = os.getenv("SHELL") + proc = subprocess.run( + [bash, '-c', cmd], + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + return np.array(proc.stdout.split()) + + +# -------------------------------------------------------------------------------------------------------- +# Functions to run "module" commands +# -------------------------------------------------------------------------------------------------------- + +def module(*args, filter_fn=lambda x: x) -> np.ndarray: + """ + Function to run "module" commands. + + @param args: Extra arguments for the module command. + @param filter_fn: Filter function on the output. + @return: Array with the output of the module command. + """ + lmod = os.getenv('LMOD_CMD') + proc = subprocess.run( + [lmod, "python", "--terse"] + list(args), + encoding="utf-8", + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + exec(proc.stdout) + return filter_fn(np.array(proc.stderr.strip().split("\n"))) + + +def module_avail(name: str = "", filter_fn=lambda x: x) -> np.ndarray: + """ + Function to run "module avail" commands. + + @param name: Module name, or empty string to return all available modules. + @param filter_fn: Filter on the output. + @return: List of all available modules of name, or all if name is not given. + """ + return module("avail", name, filter_fn=filter_fn) + + +def module_swap(name: str) -> None: + """ + Function to run "module swap" commands. + + @param name: Name of module you want to swap to. + """ + module("swap", name) + + +def module_use(path: str) -> None: + """ + Function to run "module use" commands. + + @param path: Path to the directory with all the modules you want to use. + """ + module("use", path) + + +def module_unuse(path: str) -> None: + """ + Function to run "module unuse" commands. + + @param path: Path to the directory with all the modules you want to unuse. + """ + module("unuse", path) + + +def module_whatis(name: str) -> dict: + """ + Function to run "module whatis" commands. + + @param name: Name of module you want the whatis info for. + """ + whatis = {} + data = module("show", name) + for line in data[np.char.startswith(data, "whatis")]: + content = re.sub(pattern=r'whatis\((.*)\)', repl='\\1', string=line).strip('"') + key, value = tuple(content.split(":", maxsplit=1)) + whatis[key.strip()] = value.strip() + return whatis + + +def module_info(info: str) -> dict: + """ + Function to parse through lua file. + + @param info: String with the contents of the lua file. + """ + whatis = {} + data = np.array(info.split("\n")) + # index of start description to handle multi lined description + i = np.flatnonzero(np.char.startswith(data, "whatis([==[Description"))[0] + if np.char.endswith(data[i], "]==])"): + content = re.sub(pattern=r'whatis\(\[==\[(.*)\]==\]\)', repl='\\1', string=data[i]).strip('"') + else: + description = re.sub(pattern=r'whatis\(\[==\[(.*)', repl='\\1', string=data[i]).strip('"') + while not np.char.endswith(data[i], "]==])"): + i += 1 + description += data[i] + content = re.sub(pattern=r'(.*)\]==\]\)', repl='\\1', string=description).strip('"') + key, value = tuple(content.split(":", maxsplit=1)) + whatis[key.strip()] = value.strip() + + for line in data[np.char.startswith(data, "whatis")]: + if not np.char.startswith(line, "whatis([==[Description"): + content = re.sub(pattern=r'whatis\(\[==\[(.*)\]==\]\)', repl='\\1', string=line).strip('"') + key, value = tuple(content.split(":", maxsplit=1)) + whatis[key.strip()] = value.strip() + return whatis + + +# -------------------------------------------------------------------------------------------------------- +# Fetch data EESSI +# -------------------------------------------------------------------------------------------------------- + +def filter_fn_eessi_modules(data: np.ndarray) -> np.ndarray: + """ + Filter function for the output of all software modules for EESSI (excl. 'target'). + @param data: Output + @return: Filtered output + """ + return data[~np.char.endswith(data, ":")] + + +def targets_eessi() -> np.ndarray: + """ + Returns all the target names of EESSI. + @return: target names + """ + commands = [ + "find /cvmfs/software.eessi.io/versions/2023.06/software/linux/*/* -maxdepth 0 \\( ! -name 'intel' -a ! " + "-name 'amd' \\) -type d", + 'find /cvmfs/software.eessi.io/versions/2023.06/software/linux/*/{amd,intel}/* -maxdepth 0 -type d' + ] + targets = np.array([]) + + for command in commands: + targets = np.concatenate([targets, bash_command(command)]) + + return targets + + +def modules_eessi() -> dict: + """ + Returns names of all software module that are installed on EESSI. + They are grouped by target. + @return: Dictionary with all the modules per target + """ + print("Start collecting modules:") + data = {} + module_unuse(os.getenv('MODULEPATH')) + for target in targets_eessi(): + print(f"\t Collecting available modules for {target}... ", end="", flush=True) + module_use(target + "/modules/all/") + data[target] = module_avail(filter_fn=filter_fn_eessi_modules) + print(f"found {len(data[target])} modules!") + module_unuse(os.getenv('MODULEPATH')) + + print("All data collected!\n") + return data + + +def get_extra_info_eessi(json_data) -> dict: + """ + add Description, homepage and a list of extensions (only for software with extensions) + @return: Dictionary with all the modules and their site_packages + """ + modules = json_data['software'] + for software in modules: + for mod in modules[software]['versions']: + if software == "Java": + # TODO handle specific naming schema for Java + # code cannot handle "Java/11(@Java/11.0.20)" + continue + base_path = modules[software]['versions'][mod]['targets'][0] + '/modules/all/' + path = base_path + mod + ".lua" + f = open(path, 'r') + info = f.read() + if info != "": + whatis = module_info(info) + json_data['software'][software]['description'] = whatis['Description'] + if "Homepage" in whatis.keys(): + json_data['software'][software]['homepage'] = whatis['Homepage'] + if "Extensions" in whatis.keys(): + json_data["software"][software]["versions"][mod]["extensions"] = whatis['Extensions'] + return json_data + + +# -------------------------------------------------------------------------------------------------------- +# Util functions +# -------------------------------------------------------------------------------------------------------- + +def analyze_module(mod: str) -> Tuple: + return ( + mod.split("/", 1)[0], + mod.split("/", 1)[1] if "/" in mod else "" + ) + + +def mod_names_to_software_names(mod_list: np.ndarray) -> np.ndarray: + """ + Convert a list of module names to a list of the software names. + + @param mod_list: List of the module names + @return: List of the corresponding software names + """ + return np.unique([analyze_module(mod)[0] for mod in mod_list]) + + +def get_unique_software_names(data: Union[dict, list, np.ndarray]) -> Union[dict, list, np.ndarray]: + """ + Simplify list of modules by removing versions and duplicates. + + @param data: List of modules + @return: List of software names. + """ + + if isinstance(data, dict): + simplified_data = {target: mod_names_to_software_names(data[target]) for target in data} + else: + simplified_data = mod_names_to_software_names(data) + + return simplified_data + + +def dict_sort(dictionary: dict) -> dict: + """ + Sort a dictionary by key. + + @param dictionary: A dictionary + @return: Sorted dictionary + """ + return dict(natsorted(dictionary.items())) + + +# -------------------------------------------------------------------------------------------------------- +# Generate detailed markdown +# -------------------------------------------------------------------------------------------------------- + +def generate_software_table_data(software_data: dict, targets: list) -> list: + """ + Construct the data for the detailed software table. + + @param software_data: Software specific data. + @param targets: List with all the target names + @return: 1D list with all the data for the table + """ + #TODO: add same structure as https://github.com/laraPPr/EESSI_docs/blob/test_add_script_generate_software/docs/available_software/overview.md to table + table_data = [" "] + [target[57:] for target in targets] + + for module_name, available in list(software_data.items())[::-1]: + row = [module_name] + + for target in targets: + row += ("x" if target in available["targets"] else "-") + table_data += row + + return table_data + + +def generate_software_detail_page( + software_name: str, + software_data: dict, + generated_time: str, + targets: list, + path: str +) -> None: + """ + Generate one software specific detail page. + + @param software_name: Name of the software + @param software_data: Additional information about the software (version, etc...) + @param generated_time: Timestamp when the data was generated + @param targets: List with all the target names + @param path: Path of the directory where the detailed page will be created. + """ + sorted_versions = dict_sort(software_data["versions"]) + newest_version = list(sorted_versions.keys())[-1] + + filename = f"{path}/{software_name}.md" + md_file = MdUtils(file_name=filename, title=f"{software_name}") + if 'description' in software_data.keys(): + description = software_data['description'] + md_file.new_paragraph(f"{description}") + if 'homepage' in software_data.keys(): + homepage = software_data['homepage'] + md_file.new_paragraph(f"{homepage}") + + md_file.new_header(level=1, title="Available modules") + + md_file.new_paragraph(f"The overview below shows which {software_name} installations are available per " + f"target architecture in EESSI, ordered based on software version (new to old).") + md_file.new_paragraph(f"To start using {software_name}, load one of these modules using a `module load` command " + f"like:") + md_file.insert_code(f"module load {newest_version}", language="shell") + md_file.new_paragraph(f"(This data was automatically generated on {generated_time})", bold_italics_code="i") + md_file.new_line() + + md_file.new_table( + columns=len(targets) + 1, + rows=len(sorted_versions) + 1, + text=generate_software_table_data(sorted_versions, targets) + ) + + for version, details in list(sorted_versions.items())[::-1]: + if 'extensions' in details: + md_file.new_paragraph(f"### {version}") + md_file.new_paragraph("This is a list of extensions included in the module:") + packages = details['extensions'] + md_file.new_paragraph(f"{packages}") + + md_file.create_md_file() + + # Remove the TOC + with open(filename) as f: + read_data = f.read() + with open(filename, 'w') as f: + f.write("---\nhide:\n - toc\n---\n" + read_data) + + +def generate_detail_pages(json_path, dest_path) -> None: + """ + Generate all the detailed pages for all the software that is available. + """ + + with open(json_path) as json_data: + data = json.load(json_data) + + all_targets = data["targets"] + for software, content in data["software"].items(): + generate_software_detail_page(software, content, data["time_generated"], all_targets, dest_path) + + +# -------------------------------------------------------------------------------------------------------- +# Generate overview markdown +# -------------------------------------------------------------------------------------------------------- + +def generate_table_data(avail_mods: dict) -> Tuple[np.ndarray, int, int]: + """ + Generate data that can be used to construct a MarkDown table. + + @param avail_mods: Available modules + @return: Returns tuple (Table data, #col, #row) + """ + avail_mods = get_unique_software_names(avail_mods) + all_modules = get_unique_software_names(np.concatenate(list(avail_mods.values()))) + + final = np.array([" "]) + final = np.append(final, list(avail_mods.keys())) + + for package in all_modules: + final = np.append(final, package) + + for target in avail_mods: + final = np.append(final, "X" if package in avail_mods[target] else " ") + + return final, len(avail_mods.keys()) + 1, len(all_modules) + 1 + + +def generate_module_table(data: dict, md_file: MdUtils) -> None: + """ + Generate the general table of the overview. + + @param data: Dict with all the data. Keys are the target names. + @param md_file: MdUtils object. + """ + print("Generating markdown table... ", end="", flush=True) + structured, col, row = generate_table_data(data) + md_file.new_table(columns=col, rows=row, text=list(structured), text_align='center') + print("Done!") + + +def generate_markdown_overview(modules: dict) -> None: + """ + Generate the general overview in a markdown file. + It generates a list of all the available software and indicates for which target it is available. + """ + md_fn = 'module_overview.md' + md_file = MdUtils(file_name=md_fn, title='Overview of available modules per target architecture in EESSI') + generate_module_table(modules, md_file) + md_file.create_md_file() + print(f"Module overview created at {md_fn}") + + +# -------------------------------------------------------------------------------------------------------- +# Generate JSON +# -------------------------------------------------------------------------------------------------------- +# ----------- +# OVERVIEW +# ----------- + +# FORMAT OVERVIEW JSON +# { +# "targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"], +# "modules": { +# "Markov": [1, 0], +# "cfd": [1, 1], +# "llm": [0, 1], +# "science": [1, 1] +# } +# } +def generate_json_overview_data(modules: dict) -> dict: + """ + Generate the data for the json overview in the above format. + + @param modules: Dictionary with all the modules per target. Keys are the target names. + @return: Dictionary with the required JSON structure. + + """ + json_data = { + "targets": list(modules.keys()), + "modules": {}, + "time_generated": time.strftime("%a, %d %b %Y at %H:%M:%S %Z") + } + avail_software = get_unique_software_names(modules) + all_software = get_unique_software_names(np.concatenate(list(modules.values()))) + + # creates a list of booleans for each software that indicates + # if the software is available for the corresponding target. + for soft in all_software: + available = [] + for target in json_data["targets"]: + available.append(int(soft in avail_software[target])) + json_data["modules"][soft] = available + return json_data + + +def generate_json_overview(modules: dict, path_data_dir: str) -> str: + """ + Generate the overview in a JSON format. + + @param modules: Dictionary with all the modules per target. Keys are the target names. + @param path_data_dir: Path to the directory where the JSON will be placed. + @return: Absolute path to the json file. + """ + + # get data + json_data = generate_json_overview_data(modules) + + filepath = os.path.join(path_data_dir, "json_data.json") + # write it to a file + with open(filepath, 'w') as outfile: + json.dump(json_data, outfile) + + return filepath + + +# ----------- +# DETAILED +# ----------- + +# FORMAT DETAILED JSON: +# +# { +# "targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"], +# "software": { +# "cfd": { +# "targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.e essi.io/versions/2023.06/software/linux/x86_64/amd/zen2"], +# "versions": { +# "2.3.1": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"], +# "2.3.2": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/so ftware.e essi.io/versions/2023.06/software/linux/x86_64/amd/zen2"] +# } +# } +# } +# } + +def generate_json_detailed_data(modules: dict) -> dict: + """ + Generate the data for the detailed JSON in the above format. + + @param modules: Dictionary with all the modules per target. Keys are the target names. + @return: Dictionary with the required JSON structure. + """ + json_data = { + "targets": list(modules.keys()), + "software": {}, + "time_generated": time.strftime("%a, %d %b %Y at %H:%M:%S %Z") + } + + # Loop over every module in every target + for target in modules: + for mod in modules[target]: + software, version = analyze_module(mod) + + # Exclude modules with no version + if version != "": + # If the software is not yet present, add it. + if software not in json_data["software"]: + json_data["software"][software] = { + "targets": [], + "versions": {} + } + + # If the version is not yet present, add it. + if mod not in json_data["software"][software]["versions"]: + json_data["software"][software]["versions"][mod] = {'targets': []} + + # If the target is not yet present, add it. + if target not in json_data["software"][software]["targets"]: + json_data["software"][software]["targets"].append(target) + + # If the target is not yet present, add it. + if target not in json_data["software"][software]["versions"][mod]["targets"]: + json_data["software"][software]["versions"][mod]["targets"].append(target) + + return json_data + + +def generate_json_detailed(json_data: dict, path_data_dir: str) -> str: + """ + Generate the detailed JSON. + + @param modules: Dictionary with all the modules per target. Keys are the target names. + @param path_data_dir: Path to the directory where the JSON will be placed. + @return: Absolute path to the json file. + """ + filepath = os.path.join(path_data_dir, "json_data_detail.json") + with open(filepath, 'w') as outfile: + json.dump(json_data, outfile) + + return filepath + + +if __name__ == '__main__': + main() diff --git a/scripts/available_software/requirements.txt b/scripts/available_software/requirements.txt new file mode 100644 index 0000000000..1b7478cc43 --- /dev/null +++ b/scripts/available_software/requirements.txt @@ -0,0 +1,3 @@ +mdutils +numpy +natsort \ No newline at end of file diff --git a/scripts/available_software/requirements_tests.txt b/scripts/available_software/requirements_tests.txt new file mode 100644 index 0000000000..aad120b7c4 --- /dev/null +++ b/scripts/available_software/requirements_tests.txt @@ -0,0 +1,2 @@ +flake8 +pytest \ No newline at end of file diff --git a/scripts/available_software/test.sh b/scripts/available_software/test.sh new file mode 100755 index 0000000000..33e8376a81 --- /dev/null +++ b/scripts/available_software/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +PYTHONPATH=$PWD:$PYTHONPATH pytest -v -s diff --git a/scripts/available_software/tests/data/bash_mock.sh b/scripts/available_software/tests/data/bash_mock.sh new file mode 100755 index 0000000000..5f0e1090f1 --- /dev/null +++ b/scripts/available_software/tests/data/bash_mock.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Return an error when a variable is not set. +set -u + + +# example: /bin/bash -c "echo hello" +bash="$0" +cflag="$1" +cmd="$2" + +# Emulate find command. +if echo "$cmd" | grep -q -E "find"; then + if echo "$cmd" | grep -q -E "amd,intel"; then + cat "${MOCK_FILE_AVAIL_TARGET_AMD_INTEL}" >&1 + else + cat "${MOCK_FILE_AVAIL_TARGET}" >&1 + fi +fi diff --git a/scripts/available_software/tests/data/data_avail_simple_generic.txt b/scripts/available_software/tests/data/data_avail_simple_generic.txt new file mode 100644 index 0000000000..de988a5679 --- /dev/null +++ b/scripts/available_software/tests/data/data_avail_simple_generic.txt @@ -0,0 +1,14 @@ +/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/generic/modules/all: +cfd/1.0 +cfd/2.0 +cfd/24 +cfd/5.0 +cfd/2.0afqsdf +Markov/hidden-1.0.5 +Markov/hidden-1.0.10 +Markov/ +science/ +science/5.3.0 +science/5.3.0 +science/5.3.0 +science/7.2.0 diff --git a/scripts/available_software/tests/data/data_avail_simple_zen2.txt b/scripts/available_software/tests/data/data_avail_simple_zen2.txt new file mode 100644 index 0000000000..482528b56d --- /dev/null +++ b/scripts/available_software/tests/data/data_avail_simple_zen2.txt @@ -0,0 +1,16 @@ +/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all: +cfd/1.0 +cfd/2.0 +cfd/3.0 +cfd/24 +cfd/ +cfd/5.0 +cfd/2.0afqsdf +llm/20230627 +llm/20230627 +llm/20230627 +science/ +science/5.3.0 +science/5.3.0 +science/5.3.0 +science/7.2.0 diff --git a/scripts/available_software/tests/data/data_avail_target_amd_intel.txt b/scripts/available_software/tests/data/data_avail_target_amd_intel.txt new file mode 100644 index 0000000000..adf051fdf0 --- /dev/null +++ b/scripts/available_software/tests/data/data_avail_target_amd_intel.txt @@ -0,0 +1 @@ +/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2 diff --git a/scripts/available_software/tests/data/data_avail_target_simple.txt b/scripts/available_software/tests/data/data_avail_target_simple.txt new file mode 100644 index 0000000000..e622f8c27e --- /dev/null +++ b/scripts/available_software/tests/data/data_avail_target_simple.txt @@ -0,0 +1 @@ +/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic diff --git a/scripts/available_software/tests/data/data_show_science.txt b/scripts/available_software/tests/data/data_show_science.txt new file mode 100644 index 0000000000..e009c98580 --- /dev/null +++ b/scripts/available_software/tests/data/data_show_science.txt @@ -0,0 +1,34 @@ +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + /apps/gent/RHEL8/zen2-ib/modules/all/SciPy-bundle/2022.05-foss-2022a.lua: +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +help([[ +Description +=========== +Bundle for scientific software + + +More information +================ + - Homepage: https://science.com/ + + +Included extensions +=================== +beniget-0.4.1, Bottleneck-1.3.4 +]]) +whatis("Description: Bundle for scientific software") +whatis("Homepage: https://science.com/") +whatis("URL: https://science.com/") +whatis("Extensions: ext-1.2.3, ext-2.3.4") +conflict("SciPy-bundle") +prepend_path("CMAKE_PREFIX_PATH","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a") +prepend_path("LIBRARY_PATH","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a/lib") +prepend_path("PATH","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a/bin") +setenv("EBROOTSCIPYMINBUNDLE","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a") +setenv("EBVERSIONSCIPYMINBUNDLE","2022.05") +setenv("EBDEVELSCIPYMINBUNDLE","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a/easybuild/SciPy-bundle-2022.05-foss-2022a-easybuild-devel") +prepend_path("PYTHONPATH","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a/lib/python3.10/site-packages") +prepend_path("CPATH","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a/lib/python3.10/site-packages/numpy/core/include") +prepend_path("LD_LIBRARY_PATH","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a/lib/python3.10/site-packages/numpy/core/lib") +prepend_path("LIBRARY_PATH","/apps/gent/RHEL8/zen2-ib/software/SciPy-bundle/2022.05-foss-2022a/lib/python3.10/site-packages/numpy/core/lib") +setenv("EBEXTSLISTSCIPYMINBUNDLE","numpy-1.22.3,ply-3.11,gast-0.5.3,beniget-0.4.1,pythran-0.11.0,scipy-1.8.1,mpi4py-3.1.3,numexpr-2.8.1,Bottleneck-1.3.4,pandas-1.4.2,mpmath-1.2.1,deap-1.3.1") diff --git a/scripts/available_software/tests/data/data_swap_generic.txt b/scripts/available_software/tests/data/data_swap_generic.txt new file mode 100644 index 0000000000..6fc8f8348a --- /dev/null +++ b/scripts/available_software/tests/data/data_swap_generic.txt @@ -0,0 +1 @@ +os.environ["MOCK_FILE_AVAIL"] = os.getenv('TESTS_PATH') + "/data/data_avail_simple_generic.txt" diff --git a/scripts/available_software/tests/data/data_swap_zen2.txt b/scripts/available_software/tests/data/data_swap_zen2.txt new file mode 100644 index 0000000000..99f9e4ef6f --- /dev/null +++ b/scripts/available_software/tests/data/data_swap_zen2.txt @@ -0,0 +1 @@ +os.environ["MOCK_FILE_AVAIL"] = os.getenv('TESTS_PATH') + "/data/data_avail_simple_zen2.txt" diff --git a/scripts/available_software/tests/data/lmod_mock.sh b/scripts/available_software/tests/data/lmod_mock.sh new file mode 100755 index 0000000000..8fecc18b37 --- /dev/null +++ b/scripts/available_software/tests/data/lmod_mock.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Return an error when a variable is not set. +set -u + + +# example: $LMOD_CMD python --terse avail cluster/ +python="$1" +terse="$2" +mod_cmd="$3" +mod_args="${4:-}" + +# Emulated avail command. +if [ "$mod_cmd" = "avail" ]; then + cat "${MOCK_FILE_AVAIL}" >&2 + +# Emulated swap command. +elif [ "$mod_cmd" = "swap" ]; then + # extract the target name from the 4th argument + target=$(echo "$mod_args" | cut -d "/" -f 1) + target_name=$(echo "$mod_args" | cut -d "/" -f 2) + + if [ "$target" = "target" ]; then + # Substitute TARGET by the target_name + cat ${MOCK_FILE_SWAP/TARGET/${target_name}} >&1 + else + echo "${mod_args} is not a target." >&2 + exit 1 + fi + + +# Emulate show command +elif [ "$mod_cmd" = "show" ]; then + cat "${MOCK_FILE_SHOW}" >&2 + + +elif [ "$mod_cmd" = "use" ]; then + cvmfs=$(echo "$mod_args" | cut -d "/" -f 2) + repo=$(echo "$mod_args" | cut -d "/" -f 3) + if [ "cvmfs" = "cvmfs" ]; then + if echo "$mod_args" | grep -q -E "amd"; then + target_name=$(echo "$mod_args" | cut -d "/" -f 10) + # Substitute TARGET by the target_name + cat ${MOCK_FILE_SWAP/TARGET/${target_name}} >&1 + elif echo "$mod_args" | grep -q -E "intel"; then + target_name=$(echo "$mod_args" | cut -d "/" -f 10) + # Substitute TARGET by the target_name + cat ${MOCK_FILE_SWAP/TARGET/${target_name}} >&1 + else + target_name=$(echo "$mod_args" | cut -d "/" -f 9) + # Substitute TARGET by the target_name + cat ${MOCK_FILE_SWAP/TARGET/${target_name}} >&1 + fi + else + echo "${mod_arg} in not a cvmfs repo" + exit 1 + fi + + +else + echo "Module subcommand '${mod_cmd}' not supported yet in $0" >&2 + exit 1 +fi diff --git a/scripts/available_software/tests/data/test_json_simple_sol.json b/scripts/available_software/tests/data/test_json_simple_sol.json new file mode 100644 index 0000000000..5d14435a70 --- /dev/null +++ b/scripts/available_software/tests/data/test_json_simple_sol.json @@ -0,0 +1 @@ +{"time_generated":"dummy", "targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"], "modules": {"Markov": [1, 0], "cfd": [1, 1], "llm": [0, 1], "science": [1, 1]}} diff --git a/scripts/available_software/tests/data/test_json_simple_sol_detail.json b/scripts/available_software/tests/data/test_json_simple_sol_detail.json new file mode 100644 index 0000000000..e5df36b947 --- /dev/null +++ b/scripts/available_software/tests/data/test_json_simple_sol_detail.json @@ -0,0 +1 @@ +{"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"], "software": {"cfd": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"], "versions": {"cfd/1.0": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}, "cfd/2.0": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}, "cfd/24": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}, "cfd/5.0": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}, "cfd/2.0afqsdf": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}, "cfd/3.0": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}}}, "Markov": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"], "versions": {"Markov/hidden-1.0.5": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"]}, "Markov/hidden-1.0.10": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"]}}}, "science": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"], "versions": {"science/5.3.0": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}, "science/7.2.0": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}}}, "llm": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"], "versions": {"llm/20230627": {"targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]}}}}, "time_generated": "Thu, 31 Aug 2023 at 14:00:22 CEST"} diff --git a/scripts/available_software/tests/data/test_md_simple_sol.md b/scripts/available_software/tests/data/test_md_simple_sol.md new file mode 100644 index 0000000000..88812fe34e --- /dev/null +++ b/scripts/available_software/tests/data/test_md_simple_sol.md @@ -0,0 +1,10 @@ + +Overview Modules +================ + +| |/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic|/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2| +| :---: | :---: | :---: | +|Markov|X| | +|cfd|X|X| +|llm| |X| +|science|X|X| diff --git a/scripts/available_software/tests/test_data.py b/scripts/available_software/tests/test_data.py new file mode 100644 index 0000000000..6388736eef --- /dev/null +++ b/scripts/available_software/tests/test_data.py @@ -0,0 +1,30 @@ +import os +from available_software import modules_eessi, get_unique_software_names + + +class TestData: + # --------------------------- + # Class level setup/teardown + # --------------------------- + path = os.path.dirname(os.path.realpath(__file__)) + + @classmethod + def setup_class(cls): + os.environ["TESTS_PATH"] = cls.path + os.environ["LMOD_CMD"] = cls.path + "/data/lmod_mock.sh" + os.environ["SHELL"] = cls.path + "/data/bash_mock.sh" + os.environ["MOCK_FILE_SWAP"] = cls.path + "/data/data_swap_TARGET.txt" + os.environ["MOCK_FILE_AVAIL_TARGET"] = cls.path + "/data/data_avail_target_simple.txt" + os.environ["MOCK_FILE_AVAIL_TARGET_AMD_INTEL"] = cls.path + "/data/data_avail_target_amd_intel.txt" + + # --------------------------- + # Module tests + # --------------------------- + + def test_data_eessi(self): + sol = modules_eessi() + assert len(sol) == 2 + assert len(sol["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"]) == 13 + assert len(sol["/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]) == 15 + assert list(get_unique_software_names(sol["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"])) == ["Markov", "cfd", "science"] + assert list(get_unique_software_names(sol["/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"])) == ["cfd", "llm", "science"] diff --git a/scripts/available_software/tests/test_json.py b/scripts/available_software/tests/test_json.py new file mode 100644 index 0000000000..957f30f468 --- /dev/null +++ b/scripts/available_software/tests/test_json.py @@ -0,0 +1,74 @@ +from available_software import (generate_json_overview_data, + generate_json_overview, + modules_eessi, + generate_json_detailed, + generate_json_detailed_data) +import os +import json + + +class TestJSON: + # --------------------------- + # Class level setup/teardown + # --------------------------- + + path = os.path.dirname(os.path.realpath(__file__)) + + @classmethod + def setup_class(cls): + os.environ["TESTS_PATH"] = cls.path + os.environ["LMOD_CMD"] = cls.path + "/data/lmod_mock.sh" + os.environ["MOCK_FILE_SWAP"] = cls.path + "/data/data_swap_TARGET.txt" + os.environ["MOCK_FILE_AVAIL_TARGET"] = cls.path + "/data/data_avail_target_simple.txt" + os.environ["MOCK_FILE_SHOW"] = cls.path + "/data/data_show_science.txt" + + @classmethod + def teardown_class(cls): + if os.path.exists("json_data.json"): + os.remove("json_data.json") + os.remove("json_data_detail.json") + + # --------------------------- + # Markdown tests + # --------------------------- + + def test_json_generate_simple(self): + modules = modules_eessi() + json_data = generate_json_overview_data(modules) + assert len(json_data.keys()) == 3 + assert list(json_data["targets"]) == ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"] + assert json_data["modules"] == { + "Markov": [1, 0], + "cfd": [1, 1], + "llm": [0, 1], + "science": [1, 1] + } + + def test_json_simple(self): + modules = modules_eessi() + json_path = generate_json_overview(modules, ".") + with open(json_path) as json_data: + data_generated = json.load(json_data) + + with open(self.path + "/data/test_json_simple_sol.json") as json_data: + data_solution = json.load(json_data) + + assert len(data_generated) == 3 + assert data_generated["modules"] == data_solution["modules"] + assert data_generated["targets"] == data_solution["targets"] + + def test_json_detail_simple(self): + modules = modules_eessi() + json_data = generate_json_detailed_data(modules) + json_path = generate_json_detailed(json_data, ".") + assert os.path.exists("json_data_detail.json") + + with open(json_path) as json_data: + data_generated = json.load(json_data) + + with open(self.path + "/data/test_json_simple_sol_detail.json") as json_data: + data_solution = json.load(json_data) + + assert len(data_generated) == 3 + assert data_generated["targets"] == data_solution["targets"] + assert data_generated["software"] == data_solution["software"] diff --git a/scripts/available_software/tests/test_md.py b/scripts/available_software/tests/test_md.py new file mode 100644 index 0000000000..007824fbf0 --- /dev/null +++ b/scripts/available_software/tests/test_md.py @@ -0,0 +1,43 @@ +from mdutils.mdutils import MdUtils +from available_software import get_unique_software_names, modules_eessi, generate_table_data, generate_module_table +import os +import filecmp + + +class TestMarkdown: + # --------------------------- + # Class level setup/teardown + # --------------------------- + + path = os.path.dirname(os.path.realpath(__file__)) + + @classmethod + def setup_class(cls): + os.environ["TESTS_PATH"] = cls.path + os.environ["LMOD_CMD"] = cls.path + "/data/lmod_mock.sh" + os.environ["MOCK_FILE_SWAP"] = cls.path + "/data/data_swap_TARGET.txt" + os.environ["MOCK_FILE_AVAIL_TARGET"] = cls.path + "/data/data_avail_target_simple.txt" + + @classmethod + def teardown_class(cls): + if os.path.exists("test_simple.md"): + os.remove("test_simple.md") + + # --------------------------- + # Markdown tests + # --------------------------- + + def test_table_generate_simple(self): + simple_data = get_unique_software_names(modules_eessi()) + table_data, col, row = generate_table_data(simple_data) + assert col == 3 + assert row == 5 + assert len(table_data) == 15 + + def test_md_simple(self): + md_file = MdUtils(file_name='test_simple', title='Overview Modules') + simple_data = get_unique_software_names(modules_eessi()) + generate_module_table(simple_data, md_file) + md_file.create_md_file() + assert os.path.exists("test_simple.md") + assert filecmp.cmp(self.path + "/data/test_md_simple_sol.md", "test_simple.md") diff --git a/scripts/available_software/tests/test_module.py b/scripts/available_software/tests/test_module.py new file mode 100644 index 0000000000..c7368b18b7 --- /dev/null +++ b/scripts/available_software/tests/test_module.py @@ -0,0 +1,43 @@ +import os +from available_software import module_avail, filter_fn_eessi_modules, module_swap, module_whatis + + +class TestModule: + # --------------------------- + # Class level setup/teardown + # --------------------------- + path = os.path.dirname(os.path.realpath(__file__)) + + @classmethod + def setup_class(cls): + os.environ["TESTS_PATH"] = cls.path + os.environ["LMOD_CMD"] = cls.path + "/data/lmod_mock.sh" + + # --------------------------- + # Module tests + # --------------------------- + + def test_avail(self): + os.environ["MOCK_FILE_AVAIL"] = self.path + "/data/data_avail_simple_zen2.txt" + output = module_avail() + assert len(output) == 16 + + def test_avail_filtered(self): + os.environ["MOCK_FILE_AVAIL"] = self.path + "/data/data_avail_simple_zen2.txt" + output = module_avail(filter_fn=filter_fn_eessi_modules) + assert len(output) == 15 + assert list(output) == [ + "cfd/1.0", "cfd/2.0", "cfd/3.0", "cfd/24", "cfd/", "cfd/5.0", + "cfd/2.0afqsdf", "llm/20230627", "llm/20230627", "llm/20230627", "science/", + "science/5.3.0", "science/5.3.0", "science/5.3.0", "science/7.2.0" + ] + + def test_whatis(self): + os.environ["MOCK_FILE_SHOW"] = self.path + "/data/data_show_science.txt" + data = module_whatis("science") + assert data == { + "Description": "Bundle for scientific software", + "Homepage": "https://science.com/", + "URL": "https://science.com/", + "Extensions": "ext-1.2.3, ext-2.3.4" + }