From 77a739d2d9e65cd700c704a83c43b6874539fa8c Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Mon, 20 Jan 2025 16:20:30 +1100 Subject: [PATCH 01/13] Example cli for finding orbits --- pyproject.toml | 3 +++ sar_antarctica/nci/cli.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 sar_antarctica/nci/cli.py diff --git a/pyproject.toml b/pyproject.toml index 7bb7b36..70237c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ version = "0.1" # TODO base this on files in proje t Homepage = "https://github.com/GeoscienceAustralia/sar-antarctica.git" # Documentation = "XXX" +[project.scripts] +find-orbits = "sar_antarctica.nci.cli:find_orbits_for_scene" + [tool.pytest.ini_options] testpaths = ["tests/*"] diff --git a/sar_antarctica/nci/cli.py b/sar_antarctica/nci/cli.py new file mode 100644 index 0000000..85b4669 --- /dev/null +++ b/sar_antarctica/nci/cli.py @@ -0,0 +1,31 @@ +import click + +from sar_antarctica.nci.filesystem import get_orbits_nci +from sar_antarctica.nci.preparation.orbits import ( + filter_orbits_to_cover_time_window, +) +from sar_antarctica.nci.preparation.scenes import ( + parse_scene_file_sensor, + parse_scene_file_dates, +) + + +@click.command() +@click.argument("scene") +def find_orbits_for_scene(scene: str): + sensor = parse_scene_file_sensor(scene) + start_time, stop_time = parse_scene_file_dates(scene) + + poe_paths = get_orbits_nci("POE", sensor) + relevent_poe_paths = filter_orbits_to_cover_time_window( + poe_paths, start_time, stop_time + ) + for orbit in relevent_poe_paths: + print(orbit["orbit"]) + + res_paths = get_orbits_nci("RES", sensor) + relevant_res_paths = filter_orbits_to_cover_time_window( + res_paths, start_time, stop_time + ) + for orbit in relevant_res_paths: + print(orbit["orbit"]) From c1e2a5a1b4549555cd46975984805389675d2ef9 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Tue, 21 Jan 2025 12:54:09 +1100 Subject: [PATCH 02/13] Add cli functionality for scene processing with gamma + pyroSAR --- pyproject.toml | 1 + sar_antarctica/nci/cli.py | 31 +++++ sar_antarctica/nci/filesystem.py | 11 ++ .../nci/preparation/create_config.py | 27 ++++ .../nci/processing/pyroSAR/pyrosar_geocode.py | 127 +++++++++--------- 5 files changed, 133 insertions(+), 64 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 70237c5..13b1e5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ Homepage = "https://github.com/GeoscienceAustralia/sar-antarctica.git" [project.scripts] find-orbits = "sar_antarctica.nci.cli:find_orbits_for_scene" +run-pyrosar-gamma-workflow = "sar_antarctica.nci.cli:run_pyrosar_gamma_workflow" [tool.pytest.ini_options] testpaths = ["tests/*"] diff --git a/sar_antarctica/nci/cli.py b/sar_antarctica/nci/cli.py index 85b4669..9abc91c 100644 --- a/sar_antarctica/nci/cli.py +++ b/sar_antarctica/nci/cli.py @@ -1,6 +1,10 @@ import click +from pathlib import Path from sar_antarctica.nci.filesystem import get_orbits_nci +from sar_antarctica.nci.preparation.create_config import ( + prepare_inputs_for_pyrosar_gamma, +) from sar_antarctica.nci.preparation.orbits import ( filter_orbits_to_cover_time_window, ) @@ -8,6 +12,33 @@ parse_scene_file_sensor, parse_scene_file_dates, ) +from sar_antarctica.nci.processing.pyroSAR.pyrosar_geocode import ( + run_pyrosar_gamma_geocode, +) + +GAMMA_LIBRARY = Path("/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712") +GAMMA_ENV = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64" +OUTPUT_DIR = Path("/g/data/yp75/projects/sar-antractica-processing") + + +@click.command() +@click.argument("scene_name", type=str) +@click.argument("spacing", type=int) +@click.argument("scaling", type=str) +def run_pyrosar_gamma_workflow(scene_name, spacing, scaling): + + scene_file, orbit_file, dem_file = prepare_inputs_for_pyrosar_gamma(scene_name) + + run_pyrosar_gamma_geocode( + scene=scene_file, + orbit=orbit_file, + dem=dem_file, + output=OUTPUT_DIR, + gamma_library=GAMMA_LIBRARY, + gamma_env=GAMMA_ENV, + geocode_spacing=spacing, + geocode_scaling=scaling, + ) @click.command() diff --git a/sar_antarctica/nci/filesystem.py b/sar_antarctica/nci/filesystem.py index e4ae90f..c161c56 100644 --- a/sar_antarctica/nci/filesystem.py +++ b/sar_antarctica/nci/filesystem.py @@ -1,6 +1,7 @@ from pathlib import Path from sar_antarctica.nci.preparation.orbits import find_orbits +from sar_antarctica.nci.preparation.dem import get_cop30_dem_for_bounds def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: @@ -45,3 +46,13 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: orbits = find_orbits(nci_orbit_directories) return orbits + + +def get_dem_nci(scene_name, scene_bounds): + OUTPUT_DEM_PATH = Path("/g/data/yp75/projects/sar-antractica-processing/data/dem") + DEM_FILE = OUTPUT_DEM_PATH / f"{scene_name}.tif" + + if not DEM_FILE.exists(): + _, _ = get_cop30_dem_for_bounds(scene_bounds, DEM_FILE, ellipsoid_heights=True) + + return DEM_FILE diff --git a/sar_antarctica/nci/preparation/create_config.py b/sar_antarctica/nci/preparation/create_config.py index bfd2b87..6c26792 100644 --- a/sar_antarctica/nci/preparation/create_config.py +++ b/sar_antarctica/nci/preparation/create_config.py @@ -7,10 +7,12 @@ from sar_antarctica.nci.preparation.scenes import ( find_scene_file_from_id, + parse_scene_file_dates, parse_scene_file_sensor, ) from sar_antarctica.nci.preparation.orbits import find_latest_orbit_for_scene from sar_antarctica.nci.preparation.dem import get_cop30_dem_for_bounds +from sar_antarctica.nci.filesystem import get_orbits_nci, get_dem_nci def write_file_paths( @@ -48,6 +50,31 @@ def write_file_paths( ) +def prepare_inputs_for_pyrosar_gamma(scene_name): + + scene_file = find_scene_file_from_id(scene_name) + + # Find orbit + sensor = parse_scene_file_sensor(scene_name) + # start_time, stop_time = parse_scene_file_dates(scene_name) + orbit_files = get_orbits_nci("POE", sensor) + orbit_file = find_latest_orbit_for_scene(scene_name, orbit_files) + + # Build DEM + scene = identify(str(scene_file)) + scene_bbox = scene.bbox().extent + scene_bounds = ( + scene_bbox["xmin"], + scene_bbox["ymin"], + scene_bbox["xmax"], + scene_bbox["ymax"], + ) + + dem_file = get_dem_nci(scene_name, scene_bounds) + + return (scene_file, orbit_file, dem_file) + + @click.command() @click.argument("scene_id", nargs=1) @click.argument("scene_config", nargs=1) diff --git a/sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py b/sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py index 31052ec..893481e 100644 --- a/sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py +++ b/sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py @@ -6,7 +6,6 @@ from pyroSAR.gamma.dem import dem_import import shutil import sys -import tomli from sar_antarctica.nci.processing.GAMMA.GAMMA_utils import set_gamma_env_variables @@ -19,65 +18,37 @@ log.setLevel(logging.INFO) -@click.command() -@click.argument("workflow_config", nargs=1) -@click.argument("scene_config", nargs=1) -def cli(workflow_config: str, scene_config: str): +def prepare_directories(processing_root: Path, scene_full_name: str, scene_outname): - # Read in config file - with open(workflow_config, "rb") as f: - workflow_config_dict = tomli.load(f) + # Set directories under the processing root + SCENE_DIR = f"data/processed_scene/{scene_full_name}" + TEMP_DIR = f"data/temp/{scene_outname}" + LOG_DIR = f"data/temp/{scene_outname}/logfiles" - with open(scene_config, "rb") as f: - scene_config_dict = tomli.load(f) + # Construct a dictionary for use + processing_directories = { + "scene": processing_root / SCENE_DIR, + "temp": processing_root / TEMP_DIR, + "logs": processing_root / LOG_DIR, + } - # Split config dicts up to ease readability - config_inputs = scene_config_dict["inputs"] - config_outputs = scene_config_dict["outputs"] - config_gamma = workflow_config_dict["gamma"] - config_geocode = workflow_config_dict["geocode"] + # Create directories if not exist + log.info("Setting directories:") + for dir_name, dir_path in processing_directories.items(): + log.info(f" {dir_name}: {dir_path}") + dir_path.mkdir(parents=True, exist_ok=True) - # Environment variables for GAMMA must be set - set_gamma_env_variables( - config_gamma["software_env_var"], config_gamma["libs_env_var"] - ) - - # Identify scene - scene_zip = Path(config_inputs["scene"]) - scene_id = scene_zip.stem - log.info(f"Scene ID: {scene_id} has the following metadata:\n{scene_zip}") - if scene_zip.exists(): - pyrosar_scene_id = identify(scene_zip) - - # Construct output scenes - data_dir = Path(config_outputs["data"]) - processed_scene_dir = ( - data_dir - / config_outputs["processed"] - / pyrosar_scene_id.outname_base(extensions=None) - ) - pyrosar_temp_dir = ( - data_dir / "temp" / pyrosar_scene_id.outname_base(extensions=None) - ) - pyrosar_temp_log_dir = pyrosar_temp_dir / "logfiles" + return processing_directories - log.info("creating directories:") - for dir in [processed_scene_dir, pyrosar_temp_dir, pyrosar_temp_log_dir]: - dir.mkdir(parents=True, exist_ok=True) - log.info(f" {dir}") - # Copy over orbit file - orbit_file = Path(config_inputs["orbit"]) - orbit_filename = orbit_file.name - shutil.copy(orbit_file, pyrosar_temp_dir / orbit_filename) +def prepare_dem_for_gamma(dem_tif: Path, temp_dir: Path, log_dir: Path) -> Path: - # Create DEM in GAMMA format - dem_tif = Path(config_inputs["dem"]) - dem_gamma = pyrosar_temp_dir / dem_tif.stem + dem_dir = dem_tif.parent + dem_name = dem_tif.stem + dem_gamma = temp_dir / dem_name if dem_gamma.exists(): log.info("DEM exists") - pass else: log.info("running DEM") @@ -85,27 +56,60 @@ def cli(workflow_config: str, scene_config: str): src=str(dem_tif), dst=str(dem_gamma), geoid=None, - logpath=str(pyrosar_temp_log_dir), - outdir=str(dem_tif.parent), + logpath=str(log_dir), + outdir=str(dem_dir), ) log.info("finished DEM") - # Run geocode process - # Note that GAMMA geocode from pyrosar produces gamma_0 RTC backscatter + return dem_gamma + + +def run_pyrosar_gamma_geocode( + scene: Path, + orbit: Path, + dem: Path, + output: Path, + gamma_library: Path, + gamma_env: str, + geocode_spacing: int, + geocode_scaling: str, +): + + # Set up environment variables for GAMMA + set_gamma_env_variables(str(gamma_library), gamma_env) + + # Identify scene + scene_name = scene.stem + pyrosar_scene_id = identify(scene) + + # Create processing directories if required + processing_directories = prepare_directories( + output, scene_name, pyrosar_scene_id.outname_base(extensions=None) + ) + + # Prepare orbit file + # Copy to temp dir to prevent pyroSAR modifying in-place + orbit_dir = processing_directories["temp"] + shutil.copy(orbit, orbit_dir / orbit.name) + + dem_gamma = prepare_dem_for_gamma( + dem, processing_directories["temp"], processing_directories["logs"] + ) + log.info("running geocode") geocode( scene=pyrosar_scene_id, dem=str(dem_gamma), - tmpdir=str(pyrosar_temp_dir), - outdir=str(processed_scene_dir), - spacing=config_geocode["spacing"], - scaling=config_geocode["scaling"], + tmpdir=str(processing_directories["temp"]), + outdir=str(processing_directories["scene"]), + spacing=geocode_spacing, + scaling=geocode_scaling, func_geoback=1, nodata=(0, -99), update_osv=False, - osvdir=str(pyrosar_temp_dir), + osvdir=str(orbit_dir), allow_RES_OSV=False, cleanup=False, export_extra=[ @@ -124,8 +128,3 @@ def cli(workflow_config: str, scene_config: str): ) log.info("finished geocode") - - -if __name__ == "__main__": - - cli() From a9c14bcf6308c5941759e2a48d7433c268fc0dd8 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:02:42 +1100 Subject: [PATCH 03/13] Add cli to submit jobs leveraging revised cli --- pyproject.toml | 1 + sar_antarctica/nci/cli.py | 26 +++++++++ .../nci/submission/pbs_template.txt | 12 +++++ .../submission/pyrosar_gamma/submit_job.py | 54 +++++++++++++++++++ sar_antarctica/nci/submission/utils.py | 31 +++++++++++ 5 files changed, 124 insertions(+) create mode 100644 sar_antarctica/nci/submission/pbs_template.txt create mode 100644 sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py create mode 100644 sar_antarctica/nci/submission/utils.py diff --git a/pyproject.toml b/pyproject.toml index 13b1e5e..97e9eca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ Homepage = "https://github.com/GeoscienceAustralia/sar-antarctica.git" [project.scripts] find-orbits = "sar_antarctica.nci.cli:find_orbits_for_scene" run-pyrosar-gamma-workflow = "sar_antarctica.nci.cli:run_pyrosar_gamma_workflow" +submit-pyrosar-gamma-workflow = "sar_antarctica.nci.cli:submit_pyrosar_gamma_workflow" [tool.pytest.ini_options] testpaths = ["tests/*"] diff --git a/sar_antarctica/nci/cli.py b/sar_antarctica/nci/cli.py index 9abc91c..3f89418 100644 --- a/sar_antarctica/nci/cli.py +++ b/sar_antarctica/nci/cli.py @@ -15,12 +15,38 @@ from sar_antarctica.nci.processing.pyroSAR.pyrosar_geocode import ( run_pyrosar_gamma_geocode, ) +from sar_antarctica.nci.submission.pyrosar_gamma.submit_job import submit_job GAMMA_LIBRARY = Path("/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712") GAMMA_ENV = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64" OUTPUT_DIR = Path("/g/data/yp75/projects/sar-antractica-processing") +@click.command() +@click.argument("scene_name", type=str) +@click.argument("spacing", type=int) +@click.argument("scaling", type=str) +@click.argument("ncpu", type=str) +@click.argument("mem", type=str) +@click.argument("queue", type=str) +@click.argument("project", type=str) +@click.argument("walltime", type=str) +def submit_pyrosar_gamma_workflow( + scene_name, spacing, scaling, ncpu, mem, queue, project, walltime +): + + print("processing") + pbs_parameters = { + "ncpu": ncpu, + "mem": mem, + "queue": queue, + "project": project, + "walltime": walltime, + } + + submit_job(scene_name, spacing, scaling, pbs_parameters) + + @click.command() @click.argument("scene_name", type=str) @click.argument("spacing", type=int) diff --git a/sar_antarctica/nci/submission/pbs_template.txt b/sar_antarctica/nci/submission/pbs_template.txt new file mode 100644 index 0000000..f529e17 --- /dev/null +++ b/sar_antarctica/nci/submission/pbs_template.txt @@ -0,0 +1,12 @@ +#!/bin/bash + +# PBS -l ncpus= +# PBS -l mem=GB +# PBS -q +# PBS -P +# PBS -l walltime= +# PBS -l storage= +# PBS -l wd +# PBS -o / +# PBS -e / +# PBS -N \ No newline at end of file diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py new file mode 100644 index 0000000..003a3ff --- /dev/null +++ b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py @@ -0,0 +1,54 @@ +import os +from pathlib import Path +from sar_antarctica.nci.submission.utils import populate_pbs_template + +LOG_DIR = Path("/g/data/yp75/projects/sar-antractica-processing/submission/logs") +ENVIRONMENT_COMMAND = """ + +export MAMBA_EXE=/g/data/yp75/ca6983/micromamba/bin/micromamba +export MAMBA_ROOT_PREFIX=/g/data/yp75/ca6983/micromamba +source $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh + +micromamba activate sar-antarctica + +""" + +DEFAULT_DICT = { + "ncpu": "4", + "mem": "32", + "queue": "normal", + "project": "u46", + "walltime": "02:00:00", +} + + +def submit_job( + scene: str, + spacing: int, + scaling: str, + pbs_parameters: dict[str, str] = DEFAULT_DICT, +): + + scene_script = LOG_DIR / scene / f"{scene}.sh" + scene_script.parent.mkdir(exist_ok=True, parents=True) + + pbs_script = populate_pbs_template( + pbs_parameters["ncpu"], + pbs_parameters["mem"], + pbs_parameters["queue"], + pbs_parameters["project"], + pbs_parameters["walltime"], + scene, + ) + + job_command = f"pyrosar-gamma-workflow {scene} {spacing} {scaling}" + + job_script = pbs_script + ENVIRONMENT_COMMAND + job_command + print(job_script) + + # Write updated text to pbs script + scene_script.write_text(job_script) + + # Submit script + qsub_command = f"qsub {scene_script}" + os.system(qsub_command) diff --git a/sar_antarctica/nci/submission/utils.py b/sar_antarctica/nci/submission/utils.py new file mode 100644 index 0000000..1f181f3 --- /dev/null +++ b/sar_antarctica/nci/submission/utils.py @@ -0,0 +1,31 @@ +from pathlib import Path + + +SUBMISSION_DIR = Path(__file__).resolve().parent +SUBMISSION_TEMPLATE = SUBMISSION_DIR / "pbs_template.txt" +LOG_DIR = "/g/data/yp75/projects/sar-antractica-processing/submission/logs" +STORAGE = "gdata/yp75+gdata/dg9+gdata/fj7+gdata/v10" + + +def populate_pbs_template( + ncpu: int, mem: int, queue: str, project: str, walltime: str, jobname: str +): + pbs_template = SUBMISSION_TEMPLATE.read_text() + + replace_dict = { + "": ncpu, + "": mem, + "": queue, + "": project, + "": walltime, + "": STORAGE, + "": LOG_DIR, + "": jobname, + } + + for key, value in replace_dict.items(): + pbs_template = pbs_template.replace( + key, value if isinstance(value, str) else str(value) + ) + + return pbs_template From f8ea06c38f363d6eaaacee58910a3850dd70ba0c Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:16:22 +1100 Subject: [PATCH 04/13] Remove unused functions --- .../nci/preparation/create_config.py | 96 ------------------- 1 file changed, 96 deletions(-) diff --git a/sar_antarctica/nci/preparation/create_config.py b/sar_antarctica/nci/preparation/create_config.py index 6c26792..2bced65 100644 --- a/sar_antarctica/nci/preparation/create_config.py +++ b/sar_antarctica/nci/preparation/create_config.py @@ -1,62 +1,21 @@ -import click -from pathlib import Path from pyroSAR import identify -import rasterio from sar_antarctica.nci.filesystem import get_orbits_nci from sar_antarctica.nci.preparation.scenes import ( find_scene_file_from_id, - parse_scene_file_dates, parse_scene_file_sensor, ) from sar_antarctica.nci.preparation.orbits import find_latest_orbit_for_scene -from sar_antarctica.nci.preparation.dem import get_cop30_dem_for_bounds from sar_antarctica.nci.filesystem import get_orbits_nci, get_dem_nci -def write_file_paths( - config_file: Path, - scene_file: Path, - orbit_file: Path, - dem_file: Path, - data_dir: Path, - ancillary_dir="ancillary", - processed_dir="processed_scene", -): - inputs_header = "[inputs]\n" - scene_setting = f"scene = '{str(scene_file)}'\n" - orbit_setting = f"orbit = '{str(orbit_file)}'\n" - dem_setting = f"dem = '{str(dem_file)}'\n" - - outputs_header = "[outputs]\n" - data_path_setting = f"data = '{str(data_dir)}'\n" - ancillary_setting = f"ancillary = '{ancillary_dir}'\n" - processed_setting = f"processed = '{processed_dir}'\n" - - with open(config_file, "w") as cf: - cf.writelines( - [ - inputs_header, - scene_setting, - orbit_setting, - dem_setting, - "\n", - outputs_header, - data_path_setting, - ancillary_setting, - processed_setting, - ] - ) - - def prepare_inputs_for_pyrosar_gamma(scene_name): scene_file = find_scene_file_from_id(scene_name) # Find orbit sensor = parse_scene_file_sensor(scene_name) - # start_time, stop_time = parse_scene_file_dates(scene_name) orbit_files = get_orbits_nci("POE", sensor) orbit_file = find_latest_orbit_for_scene(scene_name, orbit_files) @@ -73,58 +32,3 @@ def prepare_inputs_for_pyrosar_gamma(scene_name): dem_file = get_dem_nci(scene_name, scene_bounds) return (scene_file, orbit_file, dem_file) - - -@click.command() -@click.argument("scene_id", nargs=1) -@click.argument("scene_config", nargs=1) -def main(scene_id: str, scene_config: str): - """Generate a configuration file for a scene ID - - Parameters - ---------- - scene_id : str - ID of scene to process - scene_config : str - where to store the output configuration file - """ - print(f"Processing scene: {scene_id} \n") - - # Set the data path for outputs - data_dir = Path("/g/data/yp75/projects/sar-antractica-processing/data") - - # Path to configuration file for scene - config_file = Path(scene_config) - - # Identify location of scene on GADI - scene_file = find_scene_file_from_id(scene_id) - - # Identify location of latest orbit file on GADI - scene_sensor = parse_scene_file_sensor(scene_id) - poe_orbits = get_orbits_nci("POE", scene_sensor) - latest_poe_file = find_latest_orbit_for_scene(scene_id, poe_orbits) - - # Identify bounds of scene and use bounding box to build DEM - scene = identify(str(scene_file)) - scene_bbox = scene.bbox().extent - scene_bounds = ( - scene_bbox["xmin"], - scene_bbox["ymin"], - scene_bbox["xmax"], - scene_bbox["ymax"], - ) - - # Set path for dem and create - dem_dir = data_dir / "dem" - dem_file = dem_dir / f"{scene_id}_dem.tif" - _, _ = get_cop30_dem_for_bounds( - bounds=scene_bounds, save_path=dem_file, ellipsoid_heights=True - ) - - # Write to config file - write_file_paths(config_file, scene_file, latest_poe_file, dem_file, data_dir) - - -if __name__ == "__main__": - - main() From 471d7853fdfc24b0e29e1fcb3e4676c13cb092ea Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:20:02 +1100 Subject: [PATCH 05/13] move create config script to submission --- sar_antarctica/nci/cli.py | 2 +- .../pyrosar_gamma/prepare_input.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename sar_antarctica/nci/{preparation/create_config.py => submission/pyrosar_gamma/prepare_input.py} (100%) diff --git a/sar_antarctica/nci/cli.py b/sar_antarctica/nci/cli.py index 3f89418..f34786b 100644 --- a/sar_antarctica/nci/cli.py +++ b/sar_antarctica/nci/cli.py @@ -2,7 +2,7 @@ from pathlib import Path from sar_antarctica.nci.filesystem import get_orbits_nci -from sar_antarctica.nci.preparation.create_config import ( +from sar_antarctica.nci.submission.pyrosar_gamma.prepare_input import ( prepare_inputs_for_pyrosar_gamma, ) from sar_antarctica.nci.preparation.orbits import ( diff --git a/sar_antarctica/nci/preparation/create_config.py b/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py similarity index 100% rename from sar_antarctica/nci/preparation/create_config.py rename to sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py From bcae78894820c338c3f12f13f0c5ada23ec63e25 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:22:26 +1100 Subject: [PATCH 06/13] Removed unused script --- .../submission/pyrosar_gamma/pyrosar_gamma.py | 173 ------------------ .../pyrosar_gamma/pyrosar_gamma.txt | 27 --- 2 files changed, 200 deletions(-) delete mode 100644 sar_antarctica/nci/submission/pyrosar_gamma/pyrosar_gamma.py delete mode 100644 sar_antarctica/nci/submission/pyrosar_gamma/pyrosar_gamma.txt diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/pyrosar_gamma.py b/sar_antarctica/nci/submission/pyrosar_gamma/pyrosar_gamma.py deleted file mode 100644 index 015761a..0000000 --- a/sar_antarctica/nci/submission/pyrosar_gamma/pyrosar_gamma.py +++ /dev/null @@ -1,173 +0,0 @@ -import click -import os -from pathlib import Path -import tomli -from typing import Any - -WORKFLOW = "pyrosar_gamma" -PROCESSING_DIR = "/g/data/yp75/projects/sar-antractica-processing" - - -def get_list_of_scenes(scene_source: str) -> list[str]: - """Convert script input to list. - If a .zip file, produce a list with that. - If a .txt file, open the file, and produce a list of all .zip files. - - Parameters - ---------- - scene_source : str - The file to be processed. Either a single .zip or a .txt containing multiple .zip files - - Returns - ------- - list[str] - List of files to process - """ - - # Process a single .zip file - if scene_source.endswith(".zip"): - scene_list = [scene_source] - # Process a .txt file containing .zip files - elif scene_source.endswith(".txt"): - with open(scene_source, "r") as f: - scene_list = [line.strip() for line in f if line.strip().endswith(".zip")] - else: - scene_list = [] - - if scene_list is not None: - return scene_list - else: - raise RuntimeError( - "No valid scenes were found for processing. Expected single .zip file or .txt file containing at least one .zip file." - ) - - -def update_pbs_template( - pbs_template: str, scene_id: str, job_config: dict[str, str | dict[str, Any]] -) -> str: - """_summary_ - - Parameters - ---------- - pbs_template : str - A string containing a PBS jobscript - scene_id : str - The scene ID for the job - job_config : dict[str, str | dict[str, Any]] - Dictionary containing information on the job, main keys are - root, submission, configuration, and settings - - Returns - ------- - str - The updated PBS jobscript string with specified values replaced - """ - - """For a given PBS jobscript template, replace specified values with jobscript settings - - Parameters - ---------- - pbs_template : str - A string containing a PBS jobscript - jobscript_settings: dict - - - Returns - ------- - str - The updated PBS jobscript string with specified values replaced - """ - - processing_path = Path(job_config["root"]) - log_path = ( - processing_path - / job_config["submission"]["root"] - / job_config["submission"]["logs"] - ) - config_path = processing_path / job_config["configuration"]["root"] - - job_configuration = job_config["configuration"] - job_settings = job_config["settings"] - - workflow_config = job_settings["workflow_config"] - # Dictionary to replace placeholders in PBS text with values from configurations - replace_dict = { - "": scene_id, - "": job_settings["ncpu"], - "": job_settings["mem"], - "": job_settings["queue"], - "": job_settings["project"], - "": job_settings["walltime"], - "": job_settings["storage"], - "": log_path, - "": config_path - / job_configuration["workflow"] - / f"{workflow_config}.toml", - "": config_path / job_configuration["scene"] / f"{scene_id}.toml", - } - - for key, value in replace_dict.items(): - pbs_template = pbs_template.replace( - key, value if isinstance(value, str) else str(value) - ) - - return pbs_template - - -@click.command() -@click.argument("config_file", nargs=1) -@click.argument("scene_source", nargs=1) -def pyrosar_gamma_workflow( - config_file: str | os.PathLike, scene_source: str | os.PathLike -) -> None: - """Take an input of a single scene or file with multiple scenes and submit pyroSAR+GAMMA jobs - - Parameters - ---------- - processing_dir : str - The directory to store configuration and jobscript files - scene_source : str - The file to be processed. Either a single .zip or a .txt containing multiple .zip files - """ - - current_file_directory = Path(__file__).resolve().parent - - with open(config_file, "rb") as f: - config = tomli.load(f) - - # Extract specific configuration dictionaries - job_config = config["job"] - submission_config = job_config["submission"] - configuration_config = job_config["configuration"] - settings_config = job_config["settings"] - - # Get folder structure - processing_dir = Path(job_config["root"]) - log_dir = processing_dir / submission_config["root"] / submission_config["logs"] - - # Get scenes from source - scene_list = get_list_of_scenes(scene_source) - - for scene_path in scene_list: - # Determine scene ID from command line input and create submission script - scene_id = Path(scene_path).stem - scene_script = log_dir / scene_id / f"{scene_id}.sh" - scene_script.parent.mkdir(exist_ok=True, parents=True) - - # Read the workflow template and replace values - workflow_name = settings_config["workflow_config"] - template_file = current_file_directory / f"{workflow_name}.txt" - print(template_file) - pbs_template = template_file.read_text() - pbs_template = update_pbs_template(pbs_template, scene_id, job_config) - - # Write updated text to pbs script - scene_script.write_text(pbs_template) - - # Submit script - qsub_command = f"qsub {scene_script}" - os.system(qsub_command) - - -if __name__ == "__main__": - pyrosar_gamma_workflow() diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/pyrosar_gamma.txt b/sar_antarctica/nci/submission/pyrosar_gamma/pyrosar_gamma.txt deleted file mode 100644 index a33d8b5..0000000 --- a/sar_antarctica/nci/submission/pyrosar_gamma/pyrosar_gamma.txt +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -#PBS -l ncpus= -#PBS -l mem=GB -#PBS -q -#PBS -P -#PBS -l walltime= -#PBS -l storage= -#PBS -l wd -#PBS -o / -#PBS -e / -#PBS -N - -# Load the module -module use /g/data/yp75/modules/modulefiles -module load sar-antarctica/v0.1 - -# Activate micromamba environment -micromamba activate sar-antarctica - -# Run preparation script -cd /g/data/yp75/modules/sar-antarctica/v0.1/sar-antarctica/sar_antarctica/nci/preparation/ -python create_config.py - -# Run preocessing script -cd /g/data/yp75/modules/sar-antarctica/v0.1/sar-antarctica/sar_antarctica/nci/processing/pyroSAR/ -python pyrosar_geocode.py \ No newline at end of file From dbe31b3c124188deb0fc86f91b4f79f19028caf8 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:28:10 +1100 Subject: [PATCH 07/13] Fix issue with spacing of PBS commands in template and update submit-job to use the correct cli --- .../nci/submission/pbs_template.txt | 21 +++++++++---------- .../submission/pyrosar_gamma/submit_job.py | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/sar_antarctica/nci/submission/pbs_template.txt b/sar_antarctica/nci/submission/pbs_template.txt index f529e17..64d44af 100644 --- a/sar_antarctica/nci/submission/pbs_template.txt +++ b/sar_antarctica/nci/submission/pbs_template.txt @@ -1,12 +1,11 @@ #!/bin/bash - -# PBS -l ncpus= -# PBS -l mem=GB -# PBS -q -# PBS -P -# PBS -l walltime= -# PBS -l storage= -# PBS -l wd -# PBS -o / -# PBS -e / -# PBS -N \ No newline at end of file +#PBS -l ncpus= +#PBS -l mem=GB +#PBS -q +#PBS -P +#PBS -l walltime= +#PBS -l storage= +#PBS -l wd +#PBS -o / +#PBS -e / +#PBS -N \ No newline at end of file diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py index 003a3ff..2097f8d 100644 --- a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py +++ b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py @@ -41,7 +41,7 @@ def submit_job( scene, ) - job_command = f"pyrosar-gamma-workflow {scene} {spacing} {scaling}" + job_command = f"run-pyrosar-gamma-workflow {scene} {spacing} {scaling}" job_script = pbs_script + ENVIRONMENT_COMMAND + job_command print(job_script) From 660d9a29c893004a42883ed192a5a7399fde4fae Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:13:53 +1100 Subject: [PATCH 08/13] Update relevant functions to work with scene as path, rather than as scene name --- sar_antarctica/nci/cli.py | 27 ++++++---- sar_antarctica/nci/filesystem.py | 4 +- .../submission/pyrosar_gamma/prepare_input.py | 52 +++++++++++++------ .../submission/pyrosar_gamma/submit_job.py | 9 ++-- 4 files changed, 59 insertions(+), 33 deletions(-) diff --git a/sar_antarctica/nci/cli.py b/sar_antarctica/nci/cli.py index f34786b..054a063 100644 --- a/sar_antarctica/nci/cli.py +++ b/sar_antarctica/nci/cli.py @@ -3,7 +3,7 @@ from sar_antarctica.nci.filesystem import get_orbits_nci from sar_antarctica.nci.submission.pyrosar_gamma.prepare_input import ( - prepare_inputs_for_pyrosar_gamma, + get_orbit_and_dem, ) from sar_antarctica.nci.preparation.orbits import ( filter_orbits_to_cover_time_window, @@ -23,7 +23,10 @@ @click.command() -@click.argument("scene_name", type=str) +@click.argument( + "scene", + type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path), +) @click.argument("spacing", type=int) @click.argument("scaling", type=str) @click.argument("ncpu", type=str) @@ -32,10 +35,9 @@ @click.argument("project", type=str) @click.argument("walltime", type=str) def submit_pyrosar_gamma_workflow( - scene_name, spacing, scaling, ncpu, mem, queue, project, walltime + scene, spacing, scaling, ncpu, mem, queue, project, walltime ): - print("processing") pbs_parameters = { "ncpu": ncpu, "mem": mem, @@ -44,21 +46,24 @@ def submit_pyrosar_gamma_workflow( "walltime": walltime, } - submit_job(scene_name, spacing, scaling, pbs_parameters) + submit_job(scene, spacing, scaling, pbs_parameters) @click.command() -@click.argument("scene_name", type=str) +@click.argument( + "scene", + type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path), +) @click.argument("spacing", type=int) @click.argument("scaling", type=str) -def run_pyrosar_gamma_workflow(scene_name, spacing, scaling): +def run_pyrosar_gamma_workflow(scene, spacing, scaling): - scene_file, orbit_file, dem_file = prepare_inputs_for_pyrosar_gamma(scene_name) + orbit, dem = get_orbit_and_dem(scene) run_pyrosar_gamma_geocode( - scene=scene_file, - orbit=orbit_file, - dem=dem_file, + scene=scene, + orbit=orbit, + dem=dem, output=OUTPUT_DIR, gamma_library=GAMMA_LIBRARY, gamma_env=GAMMA_ENV, diff --git a/sar_antarctica/nci/filesystem.py b/sar_antarctica/nci/filesystem.py index c161c56..5e3b47a 100644 --- a/sar_antarctica/nci/filesystem.py +++ b/sar_antarctica/nci/filesystem.py @@ -48,9 +48,9 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: return orbits -def get_dem_nci(scene_name, scene_bounds): +def get_dem_nci(scene: Path, scene_bounds: tuple[float, float, float, float]): OUTPUT_DEM_PATH = Path("/g/data/yp75/projects/sar-antractica-processing/data/dem") - DEM_FILE = OUTPUT_DEM_PATH / f"{scene_name}.tif" + DEM_FILE = OUTPUT_DEM_PATH / f"{scene.stem}.tif" if not DEM_FILE.exists(): _, _ = get_cop30_dem_for_bounds(scene_bounds, DEM_FILE, ellipsoid_heights=True) diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py b/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py index 2bced65..a381a0b 100644 --- a/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py +++ b/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py @@ -1,26 +1,45 @@ +from datetime import datetime +from pathlib import Path from pyroSAR import identify -from sar_antarctica.nci.filesystem import get_orbits_nci - -from sar_antarctica.nci.preparation.scenes import ( - find_scene_file_from_id, - parse_scene_file_sensor, -) -from sar_antarctica.nci.preparation.orbits import find_latest_orbit_for_scene +from sar_antarctica.nci.preparation.orbits import find_latest_orbit_covering_window from sar_antarctica.nci.filesystem import get_orbits_nci, get_dem_nci -def prepare_inputs_for_pyrosar_gamma(scene_name): +def get_orbit_and_dem( + scene_file: Path, orbit_type: str | None = "POE" +) -> tuple[Path, Path]: + """For a given Sentinel-1 scene, find the relevant orbit path and DEM path. + The DEM will be created if no DEM path is found. + + Parameters + ---------- + scene_file : Path + Full path to the scene + e.g. "path/to/scene/scene_id.zip" + orbit_type : str, optional + The orbit type to get. Any of "POE", "RES" or None, by default "POE" + + Returns + ------- + tuple[Path, Path] + A tuple containing the path to the orbit file and a path to the DEM file. + e.g. ("path/to/orbit/orbitfile.EOF", "path/to/dem/demfile.tif") + """ - scene_file = find_scene_file_from_id(scene_name) + # Extract metadata + scene = identify(scene_file) + + # Isolate metadata for finding orbit + scene_sensor = scene.sensor + scene_start = datetime.strptime(scene.start, "%Y%m%dT%H%M%S") + scene_stop = datetime.strptime(scene.stop, "%Y%m%dT%H%M%S") # Find orbit - sensor = parse_scene_file_sensor(scene_name) - orbit_files = get_orbits_nci("POE", sensor) - orbit_file = find_latest_orbit_for_scene(scene_name, orbit_files) + orbit_files = get_orbits_nci(orbit_type, scene_sensor) + orbit_file = find_latest_orbit_covering_window(orbit_files, scene_start, scene_stop) - # Build DEM - scene = identify(str(scene_file)) + # Isolate metadata for creating DEM scene_bbox = scene.bbox().extent scene_bounds = ( scene_bbox["xmin"], @@ -29,6 +48,7 @@ def prepare_inputs_for_pyrosar_gamma(scene_name): scene_bbox["ymax"], ) - dem_file = get_dem_nci(scene_name, scene_bounds) + # Build DEM + dem_file = get_dem_nci(scene_file, scene_bounds) - return (scene_file, orbit_file, dem_file) + return (orbit_file, dem_file) diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py index 2097f8d..c734d76 100644 --- a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py +++ b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py @@ -23,13 +23,15 @@ def submit_job( - scene: str, + scene: Path, spacing: int, scaling: str, pbs_parameters: dict[str, str] = DEFAULT_DICT, ): - scene_script = LOG_DIR / scene / f"{scene}.sh" + scene_name = scene.stem + + scene_script = LOG_DIR / scene_name / f"{scene_name}.sh" scene_script.parent.mkdir(exist_ok=True, parents=True) pbs_script = populate_pbs_template( @@ -44,11 +46,10 @@ def submit_job( job_command = f"run-pyrosar-gamma-workflow {scene} {spacing} {scaling}" job_script = pbs_script + ENVIRONMENT_COMMAND + job_command - print(job_script) # Write updated text to pbs script scene_script.write_text(job_script) # Submit script qsub_command = f"qsub {scene_script}" - os.system(qsub_command) + # os.system(qsub_command) From 3ec0f8aac7ab97eade5ed39829f4acc64c4a5428 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:41:38 +1100 Subject: [PATCH 09/13] Update geocode to only run GRD border noise removal if scene product is GRD --- sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py b/sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py index 893481e..1b7f96d 100644 --- a/sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py +++ b/sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py @@ -99,6 +99,12 @@ def run_pyrosar_gamma_geocode( log.info("running geocode") + # Set the border removal step to pyroSAR for GRD products. Ignore otherwise + if pyrosar_scene_id.product == "GRD": + border_removal_method = "pyroSAR" + else: + border_removal_method = None + geocode( scene=pyrosar_scene_id, dem=str(dem_gamma), @@ -120,7 +126,7 @@ def run_pyrosar_gamma_geocode( "pix_ratio_geo", ], basename_extensions=None, - removeS1BorderNoiseMethod="pyroSAR", + removeS1BorderNoiseMethod=border_removal_method, refine_lut=False, rlks=None, azlks=None, From cf0eb5690f5457623d0f6c424e4f977f5efe90c2 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:43:17 +1100 Subject: [PATCH 10/13] Add function to parse scene product and create new cli to find files from ID --- pyproject.toml | 1 + sar_antarctica/nci/cli.py | 16 ++++++- sar_antarctica/nci/filesystem.py | 14 +++--- sar_antarctica/nci/preparation/scenes.py | 55 +++++++++++++++++++++++- 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97e9eca..66ff531 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ Homepage = "https://github.com/GeoscienceAustralia/sar-antarctica.git" # Documentation = "XXX" [project.scripts] +find-scene = "sar_antarctica.nci.cli:find_scene_file" find-orbits = "sar_antarctica.nci.cli:find_orbits_for_scene" run-pyrosar-gamma-workflow = "sar_antarctica.nci.cli:run_pyrosar_gamma_workflow" submit-pyrosar-gamma-workflow = "sar_antarctica.nci.cli:submit_pyrosar_gamma_workflow" diff --git a/sar_antarctica/nci/cli.py b/sar_antarctica/nci/cli.py index 054a063..227014f 100644 --- a/sar_antarctica/nci/cli.py +++ b/sar_antarctica/nci/cli.py @@ -11,6 +11,7 @@ from sar_antarctica.nci.preparation.scenes import ( parse_scene_file_sensor, parse_scene_file_dates, + find_scene_file_from_id, ) from sar_antarctica.nci.processing.pyroSAR.pyrosar_geocode import ( run_pyrosar_gamma_geocode, @@ -19,7 +20,15 @@ GAMMA_LIBRARY = Path("/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712") GAMMA_ENV = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64" -OUTPUT_DIR = Path("/g/data/yp75/projects/sar-antractica-processing") +OUTPUT_DIR = Path("/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/") + + +@click.command() +@click.argument("scene_name", type=str) +def find_scene_file(scene_name): + scene_file = find_scene_file_from_id(scene_name) + + click.echo(scene_file) @click.command() @@ -58,8 +67,13 @@ def submit_pyrosar_gamma_workflow( @click.argument("scaling", type=str) def run_pyrosar_gamma_workflow(scene, spacing, scaling): + click.echo("Preparing orbit and DEM") orbit, dem = get_orbit_and_dem(scene) + click.echo(f" Identified orbit: {orbit}") + click.echo(f" Identified DEM: {dem}") + + click.echo("Running processing") run_pyrosar_gamma_geocode( scene=scene, orbit=orbit, diff --git a/sar_antarctica/nci/filesystem.py b/sar_antarctica/nci/filesystem.py index 5e3b47a..7e8dd01 100644 --- a/sar_antarctica/nci/filesystem.py +++ b/sar_antarctica/nci/filesystem.py @@ -49,10 +49,14 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: def get_dem_nci(scene: Path, scene_bounds: tuple[float, float, float, float]): - OUTPUT_DEM_PATH = Path("/g/data/yp75/projects/sar-antractica-processing/data/dem") - DEM_FILE = OUTPUT_DEM_PATH / f"{scene.stem}.tif" + OUTPUT_DIR = Path( + "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/data/dem" + ) + if not OUTPUT_DIR.exists(): + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + dem_file = OUTPUT_DIR / f"{scene.stem}.tif" - if not DEM_FILE.exists(): - _, _ = get_cop30_dem_for_bounds(scene_bounds, DEM_FILE, ellipsoid_heights=True) + if not dem_file.exists(): + _, _ = get_cop30_dem_for_bounds(scene_bounds, dem_file, ellipsoid_heights=True) - return DEM_FILE + return dem_file diff --git a/sar_antarctica/nci/preparation/scenes.py b/sar_antarctica/nci/preparation/scenes.py index f7f4b35..849a667 100644 --- a/sar_antarctica/nci/preparation/scenes.py +++ b/sar_antarctica/nci/preparation/scenes.py @@ -2,7 +2,7 @@ from pathlib import Path import re -SCENE_DIR = Path("/g/data/fj7/Copernicus/Sentinel-1/C-SAR/GRD/") +SCENE_DIR = Path("/g/data/fj7/Copernicus/Sentinel-1/C-SAR/") def parse_scene_file_sensor(scene_id: str) -> str: @@ -37,6 +37,55 @@ def parse_scene_file_sensor(scene_id: str) -> str: return match.group(1) +def parse_scene_file_mode(scene_id: str) -> str: + raise NotImplementedError + + +def parse_scene_file_product(scene_id: str) -> str: + """Extract Sentinel-1 product string (GRDM or SLC_) from scene ID and return + shortened version (either GRD or SLC) + + Parameters + ---------- + scene_id : str + Sentinel-1 scene ID + e.g. S1A_EW_GRDM_1SDH_20220612T120348_20220612T120452_043629_053582_0F6 + + Returns + ------- + str + Product string, either GRD or SLC + + Raises + ------ + ValueError + Could not find expected match of four characters containing letters/underscores. + ValueError + Identified string did not match either "GRDM" or "SLC_" + """ + + pattern = r"^S1[A-Z]_[A-Z]{2}_([A-Z_]{4})_" + + match = re.match(pattern, scene_id) + + if not match: + raise ValueError( + "No product string was found. Looking for S1X_YY_ZZZZ_ where ZZZZ can be letters or underscores." + ) + + product_string = match.group(1) + if product_string == "GRDM": + product = "GRD" + elif product_string == "SLC_": + product = "SLC" + else: + raise ValueError( + f"Expected product string to be either GRDM or SLC_, but got {product_string}." + ) + + return product + + def parse_scene_file_dates(scene_id: str) -> tuple[datetime, datetime]: """Extracts start_date and end_date from the given scene ID. @@ -93,6 +142,8 @@ def find_scene_file_from_id(scene_id: str) -> Path: Found no files -- expects one. Or another Error """ + scene_product = parse_scene_file_product(scene_id) + # Parse the scene dates -- only start date is needed for search scene_start, _ = parse_scene_file_dates(scene_id) @@ -101,7 +152,7 @@ def find_scene_file_from_id(scene_id: str) -> Path: month = scene_start.strftime("%m") # Set path on GADI and search - search_path = SCENE_DIR.joinpath(f"{year}/{year}-{month}/") + search_path = SCENE_DIR.joinpath(f"{scene_product}/{year}/{year}-{month}/") file_path = list(search_path.rglob(f"{scene_id}.zip")) # Identify file From eacc05a513f31418a2479027ac3dc5ce75a75736 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Thu, 23 Jan 2025 08:50:15 +1100 Subject: [PATCH 11/13] Update cli to accept config as a parameter, which will override function defaults, but be overwritten by command line option values --- sar_antarctica/nci/cli.py | 96 +++++++++++++++---- sar_antarctica/nci/configs/default.toml | 10 ++ .../submission/pyrosar_gamma/submit_job.py | 17 +--- 3 files changed, 88 insertions(+), 35 deletions(-) create mode 100644 sar_antarctica/nci/configs/default.toml diff --git a/sar_antarctica/nci/cli.py b/sar_antarctica/nci/cli.py index 227014f..960b6fb 100644 --- a/sar_antarctica/nci/cli.py +++ b/sar_antarctica/nci/cli.py @@ -1,5 +1,6 @@ import click from pathlib import Path +import tomli from sar_antarctica.nci.filesystem import get_orbits_nci from sar_antarctica.nci.submission.pyrosar_gamma.prepare_input import ( @@ -18,10 +19,6 @@ ) from sar_antarctica.nci.submission.pyrosar_gamma.submit_job import submit_job -GAMMA_LIBRARY = Path("/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712") -GAMMA_ENV = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64" -OUTPUT_DIR = Path("/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/") - @click.command() @click.argument("scene_name", type=str) @@ -31,20 +28,45 @@ def find_scene_file(scene_name): click.echo(scene_file) +DEFAULT_CONFIGURATION = Path(__file__).resolve().parent / "configs/default.toml" + + +def configure(ctx, param, filename): + with open(filename, "rb") as f: + configuration_dictionary = tomli.load(f) + ctx.default_map = configuration_dictionary + + @click.command() @click.argument( "scene", - type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path), + type=click.Path(exists=True, dir_okay=False, path_type=Path), +) +@click.option( + "-c", + "--config", + type=click.Path(dir_okay=False), + default=DEFAULT_CONFIGURATION, + callback=configure, + is_eager=True, + expose_value=False, + help="Read option defaults from the specified .toml file", + show_default=True, +) +@click.option("--spacing", type=int) +@click.option("--scaling", type=click.Choice(["linear", "db"])) +@click.option("--ncpu", type=str, default="4") +@click.option("--mem", type=str, default="32") +@click.option("--queue", type=str, default="normal") +@click.option("--project", type=str, default="u46") +@click.option("--walltime", type=str, default="02:00:00") +@click.option( + "--output-dir", + type=click.Path(exists=True, file_okay=False, path_type=Path), + default="/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/", ) -@click.argument("spacing", type=int) -@click.argument("scaling", type=str) -@click.argument("ncpu", type=str) -@click.argument("mem", type=str) -@click.argument("queue", type=str) -@click.argument("project", type=str) -@click.argument("walltime", type=str) def submit_pyrosar_gamma_workflow( - scene, spacing, scaling, ncpu, mem, queue, project, walltime + scene, spacing, scaling, ncpu, mem, queue, project, walltime, output_dir ): pbs_parameters = { @@ -55,17 +77,48 @@ def submit_pyrosar_gamma_workflow( "walltime": walltime, } - submit_job(scene, spacing, scaling, pbs_parameters) + log_dir = output_dir / "submission/logs" + log_dir.mkdir(parents=True, exist_ok=True) + + submit_job(scene, spacing, scaling, pbs_parameters, log_dir) @click.command() @click.argument( "scene", - type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path), + type=click.Path(exists=True, dir_okay=False, path_type=Path), +) +@click.option( + "-c", + "--config", + type=click.Path(dir_okay=False), + default=DEFAULT_CONFIGURATION, + callback=configure, + is_eager=True, + expose_value=False, + help="Read option defaults from the specified .toml file", + show_default=True, +) +@click.option("--spacing", type=int) +@click.option("--scaling", type=click.Choice(["linear", "db"])) +@click.option( + "--output-dir", + type=click.Path(exists=True, file_okay=False, path_type=Path), + default="/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/", ) -@click.argument("spacing", type=int) -@click.argument("scaling", type=str) -def run_pyrosar_gamma_workflow(scene, spacing, scaling): +@click.option( + "--gamma-lib-dir", + type=click.Path(exists=True, file_okay=False, path_type=Path), + default="/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712", +) +@click.option( + "--gamma-env-var", + type=str, + default="/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64", +) +def run_pyrosar_gamma_workflow( + scene, spacing, scaling, output_dir, gamma_lib_dir, gamma_env_var +): click.echo("Preparing orbit and DEM") orbit, dem = get_orbit_and_dem(scene) @@ -74,13 +127,14 @@ def run_pyrosar_gamma_workflow(scene, spacing, scaling): click.echo(f" Identified DEM: {dem}") click.echo("Running processing") + print(scene, spacing, scaling, output_dir, gamma_lib_dir, gamma_env_var) run_pyrosar_gamma_geocode( scene=scene, orbit=orbit, dem=dem, - output=OUTPUT_DIR, - gamma_library=GAMMA_LIBRARY, - gamma_env=GAMMA_ENV, + output=output_dir, + gamma_library=gamma_lib_dir, + gamma_env=gamma_env_var, geocode_spacing=spacing, geocode_scaling=scaling, ) diff --git a/sar_antarctica/nci/configs/default.toml b/sar_antarctica/nci/configs/default.toml new file mode 100644 index 0000000..47a110b --- /dev/null +++ b/sar_antarctica/nci/configs/default.toml @@ -0,0 +1,10 @@ +spacing = 40 +scaling = "linear" +ncpu = "4" +mem = "32" +queue = "normal" +project = "u46" +walltime = "04:00:00" +output_dir = "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/" +gamma_lib_dir = "/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712" +gamma_env_var = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64" \ No newline at end of file diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py index c734d76..b63601e 100644 --- a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py +++ b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py @@ -2,7 +2,7 @@ from pathlib import Path from sar_antarctica.nci.submission.utils import populate_pbs_template -LOG_DIR = Path("/g/data/yp75/projects/sar-antractica-processing/submission/logs") + ENVIRONMENT_COMMAND = """ export MAMBA_EXE=/g/data/yp75/ca6983/micromamba/bin/micromamba @@ -13,25 +13,14 @@ """ -DEFAULT_DICT = { - "ncpu": "4", - "mem": "32", - "queue": "normal", - "project": "u46", - "walltime": "02:00:00", -} - def submit_job( - scene: Path, - spacing: int, - scaling: str, - pbs_parameters: dict[str, str] = DEFAULT_DICT, + scene: Path, spacing: int, scaling: str, pbs_parameters: dict[str, str], log_dir ): scene_name = scene.stem - scene_script = LOG_DIR / scene_name / f"{scene_name}.sh" + scene_script = log_dir / scene_name / f"{scene_name}.sh" scene_script.parent.mkdir(exist_ok=True, parents=True) pbs_script = populate_pbs_template( From 3e61ffb3cde90c542a6e2b63f9510d38d2fc6d33 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Thu, 23 Jan 2025 09:16:45 +1100 Subject: [PATCH 12/13] Update workflow docs to include instructions on running cli --- docs/workflows/README.MD | 49 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/workflows/README.MD b/docs/workflows/README.MD index fb44709..50c8cbb 100644 --- a/docs/workflows/README.MD +++ b/docs/workflows/README.MD @@ -1,4 +1,49 @@ -## Workflows +# Workflows The repository will handle multiple workflows and backends, some of which have specific requirements. -- [pyroSAR + GAMMA](pyrosar_gamma.md) \ No newline at end of file +## Workflows +- pyroSAR + GAMMA + - [Requirements](pyrosar_gamma.md) + +## Command line interface +> **_NOTE:_** At this time, pyroSAR + GAMMA is the only workflow, and the command line interfaces have not yet been generalised to other workflows. + +The package has a number of useful command line interfaces: + +### Finding the location of a scene on the NCI +The `find-scene` command will display the location of a given scene on the NCI. +The full path to the scene is required as the input to other commands. + +Example usage: +``` +$ find-scene S1A_EW_GRDM_1SDH_20240129T091735_20240129T091828_052319_065379_0F1E + +/path/to/scene/S1A_EW_GRDM_1SDH_20240129T091735_20240129T091828_052319_065379_0F1E.zip +``` + +### Submit a workflow +This will submit a job request to the NCI based on the job parameters and file paths in the supplied config. +The [default config](../../sar_antarctica/nci/configs/default.toml) will be used if no other config is provided. + +Example usage +``` +$ submit-pyrosar-gamma-workflow /path/to/scene/S1A_EW_GRDM_1SDH_20240129T091735_20240129T091828_052319_065379_0F1E.zip +``` +This will submit a job to the NCI with the default config. +To use a different config, run the command and supply the `--config` option +``` +--config /path/to/config.toml +``` + +### Run a workflow interactively +If you are still testing a workflow, it is best to run it in an interactive session. +While in an interactive session, you can run the workflow directly. + +Example usage +``` +$ run-pyrosar-gamma-workflow /path/to/scene/S1A_EW_GRDM_1SDH_20240129T091735_20240129T091828_052319_065379_0F1E.zip +``` +To use a different config, run the command and supply the `--config` option +``` +--config /path/to/config.toml +``` \ No newline at end of file From 3d523c62cc5bab57bb5f6489d37965b128674cc1 Mon Sep 17 00:00:00 2001 From: Caitlin Adams <25995927+caitlinadams@users.noreply.github.com> Date: Thu, 23 Jan 2025 16:01:09 +1100 Subject: [PATCH 13/13] Update functions to remove hard-coding for sar_antarctica/nci/filesystem.py --- sar_antarctica/nci/cli.py | 17 +++++++++++-- sar_antarctica/nci/configs/EW.toml | 12 +++++++++ sar_antarctica/nci/configs/IW.toml | 12 +++++++++ sar_antarctica/nci/configs/default.toml | 4 ++- sar_antarctica/nci/filesystem.py | 25 +++++++++++-------- .../submission/pyrosar_gamma/prepare_input.py | 9 ++++--- .../submission/pyrosar_gamma/submit_job.py | 15 ++++++++--- sar_antarctica/nci/submission/utils.py | 11 +++++--- 8 files changed, 81 insertions(+), 24 deletions(-) create mode 100644 sar_antarctica/nci/configs/EW.toml create mode 100644 sar_antarctica/nci/configs/IW.toml diff --git a/sar_antarctica/nci/cli.py b/sar_antarctica/nci/cli.py index 960b6fb..ef344a5 100644 --- a/sar_antarctica/nci/cli.py +++ b/sar_antarctica/nci/cli.py @@ -101,6 +101,10 @@ def submit_pyrosar_gamma_workflow( ) @click.option("--spacing", type=int) @click.option("--scaling", type=click.Choice(["linear", "db"])) +@click.option( + "--orbit-dir", type=click.Path(exists=True, file_okay=False, path_type=Path) +) +@click.option("--orbit-type", type=click.Choice(["POE", "RES", "either"])) @click.option( "--output-dir", type=click.Path(exists=True, file_okay=False, path_type=Path), @@ -117,11 +121,20 @@ def submit_pyrosar_gamma_workflow( default="/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64", ) def run_pyrosar_gamma_workflow( - scene, spacing, scaling, output_dir, gamma_lib_dir, gamma_env_var + scene, + spacing, + scaling, + orbit_dir, + orbit_type, + output_dir, + gamma_lib_dir, + gamma_env_var, ): click.echo("Preparing orbit and DEM") - orbit, dem = get_orbit_and_dem(scene) + dem_output_dir = output_dir / "data/dem" + + orbit, dem = get_orbit_and_dem(scene, dem_output_dir, orbit_dir, orbit_type) click.echo(f" Identified orbit: {orbit}") click.echo(f" Identified DEM: {dem}") diff --git a/sar_antarctica/nci/configs/EW.toml b/sar_antarctica/nci/configs/EW.toml new file mode 100644 index 0000000..59bb9b4 --- /dev/null +++ b/sar_antarctica/nci/configs/EW.toml @@ -0,0 +1,12 @@ +spacing = 40 +scaling = "linear" +ncpu = "4" +mem = "64" +queue = "normal" +project = "u46" +walltime = "02:00:00" +orbit_dir = "/g/data/fj7/Copernicus/Sentinel-1/" +orbit_type = "POE" +output_dir = "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/" +gamma_lib_dir = "/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712" +gamma_env_var = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64" \ No newline at end of file diff --git a/sar_antarctica/nci/configs/IW.toml b/sar_antarctica/nci/configs/IW.toml new file mode 100644 index 0000000..73e9501 --- /dev/null +++ b/sar_antarctica/nci/configs/IW.toml @@ -0,0 +1,12 @@ +spacing = 10 +scaling = "linear" +ncpu = "4" +mem = "128" +queue = "normal" +project = "u46" +walltime = "02:00:00" +orbit_dir = "/g/data/fj7/Copernicus/Sentinel-1/" +orbit_type = "POE" +output_dir = "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma_IW/" +gamma_lib_dir = "/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712" +gamma_env_var = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64" \ No newline at end of file diff --git a/sar_antarctica/nci/configs/default.toml b/sar_antarctica/nci/configs/default.toml index 47a110b..4231cc4 100644 --- a/sar_antarctica/nci/configs/default.toml +++ b/sar_antarctica/nci/configs/default.toml @@ -4,7 +4,9 @@ ncpu = "4" mem = "32" queue = "normal" project = "u46" -walltime = "04:00:00" +walltime = "02:00:00" +orbit_dir = "/g/data/fj7/Copernicus/Sentinel-1/" +orbit_type = "POE" output_dir = "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/" gamma_lib_dir = "/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712" gamma_env_var = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64" \ No newline at end of file diff --git a/sar_antarctica/nci/filesystem.py b/sar_antarctica/nci/filesystem.py index 7e8dd01..5f1a8de 100644 --- a/sar_antarctica/nci/filesystem.py +++ b/sar_antarctica/nci/filesystem.py @@ -4,7 +4,11 @@ from sar_antarctica.nci.preparation.dem import get_cop30_dem_for_bounds -def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: +def get_orbits_nci( + orbit_type: str | None, + sensor: str, + nci_orbit_dir: Path = Path("/g/data/fj7/Copernicus/Sentinel-1/"), +) -> list[Path]: """For a given orbit type and sensor, compile the relevant orbit files Parameters @@ -13,7 +17,8 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: One of 'POE', 'RES', or None. If None, both POE and RES orbits will be included sensor : str Sensor (e.g. S1A or S1B) to search. Typically extracted from the scene ID - + nci_orbit_dir : Path, optional + The path containing orbit files on the NCI, by default Path("/g/data/fj7/Copernicus/Sentinel-1/") Returns ------- list[Path] @@ -26,7 +31,6 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: """ # Constants for NCI - S1_DIR = Path("/g/data/fj7/Copernicus/Sentinel-1/") POE_DIR = "POEORB" RES_DIR = "RESORB" @@ -40,7 +44,7 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: raise ValueError("orbit_type must be one of 'POE', 'RES', or None") nci_orbit_directories = [ - S1_DIR / orbit_dir / sensor for orbit_dir in orbit_type_directories + nci_orbit_dir / orbit_dir / sensor for orbit_dir in orbit_type_directories ] orbits = find_orbits(nci_orbit_directories) @@ -48,13 +52,12 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]: return orbits -def get_dem_nci(scene: Path, scene_bounds: tuple[float, float, float, float]): - OUTPUT_DIR = Path( - "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/data/dem" - ) - if not OUTPUT_DIR.exists(): - OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - dem_file = OUTPUT_DIR / f"{scene.stem}.tif" +def get_dem_nci( + scene: Path, scene_bounds: tuple[float, float, float, float], output_dir: Path +) -> Path: + if not output_dir.exists(): + output_dir.mkdir(parents=True, exist_ok=True) + dem_file = output_dir / f"{scene.stem}.tif" if not dem_file.exists(): _, _ = get_cop30_dem_for_bounds(scene_bounds, dem_file, ellipsoid_heights=True) diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py b/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py index a381a0b..79c3b8c 100644 --- a/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py +++ b/sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py @@ -7,7 +7,10 @@ def get_orbit_and_dem( - scene_file: Path, orbit_type: str | None = "POE" + scene_file: Path, + dem_output_dir: Path, + orbit_dir: Path = Path("/g/data/fj7/Copernicus/Sentinel-1/"), + orbit_type: str | None = "POE", ) -> tuple[Path, Path]: """For a given Sentinel-1 scene, find the relevant orbit path and DEM path. The DEM will be created if no DEM path is found. @@ -36,7 +39,7 @@ def get_orbit_and_dem( scene_stop = datetime.strptime(scene.stop, "%Y%m%dT%H%M%S") # Find orbit - orbit_files = get_orbits_nci(orbit_type, scene_sensor) + orbit_files = get_orbits_nci(orbit_type, scene_sensor, orbit_dir) orbit_file = find_latest_orbit_covering_window(orbit_files, scene_start, scene_stop) # Isolate metadata for creating DEM @@ -49,6 +52,6 @@ def get_orbit_and_dem( ) # Build DEM - dem_file = get_dem_nci(scene_file, scene_bounds) + dem_file = get_dem_nci(scene_file, scene_bounds, dem_output_dir) return (orbit_file, dem_file) diff --git a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py index b63601e..e21c3d6 100644 --- a/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py +++ b/sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py @@ -15,7 +15,11 @@ def submit_job( - scene: Path, spacing: int, scaling: str, pbs_parameters: dict[str, str], log_dir + scene: Path, + spacing: int, + scaling: str, + pbs_parameters: dict[str, str], + log_dir: str, ): scene_name = scene.stem @@ -29,10 +33,13 @@ def submit_job( pbs_parameters["queue"], pbs_parameters["project"], pbs_parameters["walltime"], - scene, + scene_name, + log_dir, ) - job_command = f"run-pyrosar-gamma-workflow {scene} {spacing} {scaling}" + job_command = ( + f"run-pyrosar-gamma-workflow {scene} --spacing {spacing} --scaling {scaling}" + ) job_script = pbs_script + ENVIRONMENT_COMMAND + job_command @@ -41,4 +48,4 @@ def submit_job( # Submit script qsub_command = f"qsub {scene_script}" - # os.system(qsub_command) + os.system(qsub_command) diff --git a/sar_antarctica/nci/submission/utils.py b/sar_antarctica/nci/submission/utils.py index 1f181f3..0033a12 100644 --- a/sar_antarctica/nci/submission/utils.py +++ b/sar_antarctica/nci/submission/utils.py @@ -3,12 +3,17 @@ SUBMISSION_DIR = Path(__file__).resolve().parent SUBMISSION_TEMPLATE = SUBMISSION_DIR / "pbs_template.txt" -LOG_DIR = "/g/data/yp75/projects/sar-antractica-processing/submission/logs" STORAGE = "gdata/yp75+gdata/dg9+gdata/fj7+gdata/v10" def populate_pbs_template( - ncpu: int, mem: int, queue: str, project: str, walltime: str, jobname: str + ncpu: int, + mem: int, + queue: str, + project: str, + walltime: str, + jobname: str, + log_dir: str, ): pbs_template = SUBMISSION_TEMPLATE.read_text() @@ -19,7 +24,7 @@ def populate_pbs_template( "": project, "": walltime, "": STORAGE, - "": LOG_DIR, + "": log_dir, "": jobname, }