Update functions to remove hard-coding for sar_antarctica/nci/filesys…

…tem.py

sar_antarctica/nci/cli.py

@@ -101,6 +101,10 @@ def submit_pyrosar_gamma_workflow(
 )
 @click.option("--spacing", type=int)
 @click.option("--scaling", type=click.Choice(["linear", "db"]))
+@click.option(
+    "--orbit-dir", type=click.Path(exists=True, file_okay=False, path_type=Path)
+)
+@click.option("--orbit-type", type=click.Choice(["POE", "RES", "either"]))
 @click.option(
     "--output-dir",
     type=click.Path(exists=True, file_okay=False, path_type=Path),
@@ -117,11 +121,20 @@ def submit_pyrosar_gamma_workflow(
     default="/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64",
 )
 def run_pyrosar_gamma_workflow(
-    scene, spacing, scaling, output_dir, gamma_lib_dir, gamma_env_var
+    scene,
+    spacing,
+    scaling,
+    orbit_dir,
+    orbit_type,
+    output_dir,
+    gamma_lib_dir,
+    gamma_env_var,
 ):
 
     click.echo("Preparing orbit and DEM")
-    orbit, dem = get_orbit_and_dem(scene)
+    dem_output_dir = output_dir / "data/dem"
+
+    orbit, dem = get_orbit_and_dem(scene, dem_output_dir, orbit_dir, orbit_type)
 
     click.echo(f"    Identified orbit: {orbit}")
     click.echo(f"    Identified DEM: {dem}")

sar_antarctica/nci/configs/EW.toml

@@ -0,0 +1,12 @@
+spacing = 40
+scaling = "linear"
+ncpu = "4"
+mem = "64"
+queue = "normal"
+project = "u46"
+walltime = "02:00:00"
+orbit_dir = "/g/data/fj7/Copernicus/Sentinel-1/"
+orbit_type = "POE"
+output_dir = "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/"
+gamma_lib_dir = "/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712"
+gamma_env_var = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64"

sar_antarctica/nci/configs/IW.toml

@@ -0,0 +1,12 @@
+spacing = 10
+scaling = "linear"
+ncpu = "4"
+mem = "128"
+queue = "normal"
+project = "u46"
+walltime = "02:00:00"
+orbit_dir = "/g/data/fj7/Copernicus/Sentinel-1/"
+orbit_type = "POE"
+output_dir = "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma_IW/"
+gamma_lib_dir = "/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712"
+gamma_env_var = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64"

sar_antarctica/nci/configs/default.toml

@@ -4,7 +4,9 @@ ncpu = "4"
 mem = "32"
 queue = "normal"
 project = "u46"
-walltime = "04:00:00"
+walltime = "02:00:00"
+orbit_dir = "/g/data/fj7/Copernicus/Sentinel-1/"
+orbit_type = "POE"
 output_dir = "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/"
 gamma_lib_dir = "/g/data/dg9/GAMMA/GAMMA_SOFTWARE-20230712"
 gamma_env_var = "/g/data/yp75/projects/pyrosar_processing/sar-pyrosar-nci:/apps/fftw3/3.3.10/lib:/apps/gdal/3.6.4/lib64"

sar_antarctica/nci/filesystem.py

@@ -4,7 +4,11 @@
 from sar_antarctica.nci.preparation.dem import get_cop30_dem_for_bounds
 
 
-def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]:
+def get_orbits_nci(
+    orbit_type: str | None,
+    sensor: str,
+    nci_orbit_dir: Path = Path("/g/data/fj7/Copernicus/Sentinel-1/"),
+) -> list[Path]:
     """For a given orbit type and sensor, compile the relevant orbit files
 
     Parameters
@@ -13,7 +17,8 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]:
         One of 'POE', 'RES', or None. If None, both POE and RES orbits will be included
     sensor : str
         Sensor (e.g. S1A or S1B) to search. Typically extracted from the scene ID
-
+    nci_orbit_dir : Path, optional
+        The path containing orbit files on the NCI, by default Path("/g/data/fj7/Copernicus/Sentinel-1/")
     Returns
     -------
     list[Path]
@@ -26,7 +31,6 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]:
     """
 
     # Constants for NCI
-    S1_DIR = Path("/g/data/fj7/Copernicus/Sentinel-1/")
     POE_DIR = "POEORB"
     RES_DIR = "RESORB"
 
@@ -40,21 +44,20 @@ def get_orbits_nci(orbit_type: str | None, sensor: str) -> list[Path]:
         raise ValueError("orbit_type must be one of 'POE', 'RES', or None")
 
     nci_orbit_directories = [
-        S1_DIR / orbit_dir / sensor for orbit_dir in orbit_type_directories
+        nci_orbit_dir / orbit_dir / sensor for orbit_dir in orbit_type_directories
     ]
 
     orbits = find_orbits(nci_orbit_directories)
 
     return orbits
 
 
-def get_dem_nci(scene: Path, scene_bounds: tuple[float, float, float, float]):
-    OUTPUT_DIR = Path(
-        "/g/data/yp75/projects/sar-antractica-processing/pyrosar_gamma/data/dem"
-    )
-    if not OUTPUT_DIR.exists():
-        OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-    dem_file = OUTPUT_DIR / f"{scene.stem}.tif"
+def get_dem_nci(
+    scene: Path, scene_bounds: tuple[float, float, float, float], output_dir: Path
+) -> Path:
+    if not output_dir.exists():
+        output_dir.mkdir(parents=True, exist_ok=True)
+    dem_file = output_dir / f"{scene.stem}.tif"
 
     if not dem_file.exists():
         _, _ = get_cop30_dem_for_bounds(scene_bounds, dem_file, ellipsoid_heights=True)

sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py

@@ -7,7 +7,10 @@
 
 
 def get_orbit_and_dem(
-    scene_file: Path, orbit_type: str | None = "POE"
+    scene_file: Path,
+    dem_output_dir: Path,
+    orbit_dir: Path = Path("/g/data/fj7/Copernicus/Sentinel-1/"),
+    orbit_type: str | None = "POE",
 ) -> tuple[Path, Path]:
     """For a given Sentinel-1 scene, find the relevant orbit path and DEM path.
     The DEM will be created if no DEM path is found.
@@ -36,7 +39,7 @@ def get_orbit_and_dem(
     scene_stop = datetime.strptime(scene.stop, "%Y%m%dT%H%M%S")
 
     # Find orbit
-    orbit_files = get_orbits_nci(orbit_type, scene_sensor)
+    orbit_files = get_orbits_nci(orbit_type, scene_sensor, orbit_dir)
     orbit_file = find_latest_orbit_covering_window(orbit_files, scene_start, scene_stop)
 
     # Isolate metadata for creating DEM
@@ -49,6 +52,6 @@ def get_orbit_and_dem(
     )
 
     # Build DEM
-    dem_file = get_dem_nci(scene_file, scene_bounds)
+    dem_file = get_dem_nci(scene_file, scene_bounds, dem_output_dir)
 
     return (orbit_file, dem_file)

sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py

@@ -15,7 +15,11 @@
 
 
 def submit_job(
-    scene: Path, spacing: int, scaling: str, pbs_parameters: dict[str, str], log_dir
+    scene: Path,
+    spacing: int,
+    scaling: str,
+    pbs_parameters: dict[str, str],
+    log_dir: str,
 ):
 
     scene_name = scene.stem
@@ -29,10 +33,13 @@ def submit_job(
         pbs_parameters["queue"],
         pbs_parameters["project"],
         pbs_parameters["walltime"],
-        scene,
+        scene_name,
+        log_dir,
     )
 
-    job_command = f"run-pyrosar-gamma-workflow {scene} {spacing} {scaling}"
+    job_command = (
+        f"run-pyrosar-gamma-workflow {scene} --spacing {spacing} --scaling {scaling}"
+    )
 
     job_script = pbs_script + ENVIRONMENT_COMMAND + job_command
 
@@ -41,4 +48,4 @@ def submit_job(
 
     # Submit script
     qsub_command = f"qsub {scene_script}"
-    # os.system(qsub_command)
+    os.system(qsub_command)

sar_antarctica/nci/submission/utils.py

@@ -3,12 +3,17 @@
 
 SUBMISSION_DIR = Path(__file__).resolve().parent
 SUBMISSION_TEMPLATE = SUBMISSION_DIR / "pbs_template.txt"
-LOG_DIR = "/g/data/yp75/projects/sar-antractica-processing/submission/logs"
 STORAGE = "gdata/yp75+gdata/dg9+gdata/fj7+gdata/v10"
 
 
 def populate_pbs_template(
-    ncpu: int, mem: int, queue: str, project: str, walltime: str, jobname: str
+    ncpu: int,
+    mem: int,
+    queue: str,
+    project: str,
+    walltime: str,
+    jobname: str,
+    log_dir: str,
 ):
     pbs_template = SUBMISSION_TEMPLATE.read_text()
 
@@ -19,7 +24,7 @@ def populate_pbs_template(
         "<PROJECT>": project,
         "<WALLTIME>": walltime,
         "<STORAGE>": STORAGE,
-        "<LOGDIR>": LOG_DIR,
+        "<LOGDIR>": log_dir,
         "<JOBNAME>": jobname,
     }