Batch processing (#392)

* Fix QGIS plugin history handling

bin/grass/batch_process.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+
+def run_process(params, epsg):
+    from smoderp2d.runners.grass import GrassGisRunner
+
+    runner = GrassGisRunner()
+    runner.create_location(f'EPSG:{epsg}')
+    runner.set_options(params)
+    runner.import_data()
+    runner.run()
+    runner.finish()
+
+def main(params, epsg):
+    from smoderp2d.exceptions import ProviderError
+
+    try:
+        run_process(params, epsg)
+    except ProviderError as e:
+        print(f'ERORR: {e}', file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog='batch_process',
+        description='Run SMODERP2D as batch process'
+    )
+    parser.add_argument('--elevation', required=True)
+    parser.add_argument('--soil', required=True)
+    parser.add_argument('--soil_type_fieldname', required=True)
+    parser.add_argument('--vegetation', required=True)
+    parser.add_argument('--vegetation_type_fieldname', required=True)
+    parser.add_argument('--rainfall_file', required=True)
+    parser.add_argument('--end_time', required=True)
+    parser.add_argument('--maxdt', required=True)
+    parser.add_argument('--table_soil_vegetation', required=True)
+    parser.add_argument('--table_soil_vegetation_fieldname', required=True)
+    parser.add_argument('--output', required=True)    
+    parser.add_argument('--points')
+    parser.add_argument('--points_fieldname')
+    parser.add_argument('--streams')
+    parser.add_argument('--channel_properties_table')
+    parser.add_argument('--streams_channel_type_fieldname')
+    parser.add_argument('--flow_direction', default='single')
+    parser.add_argument('--wave', default='kinematic')
+    parser.add_argument('--generate_temporary', action='store_true')
+    parser.add_argument('--epsg', default=5514)
+
+    args = parser.parse_args()
+
+    main({
+        'elevation': args.elevation,
+        'soil': args.soil,
+        'soil_type_fieldname': args.soil_type_fieldname,
+        'vegetation': args.vegetation,
+        'vegetation_type_fieldname': args.vegetation_type_fieldname,
+        'rainfall_file': args.rainfall_file,
+        'end_time': args.end_time,
+        'maxdt': args.maxdt,
+        'output': args.output,
+        'points': args.points,
+        'points_fieldname': args.points_fieldname,
+        'streams': args.streams,
+        'table_soil_vegetation': args.table_soil_vegetation,
+        'table_soil_vegetation_fieldname': args.table_soil_vegetation_fieldname,
+        'channel_properties_table': args.channel_properties_table,
+        'streams_channel_type_fieldname': args.streams_channel_type_fieldname,
+        'flow_direction': args.flow_direction,
+        'wave': args.wave,
+        'generate_temporary': bool(args.generate_temporary)
+    }, epsg=args.epsg
+         )

bin/grass/batch_process_csv.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import sys
+import argparse
+import csv
+
+from batch_process import run_process
+
+from smoderp2d.core.general import Globals, GridGlobals
+
+def get_params_epsg(params):
+    epsg = params.pop('epsg')
+
+    return params, int(epsg)
+
+def run_process_single(params, epsg):
+    # run single process
+    print('-' * 80)
+    print(f'Run process with {params}...')
+    print('-' * 80)
+    run_process(params, epsg)
+    # reset global variables
+    Globals.reset()
+    GridGlobals.reset()
+
+def main(csv_file, workers):
+    # collect params
+    params = []
+    with open(csv_file, newline='') as fd:
+        reader = csv.DictReader(fd, delimiter=';')
+        for row in reader:
+            if row['epsg'].startswith('#'):
+                # skip commented rows
+                continue
+            for k, v in row.items():
+                if v in ('true', 'false'):
+                    row[k] = True if v.lower() == 'true' else False
+            params.append(row)
+
+    # check for duplicated output paths
+    output_paths = set(p['output'] for p in params)
+    if len(params) != len(output_paths):
+        sys.exit("ERROR: Duplicated output paths detected")
+
+    # run processes
+    if workers > 1:
+        from joblib import Parallel, delayed
+
+        Parallel(n_jobs=workers, backend="multiprocessing", verbose=10)(
+            delayed(run_process_single)(*get_params_epsg(p)) for p in params
+        )
+    else:
+        for p in params:
+            run_process_single(*get_params_epsg(p))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog='batch_process_csv',
+        description='Run SMODERP2D as batch processes from CSV',
+        epilog=r'Usage: PYTHONPATH=`pwd` python3 bin/grass/batch_process_csv.py --csv \ bin/grass/batch_process.csv'
+    )
+
+    parser.add_argument('--csv', required=True)
+    parser.add_argument('--workers', default=1)
+
+    args = parser.parse_args()
+
+    main(args.csv, int(args.workers))

bin/qgis/smoderp2d-plugin/custom_widgets.py

@@ -16,42 +16,28 @@ def __init__(self, *args, **kwargs):
         self.params_dict = {}
         super().__init__(*args, **kwargs)
 
-    def saveHistory(self, params, maps):
+    def saveHistory(self, params):
         """Save historical parameterization into class variables.
 
         :param params: parameters from the current run
-        :param maps: maps from the current run
         """
         instance = QgsProject.instance()
-        map_names = {
-            i: instance.mapLayersByName(os.path.split(os.path.splitext(j)[0])[1]) for i, j in maps.items()
-        }
-        map_names = {
-            i: None if len(j) == 0 else j[0] for i, j in map_names.items()
-        }
-        self.params_dict.update({
-            'elevation': map_names['elevation'],
-            'soil': map_names['soil'],
-            'soil_type_fieldname': params['soil_type_fieldname'],
-            'vegetation': map_names['vegetation'],
-            'vegetation_type_fieldname': params['vegetation_type_fieldname'],
-            'points': map_names['points'],
-            'points_fieldname': params['points_fieldname'],
-            'streams': map_names['streams'],
-            'rainfall_file': params['rainfall_file'],
-            'table_soil_vegetation': map_names['table_soil_vegetation'],
-            'table_soil_vegetation_fieldname': params[
-                'table_soil_vegetation_fieldname'
-            ],
-            'channel_properties_table': map_names['channel_properties_table'],
-            'streams_channel_type_fieldname': params[
-                'streams_channel_type_fieldname'
-            ],
-            'output': params['output'],
-            'end_time': params['end_time'],
-            'flow_direction': params['flow_direction'],
-            'wave': params['wave'],
-            'generate_temporary': params['generate_temporary']
-        })
-
-        self.setToolTip(str(self.params_dict))
+
+        # collect map layers
+        map_layers = {}
+        for lyr in instance.mapLayers().values():
+            map_layers[lyr.source().split('|')[0]] = lyr
+
+        # add tooltip
+        self.setToolTip(str(params))
+
+        # store params
+        self.params_dict.update(params)
+
+        # update spatial data related items
+        for key in ("elevation", "soil", "vegetation", "points", "streams",
+                    "table_soil_vegetation", "channel_properties_table"):
+            if params[key]:
+                self.params_dict[key] = map_layers.get(params[key], None)
+            else:
+                self.params_dict[key] = None