Improve parallelization of batch processing (#404)

- install r.hydrodem in advance
- reset logging after critical error

bin/grass/batch_process.py

@@ -4,24 +4,27 @@
 import sys
 import argparse
 
+from smoderp2d.runners.grass import GrassGisRunner
+from smoderp2d.exceptions import ProviderError, MaxIterationExceeded
+
 def run_process(params, epsg):
-    from smoderp2d.runners.grass import GrassGisRunner
+    retcode = 0
+    try:
+        runner = GrassGisRunner()
+        runner.create_location(f'EPSG:{epsg}')
+        runner.set_options(params)
+        runner.import_data()
+        runner.run()
+    except (ProviderError, MaxIterationExceeded) as e:
+        print(f'ERORR: {e}', file=sys.stderr)
+        retcode = 1
 
-    runner = GrassGisRunner()
-    runner.create_location(f'EPSG:{epsg}')
-    runner.set_options(params)
-    runner.import_data()
-    runner.run()
     runner.finish()
 
-def main(params, epsg):
-    from smoderp2d.exceptions import ProviderError
+    return retcode
 
-    try:
-        run_process(params, epsg)
-    except ProviderError as e:
-        print(f'ERORR: {e}', file=sys.stderr)
-        sys.exit(1)
+def main(params, epsg):
+    sys.exit(run_process(params, epsg))
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(

bin/grass/batch_process_csv.py

@@ -4,9 +4,9 @@
 import argparse
 import csv
 
-from batch_process import run_process
+from smoderp2d.exceptions import ProviderError, MaxIterationExceeded
 
-from smoderp2d.core.general import Globals, GridGlobals
+from batch_process import run_process
 
 def get_params_epsg(params):
     epsg = params.pop('epsg')
@@ -18,10 +18,8 @@ def run_process_single(params, epsg):
     print('-' * 80)
     print(f'Run process with {params}...')
     print('-' * 80)
-    run_process(params, epsg)
-    # reset global variables
-    Globals.reset()
-    GridGlobals.reset()
+
+    return run_process(params, epsg)
 
 def main(csv_file, workers):
     # collect params

bin/grass/r.smoderp2d/r.smoderp2d.py

@@ -137,7 +137,7 @@
 
 from smoderp2d.runners.grass import GrassGisRunner
 from smoderp2d.providers.base import WorkflowMode
-from smoderp2d.exceptions import ProviderError
+from smoderp2d.exceptions import ProviderError, MaxIterationExceeded
 
 if __name__ == "__main__":
     options, flags = gs.parser()
@@ -147,9 +147,7 @@
         runner = GrassGisRunner()
 
         runner.set_options(options)
-
-        sys.exit(
-            runner.run()
-        )
-    except ProviderError as e:
+        runner.run()
+        runner.finish()
+    except (ProviderError, MaxIterationExceeded) as e:
         gs.fatal(e)

bin/qgis/smoderp2d-plugin/smoderp_2D_dockwidget.py

@@ -46,7 +46,7 @@
 from smoderp2d.runners.qgis import QGISRunner
 from smoderp2d.core.general import Globals, GridGlobals
 from smoderp2d.providers import Logger
-from smoderp2d.exceptions import ProviderError, ComputationAborted
+from smoderp2d.exceptions import ProviderError, ComputationAborted, MaxIterationExceeded
 from bin.base import arguments, sections
 
 from .custom_widgets import HistoryWidget
@@ -87,7 +87,7 @@ def run(self):
             self.runner.set_options(self.input_params)
             self.runner.import_data()
             self.runner.run()
-        except (ProviderError, ImportError) as e:
+        except (ProviderError, ImportError, MaxIterationExceeded) as e:
             self.error = e
             self.finish_msg_level = Qgis.Critical
             return False

tests/batch/batch_process_csv.bat

@@ -5,6 +5,9 @@ call "%~dp0\init_windows_env.bat"
 rem change current directory to smpdepr2d root directory
 cd /d %SMODERP2D_PATH%
 
+rem install r.hydrodem
+call grass%GRASS_VERSION% --tmp-location EPSG:5514 --exec g.extension extension=r.hydrodem
+
 rem run batch process
 python3 %SMODERP2D_PATH%\bin\grass\batch_process_csv.py ^
         --csv .\tests\batch\batch_process.csv ^

tests/batch/batch_process_csv_linux.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+# install r.hydrodem
+grass --tmp-location EPSG:5514 --exec g.extension extension=r.hydrodem
+
+# run batch process
 PYTHONPATH=`pwd` python3 ./bin/grass/batch_process_csv.py \
           --csv ./tests/batch/batch_process.csv \
           --workers 1