v1.22.0 - Dec 1, 2023

- Add support for list of python paths in command line and tests
- Fix reporter.base for Custom reporter from files and tests
- work around for urllib3 v2.0 drop support for OpenSSL<1.1.1
- Some test fixes and updates
- Docs updates
- work around for urllib3 v2.0 drop support for OpenSSL<1.1.1
- Custom reporter from files tests and fix in reporter.base
- Add support for list of python paths in command line
- Update faq.rst with non-disease-specific questions from current emodp…
- add an unit style test for add_py_path()
- Add link to source code in docs
- Update installation.rst
- Fix install doc
- Merge v1.21 and on to master
- Update Jenkinsfile with jenkins_user
- Removed deprecated params, added required for RTD doc builds

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.21.1
+current_version = 1.22.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[\+a-z]+)\.(?P<build>\d+))?

.readthedocs.yml

@@ -5,6 +5,12 @@
 # Required
 version: 2
 
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
@@ -17,11 +23,9 @@ sphinx:
 # Optionally build your docs in additional formats such as PDF
 formats:
   - htmlzip
-  - pdf
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-   version: 3.7
    install:
       - requirements: requirements.txt
       - requirements: docs/requirements.txt

Jenkinsfile

@@ -69,16 +69,18 @@ podTemplate(
 				// In python 3.9 environment, dataclasses should be installed at this point.
 				//sh "pip3 install dataclasses"
 				sh "pip3 install idmtools_test --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple"
+                // withCredentials([string(credentialsId: 'idm_bamboo_user', variable: 'user'), string(credentialsId: 'idm_bamboo_user_password', variable: 'password')]) {
+				//		sh 'pip3 install emod_generic==0.0.21 --index-url=https://$user:$password@packages.idmod.org/api/pypi/pypi-staging/simple --force-reinstall --no-cache-dir'
+				//	} 
 				sh "pip3 install emod_generic --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple"
 				sh 'pip3 install keyrings.alt'
 				sh "pip3 install pytest-xdist"
 				sh "pip3 freeze"
 			}
 			stage('Login and Test') {
-				withCredentials([string(credentialsId: 'Comps_emodpy_user', variable: 'user'), string(credentialsId: 'Comps_emodpy_password', variable: 'password'),
-				                 string(credentialsId: 'Bamboo_id', variable: 'bamboo_user'), string(credentialsId: 'Bamboo', variable: 'bamboo_password')]) {
-					sh 'python3 ".dev_scripts/create_auth_token_args.py" --comps_url https://comps2.idmod.org --username $user --password $password'
-				}
+				withCredentials([usernamePassword(credentialsId: 'comps2_jenkins_user', usernameVariable: 'COMPS2_USERNAME', passwordVariable: 'COMPS2_PASSWORD')]) {
+        		    		sh 'python3 .dev_scripts/create_auth_token_args.py --comps_url https://comps2.idmod.org --username $COMPS2_USERNAME --password $COMPS2_PASSWORD'
+            			}
 				echo "Running Emodpy Tests"
 				dir('tests') {
 					sh "pytest -n 0 test_download_from_package.py"

docs/basic-installation.rst

@@ -6,27 +6,30 @@ Follow the steps below if you will use |IT_s| to run and analyze simulations, bu
 source code changes.
 
 #.  Open a command prompt and create a virtual environment in any directory you choose. The
-    command below names the environment "emodpy", but you may use any desired name::
+    command below names the environment "emodpy", but you may use any desired name, and any 
+    available path you prefer::
 
-        python -m venv emodpy
+        python -m venv /path/to/venv/root/emodpy
 
 #.  Activate the virtual environment:
 
         * On Windows, enter the following::
 
-            emodpy\Scripts\activate
+            \path\to\venv\root\emodpy\Scripts\activate
 
         * On Linux, enter the following::
 
-            source emodpy/bin/activate
+            source /path/to/venv/root/emodpy/bin/activate
 
-#.  Install |IT_s| packages::
+#.  Install |IT_s| packages. ::
 
         pip install emodpy --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
 
-#.  Verify installation by pulling up |IT_s| help::
+    (It's strongly recommended that you edit your pip.ini or pip.conf so you don't have to specificy --index-url.)
 
-        emodpy --help
+#.  Verify installation by doing a test import::
+
+        python -c 'import emodpy'
 
 #.  When you are finished, deactivate the virtual environment by entering the following at a command prompt::
 

docs/conf.py

@@ -59,7 +59,27 @@
     'plantweb.directive',
     'sphinxcontrib.programoutput',
     'sphinx.ext.intersphinx',
-    'sphinxext.remoteliteralinclude'
+    'sphinxext.remoteliteralinclude',
+    'sphinx.ext.viewcode',
+    'sphinx_search.extension', # search across multiple docsets in domain
+    'myst_parser', # source files written in MD or RST
+]
+
+myst_enable_extensions = [
+    "amsmath",
+    "attrs_inline",
+    "colon_fence",
+    "deflist",
+    "dollarmath",
+    "fieldlist",
+    "html_admonition",
+    "html_image",
+    "linkify",
+    "replacements",
+    "smartquotes",
+    "strikethrough",
+    "substitution",
+    "tasklist",
 ]
 
 plantuml = 'plantweb'
@@ -81,8 +101,7 @@
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ['.rst', '.md']
 
 # The encoding of source files.
 #
@@ -115,7 +134,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -207,13 +226,13 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
+
 html_static_path = ['_static']
 
-html_context = {
-    'css_files': [
-        '_static/theme_overrides.css'
-        ]
-}
+html_css_files = ['theme_overrides.css']
+
+html_js_files = ['show_block_by_os.js'] 
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the docs.
@@ -270,6 +289,22 @@
 #
 html_use_opensearch = 'www.idmod.org/docs/'
 
+# -- RTD Sphinx search for searching across the entire domain, default parent -------------
+
+if os.environ.get('READTHEDOCS') == 'True':
+
+    search_project_parent = "institute-for-disease-modeling-idm"
+    search_project = os.environ["READTHEDOCS_PROJECT"]
+    search_version = os.environ["READTHEDOCS_VERSION"]
+
+    rtd_sphinx_search_default_filter = f"subprojects:{search_project_parent}/{search_version}"
+
+    rtd_sphinx_search_filters = {
+        "Search this project": f"project:{search_project}/{search_version}",
+        "Search all IDM docs": f"subprojects:{search_project_parent}/{search_version}",
+    }
+
+
 # This is the file name suffix for HTML files (e.g. ".xhtml").
 # html_file_suffix = None
 
@@ -399,7 +434,6 @@
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
                        'emod_api': ('https://docs.idmod.org/projects/emod-api/en/latest/', None),
-                       'emodpy_covid': ('https://docs.idmod.org/projects/emodpy-covid/en/latest/', None),
                        'emodpy_generic': ('https://docs.idmod.org/projects/emodpy-generic/en/latest/', None),
                        'emodpy_malaria': ('https://docs.idmod.org/projects/emodpy-generic/en/latest/', None),
                        'emodpy_measles': ('https://docs.idmod.org/projects/emodpy-measles/en/latest/', None),

docs/faq.rst

@@ -136,3 +136,23 @@ you would change it to something like::
     platform = Platform( "SLURM", num_cores=4 )
 
 to run with 4 cores.
+
+
+What does "DTK" stand for?
+=========================================
+Disease Transmission Kernel. This was the early internal name of EMOD.
+
+
+What is a "parameter sweep"?
+=========================================
+When the docs refer to a "parameter sweep", it usually means an experiment consisting of a multiple simulations where almost all the input values are the same except for a single parameter. The parameter being swept will have different values across a range, possibly the min to the max, but any range of interest to the modeler. Parameter sweeps can be very useful for just learning the sensitivity of a given parameter, or as a form of manual calibration. A "1-D parameter sweep" is where you just sweep over a single parameter. You can also do "2-D parameter sweeps", where you sweep over two parameters at once, and so on. But these of course require more simulations and fancier visualization.
+
+A special kind of parameter sweep is sweeping over Run_Number which is the random number seed. This kind of sweep gives you a sense of the model to general stochasticity, given your other inputs.
+
+You can sweep over config, demographics, or campaign parameters.
+
+
+Is there any place where I can see which parameters are taken from distributions and what type of distributions are they?
+===========================================================================================================================
+Any parameter that is being set from a distribution will have the distribution type in the name. E.g., Base_Infectivity_Gaussian_Mean tells you that this value is being drawn from a Gaussian distribution. If you don't see any distribution name in the parameter name, it's just fixed at that parameter value.
+

docs/index.rst

@@ -7,7 +7,7 @@ streamline user interactions with |EMOD_s| and |IT_s|. Additional
 functionality  for interacting with |EMOD_s| is provided in the
 :doc:`emod_api:emod_api` and  :doc:`idmtools:api/idmtools_index` packages.
 
-See :doc:`idmtools:index` for a diagram showing how |IT_s| and each of the
+See :doc:`idmtools:api/idmtools_index` for a diagram showing how |IT_s| and each of the
 related packages are used in an end-to-end workflow using |EMOD_s| as the
 disease transmission model.
 

docs/installation.rst

@@ -13,8 +13,6 @@ installation method you choose, the prerequisites are the same.
 Prerequisites
 =============
 
-* Windows 10 Pro or Enterprise
-
 * |Python_supp| (https://www.python.org/downloads/release)
 
 * Python virtual environments
@@ -28,4 +26,4 @@ Prerequisites
 .. toctree::
 
     basic-installation
-    dev-installation
+    dev-installation

docs/requirements.txt

@@ -1,7 +1,12 @@
-sphinx-rtd-theme~=0.5
+sphinxext.remoteliteralinclude~=0.4.0
+sphinx~=6.2.1
+sphinx-rtd-theme~=1.2.2
 sphinxcontrib-napoleon~=0.7
-sphinx~=4.4.0
-plantweb~=1.2
-sphinxcontrib-programoutput~=0.16
-sphinxext.remoteliteralinclude
-sphinx-copybutton~=0.4.0
+plantweb~=1.2.1
+sphinxcontrib-programoutput~=0.17
+nbsphinx~=0.9.2
+jupyterlab~=4.0.2
+myst-parser~=2.0.0
+readthedocs-sphinx-search~=0.3.1
+sphinx-copybutton~=0.5.2
+pygithub~=1.57

emodpy/__init__.py

@@ -1,4 +1,4 @@
 # flake8: noqa F821
 from emodpy.defaults.iemod_default import IEMODDefault
 
-__version__ = "1.21.1"
+__version__ = "1.22.0"

emodpy/emod_task.py

@@ -94,6 +94,7 @@ class EMODTask(ITask):
 
     #: Add --python-script-path to command line
     use_embedded_python: bool = True
+    py_path_list: list = field(default_factory=lambda: [])
     is_linux: bool = False
     implicit_configs: list = field(default_factory=lambda: [])
     sif_filename: str = None
@@ -103,6 +104,7 @@ def __post_init__(self):
         from emodpy.utils import download_eradication
         super().__post_init__()
         self.executable_name = "Eradication"
+        self.py_path_list.append("./Assets/python")
         if self.eradication_path is not None:
             self.executable_name = os.path.basename(self.eradication_path)
             if urlparse(self.eradication_path).scheme in ('http', 'https'):
@@ -471,13 +473,24 @@ def set_command_line(self) -> NoReturn:
             )
         else:
             self.command = CommandLine(f"Assets/{self.executable_name}", "--config", f"{self.config_file_name}", "--dll-path", "./Assets")
+
         if self.use_embedded_python:    # This should be the always-use case but we're not quite there yet.
-            self.command._options.update({"--python-script-path": "./Assets/python"})
+            list_sep = ";"
+            self.command._options.update({"--python-script-path": list_sep.join(self.py_path_list)})
 
         # We do this here because CommandLine tries to be smart and quote input_path, but it isn't quite right...
         self.command.add_raw_argument("--input-path")
         self.command.add_raw_argument(input_path)
 
+    def add_py_path(self, path_to_add) -> NoReturn:
+        """
+        Add path to list of python paths prepended to sys.path in embedded interpreter
+
+        Returns:
+
+        """
+        self.py_path_list.append(path_to_add)
+
     def set_sif(self, path_to_sif, platform=None) -> NoReturn:
         """
         Set the Singularity Image File.

emodpy/reporters/base.py

@@ -224,7 +224,10 @@ def set_task_config(self, task: 'EMODTask') -> typing.NoReturn:
 
         """
         if not self.empty:
-            task.config.parameters.Custom_Reports_Filename = "custom_reports.json"
+            if type(task.config) is dict:
+                task.config["Custom_Reports_Filename"] = "custom_reports.json"
+            else:
+                task.config.parameters.Custom_Reports_Filename = "custom_reports.json"
 
     def gather_assets(self, **kwargs) -> typing.List[Asset]:
         # Remove the unused dlls from the folder

requirements.txt

@@ -10,3 +10,4 @@ astunparse
 bs4
 keyring
 pyCOMPS~=2.6
+requests==2.29.0

setup.py

@@ -63,5 +63,5 @@
     setup_requires=setup_requirements,
     test_suite='tests',
     extras_require=extras,
-    version='1.21.1'
+    version='1.22.0'
 )

tests/inputs/custom_reports.json

@@ -0,0 +1,16 @@
+{
+    "Custom_Reports" : 
+    {
+        "Use_Explicit_Dlls" : 1,
+        "ReportNodeDemographics" :
+        {
+            "Enabled" : 1,
+            "Reports" :
+            [
+                {
+                    "Age_Bins" : [ 40.0, 80.0, 125.0 ]
+                }
+            ]
+        }
+    }
+}

tests/inputs/silly_custom_reports.json

@@ -0,0 +1,16 @@
+{
+    "Custom_Reports" : 
+    {
+        "Use_Explicit_Dlls" : 1,
+        "SillyReporterClass" :
+        {
+            "Enabled" : 1,
+            "Reports" :
+            [
+                {
+                    "Age_Bins" : [ 40.0, 80.0, 125.0 ]
+                }
+            ]
+        }
+    }
+}