diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b300b2c..ea7e005 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.21.1 +current_version = 1.22.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)((?P[\+a-z]+)\.(?P\d+))? diff --git a/.readthedocs.yml b/.readthedocs.yml index bf78b4e..e227c57 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,12 @@ # Required version: 2 +# Set the version of Python and other tools you might need +build: + os: ubuntu-20.04 + tools: + python: "3.9" + # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py @@ -17,11 +23,9 @@ sphinx: # Optionally build your docs in additional formats such as PDF formats: - htmlzip - - pdf # Optionally set the version of Python and requirements required to build your docs python: - version: 3.7 install: - requirements: requirements.txt - requirements: docs/requirements.txt diff --git a/Jenkinsfile b/Jenkinsfile index f2baad6..eff503b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,16 +69,18 @@ podTemplate( // In python 3.9 environment, dataclasses should be installed at this point. //sh "pip3 install dataclasses" sh "pip3 install idmtools_test --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple" + // withCredentials([string(credentialsId: 'idm_bamboo_user', variable: 'user'), string(credentialsId: 'idm_bamboo_user_password', variable: 'password')]) { + // sh 'pip3 install emod_generic==0.0.21 --index-url=https://$user:$password@packages.idmod.org/api/pypi/pypi-staging/simple --force-reinstall --no-cache-dir' + // } sh "pip3 install emod_generic --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple" sh 'pip3 install keyrings.alt' sh "pip3 install pytest-xdist" sh "pip3 freeze" } stage('Login and Test') { - withCredentials([string(credentialsId: 'Comps_emodpy_user', variable: 'user'), string(credentialsId: 'Comps_emodpy_password', variable: 'password'), - string(credentialsId: 'Bamboo_id', variable: 'bamboo_user'), string(credentialsId: 'Bamboo', variable: 'bamboo_password')]) { - sh 'python3 ".dev_scripts/create_auth_token_args.py" --comps_url https://comps2.idmod.org --username $user --password $password' - } + withCredentials([usernamePassword(credentialsId: 'comps2_jenkins_user', usernameVariable: 'COMPS2_USERNAME', passwordVariable: 'COMPS2_PASSWORD')]) { + sh 'python3 .dev_scripts/create_auth_token_args.py --comps_url https://comps2.idmod.org --username $COMPS2_USERNAME --password $COMPS2_PASSWORD' + } echo "Running Emodpy Tests" dir('tests') { sh "pytest -n 0 test_download_from_package.py" diff --git a/docs/basic-installation.rst b/docs/basic-installation.rst index 50ee27d..b1b8da6 100644 --- a/docs/basic-installation.rst +++ b/docs/basic-installation.rst @@ -6,27 +6,30 @@ Follow the steps below if you will use |IT_s| to run and analyze simulations, bu source code changes. #. Open a command prompt and create a virtual environment in any directory you choose. The - command below names the environment "emodpy", but you may use any desired name:: + command below names the environment "emodpy", but you may use any desired name, and any + available path you prefer:: - python -m venv emodpy + python -m venv /path/to/venv/root/emodpy #. Activate the virtual environment: * On Windows, enter the following:: - emodpy\Scripts\activate + \path\to\venv\root\emodpy\Scripts\activate * On Linux, enter the following:: - source emodpy/bin/activate + source /path/to/venv/root/emodpy/bin/activate -#. Install |IT_s| packages:: +#. Install |IT_s| packages. :: pip install emodpy --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple -#. Verify installation by pulling up |IT_s| help:: + (It's strongly recommended that you edit your pip.ini or pip.conf so you don't have to specificy --index-url.) - emodpy --help +#. Verify installation by doing a test import:: + + python -c 'import emodpy' #. When you are finished, deactivate the virtual environment by entering the following at a command prompt:: diff --git a/docs/conf.py b/docs/conf.py index 9d8406a..aeed16b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,7 +59,27 @@ 'plantweb.directive', 'sphinxcontrib.programoutput', 'sphinx.ext.intersphinx', - 'sphinxext.remoteliteralinclude' + 'sphinxext.remoteliteralinclude', + 'sphinx.ext.viewcode', + 'sphinx_search.extension', # search across multiple docsets in domain + 'myst_parser', # source files written in MD or RST +] + +myst_enable_extensions = [ + "amsmath", + "attrs_inline", + "colon_fence", + "deflist", + "dollarmath", + "fieldlist", + "html_admonition", + "html_image", + "linkify", + "replacements", + "smartquotes", + "strikethrough", + "substitution", + "tasklist", ] plantuml = 'plantweb' @@ -81,8 +101,7 @@ # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] # The encoding of source files. # @@ -115,7 +134,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -207,13 +226,13 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". + html_static_path = ['_static'] -html_context = { - 'css_files': [ - '_static/theme_overrides.css' - ] -} +html_css_files = ['theme_overrides.css'] + +html_js_files = ['show_block_by_os.js'] + # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the docs. @@ -270,6 +289,22 @@ # html_use_opensearch = 'www.idmod.org/docs/' +# -- RTD Sphinx search for searching across the entire domain, default parent ------------- + +if os.environ.get('READTHEDOCS') == 'True': + + search_project_parent = "institute-for-disease-modeling-idm" + search_project = os.environ["READTHEDOCS_PROJECT"] + search_version = os.environ["READTHEDOCS_VERSION"] + + rtd_sphinx_search_default_filter = f"subprojects:{search_project_parent}/{search_version}" + + rtd_sphinx_search_filters = { + "Search this project": f"project:{search_project}/{search_version}", + "Search all IDM docs": f"subprojects:{search_project_parent}/{search_version}", + } + + # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None @@ -399,7 +434,6 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'python': ('https://docs.python.org/3', None), 'emod_api': ('https://docs.idmod.org/projects/emod-api/en/latest/', None), - 'emodpy_covid': ('https://docs.idmod.org/projects/emodpy-covid/en/latest/', None), 'emodpy_generic': ('https://docs.idmod.org/projects/emodpy-generic/en/latest/', None), 'emodpy_malaria': ('https://docs.idmod.org/projects/emodpy-generic/en/latest/', None), 'emodpy_measles': ('https://docs.idmod.org/projects/emodpy-measles/en/latest/', None), diff --git a/docs/faq.rst b/docs/faq.rst index c17674f..af45d58 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -136,3 +136,23 @@ you would change it to something like:: platform = Platform( "SLURM", num_cores=4 ) to run with 4 cores. + + +What does "DTK" stand for? +========================================= +Disease Transmission Kernel. This was the early internal name of EMOD. + + +What is a "parameter sweep"? +========================================= +When the docs refer to a "parameter sweep", it usually means an experiment consisting of a multiple simulations where almost all the input values are the same except for a single parameter. The parameter being swept will have different values across a range, possibly the min to the max, but any range of interest to the modeler. Parameter sweeps can be very useful for just learning the sensitivity of a given parameter, or as a form of manual calibration. A "1-D parameter sweep" is where you just sweep over a single parameter. You can also do "2-D parameter sweeps", where you sweep over two parameters at once, and so on. But these of course require more simulations and fancier visualization. + +A special kind of parameter sweep is sweeping over Run_Number which is the random number seed. This kind of sweep gives you a sense of the model to general stochasticity, given your other inputs. + +You can sweep over config, demographics, or campaign parameters. + + +Is there any place where I can see which parameters are taken from distributions and what type of distributions are they? +=========================================================================================================================== +Any parameter that is being set from a distribution will have the distribution type in the name. E.g., Base_Infectivity_Gaussian_Mean tells you that this value is being drawn from a Gaussian distribution. If you don't see any distribution name in the parameter name, it's just fixed at that parameter value. + diff --git a/docs/index.rst b/docs/index.rst index d5a14ef..5280990 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ streamline user interactions with |EMOD_s| and |IT_s|. Additional functionality for interacting with |EMOD_s| is provided in the :doc:`emod_api:emod_api` and :doc:`idmtools:api/idmtools_index` packages. -See :doc:`idmtools:index` for a diagram showing how |IT_s| and each of the +See :doc:`idmtools:api/idmtools_index` for a diagram showing how |IT_s| and each of the related packages are used in an end-to-end workflow using |EMOD_s| as the disease transmission model. diff --git a/docs/installation.rst b/docs/installation.rst index 15c5dff..693b10c 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -13,8 +13,6 @@ installation method you choose, the prerequisites are the same. Prerequisites ============= -* Windows 10 Pro or Enterprise - * |Python_supp| (https://www.python.org/downloads/release) * Python virtual environments @@ -28,4 +26,4 @@ Prerequisites .. toctree:: basic-installation - dev-installation \ No newline at end of file + dev-installation diff --git a/docs/requirements.txt b/docs/requirements.txt index fe31476..9392c41 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,12 @@ -sphinx-rtd-theme~=0.5 +sphinxext.remoteliteralinclude~=0.4.0 +sphinx~=6.2.1 +sphinx-rtd-theme~=1.2.2 sphinxcontrib-napoleon~=0.7 -sphinx~=4.4.0 -plantweb~=1.2 -sphinxcontrib-programoutput~=0.16 -sphinxext.remoteliteralinclude -sphinx-copybutton~=0.4.0 +plantweb~=1.2.1 +sphinxcontrib-programoutput~=0.17 +nbsphinx~=0.9.2 +jupyterlab~=4.0.2 +myst-parser~=2.0.0 +readthedocs-sphinx-search~=0.3.1 +sphinx-copybutton~=0.5.2 +pygithub~=1.57 \ No newline at end of file diff --git a/emodpy/__init__.py b/emodpy/__init__.py index 023c0ef..df6fd19 100644 --- a/emodpy/__init__.py +++ b/emodpy/__init__.py @@ -1,4 +1,4 @@ # flake8: noqa F821 from emodpy.defaults.iemod_default import IEMODDefault -__version__ = "1.21.1" +__version__ = "1.22.0" diff --git a/emodpy/emod_task.py b/emodpy/emod_task.py index 6dfd795..678f5ca 100644 --- a/emodpy/emod_task.py +++ b/emodpy/emod_task.py @@ -94,6 +94,7 @@ class EMODTask(ITask): #: Add --python-script-path to command line use_embedded_python: bool = True + py_path_list: list = field(default_factory=lambda: []) is_linux: bool = False implicit_configs: list = field(default_factory=lambda: []) sif_filename: str = None @@ -103,6 +104,7 @@ def __post_init__(self): from emodpy.utils import download_eradication super().__post_init__() self.executable_name = "Eradication" + self.py_path_list.append("./Assets/python") if self.eradication_path is not None: self.executable_name = os.path.basename(self.eradication_path) if urlparse(self.eradication_path).scheme in ('http', 'https'): @@ -471,13 +473,24 @@ def set_command_line(self) -> NoReturn: ) else: self.command = CommandLine(f"Assets/{self.executable_name}", "--config", f"{self.config_file_name}", "--dll-path", "./Assets") + if self.use_embedded_python: # This should be the always-use case but we're not quite there yet. - self.command._options.update({"--python-script-path": "./Assets/python"}) + list_sep = ";" + self.command._options.update({"--python-script-path": list_sep.join(self.py_path_list)}) # We do this here because CommandLine tries to be smart and quote input_path, but it isn't quite right... self.command.add_raw_argument("--input-path") self.command.add_raw_argument(input_path) + def add_py_path(self, path_to_add) -> NoReturn: + """ + Add path to list of python paths prepended to sys.path in embedded interpreter + + Returns: + + """ + self.py_path_list.append(path_to_add) + def set_sif(self, path_to_sif, platform=None) -> NoReturn: """ Set the Singularity Image File. diff --git a/emodpy/reporters/base.py b/emodpy/reporters/base.py index 94f60df..e6dfefe 100644 --- a/emodpy/reporters/base.py +++ b/emodpy/reporters/base.py @@ -224,7 +224,10 @@ def set_task_config(self, task: 'EMODTask') -> typing.NoReturn: """ if not self.empty: - task.config.parameters.Custom_Reports_Filename = "custom_reports.json" + if type(task.config) is dict: + task.config["Custom_Reports_Filename"] = "custom_reports.json" + else: + task.config.parameters.Custom_Reports_Filename = "custom_reports.json" def gather_assets(self, **kwargs) -> typing.List[Asset]: # Remove the unused dlls from the folder diff --git a/requirements.txt b/requirements.txt index c6cc5ad..13777d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ astunparse bs4 keyring pyCOMPS~=2.6 +requests==2.29.0 diff --git a/setup.py b/setup.py index 49a229f..d088e9f 100644 --- a/setup.py +++ b/setup.py @@ -63,5 +63,5 @@ setup_requires=setup_requirements, test_suite='tests', extras_require=extras, - version='1.21.1' + version='1.22.0' ) diff --git a/tests/inputs/custom_reports.json b/tests/inputs/custom_reports.json new file mode 100644 index 0000000..6e18d6b --- /dev/null +++ b/tests/inputs/custom_reports.json @@ -0,0 +1,16 @@ +{ + "Custom_Reports" : + { + "Use_Explicit_Dlls" : 1, + "ReportNodeDemographics" : + { + "Enabled" : 1, + "Reports" : + [ + { + "Age_Bins" : [ 40.0, 80.0, 125.0 ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/inputs/silly_custom_reports.json b/tests/inputs/silly_custom_reports.json new file mode 100644 index 0000000..a8819a4 --- /dev/null +++ b/tests/inputs/silly_custom_reports.json @@ -0,0 +1,16 @@ +{ + "Custom_Reports" : + { + "Use_Explicit_Dlls" : 1, + "SillyReporterClass" : + { + "Enabled" : 1, + "Reports" : + [ + { + "Age_Bins" : [ 40.0, 80.0, 125.0 ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/test_emod_task.py b/tests/test_emod_task.py index aa9c824..41c25c7 100644 --- a/tests/test_emod_task.py +++ b/tests/test_emod_task.py @@ -1,10 +1,8 @@ # flake8: noqa W605,F821 -import copy import json import os from unittest.mock import Mock -import idmtools import pytest import shutil from functools import partial @@ -13,7 +11,9 @@ from emodpy import emod_task from emodpy.emod_task import EMODTask -from emodpy.emod_task import AssetCollection +from emodpy.emod_task import add_ep4_from_path, default_ep4_fn + +from emodpy.reporters.custom import ReportNodeDemographics from idmtools.entities.experiment import Experiment from idmtools.entities.simulation import Simulation @@ -231,8 +231,101 @@ def test_from_files_config_only(self): self.assertDictEqual(config, task.config) - def test_from_default(self): + def test_from_files_valid_customReport(self): + self.prepare_input_files() + + dfs.write_config_from_default_and_params(config_path=self.default_config_file, + set_fn=partial(set_param_fn, + implicit_config_set_fns=self.demog.implicits), + config_out_path=self.config_path) + + # workaround for https://github.com/InstituteforDiseaseModeling/emodpy/issues/358 + import emod_api.schema_to_class as s2c + with open(self.config_path) as conf: + config_rod = json.load(conf, object_hook=s2c.ReadOnlyDict) + #config_rod.parameters.Enable_Demographics_Builtin = 1 + #del config_rod.parameters["Demographics_Filenames"] + config_rod.parameters['Base_Infectivity_Constant'] = 1 + config_rod.parameters['Incubation_Period_Constant'] = 1 + config_rod.parameters['Infectious_Period_Constant'] = 1 + config_rod.parameters['Base_Infectivity_Distribution'] = 'CONSTANT_DISTRIBUTION' + config_rod.parameters['Incubation_Period_Distribution'] = 'CONSTANT_DISTRIBUTION' + config_rod.parameters['Infectious_Period_Distribution'] = 'CONSTANT_DISTRIBUTION' + config_rod.parameters['x_Base_Population'] = 0.001 + config_rod.parameters['Simulation_Duration'] = 10 + config_rod.parameters['Start_Time'] = 0 + + with open(self.config_path, "w") as outfile: + json.dump(config_rod, outfile, sort_keys=True, indent=4) + + custom_reports_path = os.path.join(manifest.current_directory, "inputs", "custom_reports.json") + + task = EMODTask.from_files( + eradication_path=self.eradication_path, + config_path=self.config_path, + campaign_path=self.camp_path, + demographics_paths=self.demo_path, + custom_reports_path=custom_reports_path, + asset_path=manifest.package_folder + ) + + task.pre_creation(Simulation(), self.platform) + task.gather_common_assets() + + experiment = Experiment.from_task(task, name=self.case_name) + + # Check if reporter plugins is in assets + self.assertIn(os.path.join(manifest.package_folder, 'reporter_plugins', + ReportNodeDemographics.dll_file.replace('dll', 'so')), + [a.absolute_path for a in task.common_assets.assets]) + + # check experiment common assets are as expected + experiment.pre_creation(self.platform) + self.assertEqual(len(experiment.assets), 3) + self.assertIn(self.eradication_path, [a.absolute_path for a in experiment.assets]) + self.assertIn(self.demo_path, [a.absolute_path for a in experiment.assets]) + + sim = experiment.simulations[0] + sim.pre_creation(self.platform) + self.assertEqual(len(sim.assets), 3) + self.assertIn('custom_reports.json', [a.filename for a in sim.assets]) + self.assertEqual('custom_reports.json', sim.task.config["parameters"]['Custom_Reports_Filename']) + self.assertEqual(1, len(sim.task.reporters.custom_reporters)) + print(type(sim.task.reporters.custom_reporters[0])) + # print(isinstance(ReportNodeDemographics, type(sim.task.reporters.custom_reporters[0]))) + self.assertEqual('ReportNodeDemographics', sim.task.reporters.custom_reporters[0].name) + + task.set_sif(manifest.sft_id_file) + experiment.run(wait_until_done=True) + self.assertTrue(experiment.succeeded, msg=f"Experiment {experiment.uid} failed.\n") + + for sim in experiment.simulations: + file = self.platform.get_files(sim, ["output/ReportNodeDemographics.csv"]) + report = file["output/ReportNodeDemographics.csv"].decode("utf-8") + self.assertIn("NodeID", report) + + def test_from_files_missing_customReport(self): + self.prepare_input_files() + dfs.write_config_from_default_and_params(config_path=self.default_config_file, + set_fn=partial(set_param_fn, + implicit_config_set_fns=self.demog.implicits), + config_out_path=self.config_path) + + custom_reports_path = os.path.join(manifest.current_directory, "inputs", "silly_custom_reports.json") + with self.assertRaises(Exception) as context: + task = EMODTask.from_files( + eradication_path=self.eradication_path, + config_path=self.config_path, + campaign_path=self.camp_path, + demographics_paths=self.demo_path, + custom_reports_path=custom_reports_path, + asset_path=manifest.package_folder + ) + + self.assertTrue('Could not find the reporter class' in str(context.exception)) + + def test_from_default(self): def set_param_fn(config): print("Setting params.") config.parameters.Simulation_Duration = 100 @@ -545,3 +638,29 @@ class ProcessPlatform: experiment.post_creation(fake_platform) self.assertEquals(task.sif_path, "my_sif.sif") + def test_add_py_path(self): + self.prepare_schema_and_eradication() + task = EMODTask.from_default2(config_path=None, eradication_path=self.eradication_path, + campaign_builder=None, schema_path=self.schema_path, + param_custom_cb=None, ep4_custom_cb=None, + demog_builder=None) + pyscript_path = os.path.join(manifest.current_directory, 'inputs', 'ep4') + + add_ep4_from_path(task, pyscript_path) + pypackage_path = '/python_venv/lib/python3.11/site-packages' + task.add_py_path(pypackage_path) + + virtual_path = 'venv/lib/python3.9/site-packages/' + task.add_py_path(virtual_path) + + task.set_sif(manifest.sft_id_file) + task.use_embedded_python = True + + task.pre_creation(Simulation(), self.platform) + task.gather_common_assets() + + self.assertTrue(task.use_embedded_python) + + self.assertIn(f"--python-script-path './Assets/python;{pypackage_path};{virtual_path}'", str(task.command)) + +