diff --git a/README.md b/README.md index 8e1ec84..07b6b7c 100644 --- a/README.md +++ b/README.md @@ -390,7 +390,7 @@ Now we want to select the tokens that have a specific number of cores, and start To start scanning the different design documents, for example, to execute the work with different numbers of cores, run: ``` -python core-scanner.py +python core_scanner.py ``` which will default to view `SingleCore` that was created above and filters on a core count of 1. This is equivalent to running explicitly: diff --git a/examples/core_scanner.py b/examples/core_scanner.py index 77627f2..82ac877 100644 --- a/examples/core_scanner.py +++ b/examples/core_scanner.py @@ -2,23 +2,37 @@ import logging import sys +import argparse import picasconfig from picas.picaslogger import picaslogger from picas.clients import CouchDB from picas.executers import execute -from picas.util import arg_parser + picaslogger.propagate = False -# expand the parser for the example -parser = arg_parser() -parser.add_argument("--cores", default=1, type=str, help="Number of cores for the job") -parser.set_defaults(design_doc="SingleCore") -args = parser.parse_args() +def arg_parser(): + """ + Arguments parser for optional values of the example + returns: argparse object + """ + parser = argparse.ArgumentParser(description="Arguments used in the different classes in the example.") + parser.add_argument("--design_doc", default="Monitor", type=str, help="Select the designdoc used by the actor class") + parser.add_argument("--view", default="todo", type=str, help="Select the view used by the actor class") + parser.add_argument("-v", "--verbose", action="store_true", help="Set verbose") + parser.add_argument("--cores", default=1, type=str, help="Number of cores for the job") + parser.set_defaults(design_doc="SingleCore") + + return parser + +# parse user arguments +args = arg_parser().parse_args() +# setup connection to db client = CouchDB(url=picasconfig.PICAS_HOST_URL, db=picasconfig.PICAS_DATABASE, username=picasconfig.PICAS_USERNAME, password=picasconfig.PICAS_PASSWORD) +# check if there is work available, i.e. if there are tokens in the specified view work_avail = client.is_view_nonempty(args.view, design_doc=args.design_doc) if work_avail: picaslogger.info(f"Starting a picas clients checking view {args.view} in design document {args.design_doc}") diff --git a/examples/example-template.py b/examples/example_template.py similarity index 100% rename from examples/example-template.py rename to examples/example_template.py diff --git a/examples/local_example.py b/examples/local_example.py index 415c7ed..94fc48e 100755 --- a/examples/local_example.py +++ b/examples/local_example.py @@ -24,10 +24,24 @@ from picas.iterators import TaskViewIterator from picas.iterators import EndlessViewIterator from picas.modifiers import BasicTokenModifier -from picas.util import Timer, arg_parser +from picas.util import Timer + log = logging.getLogger(__name__) + +def arg_parser(): + """ + Arguments parser for optional values of the example + returns: argparse object + """ + parser = argparse.ArgumentParser(description="Arguments used in the different classes in the example.") + parser.add_argument("--design_doc", default="Monitor", type=str, help="Select the designdoc used by the actor class") + parser.add_argument("--view", default="todo", type=str, help="Select the view used by the actor class") + parser.add_argument("-v", "--verbose", action="store_true", help="Set verbose") + return parser + + class ExampleActor(RunActor): """ The ExampleActor is the custom implementation of a RunActor that the user needs for the processing. diff --git a/picas/clients.py b/picas/clients.py index c8d6345..9ca2404 100644 --- a/picas/clients.py +++ b/picas/clients.py @@ -236,17 +236,17 @@ def is_view_nonempty(self, view, **view_params): """ # To ensure proper logging when design_doc is not passed into is_view_nonempty, # the variable is created as the default used in self.view. Otherwise the f-string below breaks on default input. - design_doc = view_params.setdefault('design_doc', "Monitor") + design_doc = view_params.setdefault('design_doc', "Monitor") try: doc = self.get_single_from_view(view, **view_params) task = Task(doc) picaslogger.debug(doc) - picaslogger.debug(task['input']) + picaslogger.debug(task['_id']) picaslogger.info(f"View {view} under design document {design_doc} is non-empty.") return True except IndexError as e: picaslogger.info(f"View {view} under design document {design_doc} is empty: {e}") return False - except ResourceNotFound as e: + except ResourceNotFound: picaslogger.info(f"Non-existing view and design document passed: {view} in {design_doc}") return False diff --git a/picas/util.py b/picas/util.py index a71b186..0be0166 100644 --- a/picas/util.py +++ b/picas/util.py @@ -2,7 +2,6 @@ @author Joris Borgdorff """ -import argparse import time from copy import deepcopy @@ -30,18 +29,6 @@ def seconds(): return int(time.time()) -def arg_parser(): - """ - Arguments parser for optional values of the example - returns: argparse object - """ - parser = argparse.ArgumentParser(description="Arguments used in the different classes in the example.") - parser.add_argument("--design_doc", default="Monitor", type=str, help="Select the designdoc used by the actor class") - parser.add_argument("--view", default="todo", type=str, help="Select the view used by the actor class") - parser.add_argument("-v", "--verbose", action="store_true", help="Set verbose") - return parser - - class Timer: """Timer class""" diff --git a/poetry.lock b/poetry.lock index 7c10f5f..0c3ad68 100644 --- a/poetry.lock +++ b/poetry.lock @@ -124,13 +124,13 @@ files = [ [[package]] name = "pytest" -version = "8.3.3" +version = "8.3.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, - {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, + {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, + {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, ] [package.dependencies] @@ -144,6 +144,22 @@ tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "setuptools" +version = "66.1.1" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setuptools-66.1.1-py3-none-any.whl", hash = "sha256:6f590d76b713d5de4e49fe4fbca24474469f53c83632d5d0fd056f7ff7e8112b"}, + {file = "setuptools-66.1.1.tar.gz", hash = "sha256:ac4008d396bc9cd983ea483cb7139c0240a07bbc74ffb6232fceffedc6cf03a8"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "stopit" version = "1.1.2" @@ -156,16 +172,46 @@ files = [ [[package]] name = "tomli" -version = "2.1.0" +version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391"}, - {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "9b0ecac70f16d2ff915749f66773e97bd8e41a755e4305c71ce98e902b03f414" +content-hash = "f8c46daf7c4bc901ed80f1b8fd47253f80a9a3fb5062fb2d7ebcacbc9943f5a9"