diff --git a/docs/config.rst b/docs/config.rst index 866352a8..6e943b21 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -42,6 +42,14 @@ config file. host_groups: # ... + python: + gc: + enable: true + threshold: [100000, 100, 100] + + procstar: + # ... + A duration is in in seconds, or you may give durations like `30s`, `10 min` (600 seconds), `1.5h` (5400 seconds), and `1 day` (86400 seconds). @@ -143,6 +151,27 @@ A single host name is effectively a host alias. my_alias: host4.example.com +Python +------ + +Apsis allocates large numbers of Python objects, but does not heavily use Python +data structures. Python's garbage collection (GC) will occasionally run for a +substantial time, which blocks Apsis and can lead to timeouts. To enable +(default) or disable GC, or adjust its thresholds: + +.. code:: yaml + + python: + gc: + enable: false + threshold: [100000, 100, 100] + +The thresholds apply only if GC is enabled. See documentation for +`gc.set_threshold() +`_ for an +explanation of these values. + + Procstar -------- diff --git a/python/apsis/config.py b/python/apsis/config.py index 596baf83..a6525d99 100644 --- a/python/apsis/config.py +++ b/python/apsis/config.py @@ -76,7 +76,7 @@ def load(path): else: path = Path(path) with open(path) as file: - cfg = yaml.load(file, Loader=yaml.BaseLoader) + cfg = yaml.load(file, Loader=yaml.SafeLoader) if cfg is None: # Empty config. cfg = {} diff --git a/python/apsis/ctl.py b/python/apsis/ctl.py index d511fbac..fed72258 100644 --- a/python/apsis/ctl.py +++ b/python/apsis/ctl.py @@ -162,11 +162,26 @@ def cmd_restart(args): # command: serve def cmd_serve(args): + # Assemble config. cfg = apsis.config.load(args.config) for ovr in args.override: name, val = ovr.split("=", 1) apsis.lib.json.set_dotted(cfg, name, val) + # Configure Python GC. + cfg_gc = cfg.get("python", {}).get("gc", {}) + (gc.enable if bool(cfg_gc.get("enabled", True)) else gc.disable)() + try: + gc_threshold = cfg_gc["threshold"] + except KeyError: + pass + else: + gc.set_threshold(*gc_threshold) + log.info( + f"GC {'enabled' if gc.isenabled() else 'disabled'}; " + f"threshold: {gc.get_threshold()}" + ) + restart = apsis.service.main.serve( cfg, host=args.host, port=args.port, debug=args.debug) diff --git a/python/apsis/lib/json.py b/python/apsis/lib/json.py index c667f559..deb9ce79 100644 --- a/python/apsis/lib/json.py +++ b/python/apsis/lib/json.py @@ -42,6 +42,35 @@ def pop(key, type=None, default=NO_DEFAULT): raise SchemaError(f"unexpected keys: {' '.join(copy)}") +def get_dotted(mapping, key, default=NO_DEFAULT): + """ + Returns the value for a dotted `key`. + + >>> m = {"foo": {"bif": 10}} + >>> get_dotted(m, "foo.bif") + 10 + >>> get_dotted(m, "bar") + Traceback (most recent call last): + ... + KeyError: 'bar' + >>> get_dotted(m, "foo.bof") + Traceback (most recent call last): + ... + KeyError: 'bof' + + """ + m = mapping + try: + for part in key.split("."): + m = m[part] + return m + except KeyError: + if default is NO_DEFAULT: + raise + else: + return default + + def set_dotted(mapping, key, value): """ Sets dotted `key` to `value` into a hierarchical `mapping`, creating diff --git a/test/manual/procstar/config.yaml b/test/manual/procstar/config.yaml index 6d7dd6c8..07bb449d 100644 --- a/test/manual/procstar/config.yaml +++ b/test/manual/procstar/config.yaml @@ -3,6 +3,10 @@ database: timeout: 0.1m jobs: jobs +python: + gc: + enabled: False + procstar: agent: enable: true