Merge pull request #430 from alexhsamuel/feature/cfg-gc

configure Python GC

docs/config.rst

@@ -42,6 +42,14 @@ config file.
     host_groups:
       # ...
 
+    python:
+      gc:
+        enable: true
+        threshold: [100000, 100, 100]
+
+    procstar:
+      # ...
+
 
 A duration is in in seconds, or you may give durations like `30s`, `10 min`
 (600 seconds), `1.5h` (5400 seconds), and `1 day` (86400 seconds).
@@ -143,6 +151,27 @@ A single host name is effectively a host alias.
       my_alias: host4.example.com
 
 
+Python
+------
+
+Apsis allocates large numbers of Python objects, but does not heavily use Python
+data structures.  Python's garbage collection (GC) will occasionally run for a
+substantial time, which blocks Apsis and can lead to timeouts.  To enable
+(default) or disable GC, or adjust its thresholds:
+
+.. code:: yaml
+
+    python:
+      gc:
+        enable: false
+        threshold: [100000, 100, 100]
+
+The thresholds apply only if GC is enabled.  See documentation for
+`gc.set_threshold()
+<https://docs.python.org/3/library/gc.html#gc.set_threshold>`_ for an
+explanation of these values.
+
+
 Procstar
 --------
 

python/apsis/config.py

@@ -76,7 +76,7 @@ def load(path):
     else:
         path = Path(path)
         with open(path) as file:
-            cfg = yaml.load(file, Loader=yaml.BaseLoader)
+            cfg = yaml.load(file, Loader=yaml.SafeLoader)
             if cfg is None:
                 # Empty config.
                 cfg = {}

python/apsis/ctl.py

@@ -162,11 +162,26 @@ def cmd_restart(args):
     # command: serve
 
     def cmd_serve(args):
+        # Assemble config.
         cfg = apsis.config.load(args.config)
         for ovr in args.override:
             name, val = ovr.split("=", 1)
             apsis.lib.json.set_dotted(cfg, name, val)
 
+        # Configure Python GC.
+        cfg_gc = cfg.get("python", {}).get("gc", {})
+        (gc.enable if bool(cfg_gc.get("enabled", True)) else gc.disable)()
+        try:
+            gc_threshold = cfg_gc["threshold"]
+        except KeyError:
+            pass
+        else:
+            gc.set_threshold(*gc_threshold)
+        log.info(
+            f"GC {'enabled' if gc.isenabled() else 'disabled'}; "
+            f"threshold: {gc.get_threshold()}"
+        )
+
         restart = apsis.service.main.serve(
             cfg, host=args.host, port=args.port, debug=args.debug)
 

python/apsis/lib/json.py

@@ -42,6 +42,35 @@ def pop(key, type=None, default=NO_DEFAULT):
         raise SchemaError(f"unexpected keys: {' '.join(copy)}")
 
 
+def get_dotted(mapping, key, default=NO_DEFAULT):
+    """
+    Returns the value for a dotted `key`.
+
+      >>> m = {"foo": {"bif": 10}}
+      >>> get_dotted(m, "foo.bif")
+      10
+      >>> get_dotted(m, "bar")
+      Traceback (most recent call last):
+      ...
+      KeyError: 'bar'
+      >>> get_dotted(m, "foo.bof")
+      Traceback (most recent call last):
+      ...
+      KeyError: 'bof'
+
+    """
+    m = mapping
+    try:
+        for part in key.split("."):
+            m = m[part]
+        return m
+    except KeyError:
+        if default is NO_DEFAULT:
+            raise
+        else:
+            return default
+
+
 def set_dotted(mapping, key, value):
     """
     Sets dotted `key` to `value` into a hierarchical `mapping`, creating

test/manual/procstar/config.yaml

@@ -3,6 +3,10 @@ database:
   timeout: 0.1m
 jobs: jobs
 
+python:
+  gc:
+    enabled: False
+
 procstar:
   agent:
     enable: true