Merge pull request #351 from alexhsamuel/fix/protocol-cleanup

restart test

python/apsis/lib/asyn.py

@@ -3,8 +3,6 @@
 
 import logging
 
-import logging
-
 #-------------------------------------------------------------------------------
 
 def _install():

test/int/instance.py

@@ -80,7 +80,7 @@ def start_serve(self):
 
 
     def wait_for_serve(self):
-        self.client.wait_running(1)
+        self.client.wait_running(2)
 
 
     def is_running(self):

test/int/procstar/jobs/group sleep.yaml

@@ -0,0 +1,7 @@
+params: {duration, group_id}
+
+program:
+  type: procstar
+  group_id: "{{ group_id }}"
+  argv: ["/usr/bin/sleep", "{{ duration }}"]
+

test/int/procstar/test_restart.py

@@ -0,0 +1,56 @@
+from   pathlib import Path
+import logging
+import time
+
+from   procstar_instance import ApsisService
+
+JOB_DIR = Path(__file__).parent / "jobs"
+
+#-------------------------------------------------------------------------------
+
+def _start(svc, job_id, args):
+    """
+    Starts a run for `job_id` and `args`, and waits for it to start running.
+    """
+    run_id = svc.client.schedule(job_id, args)["run_id"]
+    assert svc.wait_run(run_id, wait_states=("starting", ))["state"] == "running"
+    return run_id
+
+
+def test_restart_some():
+    """
+    Tests restarting some agents but not others, while processes run.
+    """
+    with (
+            ApsisService(job_dir=JOB_DIR) as svc,
+            svc.agent(group_id="group0") as agent0,
+            svc.agent(group_id="group1"),
+            svc.agent(group_id="group2") as agent2,
+    ):
+        svc.client.wait_running(1)
+        # Start runs in groups 1 and 2.
+        run_id0 = _start(svc, "group sleep", {"group_id": "group0", "duration": "2"})
+        run_id1 = _start(svc, "group sleep", {"group_id": "group1", "duration": "2"})
+
+        # Restart agent 2.
+        agent2.restart()
+
+        # The runs should be unaffected.
+        assert svc.client.get_run(run_id0)["state"] == "running"
+        assert svc.client.get_run(run_id1)["state"] == "running"
+
+        # Restart agent 0.  Run 0 should error.
+        agent0.restart()
+        time.sleep(0.1)
+        assert svc.client.get_run(run_id0)["state"] == "failure"  # SIGTERM
+        assert svc.client.get_run(run_id1)["state"] == "running"
+
+        assert svc.wait_run(run_id1)["state"] == "success"
+
+
+if __name__ == "__main__":
+    from apsis.lib import logging
+    logging.configure(level="DEBUG")
+    logging.set_log_levels()
+    test_restart_some()
+

test/int/procstar_instance.py

@@ -93,6 +93,18 @@ def close(self):
             self.proc = None
 
 
+    def restart(self, *, signum=signal.SIGTERM, keep_conn_id=False):
+        logging.info("killing Procstar agent")
+        self.proc.send_signal(signum)
+        self.proc.wait()
+        self.proc = None
+
+        if not keep_conn_id:
+            self.conn_id = str(uuid.uuid4())
+
+        self.start()
+
+
     def __enter__(self):
         self.start()
         return self