Skip to content

Commit

Permalink
Fix muted exceptions in fm_dispatch due to self SIGKILL
Browse files Browse the repository at this point in the history
This commit fixes the issue where fm_dispatch would call SIGKILL on itself before reraising the exception that caused it to fail.
  • Loading branch information
jonathan-eq committed Feb 27, 2025
1 parent 9c06e6d commit 3d3894a
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 9 deletions.
14 changes: 5 additions & 9 deletions src/_ert/forward_model_runner/fm_dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,6 @@ def _stop_reporters(
reporter.stop(exited_event=exited_event)


def sigterm_handler(_signo, _stack_frame):
signal.signal(signal.SIGTERM, signal.SIG_DFL)
os.kill(0, signal.SIGTERM)


def fm_dispatch(args):
parser = argparse.ArgumentParser(
description=(
Expand Down Expand Up @@ -207,13 +202,14 @@ def sigterm_handler(_signo, _stack_frame):

def main():
os.nice(19)
signal.signal(signal.SIGTERM, sigterm_handler)
try:
fm_dispatch(sys.argv)
except Exception as e:
except Exception as exc:
print(f"fm_dispatch failed with {exc=}")
pgid = os.getpgid(os.getpid())
os.killpg(pgid, signal.SIGTERM)
raise e
os.killpg(
pgid, signal.SIGTERM
) # This will trigger the sigterm_handler, which shuts down the reporters and SIGKILLS any remaining processes.


if __name__ == "__main__":
Expand Down
17 changes: 17 additions & 0 deletions tests/ert/unit_tests/forward_model_runner/test_fm_dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -484,3 +484,20 @@ async def wait_for_msg(msg_type):
event_from_json(zmq_server.messages[-1]).error_msg
== FORWARD_MODEL_TERMINATED_MSG
)


async def test_fm_dispatch_main_signals_sigterm_on_exception(capsys):
def mock_fm_dispatch_raises(*args):
raise RuntimeError("forward model critical error")

with (
patch("_ert.forward_model_runner.fm_dispatch.os.killpg") as mock_killpg,
patch("_ert.forward_model_runner.fm_dispatch.os.getpgid") as mock_getpgid,
patch("_ert.forward_model_runner.fm_dispatch.fm_dispatch") as mock_fm_dispatch,
):
mock_getpgid.return_value = 17
mock_fm_dispatch.side_effect = mock_fm_dispatch_raises
_ert.forward_model_runner.fm_dispatch.main()
assert "forward model critical error" in capsys.readouterr().out

mock_killpg.assert_called_with(17, signal.SIGTERM)

0 comments on commit 3d3894a

Please sign in to comment.