global: fix graceful shutdown

CHANGES.rst

@@ -12,6 +12,7 @@ Version 0.9.1 (UNRELEASED)
 - Changes uWSGI configuration to increase buffer size, add vacuum option, etc.
 - Fixes job status inconsistency by correctly setting running jobs' status to ``stopped`` when a workflow is stopped.
 - Fixes uWSGI memory consumption on systems with very high allowed number of open files.
+- Fixes uWSGI and ``consume-job-queue`` command to gracefully stop when being terminated.
 
 Version 0.9.0 (2023-01-19)
 --------------------------

Dockerfile

@@ -66,10 +66,14 @@ ENV FLASK_APP=reana_workflow_controller/app.py \
 EXPOSE 5000
 
 # Run server
+# exec is used to make sure signals are propagated to uwsgi,
+# while also allowing shell expansion
 # hadolint ignore=DL3025
-CMD uwsgi \
+CMD exec uwsgi \
  --buffer-size ${UWSGI_BUFFER_SIZE} \
  --die-on-term \
+ --hook-master-start "unix_signal:2 gracefully_kill_them_all" \
+ --hook-master-start "unix_signal:15 gracefully_kill_them_all" \
  --enable-threads \
  --http-socket 0.0.0.0:5000 \
  --master \

reana_workflow_controller/cli.py

@@ -9,6 +9,7 @@
 """REANA Workflow Controller command line interface."""
 
 import logging
+import signal
 
 import click
 from reana_commons.config import REANA_LOG_FORMAT, REANA_LOG_LEVEL
@@ -21,4 +22,12 @@
  """Consumes job queue and updates job status."""
  logging.basicConfig(level=REANA_LOG_LEVEL, format=REANA_LOG_FORMAT)
  consumer = JobStatusConsumer()
+
+ def stop_consumer(signum, frame):
+ logging.info("Stopping job status consumer...")
+ consumer.should_stop = True
+
+ signal.signal(signal.SIGTERM, stop_consumer)
+
+ logging.info("Starting job status consumer...")
  consumer.run()