Extend SSL/TLS support to st2stream and st2api - plus move common API code into common library

CHANGELOG.rst

@@ -39,6 +39,10 @@ Added
   Contributed by @amanda11
 * Ensure `.pth` files in the st2 virtualenv get loaded by pack virtualenvs. #6183
   Contributed by @cognifloyd
+* Add support for SSL/TLS to `st2api` and `st2stream` components. #6204
+  Contributed by @jk464
+* Move API code for `st2api`, `st2auth` and `st2stream` components to common library `st2common.openapi`. #6204
+  Contributed by @jk464
 
 * Added a `get_result` method to the `ExecutionResourceManager` Class for st2client
   Contributed by @skiedude

st2api/st2api/app.py

@@ -16,82 +16,39 @@
 from oslo_config import cfg
 
 from st2api import config as st2api_config
-from st2common import log as logging
-from st2common.middleware.streaming import StreamingMiddleware
-from st2common.middleware.error_handling import ErrorHandlingMiddleware
-from st2common.middleware.cors import CorsMiddleware
-from st2common.middleware.request_id import RequestIDMiddleware
-from st2common.middleware.logging import LoggingMiddleware
-from st2common.middleware.instrumentation import RequestInstrumentationMiddleware
-from st2common.middleware.instrumentation import ResponseInstrumentationMiddleware
-from st2common.router import Router
-from st2common.constants.system import VERSION_STRING
-from st2common.service_setup import setup as common_setup
-from st2common.util import spec_loader
+from st2common.openapi import app
 from st2api.validation import validate_auth_cookie_is_correctly_configured
 from st2api.validation import validate_rbac_is_correctly_configured
 
-LOG = logging.getLogger(__name__)
 
+def setup_app(config={}):
+    common_setup = {
+        "register_mq_exchanges": True,
+        "register_internal_trigger_types": True,
+        "run_migrations": True,
+    }
 
-def setup_app(config=None):
-    config = config or {}
-
-    LOG.info("Creating st2api: %s as OpenAPI app.", VERSION_STRING)
-
-    is_gunicorn = config.get("is_gunicorn", False)
-    if is_gunicorn:
-        # NOTE: We only want to perform this logic in the WSGI worker
-        st2api_config.register_opts(ignore_errors=True)
-        capabilities = {
-            "name": "api",
-            "listen_host": cfg.CONF.api.host,
-            "listen_port": cfg.CONF.api.port,
-            "type": "active",
-        }
-
-        # This should be called in gunicorn case because we only want
-        # workers to connect to db, rabbbitmq etc. In standalone HTTP
-        # server case, this setup would have already occurred.
-        common_setup(
-            service="api",
-            config=st2api_config,
-            setup_db=True,
-            register_mq_exchanges=True,
-            register_signal_handlers=True,
-            register_internal_trigger_types=True,
-            run_migrations=True,
-            service_registry=True,
-            capabilities=capabilities,
-            config_args=config.get("config_args", None),
-        )
-
-    # Additional pre-run time checks
-    validate_auth_cookie_is_correctly_configured()
-    validate_rbac_is_correctly_configured()
-
-    router = Router(
-        debug=cfg.CONF.api.debug, auth=cfg.CONF.auth.enable, is_gunicorn=is_gunicorn
-    )
+    pre_run_checks = [
+        validate_auth_cookie_is_correctly_configured,
+        validate_rbac_is_correctly_configured,
+    ]
 
-    spec = spec_loader.load_spec("st2common", "openapi.yaml.j2")
     transforms = {
         "^/api/v1/$": ["/v1"],
         "^/api/v1/": ["/", "/v1/"],
         "^/api/v1/executions": ["/actionexecutions", "/v1/actionexecutions"],
         "^/api/exp/": ["/exp/"],
     }
-    router.add_spec(spec, transforms=transforms)
-
-    app = router.as_wsgi
 
-    # Order is important. Check middleware for detailed explanation.
-    app = StreamingMiddleware(app, path_whitelist=["/v1/executions/*/output*"])
-    app = ErrorHandlingMiddleware(app)
-    app = CorsMiddleware(app)
-    app = LoggingMiddleware(app, router)
-    app = ResponseInstrumentationMiddleware(app, router, service_name="api")
-    app = RequestIDMiddleware(app)
-    app = RequestInstrumentationMiddleware(app, router, service_name="api")
-
-    return app
+    path_whitelist = ["/v1/executions/*/output*"]
+
+    return app.setup_app(
+        service_name="api",
+        app_config=st2api_config,
+        oslo_cfg=cfg.CONF.api,
+        pre_run_checks=pre_run_checks,
+        transforms=transforms,
+        common_setup_kwargs=common_setup,
+        path_whitelist=path_whitelist,
+        config=config,
+    )

st2api/st2api/cmd/api.py

@@ -13,9 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import sys
-
 # NOTE: It's important that we perform monkey patch as early as possible before any other modules
 # are important, otherwise SSL support for MongoDB won't work.
 # See https://github.com/StackStorm/st2/issues/4832 and https://github.com/gevent/gevent/issues/1016
@@ -24,15 +21,11 @@
 
 monkey_patch()
 
-import eventlet
 from oslo_config import cfg
-from eventlet import wsgi
 
 from st2common import log as logging
-from st2common.service_setup import setup as common_setup
-from st2common.service_setup import teardown as common_teardown
-from st2common.service_setup import deregister_service
 from st2api import config
+from st2common.openapi import api
 
 config.register_opts(ignore_errors=True)
 
@@ -43,65 +36,27 @@
 __all__ = ["main"]
 
 LOG = logging.getLogger(__name__)
-API = "api"
-
-# How much time to give to the request in progress to finish in seconds before killing them
-WSGI_SERVER_REQUEST_SHUTDOWN_TIME = 2
 
 
-def _setup():
-    capabilities = {
-        "name": "api",
-        "listen_host": cfg.CONF.api.host,
-        "listen_port": cfg.CONF.api.port,
-        "type": "active",
+def main():
+    common_setup = {
+        "register_mq_exchanges": True,
+        "register_internal_trigger_types": True,
     }
 
-    common_setup(
-        service=API,
-        config=config,
-        setup_db=True,
-        register_mq_exchanges=True,
-        register_signal_handlers=True,
-        register_internal_trigger_types=True,
-        service_registry=True,
-        capabilities=capabilities,
+    pre_run_checks = [
+        validate_auth_cookie_is_correctly_configured,
+        validate_rbac_is_correctly_configured,
+    ]
+
+    api.run(
+        service_name="api",
+        app_config=config,
+        cfg=cfg.CONF.api,
+        app=app,
+        log=LOG,
+        use_custom_pool=False,
+        log_output=False,
+        common_setup_kwargs=common_setup,
+        pre_run_checks=pre_run_checks,
     )
-
-    # Additional pre-run time checks
-    validate_auth_cookie_is_correctly_configured()
-    validate_rbac_is_correctly_configured()
-
-
-def _run_server():
-    host = cfg.CONF.api.host
-    port = cfg.CONF.api.port
-
-    LOG.info("(PID=%s) ST2 API is serving on http://%s:%s.", os.getpid(), host, port)
-
-    max_pool_size = eventlet.wsgi.DEFAULT_MAX_SIMULTANEOUS_REQUESTS
-    worker_pool = eventlet.GreenPool(max_pool_size)
-    sock = eventlet.listen((host, port))
-
-    wsgi.server(
-        sock, app.setup_app(), custom_pool=worker_pool, log=LOG, log_output=False
-    )
-    return 0
-
-
-def _teardown():
-    common_teardown()
-
-
-def main():
-    try:
-        _setup()
-        return _run_server()
-    except SystemExit as exit_code:
-        deregister_service(API)
-        sys.exit(exit_code)
-    except Exception:
-        LOG.exception("(PID=%s) ST2 API quit due to exception.", os.getpid())
-        return 1
-    finally:
-        _teardown()

st2api/st2api/config.py

@@ -24,32 +24,22 @@
 from oslo_config import cfg
 
 import st2common.config as common_config
-from st2common.constants.system import VERSION_STRING
-from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH
+from st2common.openapi import config
 
 CONF = cfg.CONF
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 def parse_args(args=None):
-    cfg.CONF(
-        args=args,
-        version=VERSION_STRING,
-        default_config_files=[DEFAULT_CONFIG_FILE_PATH],
-    )
+    config.parse_args(args=args)
 
 
 def register_opts(ignore_errors=False):
-    _register_common_opts(ignore_errors=ignore_errors)
-    _register_app_opts(ignore_errors=ignore_errors)
-
-
-def _register_common_opts(ignore_errors=False):
-    common_config.register_opts(ignore_errors=ignore_errors)
+    config.register_opts(_register_app_opts, ignore_errors=ignore_errors)
 
 
 def get_logging_config_path():
-    return cfg.CONF.api.logging
+    return config.get_logging_config_path(cfg.CONF.api)
 
 
 def _register_app_opts(ignore_errors=False):
@@ -76,13 +66,7 @@ def _register_app_opts(ignore_errors=False):
         pecan_opts, group="api_pecan", ignore_errors=ignore_errors
     )
 
-    logging_opts = [
-        cfg.BoolOpt("debug", default=False),
-        cfg.StrOpt(
-            "logging",
-            default="/etc/st2/logging.api.conf",
-            help="location of the logging.conf file",
-        ),
+    api_opts = config.get_base_opts("api") + [
         cfg.IntOpt(
             "max_page_size",
             default=100,
@@ -91,6 +75,4 @@ def _register_app_opts(ignore_errors=False):
         ),
     ]
 
-    common_config.do_register_opts(
-        logging_opts, group="api", ignore_errors=ignore_errors
-    )
+    common_config.do_register_opts(api_opts, group="api", ignore_errors=ignore_errors)

st2auth/st2auth/app.py

@@ -15,78 +15,28 @@
 
 from oslo_config import cfg
 
-from st2common import log as logging
-from st2common.middleware.error_handling import ErrorHandlingMiddleware
-from st2common.middleware.cors import CorsMiddleware
-from st2common.middleware.request_id import RequestIDMiddleware
-from st2common.middleware.logging import LoggingMiddleware
-from st2common.middleware.instrumentation import RequestInstrumentationMiddleware
-from st2common.middleware.instrumentation import ResponseInstrumentationMiddleware
-from st2common.router import Router
-from st2common.constants.system import VERSION_STRING
-from st2common.service_setup import setup as common_setup
-from st2common.util import spec_loader
-from st2common.util.monkey_patch import use_select_poll_workaround
+from st2common.openapi import app
 from st2auth import config as st2auth_config
 from st2auth.validation import validate_auth_backend_is_correctly_configured
 
-LOG = logging.getLogger(__name__)
 
+def setup_app(config={}):
+    common_setup = {
+        "register_mq_exchanges": False,
+        "register_internal_trigger_types": False,
+        "run_migrations": False,
+    }
 
-def setup_app(config=None):
-    config = config or {}
-
-    LOG.info("Creating st2auth: %s as OpenAPI app.", VERSION_STRING)
-
-    is_gunicorn = config.get("is_gunicorn", False)
-    if is_gunicorn:
-        # NOTE: We only want to perform this logic in the WSGI worker
-        st2auth_config.register_opts(ignore_errors=True)
-        capabilities = {
-            "name": "auth",
-            "listen_host": cfg.CONF.auth.host,
-            "listen_port": cfg.CONF.auth.port,
-            "listen_ssl": cfg.CONF.auth.use_ssl,
-            "type": "active",
-        }
-
-        # This should be called in gunicorn case because we only want
-        # workers to connect to db, rabbbitmq etc. In standalone HTTP
-        # server case, this setup would have already occurred.
-        common_setup(
-            service="auth",
-            config=st2auth_config,
-            setup_db=True,
-            register_mq_exchanges=False,
-            register_signal_handlers=True,
-            register_internal_trigger_types=False,
-            run_migrations=False,
-            service_registry=True,
-            capabilities=capabilities,
-            config_args=config.get("config_args", None),
-        )
-
-        # pysaml2 uses subprocess communicate which calls communicate_with_poll
-        if cfg.CONF.auth.sso and cfg.CONF.auth.sso_backend == "saml2":
-            use_select_poll_workaround(nose_only=False)
-
-    # Additional pre-run time checks
-    validate_auth_backend_is_correctly_configured()
-
-    router = Router(debug=cfg.CONF.auth.debug, is_gunicorn=is_gunicorn)
-
-    spec = spec_loader.load_spec("st2common", "openapi.yaml.j2")
     transforms = {"^/auth/v1/": ["/", "/v1/"]}
-    router.add_spec(spec, transforms=transforms)
-
-    app = router.as_wsgi
-
-    # Order is important. Check middleware for detailed explanation.
-    app = ErrorHandlingMiddleware(app)
-    app = CorsMiddleware(app)
-    app = LoggingMiddleware(app, router)
-    app = ResponseInstrumentationMiddleware(app, router, service_name="auth")
-    app = RequestIDMiddleware(app)
-    app = RequestInstrumentationMiddleware(app, router, service_name="auth")
 
-    return app
+    pre_run_checks = [validate_auth_backend_is_correctly_configured]
+
+    return app.setup_app(
+        service_name="auth",
+        app_config=st2auth_config,
+        oslo_cfg=cfg.CONF.auth,
+        pre_run_checks=pre_run_checks,
+        transforms=transforms,
+        common_setup_kwargs=common_setup,
+        config=config,
+    )