From 5e0dc4ddc78b687c30fa4dd4017fd60ed00b5ed2 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Fri, 30 Aug 2024 17:33:47 +0200 Subject: [PATCH] Use certificate authentication in the ssl test This sets up nginx to accept client certificates and provides a matching client certificate with DN=admin. --- .ci/gen_certs.py | 59 +++++++++ .ci/nginx.conf.j2 | 147 ++++++++++++++++++++++ .ci/run_container.sh | 28 +++-- pulp-glue/tests/conftest.py | 16 ++- pytest_pulp_cli/__init__.py | 22 +++- tests/scripts/pulp_container/test_role.sh | 14 +-- tests/scripts/pulp_file/test_role.sh | 18 +-- tests/scripts/pulpcore/test_role.sh | 16 +-- tests/scripts/test_config.sh | 2 +- 9 files changed, 280 insertions(+), 42 deletions(-) create mode 100644 .ci/gen_certs.py create mode 100644 .ci/nginx.conf.j2 diff --git a/.ci/gen_certs.py b/.ci/gen_certs.py new file mode 100644 index 000000000..5026efff8 --- /dev/null +++ b/.ci/gen_certs.py @@ -0,0 +1,59 @@ +import argparse +import os +import sys +import typing as t + +import trustme + + +def main(argv: t.Optional[t.List[str]] = None) -> None: + if argv is None: + argv = sys.argv[1:] + + parser = argparse.ArgumentParser(prog="gen_certs") + parser.add_argument( + "-d", + "--dir", + default=os.getcwd(), + help="Directory where certificates and keys are written to. Defaults to cwd.", + ) + + args = parser.parse_args(argv) + cert_dir = args.dir + + if not os.path.isdir(cert_dir): + raise ValueError(f"--dir={cert_dir} is not a directory") + + key_type = trustme.KeyType["ECDSA"] + + # Generate the CA certificate + ca = trustme.CA(key_type=key_type) + # Write the certificate the client should trust + ca_cert_path = os.path.join(cert_dir, "ca.pem") + ca.cert_pem.write_to_path(path=ca_cert_path) + + # Generate the server certificate + server_cert = ca.issue_cert("localhost", "127.0.0.1", "::1", key_type=key_type) + # Write the certificate and private key the server should use + server_key_path = os.path.join(cert_dir, "server.key") + server_cert_path = os.path.join(cert_dir, "server.pem") + server_cert.private_key_pem.write_to_path(path=server_key_path) + with open(server_cert_path, mode="w") as f: + f.truncate() + for blob in server_cert.cert_chain_pems: + blob.write_to_path(path=server_cert_path, append=True) + + # Generate the client certificate + client_cert = ca.issue_cert("admin@example.com", common_name="admin", key_type=key_type) + # Write the certificate and private key the client should use + client_key_path = os.path.join(cert_dir, "client.key") + client_cert_path = os.path.join(cert_dir, "client.pem") + client_cert.private_key_pem.write_to_path(path=client_key_path) + with open(client_cert_path, mode="w") as f: + f.truncate() + for blob in client_cert.cert_chain_pems: + blob.write_to_path(path=client_cert_path, append=True) + + +if __name__ == "__main__": + main() diff --git a/.ci/nginx.conf.j2 b/.ci/nginx.conf.j2 new file mode 100644 index 000000000..70a2e5a3d --- /dev/null +++ b/.ci/nginx.conf.j2 @@ -0,0 +1,147 @@ +# Copy from pulp-oci-images. +# Ideally we can get it upstream again. +# +# TODO: Support IPv6. +# TODO: Maybe serve multiple `location`s, not just one. + +# The "nginx" package on fedora creates this user and group. +user nginx nginx; +# Gunicorn docs suggest this value. +worker_processes 1; +daemon off; +events { + worker_connections 1024; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 +} + +http { + include mime.types; + # fallback in case we can't determine a type + default_type application/octet-stream; + sendfile on; + + # If left at the default of 1024, nginx emits a warning about being unable + # to build optimal hash types. + types_hash_max_size 4096; + + map $ssl_client_s_dn $ssl_client_s_dn_cn { + default ""; + ~CN=(?[^,]+) $CN; + } + + upstream pulp-content { + server 127.0.0.1:24816; + } + + upstream pulp-api { + server 127.0.0.1:24817; + } + + server { + # Gunicorn docs suggest the use of the "deferred" directive on Linux. + {% if https | default(false) -%} + listen 443 default_server deferred ssl; + + ssl_certificate /etc/pulp/certs/pulp_webserver.crt; + ssl_certificate_key /etc/pulp/certs/pulp_webserver.key; + ssl_session_cache shared:SSL:50m; + ssl_session_timeout 1d; + ssl_session_tickets off; + + # intermediate configuration + ssl_protocols TLSv1.2; + ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; + ssl_prefer_server_ciphers on; + + # HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months) + add_header Strict-Transport-Security max-age=15768000; + + # Configure client cert authentication + ssl_client_certificate /etc/pulp/certs/ca.pem; + ssl_verify_client optional; + {%- else -%} + listen 80 default_server deferred; + {%- endif %} + server_name $hostname; + + # The default client_max_body_size is 1m. Clients uploading + # files larger than this will need to chunk said files. + client_max_body_size 10m; + + # Gunicorn docs suggest this value. + keepalive_timeout 5; + + location {{ content_path }} { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-content; + } + + location {{ api_root }}api/v3/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_set_header Remoteuser $ssl_client_s_dn_cn; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + client_max_body_size 0; + } + + {%- if domain_enabled | default(false) %} + location ~ {{ api_root }}.+/api/v3/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + client_max_body_size 0; + } + {%- endif %} + + location /auth/login/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + } + + include pulp/*.conf; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + # most pulp static files are served through whitenoise + # http://whitenoise.evans.io/en/stable/ + } + + {%- if https | default(false) %} + # ACME http-01 tokens, i.e, for Let's Encrypt + location /.well-known/ { + try_files $uri $uri/ =404; + } + {%- endif %} + } + {%- if https | default(false) %} + server { + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; + } + {%- endif %} +} diff --git a/.ci/run_container.sh b/.ci/run_container.sh index 478b00b70..0d7440fe1 100755 --- a/.ci/run_container.sh +++ b/.ci/run_container.sh @@ -16,11 +16,12 @@ then fi export CONTAINER_RUNTIME -TMPDIR="$(mktemp -d)" +PULP_CLI_TEST_TMPDIR="$(mktemp -d)" +export PULP_CLI_TEST_TMPDIR cleanup () { "${CONTAINER_RUNTIME}" stop pulp-ephemeral && true - rm -rf "${TMPDIR}" + rm -rf "${PULP_CLI_TEST_TMPDIR}" } trap cleanup EXIT @@ -48,8 +49,8 @@ else SELINUX="" fi; -mkdir -p "${TMPDIR}/settings/certs" -cp "${BASEPATH}/settings/settings.py" "${TMPDIR}/settings" +mkdir -p "${PULP_CLI_TEST_TMPDIR}/settings/certs" +cp "${BASEPATH}/settings/settings.py" "${PULP_CLI_TEST_TMPDIR}/settings" if [ -z "${PULP_HTTPS:+x}" ] then @@ -60,10 +61,16 @@ else PROTOCOL="https" PORT="443" PULP_CONTENT_ORIGIN="https://localhost:8080/" - python3 -m trustme -d "${TMPDIR}/settings/certs" - export PULP_CA_BUNDLE="${TMPDIR}/settings/certs/client.pem" - ln -fs server.pem "${TMPDIR}/settings/certs/pulp_webserver.crt" - ln -fs server.key "${TMPDIR}/settings/certs/pulp_webserver.key" + python3 "${BASEPATH}/gen_certs.py" -d "${PULP_CLI_TEST_TMPDIR}/settings/certs" + export PULP_CA_BUNDLE="${PULP_CLI_TEST_TMPDIR}/settings/certs/ca.pem" + ln -fs server.pem "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.crt" + ln -fs server.key "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.key" + { + echo "AUTHENTICATION_BACKENDS = '@merge django.contrib.auth.backends.RemoteUserBackend'" + echo "MIDDLEWARE = '@merge django.contrib.auth.middleware.RemoteUserMiddleware'" + echo "REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES = '@merge pulpcore.app.authentication.PulpRemoteUserAuthentication'" + echo "REMOTE_USER_ENVIRON_NAME = 'HTTP_REMOTEUSER'" + } >> "${PULP_CLI_TEST_TMPDIR}/settings/settings.py" fi export PULP_CONTENT_ORIGIN @@ -75,7 +82,8 @@ export PULP_CONTENT_ORIGIN --env PULP_CONTENT_ORIGIN \ --detach \ --name "pulp-ephemeral" \ - --volume "${TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \ + --volume "${PULP_CLI_TEST_TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \ + --volume "${BASEPATH}/nginx.conf.j2:/nginx/nginx.conf.j2${SELINUX:+:Z}" \ --network bridge \ --publish "8080:${PORT}" \ "ghcr.io/pulp/pulp:${IMAGE_TAG}" @@ -105,7 +113,7 @@ done "${CONTAINER_RUNTIME}" exec "pulp-ephemeral" pulpcore-manager reset-admin-password --password password # Create pulp config -PULP_CLI_CONFIG="${TMPDIR}/settings/certs/cli.toml" +PULP_CLI_CONFIG="${PULP_CLI_TEST_TMPDIR}/settings/certs/cli.toml" export PULP_CLI_CONFIG pulp config create --overwrite --location "${PULP_CLI_CONFIG}" --base-url "${PROTOCOL}://localhost:8080" ${PULP_API_ROOT:+--api-root "${PULP_API_ROOT}"} --username "admin" --password "password" # show pulpcore/plugin versions we're using diff --git a/pulp-glue/tests/conftest.py b/pulp-glue/tests/conftest.py index 1c098a004..5a79a48a5 100644 --- a/pulp-glue/tests/conftest.py +++ b/pulp-glue/tests/conftest.py @@ -21,7 +21,13 @@ def pulp_ctx( return PulpTestContext( api_kwargs={ "base_url": settings["base_url"], - "auth_provider": BasicAuthProvider(settings.get("username"), settings.get("password")), + "auth_provider": ( + BasicAuthProvider(settings.get("username"), settings.get("password")) + if "username" in settings + else None + ), + "cert": settings.get("cert"), + "key": settings.get("key"), "debug_callback": lambda i, s: i <= verbose and print(s), }, api_root=settings.get("api_root", "pulp/"), @@ -39,7 +45,13 @@ def fake_pulp_ctx( return PulpTestContext( api_kwargs={ "base_url": settings["base_url"], - "auth_provider": BasicAuthProvider(settings.get("username"), settings.get("password")), + "auth_provider": ( + BasicAuthProvider(settings.get("username"), settings.get("password")) + if "username" in settings + else None + ), + "cert": settings.get("cert"), + "key": settings.get("key"), "debug_callback": lambda i, s: i <= verbose and print(s), }, api_root=settings.get("api_root", "pulp/"), diff --git a/pytest_pulp_cli/__init__.py b/pytest_pulp_cli/__init__.py index 4b5aa743c..f8bfd5ce1 100644 --- a/pytest_pulp_cli/__init__.py +++ b/pytest_pulp_cli/__init__.py @@ -88,13 +88,25 @@ def pulp_cli_settings() -> t.Dict[str, t.Dict[str, t.Any]]: It is most likely not useful to be included standalone. The `pulp_cli_env` fixture, however depends on it and sets $XDG_CONFIG_HOME up accordingly. """ - settings = toml.load(os.environ.get("PULP_CLI_CONFIG", "tests/cli.toml")) + pulp_cli_test_tmpdir = pathlib.Path(os.environ.get("PULP_CLI_TEST_TMPDIR", ".")) + settings = {"cli": toml.load(os.environ.get("PULP_CLI_CONFIG", "tests/cli.toml"))["cli"]} if os.environ.get("PULP_HTTPS"): - for key in settings: - settings[key]["base_url"] = settings[key]["base_url"].replace("http://", "https://") + settings["cli"]["base_url"] = settings["cli"]["base_url"].replace("http://", "https://") + client_cert_path = pulp_cli_test_tmpdir / "settings" / "certs" / "client.pem" + client_key_path = pulp_cli_test_tmpdir / "settings" / "certs" / "client.key" + if client_cert_path.exists(): + settings["cli"].pop("username", None) + settings["cli"].pop("password", None) + settings["cli"]["cert"] = str(client_cert_path) + if client_key_path.exists(): + settings["cli"]["key"] = str(client_key_path) + if os.environ.get("PULP_API_ROOT"): - for key in settings: - settings[key]["api_root"] = os.environ["PULP_API_ROOT"] + settings["cli"]["api_root"] = os.environ["PULP_API_ROOT"] + + settings["cli-noauth"] = { + k: v for k, v in settings["cli"].items() if k not in {"username", "password", "cert", "key"} + } return settings diff --git a/tests/scripts/pulp_container/test_role.sh b/tests/scripts/pulp_container/test_role.sh index a614f1b27..fb1b2df14 100755 --- a/tests/scripts/pulp_container/test_role.sh +++ b/tests/scripts/pulp_container/test_role.sh @@ -18,17 +18,17 @@ expect_succ pulp user create --username "clitest" --password "${USERPASS}" expect_succ pulp group create --name "clitest" expect_succ pulp group user add --group "clitest" --username "clitest" -expect_succ pulp --username clitest --password "${USERPASS}" container repository list +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" container repository list test "$(echo "${OUTPUT}" | jq -r 'length' )" = "0" -expect_deny pulp --username clitest --password "${USERPASS}" container repository create --name "clitest" +expect_deny pulp -p noauth --username clitest --password "${USERPASS}" container repository create --name "clitest" expect_succ pulp container repository create --name "clitest" REPOSITORY_HREF=$(jq -r '.pulp_href' <<<"${OUTPUT}") -expect_fail pulp --username clitest --password "${USERPASS}" container repository show --repository "clitest" -expect_fail pulp --username clitest --password "${USERPASS}" container repository show --repository "${REPOSITORY_HREF}" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" container repository show --repository "clitest" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" container repository show --repository "${REPOSITORY_HREF}" expect_succ pulp container repository role add --name "clitest" --user "clitest" --role "container.containerrepository_viewer" -expect_succ pulp --username clitest --password "${USERPASS}" container repository show --repository "clitest" -expect_succ pulp --username clitest --password "${USERPASS}" container repository show --repository "${REPOSITORY_HREF}" +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" container repository show --repository "clitest" +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" container repository show --repository "${REPOSITORY_HREF}" -expect_deny pulp --username clitest --password "${USERPASS}" container repository update --href "${REPOSITORY_HREF}" --retain-repo-versions 1 +expect_deny pulp -p noauth --username clitest --password "${USERPASS}" container repository update --href "${REPOSITORY_HREF}" --retain-repo-versions 1 diff --git a/tests/scripts/pulp_file/test_role.sh b/tests/scripts/pulp_file/test_role.sh index 4e191d9bb..d00310d4c 100755 --- a/tests/scripts/pulp_file/test_role.sh +++ b/tests/scripts/pulp_file/test_role.sh @@ -15,21 +15,21 @@ trap cleanup EXIT pulp user create --username "clitest" --password "${USERPASS}" -expect_succ pulp --username clitest --password "${USERPASS}" file repository list +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" file repository list test "$(echo "${OUTPUT}" | jq -r 'length' )" = "0" -expect_deny pulp --username clitest --password "${USERPASS}" file repository create --name "clitest" +expect_deny pulp -p noauth --username clitest --password "${USERPASS}" file repository create --name "clitest" expect_succ pulp file repository create --name "clitest" REPOSITORY_HREF=$(jq -r '.pulp_href' <<<"${OUTPUT}") -expect_fail pulp --username clitest --password "${USERPASS}" file repository show --repository "clitest" -expect_fail pulp --username clitest --password "${USERPASS}" file repository show --href "${REPOSITORY_HREF}" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" file repository show --repository "clitest" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" file repository show --href "${REPOSITORY_HREF}" expect_succ pulp file repository role add --repository "clitest" --user "clitest" --role "file.filerepository_viewer" -expect_succ pulp --username clitest --password "${USERPASS}" file repository show --repository "clitest" -expect_succ pulp --username clitest --password "${USERPASS}" file repository show --href "${REPOSITORY_HREF}" +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" file repository show --repository "clitest" +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" file repository show --href "${REPOSITORY_HREF}" -expect_deny pulp --username clitest --password "${USERPASS}" file repository update --repository "${REPOSITORY_HREF}" --retain-repo-versions 1 +expect_deny pulp -p noauth --username clitest --password "${USERPASS}" file repository update --repository "${REPOSITORY_HREF}" --retain-repo-versions 1 expect_succ pulp file repository role remove --name "clitest" --user "clitest" --role "file.filerepository_viewer" -expect_fail pulp --username clitest --password "${USERPASS}" file repository show --repository "clitest" -expect_fail pulp --username clitest --password "${USERPASS}" file repository show --repository "${REPOSITORY_HREF}" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" file repository show --repository "clitest" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" file repository show --repository "${REPOSITORY_HREF}" diff --git a/tests/scripts/pulpcore/test_role.sh b/tests/scripts/pulpcore/test_role.sh index 69b2472b5..0b4b13996 100755 --- a/tests/scripts/pulpcore/test_role.sh +++ b/tests/scripts/pulpcore/test_role.sh @@ -24,31 +24,31 @@ expect_succ pulp group create --name "clitest" GROUP_HREF=$(jq -r '.pulp_href' <<<"${OUTPUT}") expect_succ pulp group user add --group "clitest" --username "clitest" -expect_succ pulp --username clitest --password "${USERPASS}" task list +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" task list test "$(echo "${OUTPUT}" | jq -r 'length' )" = "0" -expect_fail pulp --username clitest --password "${USERPASS}" group show --name "clitest" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" group show --name "clitest" expect_succ pulp user role-assignment add --username "clitest" --role "clitest.group_viewer" --object "${GROUP_HREF}" expect_succ pulp user role-assignment list --username "clitest" --role "clitest.group_viewer" --object "${GROUP_HREF}" test "$(echo "${OUTPUT}" | jq -r 'length' )" = "1" expect_succ pulp user role-assignment list --username "clitest" --role-in "clitest.group_viewer" --object "${GROUP_HREF}" test "$(echo "${OUTPUT}" | jq -r 'length' )" = "1" -expect_succ pulp --username clitest --password "${USERPASS}" group show --name "clitest" +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" group show --name "clitest" expect_succ pulp user role-assignment remove --username "clitest" --role "clitest.group_viewer" --object "${GROUP_HREF}" -expect_fail pulp --username clitest --password "${USERPASS}" group show --name "clitest" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" group show --name "clitest" expect_succ pulp group role-assignment add --group "clitest" --role "clitest.group_viewer" --object "${GROUP_HREF}" expect_succ pulp group role-assignment list --group "clitest" --role "clitest.group_viewer" --object "${GROUP_HREF}" test "$(echo "${OUTPUT}" | jq -r 'length' )" = "1" expect_succ pulp group role-assignment list --group "clitest" --role-in "clitest.group_viewer" --object "${GROUP_HREF}" test "$(echo "${OUTPUT}" | jq -r 'length' )" = "1" -expect_succ pulp --username clitest --password "${USERPASS}" group show --name "clitest" +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" group show --name "clitest" expect_succ pulp group role-assignment remove --group "clitest" --role "clitest.group_viewer" --object "${GROUP_HREF}" -expect_fail pulp --username clitest --password "${USERPASS}" group show --name "clitest" +expect_fail pulp -p noauth --username clitest --password "${USERPASS}" group show --name "clitest" expect_succ pulp user role-assignment add --username "clitest" --role "core.group_creator" --object "" -expect_succ pulp --username clitest --password "${USERPASS}" group create --name "clitest2" -expect_succ pulp --username clitest --password "${USERPASS}" group role my-permissions --name "clitest2" +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" group create --name "clitest2" +expect_succ pulp -p noauth --username clitest --password "${USERPASS}" group role my-permissions --name "clitest2" expect_succ pulp role destroy --name "clitest.group_viewer" diff --git a/tests/scripts/test_config.sh b/tests/scripts/test_config.sh index f90bd384e..c564729f5 100755 --- a/tests/scripts/test_config.sh +++ b/tests/scripts/test_config.sh @@ -33,7 +33,7 @@ expect_fail pulp --key "/some/path" task list # CONFIG PROFILE cp "$bad_settings" "$profile_settings" -sed -e 's/\[cli\]/[cli-profile1]/' "$good_settings" >> "$profile_settings" +sed -e 's/\[cli\(.*\)\]/[cli-profile1\1]/' "$good_settings" >> "$profile_settings" expect_fail pulp --config "$profile_settings" task list expect_succ pulp --config "$profile_settings" --profile profile1 task list