Enable tls in one test runner

.ci/container_setup.d/10-install-signing-service.sh

@@ -2,8 +2,8 @@
 
 set -eu
 
-pulp --refresh-api --base-url "http://localhost:8080" ${PULP_API_ROOT:+--api-root "${PULP_API_ROOT}"} --username "admin" --password "password" debug has-plugin --name "deb" && HAS_DEB=true || HAS_DEB=""
-pulp --base-url "http://localhost:8080" ${PULP_API_ROOT:+--api-root "${PULP_API_ROOT}"} --username "admin" --password "password" debug has-plugin --name "ansible" && HAS_ANSIBLE=true || HAS_ANSIBLE=""
+pulp --config "${PULP_CLI_CONFIG}" debug has-plugin --name "deb" && HAS_DEB=true || HAS_DEB=""
+pulp --config "${PULP_CLI_CONFIG}" debug has-plugin --name "ansible" && HAS_ANSIBLE=true || HAS_ANSIBLE=""
 if [ "$HAS_DEB" ] || [ "$HAS_ANSIBLE" ]
 then
   echo "Setup the signing services"

.ci/container_setup.d/README

@@ -0,0 +1,7 @@
+Files in this directory of the form '<number>-<name>.sh' are executed in alphabetical order.
+They can assume to be provided with the following environmnent variables:
+* PULP_CLI_CONFIG a path to a config file for the ci container
+* CONTAINER_RUNTIME the command for interacting with containers
+* BASE_PATH the directory the 'run_container.sh' script lives in
+
+Also a running container named 'pulp-ephemeral'.

.ci/run_container.sh

@@ -16,6 +16,16 @@ then
 fi
 export CONTAINER_RUNTIME
 
+TMPDIR="$(mktemp -d)"
+
+cleanup () {
+  "${CONTAINER_RUNTIME}" stop pulp-ephemeral && true
+  rm -rf "${TMPDIR}"
+}
+
+trap cleanup EXIT
+trap cleanup INT
+
 if [ -z "${KEEP_CONTAINER:+x}" ]
 then
   RM="yes"
@@ -38,12 +48,36 @@ else
     SELINUX=""
 fi;
 
-"${CONTAINER_RUNTIME}" run ${RM:+--rm} --env S6_KEEP_ENV=1 ${PULP_API_ROOT:+--env PULP_API_ROOT} --detach --name "pulp-ephemeral" --volume "${BASEPATH}/settings:/etc/pulp${SELINUX:+:Z}" --publish "8080:80" "ghcr.io/pulp/pulp:${IMAGE_TAG}"
+mkdir -p "${TMPDIR}/settings/certs"
+cp "${BASEPATH}/settings/settings.py" "${TMPDIR}/settings"
 
-# shellcheck disable=SC2064
-trap "${CONTAINER_RUNTIME} stop pulp-ephemeral" EXIT
-# shellcheck disable=SC2064
-trap "${CONTAINER_RUNTIME} stop pulp-ephemeral" INT
+if [ -z "${PULP_HTTPS:+x}" ]
+then
+  PROTOCOL="http"
+  PORT="80"
+  PULP_CONTENT_ORIGIN="http://localhost:8080/"
+else
+  PROTOCOL="https"
+  PORT="443"
+  PULP_CONTENT_ORIGIN="https://localhost:8080/"
+  python3 -m trustme -d "${TMPDIR}/settings/certs"
+  export PULP_CA_BUNDLE="${TMPDIR}/settings/certs/client.pem"
+  ln -fs server.pem "${TMPDIR}/settings/certs/pulp_webserver.crt"
+  ln -fs server.key "${TMPDIR}/settings/certs/pulp_webserver.key"
+fi
+export PULP_CONTENT_ORIGIN
+
+"${CONTAINER_RUNTIME}" \
+  run ${RM:+--rm} \
+  --env S6_KEEP_ENV=1 \
+  ${PULP_HTTPS:+--env PULP_HTTPS} \
+  ${PULP_API_ROOT:+--env PULP_API_ROOT} \
+  --env PULP_CONTENT_ORIGIN \
+  --detach \
+  --name "pulp-ephemeral" \
+  --volume "${TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \
+  --publish "8080:${PORT}" \
+  "ghcr.io/pulp/pulp:${IMAGE_TAG}"
 
 echo "Wait for pulp to start."
 for counter in $(seq 40 -1 0)
@@ -58,23 +92,27 @@ do
   fi
 
   sleep 3
-  if curl --fail "http://localhost:8080${PULP_API_ROOT:-/pulp/}api/v3/status/" > /dev/null 2>&1
+  if curl --insecure --fail "${PROTOCOL}://localhost:8080${PULP_API_ROOT:-/pulp/}api/v3/status/" > /dev/null 2>&1
   then
     echo "SUCCESS."
     break
   fi
   echo "."
 done
 
-# show pulpcore/plugin versions we're using
-curl -s "http://localhost:8080${PULP_API_ROOT:-/pulp/}api/v3/status/" | jq '.versions|map({key: .component, value: .version})|from_entries'
-
 # Set admin password
 "${CONTAINER_RUNTIME}" exec "pulp-ephemeral" pulpcore-manager reset-admin-password --password password
 
+# Create pulp config
+PULP_CLI_CONFIG="${TMPDIR}/settings/certs/cli.toml"
+export PULP_CLI_CONFIG
+pulp config create --overwrite --location "${PULP_CLI_CONFIG}" --base-url "${PROTOCOL}://localhost:8080" ${PULP_API_ROOT:+--api-root "${PULP_API_ROOT}"} --username "admin" --password "password"
+# show pulpcore/plugin versions we're using
+pulp --config "${PULP_CLI_CONFIG}" --refresh-api status
+
 if [ -d "${BASEPATH}/container_setup.d/" ]
 then
-  run-parts --regex '^[0-9]+-[-_[:alnum:]]*\.sh$' "${BASEPATH}/container_setup.d/"
+  run-parts --exit-on-error --regex '^[0-9]+-[-_[:alnum:]]*\.sh$' "${BASEPATH}/container_setup.d/"
 fi
 
 PULP_LOGGING="${CONTAINER_RUNTIME}" "$@"

.ci/settings/settings.py

@@ -1,4 +1,3 @@
-CONTENT_ORIGIN = "http://localhost:8080/"
 ALLOWED_EXPORT_PATHS = ["/tmp"]
 ORPHAN_PROTECTION_TIME = 0
 ANALYTICS = False

.github/workflows/test.yml

@@ -18,6 +18,7 @@ jobs:
         include:
           - image_tag: "nightly"
             pulp_api_root: "/relocated/djnd/"
+            pulp_https: true
             python: "3.11"
           - image_tag: "latest"
             python: "3.11"
@@ -65,5 +66,6 @@ jobs:
           IMAGE_TAG: ${{ matrix.image_tag }}
           FROM_TAG: ${{ matrix.from_tag }}
           CONTAINER_FILE: ${{ matrix.container_file }}
+          PULP_HTTPS: ${{ matrix.pulp_https }}
           PULP_API_ROOT: ${{ matrix.pulp_api_root }}
         run: .ci/run_container.sh make test

.gitignore

@@ -3,7 +3,6 @@ __pycache__/
 build/
 tests/cli.toml
 pytest_pulp_cli/GPG-PRIVATE-KEY-fixture-signing
-/.ci/settings/certs
 site/
 dist/
 *.po~

cookiecutter/ci/{{ cookiecutter.__project_name }}/.ci/container_setup.d/README

@@ -0,0 +1,7 @@
+Files in this directory of the form '<number>-<name>.sh' are executed in alphabetical order.
+They can assume to be provided with the following environmnent variables:
+* PULP_CLI_CONFIG a path to a config file for the ci container
+* CONTAINER_RUNTIME the command for interacting with containers
+* BASE_PATH the directory the 'run_container.sh' script lives in
+
+Also a running container named 'pulp-ephemeral'.

cookiecutter/ci/{{ cookiecutter.__project_name }}/.ci/run_container.sh

@@ -16,6 +16,16 @@ then
 fi
 export CONTAINER_RUNTIME
 
+TMPDIR="$(mktemp -d)"
+
+cleanup () {
+  "${CONTAINER_RUNTIME}" stop pulp-ephemeral && true
+  rm -rf "${TMPDIR}"
+}
+
+trap cleanup EXIT
+trap cleanup INT
+
 if [ -z "${KEEP_CONTAINER:+x}" ]
 then
   RM="yes"
@@ -38,12 +48,36 @@ else
     SELINUX=""
 fi;
 
-"${CONTAINER_RUNTIME}" run ${RM:+--rm} --env S6_KEEP_ENV=1 ${PULP_API_ROOT:+--env PULP_API_ROOT} --detach --name "pulp-ephemeral" --volume "${BASEPATH}/settings:/etc/pulp${SELINUX:+:Z}" --publish "8080:80" "ghcr.io/pulp/pulp:${IMAGE_TAG}"
+mkdir -p "${TMPDIR}/settings/certs"
+cp "${BASEPATH}/settings/settings.py" "${TMPDIR}/settings"
 
-# shellcheck disable=SC2064
-trap "${CONTAINER_RUNTIME} stop pulp-ephemeral" EXIT
-# shellcheck disable=SC2064
-trap "${CONTAINER_RUNTIME} stop pulp-ephemeral" INT
+if [ -z "${PULP_HTTPS:+x}" ]
+then
+  PROTOCOL="http"
+  PORT="80"
+  PULP_CONTENT_ORIGIN="http://localhost:8080/"
+else
+  PROTOCOL="https"
+  PORT="443"
+  PULP_CONTENT_ORIGIN="https://localhost:8080/"
+  python3 -m trustme -d "${TMPDIR}/settings/certs"
+  export PULP_CA_BUNDLE="${TMPDIR}/settings/certs/client.pem"
+  ln -fs server.pem "${TMPDIR}/settings/certs/pulp_webserver.crt"
+  ln -fs server.key "${TMPDIR}/settings/certs/pulp_webserver.key"
+fi
+export PULP_CONTENT_ORIGIN
+
+"${CONTAINER_RUNTIME}" \
+  run ${RM:+--rm} \
+  --env S6_KEEP_ENV=1 \
+  ${PULP_HTTPS:+--env PULP_HTTPS} \
+  ${PULP_API_ROOT:+--env PULP_API_ROOT} \
+  --env PULP_CONTENT_ORIGIN \
+  --detach \
+  --name "pulp-ephemeral" \
+  --volume "${TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \
+  --publish "8080:${PORT}" \
+  "ghcr.io/pulp/pulp:${IMAGE_TAG}"
 
 echo "Wait for pulp to start."
 for counter in $(seq 40 -1 0)
@@ -58,23 +92,27 @@ do
   fi
 
   sleep 3
-  if curl --fail "http://localhost:8080${PULP_API_ROOT:-/pulp/}api/v3/status/" > /dev/null 2>&1
+  if curl --insecure --fail "${PROTOCOL}://localhost:8080${PULP_API_ROOT:-/pulp/}api/v3/status/" > /dev/null 2>&1
   then
     echo "SUCCESS."
     break
   fi
   echo "."
 done
 
-# show pulpcore/plugin versions we're using
-curl -s "http://localhost:8080${PULP_API_ROOT:-/pulp/}api/v3/status/" | jq '.versions|map({key: .component, value: .version})|from_entries'
-
 # Set admin password
 "${CONTAINER_RUNTIME}" exec "pulp-ephemeral" pulpcore-manager reset-admin-password --password password
 
+# Create pulp config
+PULP_CLI_CONFIG="${TMPDIR}/settings/certs/cli.toml"
+export PULP_CLI_CONFIG
+pulp config create --overwrite --location "${PULP_CLI_CONFIG}" --base-url "${PROTOCOL}://localhost:8080" ${PULP_API_ROOT:+--api-root "${PULP_API_ROOT}"} --username "admin" --password "password"
+# show pulpcore/plugin versions we're using
+pulp --config "${PULP_CLI_CONFIG}" --refresh-api status
+
 if [ -d "${BASEPATH}/container_setup.d/" ]
 then
-  run-parts --regex '^[0-9]+-[-_[:alnum:]]*\.sh$' "${BASEPATH}/container_setup.d/"
+  run-parts --exit-on-error --regex '^[0-9]+-[-_[:alnum:]]*\.sh$' "${BASEPATH}/container_setup.d/"
 fi
 
 PULP_LOGGING="${CONTAINER_RUNTIME}" "$@"

cookiecutter/ci/{{ cookiecutter.__project_name }}/.ci/settings/settings.py

@@ -0,0 +1,4 @@
+ALLOWED_EXPORT_PATHS = ["/tmp"]
+ORPHAN_PROTECTION_TIME = 0
+ANALYTICS = False
+ALLOWED_CONTENT_CHECKSUMS = ["sha1", "sha256", "sha512"]

cookiecutter/ci/{{ cookiecutter.__project_name }}/.github/workflows/test.yml

@@ -46,6 +46,7 @@ jobs:
           IMAGE_TAG: ${{ matrix.image_tag }}
           FROM_TAG: ${{ matrix.from_tag }}
           CONTAINER_FILE: ${{ matrix.container_file }}
+          PULP_HTTPS: ${{ matrix.pulp_https }}
           PULP_API_ROOT: ${{ matrix.pulp_api_root }}
           {%- endraw %}
         run: .ci/run_container.sh make test

cookiecutter/ci/{{ cookiecutter.__project_name }}/.gitignore

@@ -3,7 +3,6 @@ __pycache__/
 build/
 tests/cli.toml
 pytest_pulp_cli/GPG-PRIVATE-KEY-fixture-signing
-/.ci/settings/certs
 site/
 dist/
 *.po~

pytest_pulp_cli/__init__.py

@@ -88,7 +88,10 @@ def pulp_cli_settings(tmp_path_factory: pytest.TempPathFactory) -> t.Tuple[pathl
     It is most likely not useful to be included standalone.
     The `pulp_cli_env` fixture, however depends on it and sets $XDG_CONFIG_HOME up accordingly.
     """
-    settings = toml.load("tests/cli.toml")
+    settings = toml.load(os.environ.get("PULP_CLI_CONFIG", "tests/cli.toml"))
+    if os.environ.get("PULP_HTTPS"):
+        for key in settings:
+            settings[key]["base_url"] = settings[key]["base_url"].replace("http://", "https://")
     if os.environ.get("PULP_API_ROOT"):
         for key in settings:
             settings[key]["api_root"] = os.environ["PULP_API_ROOT"]

test_requirements.txt

@@ -2,6 +2,7 @@
 pytest>=7.0.0,<=8.2.2
 pytest-subtests>=0.12.0,<=0.12.1
 python-gnupg==0.5.2
+trustme>=1.1.0,<1.2
 
 # No pinning here, because we only switch on optional dependencies here.
 pygments

tests/scripts/pulp_file/test_distribution.sh

@@ -15,9 +15,14 @@ trap cleanup EXIT
 
 if [ "$VERIFY_SSL" = "false" ]
 then
-  curl_opt="-k"
+  curl_opt=("-k")
 else
-  curl_opt=""
+  if [ "${PULP_CA_BUNDLE:+x}" ]
+  then
+    curl_opt=("--cacert" "${PULP_CA_BUNDLE}")
+  else
+    curl_opt=()
+  fi
 fi
 
 expect_succ pulp file remote create --name "cli_test_file_remote" --url "$FILE_REMOTE_URL"
@@ -57,6 +62,6 @@ base_url="$(echo "$OUTPUT" | jq -r .[0].base_url)"
 expect_succ pulp file distribution list --base-path-contains "CLI"
 test "$(echo "$OUTPUT" | jq -r length)" -gt 0
 
-expect_succ curl "$curl_opt" --head --fail "${base_url}1.iso"
+expect_succ curl "${curl_opt[@]}" --head --fail "${base_url}1.iso"
 
 expect_succ pulp file distribution destroy --distribution "cli_test_file_distro"

tests/scripts/pulp_python/test_distribution.sh

@@ -15,9 +15,14 @@ trap cleanup EXIT
 
 if [ "$VERIFY_SSL" = "false" ]
 then
-  curl_opt="-k"
+  curl_opt=("-k")
 else
-  curl_opt=""
+  if [ "${PULP_CA_BUNDLE:+x}" ]
+  then
+    curl_opt=("--cacert" "${PULP_CA_BUNDLE}")
+  else
+    curl_opt=()
+  fi
 fi
 
 expect_succ pulp python remote create --name "cli_test_python_remote" --url "$PYTHON_REMOTE_URL" --includes '["shelf-reader"]'
@@ -39,7 +44,7 @@ expect_succ pulp python distribution update \
   --base-path "cli_test_python_distro" \
   --publication "$PUBLICATION_HREF"
 
-expect_succ curl "$curl_opt" --head --fail "$PULP_BASE_URL/pypi/cli_test_python_distro/simple/"
+expect_succ curl "${curl_opt[@]}" --head --fail "$PULP_BASE_URL/pypi/cli_test_python_distro/simple/"
 expect_succ pulp python distribution update \
 --name "cli_test_python_distro" \
 --repository "cli_test_python_repository" \

tests/scripts/pulp_rpm/test_rpm_sync_publish.sh

@@ -18,9 +18,14 @@ trap cleanup EXIT
 
 if [ "$VERIFY_SSL" = "false" ]
 then
-  curl_opt="-k"
+  curl_opt=("-k")
 else
-  curl_opt=""
+  if [ "${PULP_CA_BUNDLE:+x}" ]
+  then
+    curl_opt=("--cacert" "${PULP_CA_BUNDLE}")
+  else
+    curl_opt=()
+  fi
 fi
 
 # Add content using JSON file
@@ -83,9 +88,9 @@ DISTRIBUTION_BASE_URL=$(echo "$OUTPUT" | jq -r .base_url)
 
 if pulp debug has-plugin --name "rpm" --specifier "<3.23.0"
 then
-  expect_succ curl "$curl_opt" --head --fail "${DISTRIBUTION_BASE_URL}config.repo"
+  expect_succ curl "${curl_opt[@]}" --head --fail "${DISTRIBUTION_BASE_URL}config.repo"
 else
-  expect_fail curl "$curl_opt" --head --fail "${DISTRIBUTION_BASE_URL}config.repo"
+  expect_fail curl "${curl_opt[@]}" --head --fail "${DISTRIBUTION_BASE_URL}config.repo"
 fi
 
 if pulp debug has-plugin --name "rpm" --specifier ">=3.23.0"
@@ -95,7 +100,7 @@ then
     --publication "$PUBLICATION_HREF" \
     --generate-repo-config
   DISTRIBUTION_BASE_URL=$(echo "$OUTPUT" | jq -r .base_url)
-  expect_succ curl "$curl_opt" --head --fail "${DISTRIBUTION_BASE_URL}config.repo"
+  expect_succ curl "${curl_opt[@]}" --head --fail "${DISTRIBUTION_BASE_URL}config.repo"
   expect_succ pulp rpm distribution destroy --name "cli_test_rpm_distro2"
 fi