Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow per cluster init #1845

Merged
merged 1 commit into from
Nov 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/end-to-end.yml
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ jobs:
CK8S_ENVIRONMENT_NAME: pipeline-exoscale
CK8S_CLOUD_PROVIDER: exoscale
CK8S_FLAVOR: dev
run: ./apps/bin/ck8s init
run: ./apps/bin/ck8s init both
id: initialize-apps

- name: Create buckets
Expand Down
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,16 @@ Assuming you already have everything needed to install the apps, this is what yo
See [compliantkubernetes.io](https://compliantkubernetes.io/) if you are uncertain about what order you should do things in.

```bash
./bin/ck8s init
./bin/ck8s init both
```

> [!NOTE]
> It is possible to initialize `wc` and `sc` clusters separately by replacing `both` when running the `init` command:
> ```bash
> ./bin/ck8s init wc
> ./bin/ck8s init sc
> ```

1. Edit the configuration files that have been initialized in the configuration path.
Make sure that the `objectStorage` values are set in `common-config.yaml` or `sc-config.yaml` and `wc-config.yaml`, as well as required credentials in `secrets.yaml` according to your `objectStorage.type`.
The type may already be set in the default configuration found in the `defaults/` directory depending on your selected cloud provider.
Expand Down
12 changes: 9 additions & 3 deletions bin/ck8s
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ source "${here}/common.bash"

usage() {
echo "COMMANDS:" 1>&2
echo " init [--generate-new-secrets] initialize the config path" 1>&2
echo " init <wc|sc|both> [--generate-new-secrets] initialize the config path" 1>&2
echo " bootstrap <wc|sc> bootstrap the cluster" 1>&2
echo " apps <wc|sc> [--sync] [--skip-template-validate] deploy the applications" 1>&2
echo " apply <wc|sc> [--sync] [--skip-template-validate] bootstrap and apps" 1>&2
Expand Down Expand Up @@ -58,7 +58,9 @@ done

case "${1}" in
init)
[[ "${2}" =~ ^(wc|sc|both)$ ]] || usage
check_tools
export CK8S_CLUSTER="${2}"
"${here}/init.bash" "${GEN_NEW_SECRETS}"
;;
bootstrap)
Expand Down Expand Up @@ -92,7 +94,8 @@ case "${1}" in
[[ "${3}" =~ ^(v[0-9]+\.[0-9]+)$ ]] || usage
[[ "${4}" =~ ^(prepare|apply)$ ]] || usage
check_tools
"${here}/upgrade.bash" "${2}" "${3}" "${4}"
export CK8S_CLUSTER="${2}"
"${here}/upgrade.bash" "${3}" "${4}"
;;
team)
case "${2}" in
Expand Down Expand Up @@ -159,7 +162,10 @@ case "${1}" in
"${here}/update-ips.bash" "${2}" "${3}"
;;
fix-psp-violations)
"${here}/fix-psp-violations.bash" "${2}"
[[ "${2}" =~ ^(wc|sc)$ ]] || usage
check_tools
export CK8S_CLUSTER="${2}"
"${here}/fix-psp-violations.bash"
;;
clean)
"${here}/clean.bash" "${2}"
Expand Down
35 changes: 12 additions & 23 deletions bin/fix-psp-violations.bash
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
#!/usr/bin/env bash
anders-elastisys marked this conversation as resolved.
Show resolved Hide resolved

usage() {
echo "Usage: fix-psp-violoations <wc|sc>" >&2
exit 1
}
: "${CK8S_CLUSTER:?Missing CK8S_CLUSTER}"

ROOT="$(readlink -f "$(dirname "${0}")/../")"

Expand All @@ -17,10 +14,9 @@ exempt_namepsaces=("rook-ceph")
function set_violating_resources() {
results_mult_uniq=""
results=()
cluster="$1"

# Get violations for PSPs
violations=$(kubectl_do "$cluster" get constraints -o yaml | yq4 '[.items[] | select(.kind == "K8sPSP*") | .status.violations[]]')
violations=$(kubectl_do "${CK8S_CLUSTER}" get constraints -o yaml | yq4 '[.items[] | select(.kind == "K8sPSP*") | .status.violations[]]')

# Build array of maps with relevant information
resources=$(echo "$violations" | yq4 '.[] |[{"name": .name, "namespace": .namespace}]' | yq4 'unique_by(.name,.namespace)')
Expand All @@ -32,7 +28,7 @@ function set_violating_resources() {
namespace=$(echo "$resource" | yq4 e '.namespace')
pod_name=$(echo "$resource" | yq4 e '.name')

owner_reference=$(kubectl_do "$cluster" -n "$namespace" get pod "$pod_name" --ignore-not-found=true -oyaml | yq4 '.metadata.ownerReferences.[0]')
owner_reference=$(kubectl_do "${CK8S_CLUSTER}" -n "$namespace" get pod "$pod_name" --ignore-not-found=true -oyaml | yq4 '.metadata.ownerReferences.[0]')

# Skip standalone Pods and stale references
if [ "$owner_reference" = "null" ] || [ -z "$owner_reference" ]; then continue; fi
Expand All @@ -45,7 +41,7 @@ function set_violating_resources() {

# Get owner of ReplicaSets
if [ "$owner_kind" == "ReplicaSet" ]; then
owner_reference=$(kubectl_do "$cluster" -n "$namespace" get rs "$owner_name" --ignore-not-found=true -oyaml | yq4 '.metadata.ownerReferences.[0]')
owner_reference=$(kubectl_do "${CK8S_CLUSTER}" -n "$namespace" get rs "$owner_name" --ignore-not-found=true -oyaml | yq4 '.metadata.ownerReferences.[0]')

# Skip standalone ReplicaSets and stale references
if [ "$owner_reference" = "null" ] || [ -z "$owner_reference" ]; then continue; fi
Expand All @@ -63,38 +59,31 @@ function set_violating_resources() {

function is_customer_namespace() {
namespace="$1"
cluster="$2"
operator_ns_regex="^($(kubectl_do "$cluster" get ns -l owner=operator '-ojsonpath={.items[*].metadata.name}' | sed 's/ /|/g'))$"
operator_ns_regex="^($(kubectl_do "${CK8S_CLUSTER}" get ns -l owner=operator '-ojsonpath={.items[*].metadata.name}' | sed 's/ /|/g'))$"

if [[ "$namespace" =~ $operator_ns_regex ]]; then return 1; fi
}

function restart_violating_resources() {
IFS=$'\n'
cluster="$1"
# shellcheck disable=SC2128
for entry in $results_mult_uniq; do
kind=$(echo "$entry" | yq4 .kind)
name=$(echo "$entry" | yq4 .name)
namespace=$(echo "$entry" | yq4 .namespace)

# shellcheck disable=SC2076
if [[ "${exempt_namepsaces[*]}" =~ "${namespace}" ]] || is_customer_namespace "$namespace" "$cluster"; then
log_warn "$kind/$name in $namespace for cluster $cluster requires manual restart"
if [[ "${exempt_namepsaces[*]}" =~ "${namespace}" ]] || is_customer_namespace "$namespace"; then
log_warn "$kind/$name in $namespace for cluster ${CK8S_CLUSTER} requires manual restart"
else
if [[ -n "$(kubectl_do "$cluster" get "$kind" "$name" -n "$namespace" --ignore-not-found=true -oname)" ]]; then
log_info "Will trigger a rollout restart of $kind/$name in $namespace for cluster $cluster"
kubectl_do "$cluster" rollout restart "$kind" "$name" -n "$namespace"
if [[ -n "$(kubectl_do "${CK8S_CLUSTER}" get "$kind" "$name" -n "$namespace" --ignore-not-found=true -oname)" ]]; then
log_info "Will trigger a rollout restart of $kind/$name in $namespace for cluster ${CK8S_CLUSTER}"
kubectl_do "${CK8S_CLUSTER}" rollout restart "$kind" "$name" -n "$namespace"
fi
fi

done
}

cluster="${1}"

if [[ $cluster != "wc" && $cluster != "sc" ]]; then
usage
fi
set_violating_resources "$cluster"
restart_violating_resources "$cluster"
set_violating_resources
restart_violating_resources
Loading