diff --git a/.github/actions/install-azure-cli/action.yml b/.github/actions/install-azure-cli/action.yml new file mode 100644 index 000000000..20e2709eb --- /dev/null +++ b/.github/actions/install-azure-cli/action.yml @@ -0,0 +1,14 @@ +name: Install azure-cli +description: Installs the azure cli. +runs: + using: composite + steps: + - name: install azure-cli + run: | + sudo apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg + curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null + AZ_REPO=$(lsb_release -cs) + echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + sudo apt-get update + sudo apt-get install -y azure-cli + shell: bash diff --git a/.github/workflows/aro-hcp-dev-what-if.yml b/.github/workflows/aro-hcp-dev-what-if.yml new file mode 100644 index 000000000..703d114b7 --- /dev/null +++ b/.github/workflows/aro-hcp-dev-what-if.yml @@ -0,0 +1,39 @@ +--- +name: ARO HCP Integrated DEV Bicep What-If +env: + DEPLOY_ENV: dev +on: + pull_request: + branches: + - main + paths: + - 'dev-infrastructure/**/*.bicep' + - 'dev-infrastructure/**/*.bicepparam' + - 'dev-infrastructure/Makefile' + - '.github/workflows/aro-hcp-dev-bicep-what-if.yml' + +jobs: + what-if: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + fetch-depth: 1 + + - name: 'Az CLI login' + uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Install Azure CLI + uses: ./.github/actions/install-azure-cli + + - name: 'Deployment What If' + run: | + cd dev-infrastructure/ + make what-if diff --git a/.github/workflows/bicep-what-if.yml b/.github/workflows/bicep-what-if.yml deleted file mode 100644 index bc4c5f9c8..000000000 --- a/.github/workflows/bicep-what-if.yml +++ /dev/null @@ -1,114 +0,0 @@ ---- -name: Bicep What If -env: - REGION: westus3 - REGIONAL_RESOURCEGROUP: aro-hcp-dev-westus3 - SC_RESOURCEGROUP: aro-hcp-dev-westus3-sc - MC_RESOURCEGROUP: aro-hcp-dev-westus3-mc-1 - IMAGE_SYNC_RESOURCEGROUP: aro-hcp-dev-image-sync - GLOBAL_RESOURCEGROUP: global - ARO_HCP_IMAGE_ACR: arohcpdev -on: - pull_request: - branches: - - main - paths: - - 'dev-infrastructure/**/*.bicep' - - 'dev-infrastructure/**/*.bicepparam' - - '.github/workflows/bicep-what-if.yml' - - '.github/workflows/aro-hcp-dev-env-cd.yml' - -jobs: - what-if: - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - # TODO: We don't have `make` in the azure/cli task so a lot of this is duplicated in dev-infrastructure/Makefile - # we should run our own container or find a solution to bring them closer - - name: 'Deployment What If' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${SC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${MC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${REGIONAL_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${IMAGE_SYNC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # Image Sync - az deployment group what-if \ - --name "image-sync" \ - --resource-group ${IMAGE_SYNC_RESOURCEGROUP} \ - --template-file templates/image-sync.bicep \ - --parameters \ - configurations/mvp-image-sync.bicepparam - - # ACR - az deployment group what-if \ - --name "dev-acr-${GITHUB_RUN_ID}" \ - --resource-group "${GLOBAL_RESOURCEGROUP}" \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-acr.bicepparam - - # OCP ACR - az deployment group what-if \ - --name "dev-ocp-acr-${GITHUB_RUN_ID}" \ - --resource-group ${GLOBAL_RESOURCEGROUP} \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-ocp-acr.bicepparam - - # SVC ACR - az deployment group what-if \ - --name "dev-svc-acr-${GITHUB_RUN_ID}" \ - --resource-group ${GLOBAL_RESOURCEGROUP} \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-svc-acr.bicepparam - - # region infra - az deployment group what-if \ - --name "region-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file templates/region.bicep \ - --parameters configurations/mvp-region.bicepparam \ - --parameters regionalDNSSubdomain="${REGION}" - - # service cluster - az deployment group what-if \ - --name "svc-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/svc-cluster.bicep \ - --parameters configurations/mvp-svc-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - # management cluster - az deployment group what-if \ - --name "mgmt-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${MC_RESOURCEGROUP}" \ - --template-file templates/mgmt-cluster.bicep \ - --parameters configurations/mvp-mgmt-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - # metrics infrastructure - az deployment group what-if \ - --name "metrics-infra-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file modules/metrics/metrics.bicep \ - --parameters configurations/mvp-metrics.bicepparam \ - --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}" diff --git a/.github/workflows/cs-integ-bicep-what-if.yml b/.github/workflows/cs-integ-bicep-what-if.yml deleted file mode 100644 index 9ec5c6d0c..000000000 --- a/.github/workflows/cs-integ-bicep-what-if.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: CS Integration Bicep What If -env: - REGION: westus3 - REGIONAL_RESOURCEGROUP: cs-integ-westus3 - SC_RESOURCEGROUP: cs-integ-westus3-sc - MC_RESOURCEGROUP: cs-integ-westus3-mc-1 - ARO_HCP_IMAGE_ACR: arohcpdev -on: - pull_request: - branches: - - main - paths: - - 'dev-infrastructure/**/*.bicep' - - 'dev-infrastructure/**/*.bicepparam' - - '.github/workflows/cs-integ-bicep-what-if.yml' - - '.github/workflows/cs-integration-env-cd.yml' - -jobs: - what-if: - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - # TODO: We don't have `make` in the azure/cli task so a lot of this is duplicated in dev-infrastructure/Makefile - # we should run our own container or find a solution to bring them closer - - name: 'Deployment What If' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${SC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${MC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${REGIONAL_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # region infra - az deployment group what-if \ - --name "region-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file templates/region.bicep \ - --parameters configurations/cs-integ-region.bicepparam \ - --parameters regionalDNSSubdomain="${REGION}" - - # service cluster - az deployment group what-if \ - --name "svc-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/svc-cluster.bicep \ - --parameters configurations/cs-integ-svc-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - # management cluster - az deployment group what-if \ - --name "mgmt-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${MC_RESOURCEGROUP}" \ - --template-file templates/mgmt-cluster.bicep \ - --parameters configurations/cs-integ-mgmt-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - # metrics infrastructure - az deployment group what-if \ - --name "metrics-infra-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file modules/metrics/metrics.bicep \ - --parameters configurations/cs-integ-metrics.bicepparam \ - --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}" diff --git a/.github/workflows/cs-pr-what-if.yml b/.github/workflows/cs-pr-what-if.yml new file mode 100644 index 000000000..0c073d04c --- /dev/null +++ b/.github/workflows/cs-pr-what-if.yml @@ -0,0 +1,39 @@ +--- +name: CS PR Check Environment Bicep What-If +env: + DEPLOY_ENV: cs-pr +on: + pull_request: + branches: + - main + paths: + - 'dev-infrastructure/**/*.bicep' + - 'dev-infrastructure/**/*.bicepparam' + - 'dev-infrastructure/Makefile' + - '.github/workflows/cs-pr-bicep-what-if.yml' + +jobs: + what-if: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + fetch-depth: 1 + + - name: 'Az CLI login' + uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Install Azure CLI + uses: ./.github/actions/install-azure-cli + + - name: 'Deployment What If' + run: | + cd dev-infrastructure/ + make what-if diff --git a/.github/workflows/yaml-lint.yml b/.github/workflows/yaml-lint.yml index c8a75d1fd..6fd927b88 100644 --- a/.github/workflows/yaml-lint.yml +++ b/.github/workflows/yaml-lint.yml @@ -22,4 +22,4 @@ jobs: - name: Linters uses: oxsecurity/megalinter/flavors/ci_light@v8 env: - FILTER_REGEX_EXCLUDE: 'hypershiftoperator/deploy/base/|hypershiftoperator/deploy/crds/|maestro/deploy/helm/|acm/deploy/helm/multicluster-engine-config/charts/policy/charts' + FILTER_REGEX_EXCLUDE: 'hypershiftoperator/deploy/base/|hypershiftoperator/deploy/crds/|maestro/server/helm/templates/allow-cluster-service.authorizationpolicy.yaml|acm/deploy/helm/multicluster-engine-config/charts/policy/charts' diff --git a/.yamllint.yml b/.yamllint.yml index e02c0be71..22af28228 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -5,7 +5,6 @@ yaml-files: - '.yamllint' rules: - braces: enable brackets: enable colons: enable commas: enable diff --git a/Makefile b/Makefile index cc50cf4c6..e09b7c40b 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ SHELL = /bin/bash # https://github.com/containers/image?tab=readme-ov-file#building GOTAGS?='containers_image_openpgp' TOOLS_BIN_DIR := tooling/bin +DEPLOY_ENV ?= personal-dev .DEFAULT_GOAL := all @@ -24,4 +25,96 @@ lint: $(GOLANGCI_LINT) fmt: $(GOIMPORTS) $(GOIMPORTS) -w -local github.com/Azure/ARO-HCP $(shell go list -f '{{.Dir}}' -m | xargs) -.PHONY: all clean lint test fmt +# +# Infra +# + +infra.svc: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make svc.init + +infra.mgmt: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make mgmt.init + +infra.imagesync: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make imagesync + +infra.all: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make infra + +infra.svc.clean: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) svc.clean + +infra.mgmt.clean: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) mgmt.clean + +infra.region.clean: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) region.clean + +infra.imagesync.clean: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) imagesync.clean + +infra.clean: + cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) clean + +# +# Cluster Service +# + +cs.deploy: + ./svc-deploy.sh $(DEPLOY_ENV) cluster-service svc + +# +# Maestro +# + +maestro.server.deploy: + ./svc-deploy.sh $(DEPLOY_ENV) maestro/server svc + +maestro.agent.deploy: + ./svc-deploy.sh $(DEPLOY_ENV) maestro/agent mgmt + +maestro.registration.deploy: + ./svc-deploy.sh $(DEPLOY_ENV) maestro/registration mgmt + +maestro: maestro.server.deploy maestro.agent.deploy maestro.registration.deploy + +# +# Resource Provider +# + +rp.frontend.deploy: + ./svc-deploy.sh $(DEPLOY_ENV) frontend svc + +rp.backend.deploy: + ./svc-deploy.sh $(DEPLOY_ENV) backend svc + +# +# ACM +# + +acm.deploy: + ./svc-deploy.sh $(DEPLOY_ENV) acm svc + +# +# Hypershift +# + +hypershift.deploy: + ./svc-deploy.sh $(DEPLOY_ENV) hypershift svc + +# +# Deploy ALL components +# + +deploy.svc.all: maestro.server.deploy maestro.registration.deploy cs.deploy rp.frontend.deploy rp.backend.deploy + +deploy.mgmt.all: maestro.agent.deploy acm.deploy hypershift.deploy + +deploy.all: deploy.svc.all deploy.mgmt.all + +list: + @grep '^[^#[:space:]].*:' Makefile +.PHONY: list + + +.PHONY: all clean lint test fmt deploy.all deploy.svc.all deploy.mgmt.all acm.deploy hypershift.deploy rp.frontend.deploy rp.backend.deploy maestro.server.deploy maestro.agent.deploy maestro.registration.deploy maestro infra.svc infra.mgmt infra.imagesync infra.all infra.clean infra.region.clean infra.svc.clean infra.mgmt.clean diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 000000000..aee2e4ce1 --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1 @@ +config.mk diff --git a/backend/Makefile b/backend/Makefile index a5d7f1a57..ed4ab34d2 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -1,19 +1,11 @@ SHELL = /bin/bash - -# for deploying backend into private aks cluster via invoke command -# these values must be set -AKSCONFIG ?= svc-cluster -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk +DEPLOY_ENV ?= personal-dev +$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk COMMIT = $(shell git rev-parse --short=7 HEAD) ARO_HCP_BASE_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io ARO_HCP_BACKEND_IMAGE ?= $(ARO_HCP_BASE_IMAGE)/arohcpbackend:$(COMMIT) -CLUSTER_NAME ?= -DEPLOYMENTNAME=$(RESOURCEGROUP) - -# dev-infrastructure defines this as REGION -LOCATION ?= ${REGION} backend: go build -o aro-hcp-backend . @@ -38,14 +30,13 @@ deploy: -g ${RESOURCEGROUP} \ -n backend \ --query clientId);\ - DB_NAME=$(shell az cosmosdb list -g ${RESOURCEGROUP} | jq -r '.[].name') DB_NAME=$${DB_NAME:-"none"};\ - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ + ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ kubectl create namespace aro-hcp --dry-run=client -o json | kubectl apply -f - && \ kubectl label namespace aro-hcp "istio.io/rev=$${ISTO_VERSION}" --overwrite=true && \ helm upgrade --install aro-hcp-backend-dev \ deploy/helm/backend/ \ - --set configMap.databaseName=$${DB_NAME} \ - --set configMap.databaseUrl="https://$${DB_NAME}.documents.azure.com:443/" \ + --set configMap.databaseName=${DB_NAME} \ + --set configMap.databaseUrl="https://${DB_NAME}.documents.azure.com:443/" \ --set configMap.backendMiClientId="$${BACKEND_MI_CLIENT_ID}" \ --set serviceAccount.workloadIdentityClientId="$${BACKEND_MI_CLIENT_ID}" \ --set configMap.currentVersion=${ARO_HCP_BACKEND_IMAGE} \ diff --git a/backend/config.tmpl.mk b/backend/config.tmpl.mk new file mode 100644 index 000000000..9b959270f --- /dev/null +++ b/backend/config.tmpl.mk @@ -0,0 +1,5 @@ +ARO_HCP_IMAGE_ACR ?= {{ .acrName }} +LOCATION ?= {{ .region }} +RESOURCEGROUP ?= {{ .serviceClusterRG }} +AKS_NAME ?= {{ .aksName }} +DB_NAME ?= {{ .frontendCosmosDBName }} diff --git a/cluster-service/.gitignore b/cluster-service/.gitignore index c76242eef..1e48811d8 100644 --- a/cluster-service/.gitignore +++ b/cluster-service/.gitignore @@ -1 +1,2 @@ deploy/tmp-provisioning-shard.yml +config.mk diff --git a/cluster-service/Makefile b/cluster-service/Makefile index 2eef61048..44a745d04 100644 --- a/cluster-service/Makefile +++ b/cluster-service/Makefile @@ -1,56 +1,44 @@ SHELL = /bin/bash -AKSCONFIG ?= svc-cluster - -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk - -CONSUMER_NAME ?= $(shell az aks list --query "[?tags.clusterType == 'mgmt-cluster' && starts_with(resourceGroup, '$(REGIONAL_RESOURCEGROUP)')].resourceGroup" -o tsv) -KEYVAULT_NAME ?= $(shell az keyvault list --query "[?tags.aroHCPPurpose=='service'].name" -g ${SVC_KV_RESOURCEGROUP} --output tsv) -FPA_CERT_NAME ?= firstPartyCert -AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= "57e54810-3138-4f38-bd3b-29cb33f4c358" +DEPLOY_ENV ?= personal-dev +$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk deploy: - ZONE_RESOURCE_ID=$(shell az network dns zone list -g ${REGIONAL_RESOURCEGROUP} --query "[?zoneType=='Public'].id" -o tsv) && \ - sed -e "s#ZONE_RESOURCE_ID#$${ZONE_RESOURCE_ID}#g" -e "s/REGION/${REGION}/g" -e "s/CONSUMER_NAME/${CONSUMER_NAME}/g" deploy/mvp-provisioning-shards.yml > deploy/tmp-provisioning-shard.yml - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ + sed -e "s#ZONE_RESOURCE_ID#${ZONE_RESOURCE_ID}#g" -e "s/REGION/${REGION}/g" -e "s/CONSUMER_NAME/${CONSUMER_NAME}/g" deploy/mvp-provisioning-shards.yml > deploy/tmp-provisioning-shard.yml + + ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ oc process --local -f deploy/openshift-templates/arohcp-namespace-template.yml \ -p ISTIO_VERSION=$${ISTO_VERSION} | oc apply -f - kubectl apply -f deploy/istio.yml + oc process --local -f deploy/openshift-templates/arohcp-db-template.yml | oc apply -f - oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \ -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/tmp-provisioning-shard.yml)" | oc apply -f - - AZURE_CS_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n clusters-service \ - --query clientId) && \ - AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} && \ - CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "service-kv-aro-hcp-dev" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \ + + AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \ + CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "${SERVICE_KV}" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \ TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account list --query "[?starts_with(name, 'arohcpoidc')].primaryEndpoints.blob" -g ${REGIONAL_RESOURCEGROUP} -o tsv) && \ - OIDC_WEB_ENDPOINT=$(shell az storage account list --query "[?starts_with(name, 'arohcpoidc')].primaryEndpoints.web" -g ${REGIONAL_RESOURCEGROUP} -o tsv) && \ + OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.blob -o tsv) && \ + OIDC_WEB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.web -o tsv) && \ oc process --local -f deploy/openshift-templates/arohcp-service-template.yml \ -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \ -p TENANT_ID=$${TENANT_ID} \ -p REGION=${REGION} \ - -p SERVICE_KEYVAULT_NAME=${KEYVAULT_NAME} \ + -p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \ -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \ - -p IMAGE_REGISTRY=${ARO_HCP_IMAGE_ACR}.azurecr.io \ - -p IMAGE_REPOSITORY=app-sre/uhc-clusters-service \ - -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=$${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ + -p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \ + -p IMAGE_REPOSITORY=${IMAGE_REPO} \ + -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ -p FPA_CERT_NAME=${FPA_CERT_NAME} \ - -p IMAGE_TAG=cf23767 | oc apply -f - + -p IMAGE_TAG=${IMAGE_TAG} | oc apply -f - -deploy-integ: - AZURE_CS_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n clusters-service \ - --query clientId) && \ +deploy-pr-env-deps: + AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \ oc process --local -f deploy/integration/cluster-service-namespace.yaml \ - -p CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} | oc apply -f - + -p CLIENT_ID=${AZURE_CS_MI_CLIENT_ID} | oc apply -f - # for local development provision-shard: - @ZONE_RESOURCE_ID=$(shell az network dns zone list -g ${REGIONAL_RESOURCEGROUP} --query "[?zoneType=='Public'].id" -o tsv) && \ - sed -e "s#ZONE_RESOURCE_ID#$${ZONE_RESOURCE_ID}#g" -e "s/REGION/${REGION}/g" -e "s/CONSUMER_NAME/${CONSUMER_NAME}/g" deploy/dev-provisioning-shards.yml + sed -e "s#ZONE_RESOURCE_ID#${ZONE_RESOURCE_ID}#g" -e "s/REGION/${REGION}/g" -e "s/CONSUMER_NAME/${CONSUMER_NAME}/g" deploy/dev-provisioning-shards.yml .PHONY: deploy deploy-integ provision-shard diff --git a/cluster-service/config.tmpl.mk b/cluster-service/config.tmpl.mk new file mode 100644 index 000000000..e73cc967b --- /dev/null +++ b/cluster-service/config.tmpl.mk @@ -0,0 +1,13 @@ +REGION ?= {{ .region }} +CONSUMER_NAME ?= {{ .maestroConsumerName }} +RESOURCEGROUP ?= {{ .serviceClusterRG }} +REGIONAL_RESOURCEGROUP ?= {{ .regionRG }} +AKS_NAME ?= {{ .aksName }} +SERVICE_KV ?= {{ .serviceKeyVaultName }} +OIDC_STORAGE_ACCOUNT ?= {{ .oidcStorageAccountName }} +IMAGE_REPO ?= {{ .clusterServiceImageRepo }} +IMAGE_TAG ?= {{ .clusterServiceImageTag }} +ACR_NAME ?= {{ .acrName }} +AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= {{ .firstPartyAppClientId }} +FPA_CERT_NAME ?= firstPartyCert +ZONE_RESOURCE_ID ?= $(shell az network dns zone show -n {{ .regionalDNSSubdomain }}.{{ .baseDnsZoneName }} -g {{ .regionRG }} --query id -o tsv) diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 000000000..34d197cac --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,151 @@ +defaults: + region: {{ .ctx.region }} + # Resourcegroups + globalRG: global + regionRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp}} + serviceClusterRG: hcp-underlay-{{ .ctx.region }}-svc-{{ .ctx.regionStamp }} + managementClusterRG: hcp-underlay-{{ .ctx.region }}-mgmt-{{ .ctx.regionStamp }}-{{ .ctx.cxStamp }} + imageSyncRG: hcp-underlay-{{ .ctx.region }}-imagesync-{{ .ctx.regionStamp }} + + # General AKS config + kubernetesVersion: 1.30.5 + istioVersion: "['asm-1-22']" + vnetAddressPrefix: "10.128.0.0/14" + persist: true + subnetPrefix: "10.128.8.0/21" + podSubnetPrefix: "10.128.64.0/18" + aksName: aro-hcp-aks + + # ACR + serviceComponentAcrResourceGroups: global + + # SVC cluster specifics + svcEtcdKVName: {{ azureKeyVaultName "aro-hcp-etcd" 5 .ctx.region .ctx.regionStamp }} + svcEtcdKVSoftDelete: true + + # MGMT cluster specifics + mgmtEtcdKVName: {{ azureKeyVaultName "aro-hcp-etcd" 5 .ctx.region .ctx.regionStamp .ctx.cxStamp }} + mgmtEtcdKVSoftDelete: true + + # Frontend + frontendCosmosDBDeploy: true + frontendCosmosDBDisableLocalAuth: true + frontendCosmosDBName: {{ azureCosmosDBName "aro-hcp-rp" 5 .ctx.region .ctx.regionStamp }} + + # Maestro + maestroKeyVaultName: {{ azureKeyVaultName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroEventgridName: {{ azureEventGridName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroEventGridMaxClientSessionsPerAuthName: '4' + maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com' + maestroPostgresName: {{ azurePostgresName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroPostgresServerVersion: '15' + maestroPostgresServerStorageSizeGB: '32' + maestroPostgresDeploy: true + maestroPostgresPrivate: true + maestroRestrictIstioIngress: true + maestroConsumerName: hcp-underlay-{{ .ctx.region }}-mgmt-{{ .ctx.regionStamp }}-{{ .ctx.cxStamp }} + + # Cluster Service + clusterServicePostgresName: {{ azurePostgresName "cs" 5 .ctx.region .ctx.regionStamp }} + clusterServicePostgresDeploy: true + clusterServicePostgresPrivate: true + clusterServiceAcrRG: global + + # Image Sync + imageSyncAcrRG: global + imageSyncEnvironmentName: aro-hcp-image-sync + imageSyncRepositories: '{registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/pstefans/controlplaneoperator,quay.io/app-sre/uhc-clusters-service}' + + # Service KeyVault + serviceKeyVaultName: {{ azureKeyVaultName "svc-kv" 5 .ctx.region .ctx.regionStamp }} + serviceKeyVaultRG: hcp-underlay-{{ .ctx.region }}-svc-{{ .ctx.regionStamp }} + serviceKeyVaultRegion: {{ .ctx.region }} + serviceKeyVaultSoftDelete: true + serviceKeyVaultPrivate: true + + # DNS + baseDnsZoneRG: 'global' +clouds: + fairfax: + defaults: + public: + defaults: + environments: + rh-dev-tmpl: + # this configuration serves as a template for for all RH DEV subscription deployments + # the following vars need approprivate overrides: + # - regionalDNSSubdomain + default: &rh-dev-tmpl + # DNS + baseDnsZoneName: 'hcp.osadev.cloud' + # MGMTM AKS nodepools - big enough for 2 HCPs + mgmtSystemAgentPoolMinCount: 1 + mgmtSystemAgentPoolMaxCount: 4 + mgmtSystemAgentPoolVmSize: 'Standard_D2s_v3' + mgmtSystemAgentPoolOsDiskSizeGB: 32 + mgmtUserAgentPoolMinCount: 1 + mgmtUserAgentPoolMaxCount: 6 + mgmtUserAgentPoolVmSize: 'Standard_D4s_v3' + mgmtUserAgentPoolOsDiskSizeGB: 100 + mgmtUserAgentPoolAzCount: 3 + # 1P app + firstPartyAppClientId: 57e54810-3138-4f38-bd3b-29cb33f4c358 + # Maestro + maestroPostgresDeploy: false + maestroImageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro + maestroImageTag: ea066c250a002f0cc458711945165591bc9f6d3f + # Cluster Service + clusterServicePostgresDeploy: false + clusterServiceImageTag: cf23767 + clusterServiceImageRepo: app-sre/uhc-clusters-service + # Shared SVC KV + serviceKeyVaultName: 'aro-hcp-dev-svc-kv' + serviceKeyVaultRG: 'global' + serviceKeyVaultRegion: 'westus3' + serviceKeyVaultPrivate: false + # disable soft delete on etcd KVs in DEV + svcEtcdKVSoftDelete: false + mgmtEtcdKVSoftDelete: false + # Shared ACRs + acrName: arohcpdev + svcAcrName: arohcpsvcdev + ocpAcrName: arohcpocpdev + # Shared Image Sync + imageSyncRG: hcp-underlay-westus3-imagesync-dev + # OIDC + oidcStorageAccountName: {{ azureStorageAccountName "arohcpoidc" 5 .ctx.region .ctx.regionStamp }} + # Metrics + monitoringWorkspaceName: 'aro-hcp-monitor-{{ uniqueString 5 .ctx.region .ctx.regionStamp}}' + grafanaName: 'aro-hcp-grafana-{{ uniqueString 5 .ctx.region .ctx.regionStamp}}' + monitoringMsiName: 'aro-hcp-metrics-msi-{{ uniqueString 5 .ctx.region .ctx.regionStamp }}' + grafanaAdminGroupPrincipalId: 6b6d3adf-8476-4727-9812-20ffdef2b85c + # DEVOPS MSI + aroDevopsMsiId: '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' + dev: + # this is the integrated DEV environment + defaults: + <<: *rh-dev-tmpl + # MGMTM AKS nodepools - big enough for multiple HCPs + mgmtUserAgentPoolMinCount: 2 + mgmtUserAgentPoolMaxCount: 12 + # DNS + regionalDNSSubdomain: '{{ .ctx.region }}' + cs-pr: + # this is the cluster service PR check and full cycle test environment + defaults: + <<: *rh-dev-tmpl + # MGMTM AKS nodepools - big enough for multiple HCPs + mgmtUserAgentPoolMinCount: 2 + mgmtUserAgentPoolMaxCount: 12 + # DNS + regionalDNSSubdomain: '{{ .ctx.region }}-cs' + # Maestro + maestroRestrictIstioIngress: false + personal-dev: + # this is the personal DEV environment + defaults: + <<: *rh-dev-tmpl + # saving money! + persist: false + # DNS + regionalDNSSubdomain: '{{ .ctx.region }}-{{ uniqueString 5 .ctx.region .ctx.regionStamp }}' diff --git a/dev-infrastructure/.gitignore b/dev-infrastructure/.gitignore new file mode 100644 index 000000000..36806b6a7 --- /dev/null +++ b/dev-infrastructure/.gitignore @@ -0,0 +1,9 @@ +configurations/mgmt-cluster.bicepparam +configurations/svc-cluster.bicepparam +configurations/region.bicepparam +configurations/metrics.bicepparam +configurations/acr-svc.bicepparam +configurations/acr-ocp.bicepparam +configurations/acr.bicepparam +configurations/image-sync.bicepparam +config.mk diff --git a/dev-infrastructure/Makefile b/dev-infrastructure/Makefile index 56e199424..28e78cffd 100644 --- a/dev-infrastructure/Makefile +++ b/dev-infrastructure/Makefile @@ -1,17 +1,21 @@ SHELL = /bin/bash - -CONFIG_PROFILE ?= dev -include configurations/$(CONFIG_PROFILE).mk - -CURRENTUSER = $(shell az ad signed-in-user show | jq -r '.id') -DEPLOYMENTNAME = $(RESOURCEGROUP) -DEPLOYMENTNAMEREGION = $(REGIONAL_RESOURCEGROUP)-REGION +DEPLOY_ENV ?= personal-dev +$(shell ./create-config.sh $(DEPLOY_ENV)) +include config.mk # Set SKIP_CONFIRM to a non-empty value to skip "what-if" confirmation prompts. ifndef SKIP_CONFIRM PROMPT_TO_CONFIRM = "--confirm-with-what-if" endif +MGMT_KUBECONFIG_FILE ?= ${HOME}/.kube/${MGMT_RESOURCEGROUP}.kubeconfig +SVC_KUBECONFIG_FILE ?= ${HOME}/.kube/${SVC_RESOURCEGROUP}.kubeconfig +MGMG_RG_DEPLOYMENT_NAME ?= "mgmt" +SVC_RG_DEPLOYMENT_NAME ?= "svc" +IMAGE_SYNC_RG_DEPLOYMENT_NAME ?= "image-sync" +REGIONAL_RG_DEPLOYMENT_NAME ?= "region" +METRICS_INFRA_RG_DEPLOYMENT_NAME ?= "metrics-infra" + list: @grep '^[^#[:space:]].*:' Makefile .PHONY: list @@ -42,41 +46,16 @@ feature-registration: # hardcoded to eastus as this is a subscription deployment $(PROMPT_TO_CONFIRM) .PHONY: feature-registration -rg: - @if [ "$$(az group exists --name $(RESOURCEGROUP))" = "false" ]; then \ - az group create \ - --name $(RESOURCEGROUP) \ - --location $(REGION) \ - --output none; \ - fi -.PHONY: rg - -regionalRg: - @if [ "$$(az group exists --name $(REGIONAL_RESOURCEGROUP))" = "false" ]; then \ - az group create \ - --name $(REGIONAL_RESOURCEGROUP) \ - --location $(REGION) \ - --output none; \ - fi -.PHONY: regionalRg - - -imagesyncRg: - @if [ "$$(az group exists --name $(IMAGE_SYNC_RESOURCEGROUP))" = "false" ]; then \ - az group create \ - --name $(IMAGE_SYNC_RESOURCEGROUP) \ - --location $(REGION) \ - --output none; \ - fi -.PHONY: imagesyncRg - cleanup-orphaned-rolebindings: - @scripts/cleanup-orphaned-rolebindings.sh $(RESOURCEGROUP) + @scripts/cleanup-orphaned-rolebindings.sh $(SVC_RESOURCEGROUP) + @scripts/cleanup-orphaned-rolebindings.sh $(MGMT_RESOURCEGROUP) @scripts/cleanup-orphaned-rolebindings.sh $(REGIONAL_RESOURCEGROUP) @scripts/cleanup-orphaned-rolebindings.sh ${GLOBAL_RESOURCEGROUP} + @scripts/cleanup-orphaned-rolebindings.sh ${IMAGE_SYNC_RESOURCEGROUP} .PHONY: cleanup-orphaned-rolebindings create-mock-identities: + az deployment group wait --created --name "aro-hcp-dev-mock-certificates" --resource-group $(GLOBAL_RESOURCEGROUP) --interval 10 az deployment group create \ --name "aro-hcp-dev-mock-certificates" \ --resource-group $(GLOBAL_RESOURCEGROUP) \ @@ -108,123 +87,346 @@ create-mock-identities: ./scripts/create-sp-for-rbac.sh .PHONY: create-mock-identities -image-sync: imagesyncRg +# +# Image Sync +# + +imagesync.rg: + @if [ "$$(az group exists --name $(IMAGE_SYNC_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(IMAGE_SYNC_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: imagesyncRg + +imagesync.wait: + @./ensure-no-running-deployment.sh $(IMAGE_SYNC_RESOURCEGROUP) $(IMAGE_SYNC_RG_DEPLOYMENT_NAME) +.PHONY: imagesync.wait + +imagesync: imagesync.rg imagesync.wait az deployment group create \ - --name "image-sync" \ + --name $(IMAGE_SYNC_RG_DEPLOYMENT_NAME) \ --resource-group $(IMAGE_SYNC_RESOURCEGROUP) \ --template-file templates/image-sync.bicep \ $(PROMPT_TO_CONFIRM) \ --parameters \ - configurations/mvp-image-sync.bicepparam -.PHONY: image-sync + configurations/image-sync.bicepparam +.PHONY: imagesync + +imagesync.what-if: imagesync.rg + az deployment group what-if \ + --name $(IMAGE_SYNC_RG_DEPLOYMENT_NAME) \ + --resource-group $(IMAGE_SYNC_RESOURCEGROUP) \ + --template-file templates/image-sync.bicep \ + --parameters \ + configurations/image-sync.bicepparam +.PHONY: imagesync.what-if + +imagesync.clean: + @if [ "$$(az group exists --name $(IMAGE_SYNC_RESOURCEGROUP))" = "true" ]; then \ + az group delete -g $(IMAGE_SYNC_RESOURCEGROUP); \ + fi +.PHONY: imagesync.clean + +# +# Region +# + +regional.rg: + @if [ "$$(az group exists --name $(REGIONAL_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(REGIONAL_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: regional.rg + +region.wait: + @./ensure-no-running-deployment.sh $(REGIONAL_RESOURCEGROUP) $(REGIONAL_RG_DEPLOYMENT_NAME) +.PHONY: region.wait -region: regionalRg +region: regional.rg az deployment group create \ - --name "$(DEPLOYMENTNAMEREGION)" \ + --name $(REGIONAL_RG_DEPLOYMENT_NAME) \ --resource-group $(REGIONAL_RESOURCEGROUP) \ --template-file templates/region.bicep \ $(PROMPT_TO_CONFIRM) \ --parameters \ - configurations/region.bicepparam \ - --parameters \ - currentUserId=$(CURRENTUSER) + configurations/region.bicepparam .PHONY: region -cluster: rg cleanup-orphaned-rolebindings region -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif +region.clean: + @if [ "$$(az group exists --name $(REGIONAL_RESOURCEGROUP))" = "true" ]; then \ + az group delete -g $(REGIONAL_RESOURCEGROUP); \ + fi +.PHONY: region.clean + +region.what-if: regional.rg + az deployment group what-if \ + --name "region" \ + --resource-group $(REGIONAL_RESOURCEGROUP) \ + --template-file templates/region.bicep \ + --parameters \ + configurations/region.bicepparam +.PHONY: region.what-if + +# +# Service Cluster +# + +svc.rg: + @if [ "$$(az group exists --name $(SVC_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(SVC_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: svc.rg + +svc.wait: + @./ensure-no-running-deployment.sh $(SVC_RESOURCEGROUP) $(SVC_RG_DEPLOYMENT_NAME) +.PHONY: svc.wait + +svc: svc.wait svc.rg cleanup-orphaned-rolebindings region az deployment group create \ - --name "$(DEPLOYMENTNAME)" \ - --resource-group $(RESOURCEGROUP) \ - --template-file templates/$(AKSCONFIG).bicep \ + --name "svc" \ + --resource-group $(SVC_RESOURCEGROUP) \ + --template-file templates/svc-cluster.bicep \ $(PROMPT_TO_CONFIRM) \ --parameters \ - configurations/$(AKSCONFIG).bicepparam \ + configurations/svc-cluster.bicepparam +.PHONY: svc + +svc.enable-aks-metrics: metrics-infra + @if [ "$$(az aks show --resource-group $(SVC_RESOURCEGROUP) --name ${AKS_NAME} --query 'azureMonitorProfile.metrics.enabled' -o tsv)" = "true" ]; then \ + echo "Azure Monitor metrics are already enabled."; \ + else \ + az aks update --enable-azure-monitor-metrics \ + --resource-group $(SVC_RESOURCEGROUP) \ + --name ${AKS_NAME} \ + --azure-monitor-workspace-resource-id $$(az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.monitorId.value) \ + --grafana-resource-id $$(az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.grafanaId.value); \ + fi +.PHONY: svc.enable-aks-metrics + +svc.aks.admin-access: + @scripts/aks-admin-access.sh $(SVC_RESOURCEGROUP) +.PHONY: svc.aks.admin-access + +svc.aks.kubeconfig: + @az aks get-credentials --overwrite-existing --only-show-errors -n ${AKS_NAME} -g $(SVC_RESOURCEGROUP) -f "${SVC_KUBECONFIG_FILE}" + @kubelogin convert-kubeconfig -l azurecli --kubeconfig "${SVC_KUBECONFIG_FILE}" +.PHONY: svc.aks.kubeconfig + +svc.aks.kubeconfigfile: + @echo ${SVC_KUBECONFIG_FILE} +.PHONY: svc.aks.kubeconfigfile + +svc.init: svc svc.aks.admin-access svc.aks.kubeconfig svc.enable-aks-metrics +.PHONY: svc.init + +svc.what-if: svc.rg + az deployment group what-if \ + --name "svc" \ + --resource-group $(SVC_RESOURCEGROUP) \ + --template-file templates/svc-cluster.bicep \ --parameters \ - currentUserId=$(CURRENTUSER) \ - regionalResourceGroup=$(REGIONAL_RESOURCEGROUP) -.PHONY: cluster + configurations/svc-cluster.bicepparam +.PHONY: svc.what-if -cluster-what-if: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif +svc.clean: + @if [ "$$(az group exists --name $(SVC_RESOURCEGROUP))" = "true" ]; then \ + az group delete -g $(SVC_RESOURCEGROUP); \ + fi +.PHONY: svc.clean + +# +# Management Cluster +# + +mgmt.rg: + @if [ "$$(az group exists --name $(MGMT_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(MGMT_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: mgmt.rg + +mgmt.wait: + @./ensure-no-running-deployment.sh $(MGMT_RESOURCEGROUP) $(MGMG_RG_DEPLOYMENT_NAME) +.PHONY: mgmt.wait + +mgmt: mgmt.wait mgmt.rg cleanup-orphaned-rolebindings region + az deployment group create \ + --name $(MGMG_RG_DEPLOYMENT_NAME) \ + --resource-group $(MGMT_RESOURCEGROUP) \ + --template-file templates/mgmt-cluster.bicep \ + $(PROMPT_TO_CONFIRM) \ + --parameters \ + configurations/mgmt-cluster.bicepparam +.PHONY: mgmt + +mgmt.enable-aks-metrics: metrics-infra + @if [ "$$(az aks show --resource-group $(MGMT_RESOURCEGROUP) --name ${AKS_NAME} --query 'azureMonitorProfile.metrics.enabled' -o tsv)" = "true" ]; then \ + echo "Azure Monitor metrics are already enabled."; \ + else \ + az aks update --enable-azure-monitor-metrics \ + --resource-group $(MGMT_RESOURCEGROUP) \ + --name ${AKS_NAME} \ + --azure-monitor-workspace-resource-id $$(az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.monitorId.value) \ + --grafana-resource-id $$(az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.grafanaId.value); \ + fi +.PHONY: mgmt.enable-aks-metrics + +mgmt.aks.admin-access: + @scripts/aks-admin-access.sh $(MGMT_RESOURCEGROUP) +.PHONY: mgmt.aks.admin-access + +mgmt.aks.kubeconfig: mgmt.aks.kubeconfigfile + @az aks get-credentials --overwrite-existing --only-show-errors -n ${AKS_NAME} -g $(MGMT_RESOURCEGROUP) -f "${MGMT_KUBECONFIG_FILE}" + @kubelogin convert-kubeconfig -l azurecli --kubeconfig "${MGMT_KUBECONFIG_FILE}" +.PHONY: mgmt.aks.kubeconfig + +mgmt.aks.kubeconfigfile: + @echo ${MGMT_KUBECONFIG_FILE} +.PHONY: mgmt.aks.kubeconfigfile + +mgmt.init: mgmt mgmt.aks.admin-access mgmt.aks.kubeconfig mgmt.enable-aks-metrics +.PHONY: mgmt.init + +mgmt.what-if: mgmt.rg az deployment group what-if \ - --name "$(DEPLOYMENTNAME)" \ - --resource-group $(RESOURCEGROUP) \ - --template-file templates/$(AKSCONFIG).bicep \ + --name "mgmt" \ + --resource-group $(MGMT_RESOURCEGROUP) \ + --template-file templates/mgmt-cluster.bicep \ --parameters \ - configurations/$(AKSCONFIG).bicepparam \ + configurations/mgmt-cluster.bicepparam +.PHONY: mgmt.what-if + +mgmt.clean: + @if [ "$$(az group exists --name $(MGMT_RESOURCEGROUP))" = "true" ]; then \ + az group delete -g $(MGMT_RESOURCEGROUP); \ + fi +.PHONY: mgmt.clean + +# ACR + +global.rg: + @if [ "$$(az group exists --name $(GLOBAL_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(GLOBAL_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: global.rg + +acr: global.rg + @./ensure-no-running-deployment.sh $(GLOBAL_RESOURCEGROUP) acr + az deployment group create \ + --name "acr" \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + $(PROMPT_TO_CONFIRM) \ --parameters \ - currentUserId=$(CURRENTUSER) \ -.PHONY: cluster-what-if + configurations/acr.bicepparam +.PHONY: acr -aks.admin-access: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/aks-admin-access.sh $(RESOURCEGROUP) -.PHONY: aks.admin-access +acr.what-if: global.rg + az deployment group what-if \ + --name "acr" \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + --parameters \ + configurations/acr.bicepparam +.PHONY: acr.what-if -cs-current-user-pg-connect: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/cs-current-user-pg-connect.sh $(RESOURCEGROUP) "cs-pg-" +acr-svc: global.rg + @./ensure-no-running-deployment.sh $(GLOBAL_RESOURCEGROUP) acr-svc + az deployment group create \ + --name "acr-svc" \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + $(PROMPT_TO_CONFIRM) \ + --parameters \ + configurations/acr-svc.bicepparam +.PHONY: acr-svc + +acr-svc.what-if: global.rg + az deployment group what-if \ + --name "acr-svc" \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + --parameters \ + configurations/acr-svc.bicepparam +.PHONY: acr-svc.what-if + +acr-ocp: global.rg + @./ensure-no-running-deployment.sh $(GLOBAL_RESOURCEGROUP) acr-ocp + az deployment group create \ + --name "acr-ocp" \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + $(PROMPT_TO_CONFIRM) \ + --parameters \ + configurations/acr-ocp.bicepparam +.PHONY: acr-ocp + +acr-ocp.what-if: global.rg + az deployment group what-if \ + --name "acr-ocp" \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + --parameters \ + configurations/acr-ocp.bicepparam +.PHONY: acr-ocp.what-if + +# +# Postgres Authentication Helpers +# + +cs-current-user-pg-connect: + @scripts/cs-current-user-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) .PHONY: cs-current-user-pg-connect -cs-miwi-pg-connect: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/cs-miwi-pg-connect.sh $(RESOURCEGROUP) "cs-pg-" "clusters-service" "cluster-service" "clusters-service" +cs-miwi-pg-connect: + @scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) "clusters-service" "cluster-service" "clusters-service" .PHONY: cs-miwi-pg-connect -maestro-current-user-pg-connect: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/cs-current-user-pg-connect.sh $(RESOURCEGROUP) "maestro-pg-" +maestro-current-user-pg-connect: + @scripts/cs-current-user-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) .PHONY: maestro-current-user-pg-connect -maestro-miwi-pg-connect: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/cs-miwi-pg-connect.sh $(RESOURCEGROUP) "maestro-pg-" "maestro-server" "maestro" "maestro" +maestro-miwi-pg-connect: + @scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) "maestro-server" "maestro" "maestro" .PHONY: maestro-miwi-pg-connect -metrics-infra: regionalRg - CURRENTUSER=$(CURRENTUSER) az deployment group create \ - --name "metrics-infra" \ +# +# Metrics +# + +metrics-infra: regional.rg + @./ensure-no-running-deployment.sh $(REGIONAL_RESOURCEGROUP) $(METRICS_INFRA_RG_DEPLOYMENT_NAME) + az deployment group create \ + --name $(METRICS_INFRA_RG_DEPLOYMENT_NAME) \ --resource-group $(REGIONAL_RESOURCEGROUP) \ --template-file modules/metrics/metrics.bicep \ $(PROMPT_TO_CONFIRM) \ - --parameters \ - globalResourceGroup=$(REGIONAL_RESOURCEGROUP) \ - --parameters configurations/dev-metrics.bicepparam -.PHONY: metrics-infra + --parameters configurations/metrics.bicepparam +.PHONY: metrics-infra -enable-aks-metrics: metrics-infra -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - az aks update --enable-azure-monitor-metrics \ - --resource-group $(RESOURCEGROUP) \ - --name aro-hcp-$(AKSCONFIG) \ - --azure-monitor-workspace-resource-id $(shell az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.monitorId.value) \ - --grafana-resource-id $(shell az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.grafanaId.value) -.PHONY: enable-aks-metrics - -aks.kubeconfig: -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - AKS_NAME="$$(az aks list --query "[?tags.clusterType == '$(AKSCONFIG)'].name" -ojson -g $(RESOURCEGROUP) | jq -r '.[0]')"; \ - az aks get-credentials -n "$${AKS_NAME}" -g $(RESOURCEGROUP) -f "${HOME}/.kube/$(AKSCONFIG).kubeconfig" - kubelogin convert-kubeconfig -l azurecli --kubeconfig "${HOME}/.kube/$(AKSCONFIG).kubeconfig" -.PHONY: aks.kubeconfig +metrics-infra.what-if: regional.rg + az deployment group what-if \ + --name $(METRICS_INFRA_RG_DEPLOYMENT_NAME) \ + --resource-group $(REGIONAL_RESOURCEGROUP) \ + --template-file modules/metrics/metrics.bicep \ + --parameters configurations/metrics.bicepparam +.PHONY: metrics-infra.what-if + +# +# Secrets +# secrets-download: @[ "${SECRET_SA_ACCOUNT_NAME}" ] || ( echo ">> SECRET_SA_ACCOUNT_NAME is not set"; exit 1 ) @@ -241,7 +443,15 @@ secrets-yes-i-want-to-update-the-shared-secrets: rm secrets.tar.gz .PHONY: secrets-yes-i-want-to-update-the-shared-secrets -clean: - az group delete -g $(RESOURCEGROUP) - az keyvault purge --name "$$(az keyvault list-deleted --query "[?properties.tags.resourceGroup == '$(RESOURCEGROUP)'].name" -ojson | jq -r '.[0]')" +# +# Common +# + +what-if: acr.what-if acr-svc.what-if acr-ocp.what-if region.what-if svc.what-if mgmt.what-if metrics-infra.what-if imagesync.what-if +.PHONY: what-if + +infra: region svc.init mgmt.init imagesync +.PHONY: infra + +clean: svc.clean mgmt.clean region.clean .PHONY: clean diff --git a/dev-infrastructure/config.tmpl.mk b/dev-infrastructure/config.tmpl.mk new file mode 100644 index 000000000..d31d2effc --- /dev/null +++ b/dev-infrastructure/config.tmpl.mk @@ -0,0 +1,13 @@ +REGION ?= {{ .region }} +SVC_RESOURCEGROUP ?= {{ .serviceClusterRG }} +MGMT_RESOURCEGROUP ?= {{ .managementClusterRG }} +REGIONAL_RESOURCEGROUP ?= {{ .regionRG }} +SVC_KV_RESOURCEGROUP ?= {{ .serviceKeyVaultRG }} +GLOBAL_RESOURCEGROUP ?= {{ .globalRG }} +IMAGE_SYNC_RESOURCEGROUP ?= {{ .imageSyncRG }} +IMAGE_SYNC_ENVIRONMENT ?= {{ .imageSyncEnvironmentName }} +ARO_HCP_IMAGE_ACR ?= {{ .acrName }} +REPOSITORIES_TO_SYNC ?= '{{ .imageSyncRepositories }}' +AKS_NAME ?= {{ .aksName }} +CS_PG_NAME ?= {{ .clusterServicePostgresName }} +MAESTRO_PG_NAME ?= {{ .maestroPostgresName }} diff --git a/dev-infrastructure/configurations/mvp-dev-ocp-acr.bicepparam b/dev-infrastructure/configurations/acr-ocp.tmpl.bicepparam similarity index 80% rename from dev-infrastructure/configurations/mvp-dev-ocp-acr.bicepparam rename to dev-infrastructure/configurations/acr-ocp.tmpl.bicepparam index 4cddbf50d..ecd1e1279 100644 --- a/dev-infrastructure/configurations/mvp-dev-ocp-acr.bicepparam +++ b/dev-infrastructure/configurations/acr-ocp.tmpl.bicepparam @@ -1,8 +1,8 @@ using '../templates/dev-acr.bicep' -param acrName = 'arohcpocpdev' +param acrName = '{{ .ocpAcrName }}' param acrSku = 'Premium' -param location = 'westus3' +param location = '{{ .region }}' param quayRepositoriesToCache = [ { @@ -23,4 +23,4 @@ param purgeJobs = [ } ] -param keyVaultName = 'aro-hcp-dev-global-kv' +param keyVaultName = '{{ .serviceKeyVaultName }}' diff --git a/dev-infrastructure/configurations/mvp-dev-svc-acr.bicepparam b/dev-infrastructure/configurations/acr-svc.tmpl.bicepparam similarity index 84% rename from dev-infrastructure/configurations/mvp-dev-svc-acr.bicepparam rename to dev-infrastructure/configurations/acr-svc.tmpl.bicepparam index c7c27084c..f18b587b7 100644 --- a/dev-infrastructure/configurations/mvp-dev-svc-acr.bicepparam +++ b/dev-infrastructure/configurations/acr-svc.tmpl.bicepparam @@ -1,8 +1,8 @@ using '../templates/dev-acr.bicep' -param acrName = 'arohcpsvcdev' +param acrName = '{{ .svcAcrName }}' param acrSku = 'Premium' -param location = 'westus3' +param location = '{{ .region }}' param quayRepositoriesToCache = [ { @@ -29,4 +29,4 @@ param purgeJobs = [ } ] -param keyVaultName = 'aro-hcp-dev-global-kv' +param keyVaultName = '{{ .serviceKeyVaultName }}' diff --git a/dev-infrastructure/configurations/mvp-dev-acr.bicepparam b/dev-infrastructure/configurations/acr.tmpl.bicepparam similarity index 91% rename from dev-infrastructure/configurations/mvp-dev-acr.bicepparam rename to dev-infrastructure/configurations/acr.tmpl.bicepparam index c28541b5c..f6478c853 100644 --- a/dev-infrastructure/configurations/mvp-dev-acr.bicepparam +++ b/dev-infrastructure/configurations/acr.tmpl.bicepparam @@ -1,8 +1,8 @@ using '../templates/dev-acr.bicep' -param acrName = 'arohcpdev' +param acrName = '{{ .acrName }}' param acrSku = 'Premium' -param location = 'westus3' +param location = '{{ .region }}' param quayRepositoriesToCache = [ { @@ -49,4 +49,4 @@ param purgeJobs = [ } ] -param keyVaultName = 'aro-hcp-dev-global-kv' +param keyVaultName = '{{ .serviceKeyVaultName }}' diff --git a/dev-infrastructure/configurations/cs-integ-metrics.bicepparam b/dev-infrastructure/configurations/cs-integ-metrics.bicepparam deleted file mode 100644 index 66c739799..000000000 --- a/dev-infrastructure/configurations/cs-integ-metrics.bicepparam +++ /dev/null @@ -1,7 +0,0 @@ -using '../modules/metrics/metrics.bicep' - -param grafanaName = 'cs-integ-grafana' -param msiName = 'cs-integ-metrics-msi' - -// overriden in makefile -param globalResourceGroup = '' diff --git a/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam b/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam deleted file mode 100644 index f8759cff2..000000000 --- a/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam +++ /dev/null @@ -1,32 +0,0 @@ -using '../templates/mgmt-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param vnetAddressPrefix = '10.132.0.0/14' -param subnetPrefix = '10.132.8.0/21' -param podSubnetPrefix = '10.132.64.0/18' -param aksClusterName = take('cs-integ-mgmt-cluster-${uniqueString('cs-integ-mgmt-cluster')}', 63) -param aksKeyVaultName = 'aks-kv-cs-integ-mc-1' -param systemAgentMinCount = 2 -param systemAgentMaxCount = 6 -param systemAgentVMSize = 'Standard_D2s_v3' -param aksSystemOsDiskSizeGB = 32 -param userAgentMinCount = 1 -param userAgentMaxCount = 12 -param userAgentVMSize = 'Standard_D4s_v3' -param aksUserOsDiskSizeGB = 100 -param userAgentPoolAZCount = 3 -param persist = true - -param deployMaestroConsumer = true -param maestroKeyVaultName = 'maestro-kv-cs-integ' -param maestroEventGridNamespacesName = 'maestro-eventgrid-cs-integ' -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-dev.azure.com' - -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3-cs' - -param acrPullResourceGroups = [regionalResourceGroup, 'global'] - -// These parameters are always overridden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/cs-integ-msi.bicepparam b/dev-infrastructure/configurations/cs-integ-msi.bicepparam deleted file mode 100644 index 2ebe4c9fe..000000000 --- a/dev-infrastructure/configurations/cs-integ-msi.bicepparam +++ /dev/null @@ -1,7 +0,0 @@ -using '../templates/cs-integration-msi.bicep' - -param namespaceFormatString = 'sandbox-jenkins-{0}-aro-hcp' - -param clusterServiceManagedIdentityName = 'clusters-service' - -param clusterName = take('cs-integ-svc-cluster-${uniqueString('svc-cluster')}', 63) diff --git a/dev-infrastructure/configurations/cs-integ-region.bicepparam b/dev-infrastructure/configurations/cs-integ-region.bicepparam deleted file mode 100644 index 42886d231..000000000 --- a/dev-infrastructure/configurations/cs-integ-region.bicepparam +++ /dev/null @@ -1,16 +0,0 @@ -using '../templates/region.bicep' - -param persist = true - -// dns -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3-cs' -param baseDNSZoneResourceGroup = 'global' - -// maestro -param maestroKeyVaultName = 'maestro-kv-cs-integ' -param maestroEventGridNamespacesName = 'maestro-eventgrid-cs-integ' -param maestroEventGridMaxClientSessionsPerAuthName = 4 - -// This parameter is always overriden in the Makefile -param currentUserId = '' diff --git a/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam b/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam deleted file mode 100644 index c940ec182..000000000 --- a/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam +++ /dev/null @@ -1,44 +0,0 @@ -using '../templates/svc-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param istioVersion = ['asm-1-22'] -param vnetAddressPrefix = '10.128.0.0/14' -param subnetPrefix = '10.128.8.0/21' -param podSubnetPrefix = '10.128.64.0/18' -param persist = true -param aksClusterName = take('cs-integ-svc-cluster-${uniqueString('svc-cluster')}', 63) -param aksKeyVaultName = 'aks-kv-cs-integ-sc' -param disableLocalAuth = false -param deployFrontendCosmos = true - -param maestroKeyVaultName = 'maestro-kv-cs-integ' -param maestroEventGridNamespacesName = 'maestro-eventgrid-cs-integ' -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-dev.azure.com' -param maestroPostgresServerName = 'maestro-pg-cs-integ' -param maestroPostgresServerVersion = '15' -param maestroPostgresServerStorageSizeGB = 32 -param deployMaestroPostgres = false -param maestroPostgresPrivate = false - -param deployCsInfra = false -param csPostgresServerName = 'cs-pg-cs-integ' -param clusterServicePostgresPrivate = false - -param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' -param serviceKeyVaultResourceGroup = 'global' -param serviceKeyVaultSoftDelete = true -param serviceKeyVaultPrivate = false - -param acrPullResourceGroups = ['global'] -param clustersServiceAcrResourceGroupNames = ['global'] -param imageSyncAcrResourceGroupNames = ['global'] - -param oidcStorageAccountName = 'arohcpoidccsinteg' -param aroDevopsMsiId = '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' - -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3-cs' - -// These parameters are always overridden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/cs-integ.mk b/dev-infrastructure/configurations/cs-integ.mk deleted file mode 100644 index ca1de3066..000000000 --- a/dev-infrastructure/configurations/cs-integ.mk +++ /dev/null @@ -1,6 +0,0 @@ -REGION ?= westus3 -RESOURCEGROUP ?= cs-integ-$(USER)-$(REGION)-$(AKSCONFIG) -REGIONAL_RESOURCEGROUP ?= cs-integ-$(USER)-$(REGION) -SVC_KV_RESOURCEGROUP ?= global -ARO_HCP_IMAGE_ACR ?= arohcpdev -REGIONAL_ACR_NAME ?= arohcpdev$(shell echo $(CURRENTUSER) | sha256sum | head -c 24) diff --git a/dev-infrastructure/configurations/dev-metrics.bicepparam b/dev-infrastructure/configurations/dev-metrics.bicepparam deleted file mode 100644 index e91b70fa2..000000000 --- a/dev-infrastructure/configurations/dev-metrics.bicepparam +++ /dev/null @@ -1,8 +0,0 @@ -using '../modules/metrics/metrics.bicep' - -param monitorName = 'aro-hcp-monitor-${take(uniqueString(readEnvironmentVariable('CURRENTUSER', '')), 5)}' -param grafanaName = take('aro-hcp-grafana-${uniqueString(readEnvironmentVariable('CURRENTUSER', ''))}', 23) -param msiName = 'aro-hcp-metrics-msi-${take(uniqueString(readEnvironmentVariable('CURRENTUSER', '')), 5)}' - -// overriden in makefile -param globalResourceGroup = '' diff --git a/dev-infrastructure/configurations/dev.mk b/dev-infrastructure/configurations/dev.mk deleted file mode 100644 index ce757e192..000000000 --- a/dev-infrastructure/configurations/dev.mk +++ /dev/null @@ -1,9 +0,0 @@ -REGION ?= westus3 -RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION)-$(AKSCONFIG) -REGIONAL_RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION) -SVC_KV_RESOURCEGROUP ?= global -GLOBAL_RESOURCEGROUP ?= global -IMAGE_SYNC_RESOURCEGROUP ?= aro-hcp-image-sync-$(USER)-$(REGION) -IMAGE_SYNC_ENVIRONMENT ?= image-sync-env-sxo4oqbcjiekg -ARO_HCP_IMAGE_ACR ?= arohcpdev -REPOSITORIES_TO_SYNC ?= '{registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/pstefans/controlplaneoperator,quay.io/app-sre/uhc-clusters-service}' diff --git a/dev-infrastructure/configurations/image-sync.tmpl.bicepparam b/dev-infrastructure/configurations/image-sync.tmpl.bicepparam new file mode 100644 index 000000000..943e520bd --- /dev/null +++ b/dev-infrastructure/configurations/image-sync.tmpl.bicepparam @@ -0,0 +1,12 @@ +using '../templates/image-sync.bicep' + +param containerAppEnvName = '{{ .imageSyncEnvironmentName }}' + +param acrResourceGroup = '{{ .imageSyncAcrRG }}' +param keyVaultName = 'aro-hcp-dev-global-kv' +param keyVaultResourceGroup = '{{ .serviceKeyVaultRG }}' + +param requiredSecretNames = [ + 'component-sync-pull-secret' + 'bearer-secret' +] diff --git a/dev-infrastructure/configurations/metrics.tmpl.bicepparam b/dev-infrastructure/configurations/metrics.tmpl.bicepparam new file mode 100644 index 000000000..315f5aadd --- /dev/null +++ b/dev-infrastructure/configurations/metrics.tmpl.bicepparam @@ -0,0 +1,7 @@ +using '../modules/metrics/metrics.bicep' + +param monitorName = '{{ .monitoringWorkspaceName }}' +param grafanaName = '{{ .grafanaName }}' +param msiName = '{{ .monitoringMsiName }}' +param grafanaAdminGroupPrincipalId = '{{ .grafanaAdminGroupPrincipalId }}' +param globalResourceGroup = '{{ .regionRG }}' diff --git a/dev-infrastructure/configurations/mgmt-cluster.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.bicepparam deleted file mode 100644 index f912182dc..000000000 --- a/dev-infrastructure/configurations/mgmt-cluster.bicepparam +++ /dev/null @@ -1,32 +0,0 @@ -using '../templates/mgmt-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param vnetAddressPrefix = '10.132.0.0/14' -param subnetPrefix = '10.132.8.0/21' -param podSubnetPrefix = '10.132.64.0/18' -param aksClusterName = 'aro-hcp-mgmt-cluster' -param aksKeyVaultName = take('aks-kv-mgmt-cluster-${uniqueString(currentUserId)}', 24) -param aksEtcdKVEnableSoftDelete = false -param systemAgentMinCount = 2 -param systemAgentMaxCount = 3 -param systemAgentVMSize = 'Standard_D2s_v3' -param aksSystemOsDiskSizeGB = 32 -param userAgentMinCount = 1 -param userAgentMaxCount = 6 -param userAgentVMSize = 'Standard_D4s_v3' -param aksUserOsDiskSizeGB = 100 -param userAgentPoolAZCount = 3 -param persist = false - -param deployMaestroConsumer = true -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-int.azure.com' - -param baseDNSZoneName = 'hcp.osadev.cloud' - -param acrPullResourceGroups = ['global'] - -// These parameters are always overriden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam new file mode 100644 index 000000000..3073d8b1c --- /dev/null +++ b/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam @@ -0,0 +1,30 @@ +using '../templates/mgmt-cluster.bicep' + +param kubernetesVersion = '{{ .kubernetesVersion}}' +param vnetAddressPrefix = '{{ .vnetAddressPrefix }}' +param subnetPrefix = '{{ .subnetPrefix }}' +param podSubnetPrefix = '{{ .podSubnetPrefix }}' +param aksClusterName = '{{ .aksName }}' +param aksKeyVaultName = '{{ .mgmtEtcdKVName }}' +param aksEtcdKVEnableSoftDelete = {{ .mgmtEtcdKVSoftDelete }} +param systemAgentMinCount = {{ .mgmtSystemAgentPoolMinCount}} +param systemAgentMaxCount = {{ .mgmtSystemAgentPoolMaxCount }} +param systemAgentVMSize = '{{ .mgmtSystemAgentPoolVmSize }}' +param aksSystemOsDiskSizeGB = {{ .mgmtSystemAgentPoolOsDiskSizeGB }} +param userAgentMinCount = {{ .mgmtUserAgentPoolMinCount }} +param userAgentMaxCount = {{ .mgmtUserAgentPoolMaxCount }} +param userAgentVMSize = '{{ .mgmtUserAgentPoolVmSize }}' +param aksUserOsDiskSizeGB = {{ .mgmtUserAgentPoolOsDiskSizeGB }} +param userAgentPoolAZCount = {{ .mgmtUserAgentPoolAzCount }} +param persist = {{ .persist }} + +param deployMaestroConsumer = true +param maestroKeyVaultName = '{{ .maestroKeyVaultName }}' +param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}' +param maestroCertDomain = '{{ .maestroCertDomain }}' + +param regionalDNSZoneName = '{{ .regionalDNSSubdomain}}.{{ .baseDnsZoneName }}' + +param acrPullResourceGroups = ['{{ .serviceComponentAcrResourceGroups }}'] + +param regionalResourceGroup = '{{ .regionRG }}' diff --git a/dev-infrastructure/configurations/mvp-image-sync.bicepparam b/dev-infrastructure/configurations/mvp-image-sync.bicepparam deleted file mode 100644 index 3b58c091a..000000000 --- a/dev-infrastructure/configurations/mvp-image-sync.bicepparam +++ /dev/null @@ -1,10 +0,0 @@ -using '../templates/image-sync.bicep' - -param acrResourceGroup = 'global' - -param keyVaultName = 'aro-hcp-dev-global-kv' - -param requiredSecretNames = [ - 'component-sync-pull-secret' - 'bearer-secret' -] diff --git a/dev-infrastructure/configurations/mvp-metrics.bicepparam b/dev-infrastructure/configurations/mvp-metrics.bicepparam deleted file mode 100644 index ff7efe0b4..000000000 --- a/dev-infrastructure/configurations/mvp-metrics.bicepparam +++ /dev/null @@ -1,8 +0,0 @@ -using '../modules/metrics/metrics.bicep' - -param monitorName = 'aro-hcp-monitor' -param grafanaName = 'aro-hcp-grafana' -param msiName = 'aro-hcp-metrics-msi' - -// overriden in makefile -param globalResourceGroup = '' diff --git a/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam b/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam deleted file mode 100644 index 6af5e300e..000000000 --- a/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam +++ /dev/null @@ -1,32 +0,0 @@ -using '../templates/mgmt-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param vnetAddressPrefix = '10.132.0.0/14' -param subnetPrefix = '10.132.8.0/21' -param podSubnetPrefix = '10.132.64.0/18' -param aksClusterName = take('aro-hcp-mgmt-cluster-${uniqueString('mgmt-cluster')}', 63) -param aksKeyVaultName = 'aks-kv-aro-hcp-dev-mc-1' -param systemAgentMinCount = 2 -param systemAgentMaxCount = 3 -param systemAgentVMSize = 'Standard_D2s_v3' -param aksSystemOsDiskSizeGB = 32 -param userAgentMinCount = 1 -param userAgentMaxCount = 9 -param userAgentVMSize = 'Standard_D4s_v3' -param aksUserOsDiskSizeGB = 100 -param userAgentPoolAZCount = 3 -param persist = true - -param deployMaestroConsumer = true -param maestroKeyVaultName = 'maestro-kv-aro-hcp-dev' -param maestroEventGridNamespacesName = 'maestro-eventgrid-aro-hcp-dev' -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-dev.azure.com' - -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3' - -param acrPullResourceGroups = [regionalResourceGroup, 'global'] - -// These parameters are always overridden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/mvp-region.bicepparam b/dev-infrastructure/configurations/mvp-region.bicepparam deleted file mode 100644 index 9d58a7a44..000000000 --- a/dev-infrastructure/configurations/mvp-region.bicepparam +++ /dev/null @@ -1,15 +0,0 @@ -using '../templates/region.bicep' - -param persist = true - -// dns -param baseDNSZoneName = 'hcp.osadev.cloud' -param baseDNSZoneResourceGroup = 'global' - -// maestro -param maestroKeyVaultName = 'maestro-kv-aro-hcp-dev' -param maestroEventGridNamespacesName = 'maestro-eventgrid-aro-hcp-dev' -param maestroEventGridMaxClientSessionsPerAuthName = 4 - -// This parameter is always overriden in the Makefile -param currentUserId = '' diff --git a/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam b/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam deleted file mode 100644 index d1c310ec8..000000000 --- a/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam +++ /dev/null @@ -1,44 +0,0 @@ -using '../templates/svc-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param istioVersion = ['asm-1-22'] -param vnetAddressPrefix = '10.128.0.0/14' -param subnetPrefix = '10.128.8.0/21' -param podSubnetPrefix = '10.128.64.0/18' -param persist = true -param aksClusterName = take('aro-hcp-svc-cluster-${uniqueString('svc-cluster')}', 63) -param aksKeyVaultName = 'aks-kv-aro-hcp-dev-sc' -param disableLocalAuth = false -param deployFrontendCosmos = true - -param maestroKeyVaultName = 'maestro-kv-aro-hcp-dev' -param maestroEventGridNamespacesName = 'maestro-eventgrid-aro-hcp-dev' -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-dev.azure.com' -param maestroPostgresServerName = 'maestro-pg-aro-hcp-dev' -param maestroPostgresServerVersion = '15' -param maestroPostgresServerStorageSizeGB = 32 -param deployMaestroPostgres = false -param maestroPostgresPrivate = false - -param deployCsInfra = false -param csPostgresServerName = 'cs-pg-aro-hcp-dev' -param clusterServicePostgresPrivate = false - -param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' -param serviceKeyVaultResourceGroup = 'global' -param serviceKeyVaultSoftDelete = true -param serviceKeyVaultPrivate = false - -param acrPullResourceGroups = ['global'] -param clustersServiceAcrResourceGroupNames = ['global'] -param imageSyncAcrResourceGroupNames = ['global'] - -param oidcStorageAccountName = 'arohcpoidcdev' -param aroDevopsMsiId = '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' - -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3' - -// These parameters are always overridden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/region.bicepparam b/dev-infrastructure/configurations/region.bicepparam deleted file mode 100644 index b679aea1c..000000000 --- a/dev-infrastructure/configurations/region.bicepparam +++ /dev/null @@ -1,13 +0,0 @@ -using '../templates/region.bicep' - -// dns -param baseDNSZoneName = 'hcp.osadev.cloud' -param baseDNSZoneResourceGroup = 'global' - -// maestro -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroEventGridMaxClientSessionsPerAuthName = 4 - -// These parameters are always overriden in the Makefile -param currentUserId = '' diff --git a/dev-infrastructure/configurations/region.tmpl.bicepparam b/dev-infrastructure/configurations/region.tmpl.bicepparam new file mode 100644 index 000000000..220f6192a --- /dev/null +++ b/dev-infrastructure/configurations/region.tmpl.bicepparam @@ -0,0 +1,11 @@ +using '../templates/region.bicep' + +// dns +param baseDNSZoneName = '{{ .baseDnsZoneName }}' +param baseDNSZoneResourceGroup = '{{ .baseDnsZoneRG }}' +param regionalDNSSubdomain = '{{ .regionalDNSSubdomain }}' + +// maestro +param maestroKeyVaultName = '{{ .maestroKeyVaultName }}' +param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}' +param maestroEventGridMaxClientSessionsPerAuthName = {{ .maestroEventGridMaxClientSessionsPerAuthName }} diff --git a/dev-infrastructure/configurations/svc-cluster.bicepparam b/dev-infrastructure/configurations/svc-cluster.bicepparam deleted file mode 100644 index c2df935e2..000000000 --- a/dev-infrastructure/configurations/svc-cluster.bicepparam +++ /dev/null @@ -1,45 +0,0 @@ -using '../templates/svc-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param istioVersion = ['asm-1-22'] -param vnetAddressPrefix = '10.128.0.0/14' -param subnetPrefix = '10.128.8.0/21' -param podSubnetPrefix = '10.128.64.0/18' -param persist = false -param aksClusterName = 'aro-hcp-svc-cluster' -param aksKeyVaultName = take('aks-kv-svc-cluster-${uniqueString(currentUserId)}', 24) -param aksEtcdKVEnableSoftDelete = false -param disableLocalAuth = false -param deployFrontendCosmos = false - -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-int.azure.com' -param maestroPostgresServerName = take('maestro-pg-${uniqueString(currentUserId)}', 60) -param maestroPostgresServerVersion = '15' -param maestroPostgresServerStorageSizeGB = 32 -param deployMaestroPostgres = false -param maestroPostgresPrivate = false - -param deployCsInfra = false -param csPostgresServerName = take('cs-pg-${uniqueString(currentUserId)}', 60) -param clusterServicePostgresPrivate = false - -param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' -param serviceKeyVaultResourceGroup = 'global' -param serviceKeyVaultLocation = 'westus3' -param serviceKeyVaultSoftDelete = true -param serviceKeyVaultPrivate = false - -param acrPullResourceGroups = ['global'] -param imageSyncAcrResourceGroupNames = ['global'] -param clustersServiceAcrResourceGroupNames = ['global'] - -param oidcStorageAccountName = take('arohcpoidcdev${uniqueString(currentUserId)}', 24) -param aroDevopsMsiId = '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' - -param baseDNSZoneName = 'hcp.osadev.cloud' - -// These parameters are always overriden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam new file mode 100644 index 000000000..5d53b3d57 --- /dev/null +++ b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam @@ -0,0 +1,45 @@ +using '../templates/svc-cluster.bicep' + +param kubernetesVersion = '{{ .kubernetesVersion }}' +param istioVersion = {{ .istioVersion }} +param vnetAddressPrefix = '{{ .vnetAddressPrefix }}' +param subnetPrefix = '{{ .subnetPrefix }}' +param podSubnetPrefix = '{{ .podSubnetPrefix }}' +param persist = {{ .persist }} +param aksClusterName = '{{ .aksName }}' +param aksKeyVaultName = '{{ .svcEtcdKVName }}' +param aksEtcdKVEnableSoftDelete = {{ .svcEtcdKVSoftDelete }} + +param disableLocalAuth = {{ .frontendCosmosDBDisableLocalAuth }} +param deployFrontendCosmos = {{ .frontendCosmosDBDeploy }} +param rpCosmosDbName = '{{ .frontendCosmosDBName }}' + +param maestroKeyVaultName = '{{ .maestroKeyVaultName }}' +param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}' +param maestroCertDomain = '{{ .maestroCertDomain}}' +param maestroPostgresServerName = '{{ .maestroPostgresName }}' +param maestroPostgresServerVersion = '{{ .maestroPostgresServerVersion }}' +param maestroPostgresServerStorageSizeGB = {{ .maestroPostgresServerStorageSizeGB }} +param deployMaestroPostgres = {{ .maestroPostgresDeploy }} +param maestroPostgresPrivate = {{ .maestroPostgresPrivate }} + +param deployCsInfra = {{ .clusterServicePostgresDeploy }} +param csPostgresServerName = '{{ .clusterServicePostgresName }}' +param clusterServicePostgresPrivate = {{ .clusterServicePostgresPrivate }} + +param serviceKeyVaultName = '{{ .serviceKeyVaultName }}' +param serviceKeyVaultResourceGroup = '{{ .serviceKeyVaultRG }}' +param serviceKeyVaultLocation = '{{ .serviceKeyVaultRegion }}' +param serviceKeyVaultSoftDelete = {{ .serviceKeyVaultSoftDelete }} +param serviceKeyVaultPrivate = {{ .serviceKeyVaultPrivate }} + +param acrPullResourceGroups = ['{{ .serviceComponentAcrResourceGroups }}'] +param imageSyncAcrResourceGroupNames = ['{{ .imageSyncAcrRG }}'] +param clustersServiceAcrResourceGroupNames = ['{{ .clusterServiceAcrRG }}'] + +param oidcStorageAccountName = '{{ .oidcStorageAccountName }}' +param aroDevopsMsiId = '{{ .aroDevopsMsiId }}' + +param regionalDNSZoneName = '{{ .regionalDNSSubdomain}}.{{ .baseDnsZoneName }}' + +param regionalResourceGroup = '{{ .regionRG }}' diff --git a/dev-infrastructure/create-config.sh b/dev-infrastructure/create-config.sh new file mode 100755 index 000000000..056fd4026 --- /dev/null +++ b/dev-infrastructure/create-config.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +DEPLOY_ENV=$1 +cd $(dirname "$(realpath "${BASH_SOURCE[0]}")") +../templatize.sh "$DEPLOY_ENV" config.tmpl.mk config.mk +for tmpl_file in configurations/*.tmpl.*; do + output_file="${tmpl_file/.tmpl/}" + ../templatize.sh "$DEPLOY_ENV" "$tmpl_file" "$output_file" +done diff --git a/dev-infrastructure/docs/development-setup.md b/dev-infrastructure/docs/development-setup.md index e78592c02..b9443fe30 100644 --- a/dev-infrastructure/docs/development-setup.md +++ b/dev-infrastructure/docs/development-setup.md @@ -11,16 +11,91 @@ The idea of this repo is to provide means to create a development environment th * `az` version >= 2.60, `jq`, `make`, `kubelogin` (from ), `kubectl` version >= 1.30, `helm` * `az login` with your Red Hat email * Register the needed [AFEC](https://aka.ms/afec) feature flags using `cd dev-infrastructure && make feature-registration - * __NOTE:__ This will take awhile, you will have to wait until they're in a registered state. +* __NOTE:__ This will take awhile, you will have to wait until they're in a registered state. -## Cluster creation procedure +## Infrastructure -There are a few variants to chose from when creating an AKS cluster: +This section describes how to create the infrastructure required to run ARO HCP. -* Service Cluster: Public AKS cluster with optional params that can be modified to include all Azure resources needed to run a Service cluster -* Management Cluster: Public AKS cluster with optional params that can be modified to include all Azure resources needed to run a Management cluster +The infrastructure roughly consists of two AKS clusters: -When creating a cluster, also supporting infrastructure is created, e.g. managed identities, permissions, databases, keyvaults, ... +* Service Cluster: the cluster hosting supporting services for a an ARO HCP region, e.g. the Frontend, Cluster Service, Maestro, etc. + +* Management Cluster: the cluster hosting the actual hosted controlplanes and supporting services to provision and manage them + +In addition to the clusters, the services require supporting infrastructure as well, consisting of managed identities (and their permissions), Key Vaults, Databases, Networking, DNS, Storage, ... + +All this infrastructure is managed by the bicep templates found in the `dev-infrastructure` folder. Despite the name of this folder, these templates are also being used in higher environments (MSFT INT, MSFT PROD) for infrastructure management. + +### Shared infrastructure + +Every developer creates their own set of service/management clusters, including the supporting infrastructure. This allows for independant development. Certain parts of the infrastructure are shared between developers though for various reasons (cost, ease of management, time): + +* Service Key Vault `aro-hcp-dev-svc-kv`: this KV holds various shared secrets that are the same for all developer setups (e.g. 1P app certificates, ARM helper certificates, Quay.io pullsecrets). Some of these need to be recycled occasionally so sharing them allows for a central cycle process. Access to this KV is read-only, therefore sharing is not going to result in conflicts between individual developers. See [SD-DDR-0043](https://docs.google.com/document/d/1YKnMFPFvdIuGpGC1il78O9d3WwTyiVgw7jzCpDTUlII/edit#heading=h.bupciudrwmna) for more details about this KV. + +* SVC ACR: this ACR holds mirrored service image to be used by developers. Having these mirrored only once saves time and money. The mirror process for this ACR is driven by the integrated DEV environment. Developers access this ACR read-only, therefore sharing it is not going to result in conflicts. + +* OCP ACR: this ACR holds mirrored OCP release payloads. The mirror process for this ACR is driven by the integrated DEV environment. Developers access this ACR read-only, therefore sharing it is not going to result in conflicts. + +* Image sync: since we share ACRs, we can also share the image sync deployment + +#### Shared SVC KV Secrets + +* `acm-d-componentsync-password` and `acm-d-componentsync-username` + what: credentials for the `quay.io/acm-d` organization + purpose: used for ACR caching to make ACM prerelease images available for ACR HCP + +* `quay-componentsync-password` and `quay-componentsync-password` + what: credentials for the `quay.io/app-sre` organization + purpose: used for ACR caching to make CS sandbox images available to the CS PR check environment + +* `quay-password` and `quay-username` + what: credentials for the `quay.io/openshift-release-dev` organization + purpose: we only sync stable releases with `oc-mirror` but a ACR caching rule makes + other releases like nightly available for testing purposes + +* `component-sync-pull-secret` + what: base64 encoded pull secret for container registries + purpose: used by image-sync to mirror component images + +* `bearer-secret` + what: base64 encoded access token for the `quay.io/app-sre` organization + purposes: used by image-sync to mirror component images + +* `aro-hcp-dev-sp` + what: Azure SP credentials to be used for HCPs + purpose: until managed identities are available for HCPs, this is the auth creds + for controlplane operators to interact with Azure. This SP has contributer + permissions in the subscription + +* `aro-hcp-dev-sp-cs` + what: the same Azure SP credentials as `aro-hcp-dev-sp` but formatted for CS + purpose: until the 1P mock certificate is going to be used by CS to interact + with Azure, it will use these static creds instead + +* `pull-secret` + what: pull secret for quay and redhat registries of user `aro-hcp-service-lifecycle-team+quay@redhat.com` + purpose: used by `oc-mirror` to mirror OCP release payloads into the ACR + +* `aro-hcp-dev-pull-secret` - can be removed???? + what: pull secret for quay.io and registry.redhat.io and the `arohcpdev` ACR + purpose: this was used during P1 while we still installed clusters from quay.io payloads + later it was used to for HCPs to get access to the ACR while CS was not + yet creating dedicated pull secrets for them + note: since HCPs don't pull from quay or RH registries anymore and CS now creates + dedicated pull secrets for the ACR, this should be safe to delete + +* `component-pull-secret` - can be removed???? + what: holds the same a pull secret for quay.io (same as `component-sync-pull-secret`) but + with an incomplete one for arohcpdev as well + +* `quay-pull-token` - can be removed???? + what: a quay token + purpose: unknown + +* `testing` - can be removed???? + what: foo-bar + purpose: unkown ### Grant yourself Key Vault access @@ -32,90 +107,101 @@ az role assignment create --role "Key Vault Secrets User" --assignee $(az ad sig Note: you only need to run this once. Re-runing it wont hurt, but it will not change anything. -### Create a Service Cluster +### Create infrastructure the easy way -The service cluster base configuration to use for development is `configurations/svc-cluster.bicepparam`. Depending on the personal requirements this file offers some features toggles for the main features of the service cluster and the regional resources. +To create the service cluster, management cluster and supporting infrastructure run the following command from the root of this repository. -* `deployFrontendCosmos` - set to `true` if you want a CosmosDB created for the RP + ```bash + SKIP_CONFIRM=1 make infra.all + ``` - This also includes managed identity and access permissions +Running this the first time takes around 60 minutes. -* `deployCsInfra` - set to `true` if you want CS infra to be provisioned, e.g. if you want to develop on RP and run it towards an on-cluster CS +If you only need a management cluster or service cluster for development work, consider using one of the following commands. They take less time and the resulting infrastructure costs less money - This includes a Postgres DB and access permissions to the DB and the service KeyVault, as well as the Maestro Server - and supporting infrastructure (EventGrid Namespaces instance, Postgres DB and necessary access permissions). + ```bash + SKIP_CONFIRM=1 make infra.svc + or + SKIP_CONFIRM=1 make infra.mgmt + ``` -* `persist` - if set to `true` the resourcegroup holding the cluster and the regional resources will not be deleted after a couple of days +### Updating infrastructure -Change those flags accordingly and then run the following command. Depending on the selected features, this may take a while: +To update already existing infrastructure you can run `make infra.all` again. You can also use more fine grained make tasks that finish quicker, e.g. ```bash - AKSCONFIG=svc-cluster make cluster + make infra.svc + make infra.mgmt + make infra.imagesync ``` -Enable metrics for the svc-cluster - ```bash - AKSCONFIG=svc-cluster make enable-aks-metrics - ``` +### Customizing infra deployment -### Create a Management Cluster +The basic configuration for infrastructure deployment can be found in the `config/config.yaml` file. +This file offers multiple levels of overrides depending on cloud and deployment environments. -The service cluster base configuration to use for development is `configurations/mgmt-cluster.bicepparam`. This parameter file offers feature toggles as well. +The base configuration for all Red Hat Azure Subscription based deployments can be found under `clouds.public.environments.rh-dev-tmpl`. This configures the shared infrastructure and component versions to be used in general. -* `deployMaestroConsumer` - if set to `true` deploys the required infrastructure to run a Maestro Consumer (TODO find a better name for this flag because it does not deploy the consumer itself). +The deployment environment used for personal developer infrastructure is found under `.clouds.public.environments.personal-dev`. It inherits from `rh-dev-tmpl` and defines certain overrides, the most important one being `persist: false` which results in personal dev infrastructure to be cleaned up 48h after creation. -* `persist` - if set to `true` the resourcegroup holding the cluster will not be deleted after a couple of days +### Access AKS clusters -> A Management Cluster depends on certain resources found in the resource group of the Service Cluster. Therefore, a standalone Management Cluster can't be created right now and requires a Service Cluster +Running `make infra.all` will provide you with cluster admin on your clusters and kubeconfig files being created under `~/.kube`. The kubeconfigs are named after the resource group name that holds the cluster. The term `svc` and `mgmt` used in these filesnames indicate what cluster they are for. - ```bash - AKSCONFIG=mgmt-cluster make cluster - ``` +Please not that these kubeconfig files require an active Azure CLI session (`az login`) to work properly. + +If you loose these files, you can recreate them by running -Enable metrics for the mgmt-cluster ```bash - AKSCONFIG=mgmt-cluster make enable-aks-metrics + make -f dev-infrastructure/Makefile svc.aks.admin-access svc.aks.kubeconfig + or + make -f dev-infrastructure/Makefile mgmt.aks.admin-access mgmt.aks.kubeconfig ``` -### Access AKS clusters - - ```bash - AKSCONFIG=svc-cluster make aks.admin-access # one time - AKSCONFIG=svc-cluster make aks.kubeconfig - AKSCONFIG=svc-cluster export KUBECONFIG=${HOME}/.kube/${AKSCONFIG}.kubeconfig - kubectl get ns - ``` +> Freshly granted cluster admin permissions might not be effective immediately. If you get permission denied errors on your `kubectl` commands, consider waiting a couple of minutes for the permissons to be propagated - (Replace svc with mgmt for management clusters) +### Cleanup -### Access cluster via the Azure portal or via `az aks command invoke` +To clean up the entire infrastructure of a personal dev environment, run the following command ```bash - AKSCONFIG=svc-cluster make aks.admin-access # one time - az aks command invoke ... + make infra.clean ``` -### Cleanup +There are more fine grained cleanup tasks available as well + + ```bash + make infra.svc.clean + make infra.mgmt.clean + make infra.region.clean + make infra.imagesync.clean + ``` > Please note that all resource groups not tagged with `persist=true` will be deleted by our cleanup pipeline after 48 hours -Setting the correct `AKSCONFIG`, this will cleanup all resources created in Azure +## Deploying Services quick and easy - ```bash - AKSCONFIG=svc-cluster make clean - ``` +To followup sections describe how to deploy the components individually. But if you are looking for a quick and easy way to install or update ALL components on both clusters with one command, then run this: -## Deploy Services to the service cluster + ```bash + make deploy.svc.all + make deploy.mgmt.all + ``` -> Make sure your `KUBECONFIG` points to the service cluster!!! +Or even simpler with + + ```bash + make deploy.all + ``` + +## Deploy Services to the service cluster > The service cluster has no ingress. To interact with the services you deploy use `kubectl port-forward` ### Maestro Server ```bash - cd maestro - AKSCONFIG=svc-cluster make deploy-server + make maestro.server.deploy ``` To validate, have a look at the `maestro` namespace on the service cluster. Some pod restarts are expected in the first 1 minute until the containerized DB is ready. @@ -129,66 +215,51 @@ To access the HTTP and GRPC endpoints of maestro, run ### Cluster Service -> This might not work with oc 4.17.0, please use oc 4.16.x until this is fixed in 4.17 -> -Deploy CS: +> This might not work with `oc` 4.17.0, please use oc 4.16.x until this is fixed in 4.17 + ```bash - cd cluster-service/ - make deploy + make cs.deploy ``` -To validate, have a look at the `cluster-service` namespace. +To validate, have a look at the `cluster-service` namespace or the service cluster. -### Resource Provider +### Resource Provider / Frontend The ARO-HCP resource provider consists of independent frontend and backend components. ```bash - cd frontend/ - make deploy - ``` - - ```bash - cd backend/ - make deploy + make rp.frontend.deploy + make rp.backend.deploy ``` To validate, have a look at the `aro-hcp` namespace on the service cluster. ## Deploy Services to the management cluster -> Make sure your `KUBECONFIG` points to the management cluster!!! - ### ACM ```bash - cd acm - make deploy + make acm.deploy ``` ### Hypershift Operator and External DNS ```bash - cd hypershiftoperator/ - make deploy + make hypershift.deploy ``` -## Maestro Agent +### Maestro Agent First install the agent ```bash - cd maestro - AKSCONFIG=mgmt-cluster make deploy-agent + make maestro.agent.deploy ``` Then register it with the Maestro Server -Make sure your `KUBECONFIG` points to the service cluster, then run - ```bash - cd maestro - AKSCONFIG=svc-cluster make register-agent + make maestro.registration.deploy ``` ## CS Local Development Setup @@ -495,37 +566,30 @@ Users require membership in the `aro-hcp-engineering` group to read secrets. Th az keyvault secret show --vault-name "aro-hcp-dev-svc-kv" --name "aro-hcp-dev-sp-cs" | jq .value -r > azure-creds.json ``` -In case the `aro-hcp-dev-svc-kv` KV gets recreated as part of a DEV environment recreation, the lost secrets can be replayed from the `aro-hcp-dev-global-kv` KV by ensuring you have `Secret Officer` permissions in the target KV and running - -```sh -dev-infrastructure/scripts/import-kv.sh aro-hcp-dev-global-kv aro-hcp-dev-svc-kv -``` - ### Access integrated DEV environment The integrated DEV environment is hosted in `westus3` and consists of -* the RG `aro-hcp-dev-westus3` containing shared regional resources (regional DNS zone, Maestro Eventgrid, Maestro KV) -* the RG `aro-hcp-dev-westus3-sc` the AKS service cluster and the resources required by the service components running on the SC (Postgres for Maestro Server, Postgres for Cluster Service, CosmosDB for RP, Service Key Vault, ...) -* the RG `aro-hcp-dev-westus3-mc-1` containing the AKS mgmt cluster -* the ACR `devarohcp` running in the `global` RG +* the RG `hcp-underlay-westus3-dev` containing shared regional resources (regional DNS zone, Maestro Eventgrid, Maestro KV) +* the RG `hcp-underlay-westus3-svc-dev` the AKS service cluster and the resources required by the service components running on the SC (Postgres for Maestro Server, Postgres for Cluster Service, CosmosDB for RP, Service Key Vault, ...) +* the RG `hcp-underlay-westus3-mgmt-dev-1` containing the AKS mgmt cluster +* the shared ACRs `arohcpsvcdev` and `arohcpocpdev` running in the `global` RG To access the SC run ```sh -AKSCONFIG=svc-cluster RESOURCEGROUP=aro-hcp-dev-westus3-sc make aks.admin-access # run one -AKSCONFIG=svc-cluster RESOURCEGROUP=aro-hcp-dev-westus3-sc make aks.kubeconfig -export KUBECONFIG=${HOME}/.kube/svc-cluster.kubeconfig +DEPLOY_ENV=dev make svc.aks.admin-access svc.aks.kubeconfig +export KUBECONFIG=$(DEPLOY_ENV=dev make svc.aks.kubeconfigfile) kubectl get ns ``` To access the MC run ```sh -AKSCONFIG=mgmt-cluster RESOURCEGROUP=aro-hcp-dev-westus3-mc-1 make aks.admin-access # run one -AKSCONFIG=mgmt-cluster RESOURCEGROUP=aro-hcp-dev-westus3-mc-1 make aks.kubeconfig -export KUBECONFIG=${HOME}/.kube/mgmt-cluster.kubeconfig +```sh +DEPLOY_ENV=dev make mgmt.aks.admin-access mgmt.aks.kubeconfig +export KUBECONFIG=$(DEPLOY_ENV=dev make mgmt.aks.kubeconfigfile) kubectl get ns ``` -> It might take a couple of minutes for the permissions created by `make aks.admin-access` to take effect. +> It might take a couple of minutes for the permissions created by `make xxx.aks.admin-access` to take effect. diff --git a/dev-infrastructure/ensure-no-running-deployment.sh b/dev-infrastructure/ensure-no-running-deployment.sh new file mode 100755 index 000000000..947586da3 --- /dev/null +++ b/dev-infrastructure/ensure-no-running-deployment.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +RESOURCEGROUP_NAME=$1 +DEPLOYMENT_NAME=$2 + +while [ "$(az deployment group show --name ${DEPLOYMENT_NAME} --resource-group ${RESOURCEGROUP_NAME} --query 'properties.provisioningState' -o tsv 2>/dev/null)" = "Running" ]; do \ + echo "Waiting for existing deployment ${DEPLOYMENT_NAME} in ${RESOURCEGROUP_NAME} to complete..."; \ + sleep 10; \ +done diff --git a/dev-infrastructure/modules/metrics/metrics.bicep b/dev-infrastructure/modules/metrics/metrics.bicep index 4339d00aa..a5dd65972 100644 --- a/dev-infrastructure/modules/metrics/metrics.bicep +++ b/dev-infrastructure/modules/metrics/metrics.bicep @@ -1,21 +1,21 @@ // this module is only used in dev -@description('Captures logged in users UID') -param currentUserId string = '' - @description('Metrics global resource group name') param globalResourceGroup string @description('Metrics global MSI name') -param msiName string = take('metrics-admin-${uniqueString(currentUserId)}', 20) +param msiName string @description('Metrics regional monitor name') -param monitorName string = take('aro-hcp-monitor-${uniqueString(currentUserId)}', 23) +param monitorName string @description('Metrics global Grafana name') -param grafanaName string = take('aro-hcp-grafana-${uniqueString(currentUserId)}', 23) +param grafanaName string + +@description('The admin group principal ID to manage Grafana') +param grafanaAdminGroupPrincipalId string var grafanaAdmin = { - principalId: '6b6d3adf-8476-4727-9812-20ffdef2b85c' // aro-hcp-engineering-App Developer + principalId: grafanaAdminGroupPrincipalId principalType: 'group' } diff --git a/dev-infrastructure/modules/rp-cosmos.bicep b/dev-infrastructure/modules/rp-cosmos.bicep index 2cb6bb1a0..9695aff95 100644 --- a/dev-infrastructure/modules/rp-cosmos.bicep +++ b/dev-infrastructure/modules/rp-cosmos.bicep @@ -1,6 +1,6 @@ // Constants // Maximum DB account name length is 44 -param name string = '${take(resourceGroup().name, 34)}-rp-cosmos' +param name string param disableLocalAuth bool = true // Passed Params and Overrides diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep index 5c4daec2d..868ab762b 100644 --- a/dev-infrastructure/templates/image-sync.bicep +++ b/dev-infrastructure/templates/image-sync.bicep @@ -2,13 +2,13 @@ param location string = resourceGroup().location @description('Specifies the name of the container app environment.') -param containerAppEnvName string = 'image-sync-env-${uniqueString(resourceGroup().id)}' +param containerAppEnvName string @description('Specifies the name of the log analytics workspace.') -param containerAppLogAnalyticsName string = 'containerapp-log-${uniqueString(resourceGroup().id)}' +param containerAppLogAnalyticsName string = 'containerapp-log' @description('Specifies the name of the user assigned managed identity.') -param imageSyncManagedIdentity string = 'image-sync-${uniqueString(resourceGroup().id)}' +param imageSyncManagedIdentity string = 'image-sync' @description('Resource group of the ACR containerapps will get permissions on') param acrResourceGroup string @@ -20,7 +20,7 @@ param requiredSecretNames array param keyVaultName string @description('Name of the KeyVault RG') -param keyVaultResourceGroup string = 'global' +param keyVaultResourceGroup string resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { name: containerAppLogAnalyticsName diff --git a/dev-infrastructure/templates/mgmt-cluster.bicep b/dev-infrastructure/templates/mgmt-cluster.bicep index 9899bdba6..a7addb63f 100644 --- a/dev-infrastructure/templates/mgmt-cluster.bicep +++ b/dev-infrastructure/templates/mgmt-cluster.bicep @@ -4,9 +4,6 @@ param location string = resourceGroup().location @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false -@description('Captures logged in users UID') -param currentUserId string - @description('AKS cluster name') param aksClusterName string = 'aro-hcp-aks' @@ -69,7 +66,6 @@ param deployMaestroConsumer bool param maestroCertDomain string @description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -@maxLength(24) param maestroKeyVaultName string @description('The name of the managed identity that will manage certificates in maestros keyvault.') @@ -78,13 +74,8 @@ param maestroKeyVaultCertOfficerMSIName string = '${maestroKeyVaultName}-cert-of @description('The name of the eventgrid namespace for Maestro.') param maestroEventGridNamespacesName string -@description('This is a global DNS zone name that will be the parent of regional DNS zones to host ARO HCP customer cluster DNS records') -param baseDNSZoneName string = '' - -@description('This is the region name in dev/staging/production') -param regionalDNSSubdomain string = empty(currentUserId) - ? location - : '${location}-${take(uniqueString(currentUserId), 5)}' +@description('This is a regional DNS zone') +param regionalDNSZoneName string @description('The resource group that hosts the regional zone') param regionalResourceGroup string @@ -98,7 +89,6 @@ resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { properties: { tags: { persist: toLower(string(persist)) - deployedBy: currentUserId } } } @@ -177,10 +167,10 @@ var externalDnsManagedIdentityPrincipalId = filter( )[0].uamiPrincipalID module dnsZoneContributor '../modules/dns/zone-contributor.bicep' = { - name: guid(regionalDNSSubdomain, mgmtCluster.name, 'external-dns') + name: guid(regionalDNSZoneName, mgmtCluster.name, 'external-dns') scope: resourceGroup(regionalResourceGroup) params: { - zoneName: '${regionalDNSSubdomain}.${baseDNSZoneName}' + zoneName: regionalDNSZoneName zoneContributerManagedIdentityPrincipalId: externalDnsManagedIdentityPrincipalId } } diff --git a/dev-infrastructure/templates/region.bicep b/dev-infrastructure/templates/region.bicep index 0ca81eecb..e7e984270 100644 --- a/dev-infrastructure/templates/region.bicep +++ b/dev-infrastructure/templates/region.bicep @@ -1,9 +1,6 @@ @description('Azure Region Location') param location string = resourceGroup().location -@description('Captures logged in users UID') -param currentUserId string - @description('The name of the keyvault for Maestro Eventgrid namespace certificates.') @maxLength(24) param maestroKeyVaultName string @@ -26,9 +23,7 @@ param baseDNSZoneName string @description('The resource group to deploy the base DNS zone to') param baseDNSZoneResourceGroup string = 'global' -param regionalDNSSubdomain string = empty(currentUserId) - ? location - : '${location}-${take(uniqueString(currentUserId), 5)}' +param regionalDNSSubdomain string // Tags the resource group resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { @@ -37,7 +32,6 @@ resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { properties: { tags: { persist: toLower(string(persist)) - deployedBy: currentUserId } } } diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index a572484cb..d8f6ac1ee 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -4,9 +4,6 @@ param location string = resourceGroup().location @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false -@description('Captures logged in users UID') -param currentUserId string - @description('AKS cluster name') param aksClusterName string @@ -51,6 +48,9 @@ param disableLocalAuth bool @description('Deploy ARO HCP RP Azure Cosmos DB if true') param deployFrontendCosmos bool +@description('The name of the Cosmos DB for the RP') +param rpCosmosDbName string + @description('The resourcegroup for regional infrastructure') param regionalResourceGroup string @@ -61,7 +61,6 @@ param maestroCertDomain string param maestroEventGridNamespacesName string @description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -@maxLength(24) param maestroKeyVaultName string @description('The name of the managed identity that will manage certificates in maestros keyvault.') @@ -123,13 +122,8 @@ param clustersServiceAcrResourceGroupNames array = [] @description('MSI that will be used to run the deploymentScript') param aroDevopsMsiId string -@description('This is a global DNS zone name that will be the parent of regional DNS zones to host ARO HCP customer cluster DNS records') -param baseDNSZoneName string = '' - -@description('This is the region name in dev/staging/production') -param regionalDNSSubdomain string = empty(currentUserId) - ? location - : '${location}-${take(uniqueString(currentUserId), 5)}' +@description('This is a regional DNS zone') +param regionalDNSZoneName string var clusterServiceMIName = 'clusters-service' @@ -140,7 +134,6 @@ resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { properties: { tags: { persist: toLower(string(persist)) - deployedBy: currentUserId } } } @@ -203,6 +196,7 @@ module rpCosmosDb '../modules/rp-cosmos.bicep' = if (deployFrontendCosmos) { name: 'rp_cosmos_db' scope: resourceGroup() params: { + name: rpCosmosDbName location: location aksNodeSubnetId: svcCluster.outputs.aksNodeSubnetId vnetId: svcCluster.outputs.aksVnetId @@ -315,10 +309,10 @@ module csServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep } module csDnsZoneContributor '../modules/dns/zone-contributor.bicep' = { - name: guid(regionalDNSSubdomain, svcCluster.name, 'cs') + name: guid(regionalDNSZoneName, svcCluster.name, 'cs') scope: resourceGroup(regionalResourceGroup) params: { - zoneName: '${regionalDNSSubdomain}.${baseDNSZoneName}' + zoneName: regionalDNSZoneName zoneContributerManagedIdentityPrincipalId: csManagedIdentityPrincipalId } } diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 000000000..aee2e4ce1 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1 @@ +config.mk diff --git a/frontend/Makefile b/frontend/Makefile index cfe075a53..95d1456ee 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -1,19 +1,11 @@ SHELL = /bin/bash - -# for deploying frontend into private aks cluster via invoke command -# these values must be set -AKSCONFIG ?= svc-cluster -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk +DEPLOY_ENV ?= personal-dev +$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk COMMIT = $(shell git rev-parse --short=7 HEAD) ARO_HCP_BASE_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io ARO_HCP_FRONTEND_IMAGE ?= $(ARO_HCP_BASE_IMAGE)/arohcpfrontend:$(COMMIT) -CLUSTER_NAME ?= -DEPLOYMENTNAME=$(RESOURCEGROUP) - -# dev-infrastructure defines this as REGION -LOCATION ?= ${REGION} frontend: go build -o aro-hcp-frontend . @@ -38,23 +30,22 @@ image: push: image docker push ${ARO_HCP_FRONTEND_IMAGE} -deploy: +deploy: FRONTEND_MI_CLIENT_ID=$(shell az identity show \ -g ${RESOURCEGROUP} \ -n frontend \ --query clientId);\ - DB_NAME=$(shell az cosmosdb list -g ${RESOURCEGROUP} | jq -r '.[].name') DB_NAME=$${DB_NAME:-"none"};\ - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ + ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ kubectl create namespace aro-hcp --dry-run=client -o json | kubectl apply -f - && \ kubectl label namespace aro-hcp "istio.io/rev=$${ISTO_VERSION}" --overwrite=true && \ helm upgrade --install aro-hcp-frontend-dev \ deploy/helm/frontend/ \ - --set configMap.databaseName=$${DB_NAME} \ - --set configMap.databaseUrl="https://$${DB_NAME}.documents.azure.com:443/" \ + --set configMap.databaseName=${DB_NAME} \ + --set configMap.databaseUrl="https://${DB_NAME}.documents.azure.com:443/" \ --set configMap.frontendMiClientId="$${FRONTEND_MI_CLIENT_ID}" \ --set serviceAccount.workloadIdentityClientId="$${FRONTEND_MI_CLIENT_ID}" \ --set configMap.currentVersion=${ARO_HCP_FRONTEND_IMAGE} \ - --set configMap.location=${LOCATION} \ + --set configMap.location=${LOCATION} \ --set deployment.imageName=${ARO_HCP_FRONTEND_IMAGE} \ --namespace aro-hcp diff --git a/frontend/README.md b/frontend/README.md index c63383b7a..8130b3a46 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -34,9 +34,8 @@ docker run -p 8443:8443 aro-hcp-frontend **In Cluster:** ```bash -# Requires kustomize CLI to be installed -AKSCONFIG=svc-cluster make deploy -AKSCONFIG=svc-cluster make undeploy +make deploy +make undeploy ``` > To create a cluster, follow the instructions in [development-setup.md](../dev-infrastructure/docs/development-setup.md) diff --git a/frontend/config.tmpl.mk b/frontend/config.tmpl.mk new file mode 100644 index 000000000..9b959270f --- /dev/null +++ b/frontend/config.tmpl.mk @@ -0,0 +1,5 @@ +ARO_HCP_IMAGE_ACR ?= {{ .acrName }} +LOCATION ?= {{ .region }} +RESOURCEGROUP ?= {{ .serviceClusterRG }} +AKS_NAME ?= {{ .aksName }} +DB_NAME ?= {{ .frontendCosmosDBName }} diff --git a/maestro/.gitignore b/maestro/.gitignore new file mode 100644 index 000000000..73e3765ee --- /dev/null +++ b/maestro/.gitignore @@ -0,0 +1 @@ +**/config.mk diff --git a/maestro/Makefile b/maestro/Makefile deleted file mode 100644 index b49973326..000000000 --- a/maestro/Makefile +++ /dev/null @@ -1,69 +0,0 @@ -SHELL = /bin/bash - -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk - -CONSUMER_NAME ?= $(shell az aks list --query "[?tags.clusterType == 'mgmt-cluster' && starts_with(resourceGroup, '$(REGIONAL_RESOURCEGROUP)')].resourceGroup" -o tsv) -EVENTGRID_ID = $(shell az resource list -g ${REGIONAL_RESOURCEGROUP} --resource-type "Microsoft.EventGrid/namespaces" --query "[].id" -o tsv) - -RESTRICT_ISTIO_INGRESS ?= true - -MAESTRO_BASE_IMAGE ?= quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro -MAESTRO_TAG ?= ea066c250a002f0cc458711945165591bc9f6d3f - -deploy-server: - MAESTRO_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n maestro-server \ - --query clientId) && \ - TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - MAESTRO_KV_NAME=$(shell az keyvault list --query "[?starts_with(name, 'maestro-kv')].name" -g ${REGIONAL_RESOURCEGROUP} --output tsv) && \ - EVENTGRID_CA_BASE64='$(shell cat ../dev-infrastructure/scripts/digicert-global-root-g3.crt | base64 | tr -d '\n')' && \ - EVENTGRID_HOSTNAME=$(shell az resource show --ids ${EVENTGRID_ID} --query properties.topicSpacesConfiguration.hostname -o tsv) && \ - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ - kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f - && \ - kubectl label namespace maestro "istio.io/rev=$${ISTO_VERSION}" --overwrite=true && \ - helm upgrade --install maestro-server deploy/helm/server \ - --namespace maestro \ - --set broker.host=$${EVENTGRID_HOSTNAME} \ - --set credsKeyVault.name=$${MAESTRO_KV_NAME} \ - --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ - --set azure.tenantId=$${TENANT_ID} \ - --set istio.restrictIngress=${RESTRICT_ISTIO_INGRESS} \ - --set image.base=${MAESTRO_BASE_IMAGE} \ - --set image.tag=${MAESTRO_TAG} \ - --set database.containerizedDb=true \ - --set database.ssl=disable - -deploy-agent: - MAESTRO_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n maestro-consumer \ - --query clientId -o tsv) && \ - TENANT_ID=$(shell az account show --query tenantId -o tsv) && \ - MAESTRO_KV_NAME=$(shell az keyvault list --query "[?starts_with(name, 'maestro-kv')].name" -g ${REGIONAL_RESOURCEGROUP} -o tsv) && \ - EVENTGRID_HOSTNAME=$(shell az resource show --ids ${EVENTGRID_ID} --query properties.topicSpacesConfiguration.hostname -o tsv) && \ - helm upgrade --install maestro-agent deploy/helm/agent \ - --create-namespace --namespace maestro \ - --set consumerName=${CONSUMER_NAME} \ - --set broker.host=$${EVENTGRID_HOSTNAME} \ - --set credsKeyVault.name=$${MAESTRO_KV_NAME} \ - --set credsKeyVault.secret=${CONSUMER_NAME} \ - --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ - --set azure.tenantId=$${TENANT_ID} \ - --set image.base=${MAESTRO_BASE_IMAGE} \ - --set image.tag=${MAESTRO_TAG} - -register-agent: -ifeq ($(strip ${CONSUMER_NAME}),) - $(error "Must set CONSUMER_NAME") -endif - @if ! kubectl get service maestro -n maestro > /dev/null 2>&1; then \ - echo "Error: Service 'maestro' not found in namespace 'maestro'"; \ - exit 1; \ - fi - helm upgrade --install ${CONSUMER_NAME} deploy/helm/registration \ - --namespace maestro \ - --set consumerName=${CONSUMER_NAME} - -.PHONY: deploy-server deploy-agent register-agent diff --git a/maestro/agent/Makefile b/maestro/agent/Makefile new file mode 100644 index 000000000..53facd552 --- /dev/null +++ b/maestro/agent/Makefile @@ -0,0 +1,17 @@ +SHELL = /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk + +deploy: + helm upgrade --install maestro-agent ./helm \ + --create-namespace --namespace maestro \ + --set consumerName=${CONSUMER_NAME} \ + --set broker.host=${EVENTGRID_HOSTNAME} \ + --set credsKeyVault.name=${KEYVAULT_NAME} \ + --set credsKeyVault.secret=${CONSUMER_NAME} \ + --set azure.clientId=${MAESTRO_MI_CLIENT_ID} \ + --set azure.tenantId=${TENANT_ID} \ + --set image.base=${IMAGE_BASE} \ + --set image.tag=${IMAGE_TAG} +.PHONY: deploy diff --git a/maestro/agent/config.tmpl.mk b/maestro/agent/config.tmpl.mk new file mode 100644 index 000000000..4cb6ae9dd --- /dev/null +++ b/maestro/agent/config.tmpl.mk @@ -0,0 +1,7 @@ +TENANT_ID=$(shell az account show --query tenantId --output tsv) +CONSUMER_NAME ?= {{ .maestroConsumerName }} +EVENTGRID_HOSTNAME=$(shell az resource show -n {{ .maestroEventgridName }} -g {{ .regionRG }} --resource-type "Microsoft.EventGrid/namespaces" --query properties.topicSpacesConfiguration.hostname -o tsv) +MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "{{ .managementClusterRG }}" -n maestro-consumer --query clientId -o tsv) +KEYVAULT_NAME ?= {{ .maestroKeyVaultName }} +IMAGE_BASE ?= {{ .maestroImageBase }} +IMAGE_TAG ?= {{ .maestroImageTag }} diff --git a/maestro/deploy/helm/agent/Chart.yaml b/maestro/agent/helm/Chart.yaml similarity index 100% rename from maestro/deploy/helm/agent/Chart.yaml rename to maestro/agent/helm/Chart.yaml diff --git a/maestro/deploy/helm/agent/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml b/maestro/agent/helm/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml rename to maestro/agent/helm/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrole.yaml b/maestro/agent/helm/templates/maestro-agent.agent.clusterrole.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrole.yaml rename to maestro/agent/helm/templates/maestro-agent.agent.clusterrole.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.agent.clusterrolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.agent.clusterrolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent.role.yaml b/maestro/agent/helm/templates/maestro-agent.agent.role.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent.role.yaml rename to maestro/agent/helm/templates/maestro-agent.agent.role.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent.rolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.agent.rolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent.rolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.agent.rolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.role.yaml b/maestro/agent/helm/templates/maestro-agent.agent:extension-apiserver.role.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.role.yaml rename to maestro/agent/helm/templates/maestro-agent.agent:extension-apiserver.role.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.deployment.yaml b/maestro/agent/helm/templates/maestro-agent.deployment.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.deployment.yaml rename to maestro/agent/helm/templates/maestro-agent.deployment.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.execution-admin.clusterrolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.execution-admin.clusterrolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.execution-admin.clusterrolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.execution-admin.clusterrolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.execution.clusterrolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.execution.clusterrolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.execution.clusterrolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.execution.clusterrolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro.secret.yaml b/maestro/agent/helm/templates/maestro.secret.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro.secret.yaml rename to maestro/agent/helm/templates/maestro.secret.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro.secretproviderclass.yaml b/maestro/agent/helm/templates/maestro.secretproviderclass.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro.secretproviderclass.yaml rename to maestro/agent/helm/templates/maestro.secretproviderclass.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro.serviceaccount.yaml b/maestro/agent/helm/templates/maestro.serviceaccount.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro.serviceaccount.yaml rename to maestro/agent/helm/templates/maestro.serviceaccount.yaml diff --git a/maestro/deploy/helm/agent/values.yaml b/maestro/agent/helm/values.yaml similarity index 100% rename from maestro/deploy/helm/agent/values.yaml rename to maestro/agent/helm/values.yaml diff --git a/maestro/registration/Makefile b/maestro/registration/Makefile new file mode 100644 index 000000000..d62b33184 --- /dev/null +++ b/maestro/registration/Makefile @@ -0,0 +1,14 @@ +SHELL = /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk + +deploy: + @if ! kubectl get service maestro -n maestro > /dev/null 2>&1; then \ + echo "Error: Service 'maestro' not found in namespace 'maestro'"; \ + exit 1; \ + fi + helm upgrade --install ${CONSUMER_NAME} ./helm \ + --namespace maestro \ + --set consumerName=${CONSUMER_NAME} +.PHONY: deploy diff --git a/maestro/registration/config.tmpl.mk b/maestro/registration/config.tmpl.mk new file mode 100644 index 000000000..8918876a7 --- /dev/null +++ b/maestro/registration/config.tmpl.mk @@ -0,0 +1 @@ +CONSUMER_NAME ?= {{ .maestroConsumerName }} diff --git a/maestro/deploy/helm/registration/Chart.yaml b/maestro/registration/helm/Chart.yaml similarity index 100% rename from maestro/deploy/helm/registration/Chart.yaml rename to maestro/registration/helm/Chart.yaml diff --git a/maestro/deploy/helm/registration/templates/registration.job.yaml b/maestro/registration/helm/templates/registration.job.yaml similarity index 100% rename from maestro/deploy/helm/registration/templates/registration.job.yaml rename to maestro/registration/helm/templates/registration.job.yaml diff --git a/maestro/deploy/helm/registration/templates/script.yaml b/maestro/registration/helm/templates/script.yaml similarity index 100% rename from maestro/deploy/helm/registration/templates/script.yaml rename to maestro/registration/helm/templates/script.yaml diff --git a/maestro/deploy/helm/registration/values.yaml b/maestro/registration/helm/values.yaml similarity index 100% rename from maestro/deploy/helm/registration/values.yaml rename to maestro/registration/helm/values.yaml diff --git a/maestro/server/Makefile b/maestro/server/Makefile new file mode 100644 index 000000000..0f085fc24 --- /dev/null +++ b/maestro/server/Makefile @@ -0,0 +1,20 @@ +SHELL = /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk + +deploy: + kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f - + kubectl label namespace maestro "istio.io/rev=${ISTO_VERSION}" --overwrite=true + helm upgrade --install maestro-server ./helm \ + --namespace maestro \ + --set broker.host=${EVENTGRID_HOSTNAME} \ + --set credsKeyVault.name=${KEYVAULT_NAME} \ + --set azure.clientId=${MAESTRO_MI_CLIENT_ID} \ + --set azure.tenantId=${TENANT_ID} \ + --set istio.restrictIngress=${ISTIO_RESTRICT_INGRESS} \ + --set image.base=${IMAGE_BASE} \ + --set image.tag=${IMAGE_TAG} \ + --set database.containerizedDb=${USE_CONTAINERIZED_DB} \ + --set database.ssl='${USE_DATABASE_SSL}' +.PHONY: deploy diff --git a/maestro/server/config.tmpl.mk b/maestro/server/config.tmpl.mk new file mode 100644 index 000000000..c4d48e345 --- /dev/null +++ b/maestro/server/config.tmpl.mk @@ -0,0 +1,10 @@ +TENANT_ID=$(shell az account show --query tenantId --output tsv) +MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "{{ .serviceClusterRG }}" -n maestro-server --query clientId -o tsv) +EVENTGRID_HOSTNAME=$(shell az resource show -n {{ .maestroEventgridName }} -g {{ .regionRG }} --resource-type "Microsoft.EventGrid/namespaces" --query properties.topicSpacesConfiguration.hostname -o tsv) +ISTO_VERSION=$(shell az aks show -n {{ .aksName }} -g {{ .serviceClusterRG }} --query serviceMeshProfile.istio.revisions[-1] -o tsv) +IMAGE_BASE ?= {{ .maestroImageBase }} +IMAGE_TAG ?= {{ .maestroImageTag }} +USE_CONTAINERIZED_DB ?= {{ not .maestroPostgresDeploy }} +USE_DATABASE_SSL ?= {{ ternary "enable" "disable" .maestroPostgresDeploy }} +ISTIO_RESTRICT_INGRESS ?= {{ .maestroRestrictIstioIngress }} +KEYVAULT_NAME ?= {{ .maestroKeyVaultName }} diff --git a/maestro/deploy/helm/server/Chart.yaml b/maestro/server/helm/Chart.yaml similarity index 100% rename from maestro/deploy/helm/server/Chart.yaml rename to maestro/server/helm/Chart.yaml diff --git a/maestro/deploy/helm/server/templates/allow-cluster-service.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-cluster-service.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-cluster-service.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-cluster-service.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/allow-maestro-to-db.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-maestro-to-db.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-maestro-to-db.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-maestro-to-db.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/allow-metrics.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-metrics.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-metrics.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-metrics.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/allow-nothing.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-nothing.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-nothing.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-nothing.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/grpc.service.yaml b/maestro/server/helm/templates/grpc.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/grpc.service.yaml rename to maestro/server/helm/templates/grpc.service.yaml diff --git a/maestro/deploy/helm/server/templates/health.service.yaml b/maestro/server/helm/templates/health.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/health.service.yaml rename to maestro/server/helm/templates/health.service.yaml diff --git a/maestro/deploy/helm/server/templates/http.service.yaml b/maestro/server/helm/templates/http.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/http.service.yaml rename to maestro/server/helm/templates/http.service.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.deployment.yaml b/maestro/server/helm/templates/maestro.deployment.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.deployment.yaml rename to maestro/server/helm/templates/maestro.deployment.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.peerauthentication.yaml b/maestro/server/helm/templates/maestro.peerauthentication.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.peerauthentication.yaml rename to maestro/server/helm/templates/maestro.peerauthentication.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.secret.yaml b/maestro/server/helm/templates/maestro.secret.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.secret.yaml rename to maestro/server/helm/templates/maestro.secret.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.secretproviderclass.yaml b/maestro/server/helm/templates/maestro.secretproviderclass.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.secretproviderclass.yaml rename to maestro/server/helm/templates/maestro.secretproviderclass.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.serviceaccount.yaml b/maestro/server/helm/templates/maestro.serviceaccount.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.serviceaccount.yaml rename to maestro/server/helm/templates/maestro.serviceaccount.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.servicemonitor.yaml b/maestro/server/helm/templates/maestro.servicemonitor.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.servicemonitor.yaml rename to maestro/server/helm/templates/maestro.servicemonitor.yaml diff --git a/maestro/deploy/helm/server/templates/metrics.service.yaml b/maestro/server/helm/templates/metrics.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/metrics.service.yaml rename to maestro/server/helm/templates/metrics.service.yaml diff --git a/maestro/deploy/helm/server/templates/pg.deployment.yaml b/maestro/server/helm/templates/pg.deployment.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/pg.deployment.yaml rename to maestro/server/helm/templates/pg.deployment.yaml diff --git a/maestro/deploy/helm/server/templates/pg.pvc.yaml b/maestro/server/helm/templates/pg.pvc.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/pg.pvc.yaml rename to maestro/server/helm/templates/pg.pvc.yaml diff --git a/maestro/deploy/helm/server/templates/pg.secret.yaml b/maestro/server/helm/templates/pg.secret.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/pg.secret.yaml rename to maestro/server/helm/templates/pg.secret.yaml diff --git a/maestro/deploy/helm/server/templates/pg.service.yaml b/maestro/server/helm/templates/pg.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/pg.service.yaml rename to maestro/server/helm/templates/pg.service.yaml diff --git a/maestro/deploy/helm/server/values.yaml b/maestro/server/helm/values.yaml similarity index 100% rename from maestro/deploy/helm/server/values.yaml rename to maestro/server/helm/values.yaml diff --git a/svc-deploy.sh b/svc-deploy.sh new file mode 100755 index 000000000..b9bc21e2f --- /dev/null +++ b/svc-deploy.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# deploy a service to a cluster +# ./svc-deploy +# this script expects the to contain a Makefile that takes care +# of processing any config.mk template on its own + +cd $(dirname "$(realpath "${BASH_SOURCE[0]}")") + +export DEPLOY_ENV=$1 +export DIR=$2 +export CLUSTER=$3 + +if [[ "$CLUSTER" != "svc" && "$CLUSTER" != "mgmt" ]]; then + echo "Error: CLUSTER must be either 'svc' or 'mgmt'." + exit 1 +fi + +export KUBECONFIG=$(./dev-infrastructure/make $DEPLOY_ENV $CLUSTER.aks.kubeconfigfile) + +cd $DIR +make deploy diff --git a/templatize.sh b/templatize.sh new file mode 100755 index 000000000..b309b6674 --- /dev/null +++ b/templatize.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +PROJECT_ROOT_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") + +DEPLOY_ENV=$1 +INPUT=$2 +OUTPUT=$3 + +if [ "$DEPLOY_ENV" == "personal-dev" ]; then + REGION_STAMP=$USER +else + REGION_STAMP=$DEPLOY_ENV +fi + +if [ -z "$CLOUD" ]; then + CLOUD="public" +fi + +if [ -z "$REGION" ]; then + REGION="westus3" +fi + +if [ -z "$CXSTAMP" ]; then + CXSTAMP="1" +fi + +CONFIG_FILE=${PROJECT_ROOT_DIR}/config/config.yaml +if [ -n "$INPUT" ] && [ -n "$OUTPUT" ]; then + go run ${PROJECT_ROOT_DIR}/tooling/templatize/main.go generate \ + --config-file=${CONFIG_FILE} \ + --cloud=$CLOUD \ + --deploy-env=$DEPLOY_ENV \ + --region=$REGION \ + --region-stamp=${REGION_STAMP} \ + --cx-stamp=$CXSTAMP \ + --input=$INPUT \ + --output=$OUTPUT +else + go run ${PROJECT_ROOT_DIR}/tooling/templatize/main.go inspect \ + --config-file=${CONFIG_FILE} \ + --cloud=$CLOUD \ + --deploy-env=$DEPLOY_ENV \ + --region=$REGION \ + --region-stamp=${REGION_STAMP} \ + --cx-stamp=$CXSTAMP +fi diff --git a/tooling/templatize/cmd/generate/generate_test.go b/tooling/templatize/cmd/generate/generate_test.go index abcd38ca2..2f39319d4 100644 --- a/tooling/templatize/cmd/generate/generate_test.go +++ b/tooling/templatize/cmd/generate/generate_test.go @@ -27,24 +27,22 @@ func TestExecuteTemplate(t *testing.T) { "region_maestro_keyvault": "kv", "region_eventgrid_namespace": "ns", }, - input: `param maestroKeyVaultName = '{{index . "region_maestro_keyvault"}}' -param maestroEventGridNamespacesName = '{{index . "region_eventgrid_namespace"}}' + input: `param maestroKeyVaultName = '{{ .region_maestro_keyvault }}' +param maestroEventGridNamespacesName = '{{ .region_eventgrid_namespace }}' param maestroEventGridMaxClientSessionsPerAuthName = 4`, expected: `param maestroKeyVaultName = 'kv' param maestroEventGridNamespacesName = 'ns' param maestroEventGridMaxClientSessionsPerAuthName = 4`, }, { - name: "referencing unset variable errors", // TODO: this does not error today, just gets an empty string, this is not the UX we want + name: "referencing unset variable errors", config: config.Variables{ "region_maestro_keyvault": "kv", }, - input: `param maestroKeyVaultName = '{{index . "region_maestro_keyvault"}}' -param maestroEventGridNamespacesName = '{{index . "region_eventgrid_namespace"}}' -param maestroEventGridMaxClientSessionsPerAuthName = 4`, - expected: `param maestroKeyVaultName = 'kv' -param maestroEventGridNamespacesName = '' + input: `param maestroKeyVaultName = '{{ .region_maestro_keyvault }}' +param maestroEventGridNamespacesName = '{{ .region_eventgrid_namespace }}' param maestroEventGridMaxClientSessionsPerAuthName = 4`, + expectedError: true, }, } { t.Run(testCase.name, func(t *testing.T) { diff --git a/tooling/templatize/cmd/generate/options.go b/tooling/templatize/cmd/generate/options.go index b8cc585e3..7484dd8d3 100644 --- a/tooling/templatize/cmd/generate/options.go +++ b/tooling/templatize/cmd/generate/options.go @@ -9,6 +9,7 @@ import ( "path/filepath" "text/template" + "github.com/Masterminds/sprig/v3" "github.com/spf13/cobra" options "github.com/Azure/ARO-HCP/tooling/templatize/cmd" @@ -25,7 +26,7 @@ func BindGenerationOptions(opts *RawGenerationOptions, cmd *cobra.Command) error return fmt.Errorf("failed to bind raw options: %w", err) } cmd.Flags().StringVar(&opts.Input, "input", opts.Input, "input file path") - cmd.Flags().StringVar(&opts.Output, "output", opts.Output, "output file directory") + cmd.Flags().StringVar(&opts.Output, "output", opts.Output, "output file path") for _, flag := range []string{"config-file", "input", "output"} { if err := cmd.MarkFlagFilename("config-file"); err != nil { @@ -73,11 +74,11 @@ func (o *ValidatedGenerationOptions) Complete() (*GenerationOptions, error) { inputFile := filepath.Base(o.Input) - if err := os.MkdirAll(o.Output, os.ModePerm); err != nil { + if err := os.MkdirAll(filepath.Dir(o.Output), os.ModePerm); err != nil { return nil, fmt.Errorf("failed to create output directory %s: %w", o.Output, err) } - output, err := os.Create(filepath.Join(o.Output, inputFile)) + output, err := os.Create(o.Output) if err != nil { return nil, fmt.Errorf("failed to create output file %s: %w", o.Input, err) } @@ -106,7 +107,13 @@ type GenerationOptions struct { } func (opts *GenerationOptions) ExecuteTemplate() error { - tmpl, err := template.New(opts.InputFile).ParseFS(opts.Input, opts.InputFile) + tmpl := template.New(opts.InputFile).Funcs(sprig.FuncMap()) + content, err := fs.ReadFile(opts.Input, opts.InputFile) + if err != nil { + return err + } + + tmpl, err = tmpl.Parse(string(content)) if err != nil { return err } @@ -116,5 +123,5 @@ func (opts *GenerationOptions) ExecuteTemplate() error { log.Printf("error closing output: %v\n", err) } }() - return tmpl.ExecuteTemplate(opts.Output, opts.InputFile, opts.Config) + return tmpl.Option("missingkey=error").ExecuteTemplate(opts.Output, opts.InputFile, opts.Config) } diff --git a/tooling/templatize/cmd/generate/options_test.go b/tooling/templatize/cmd/generate/options_test.go index 4a44101dd..3fd96d475 100644 --- a/tooling/templatize/cmd/generate/options_test.go +++ b/tooling/templatize/cmd/generate/options_test.go @@ -1,6 +1,7 @@ package generate import ( + "fmt" "path/filepath" "testing" @@ -15,14 +16,14 @@ func TestRawOptions(t *testing.T) { opts := &RawGenerationOptions{ RawOptions: options.RawOptions{ ConfigFile: "../../testdata/config.yaml", - Cloud: "fairfax", - DeployEnv: "prod", + Cloud: "public", + DeployEnv: "dev", Region: "uksouth", RegionStamp: "1", CXStamp: "cx", }, Input: "../../testdata/helm.sh", - Output: tmpdir, + Output: fmt.Sprintf("%s/helm.sh", tmpdir), } assert.NoError(t, generate(opts)) testutil.CompareFileWithFixture(t, filepath.Join(tmpdir, "helm.sh")) diff --git a/tooling/templatize/go.mod b/tooling/templatize/go.mod index 1bf3aef20..73bc01893 100644 --- a/tooling/templatize/go.mod +++ b/tooling/templatize/go.mod @@ -3,8 +3,29 @@ module github.com/Azure/ARO-HCP/tooling/templatize go 1.23.0 require ( + github.com/Masterminds/sprig/v3 v3.3.0 + github.com/google/go-cmp v0.6.0 github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.9.0 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/apimachinery v0.31.1 + sigs.k8s.io/yaml v1.4.0 ) -require github.com/inconshreveable/mousetrap v1.1.0 // indirect +require ( + dario.cat/mergo v1.0.1 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/crypto v0.26.0 // indirect +) diff --git a/tooling/templatize/go.sum b/tooling/templatize/go.sum index 912390a78..412df81cc 100644 --- a/tooling/templatize/go.sum +++ b/tooling/templatize/go.sum @@ -1,10 +1,57 @@ +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= +k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/tooling/templatize/internal/config/config.go b/tooling/templatize/internal/config/config.go index 7791a4973..d7004c6e3 100644 --- a/tooling/templatize/internal/config/config.go +++ b/tooling/templatize/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "fmt" "os" "text/template" @@ -47,6 +48,8 @@ func (cp *configProviderImpl) GetVariables(cloud, deployEnv string) (Variables, variables[k] = v } } + } else { + return nil, fmt.Errorf("the deployment env %s is not found under cloud %s in %s", deployEnv, cloud, cp.config) } } } @@ -66,9 +69,12 @@ func (cp *configProviderImpl) loadConfig(cloud, deployEnv string) (*VariableOver } functions := template.FuncMap{ - "azureEventGridName": naming.AzureEventGridName, - "azurePostgresName": naming.AzurePostgresName, - "azureKeyVaultName": naming.AzureKeyVaultName, + "azureEventGridName": naming.AzureEventGridName, + "azurePostgresName": naming.AzurePostgresName, + "azureKeyVaultName": naming.AzureKeyVaultName, + "azureStorageAccountName": naming.AzureStorageAccountName, + "azureCosmosDBName": naming.AzureCosmosDBName, + "uniqueString": naming.UniqueString, } // parse, execute and unmarshal the config file as a template to generate the final config file @@ -84,13 +90,15 @@ func (cp *configProviderImpl) loadConfig(cloud, deployEnv string) (*VariableOver } var tmplBytes bytes.Buffer - if err := tmpl.Execute(&tmplBytes, vars); err != nil { + if err := tmpl.Option("missingkey=error").Execute(&tmplBytes, vars); err != nil { return nil, err } currentVariableOverrides := &VariableOverrides{} if err := yaml.Unmarshal(tmplBytes.Bytes(), currentVariableOverrides); err == nil { cp.baseVariableOverrides = currentVariableOverrides + } else { + return nil, err } return cp.baseVariableOverrides, err diff --git a/tooling/templatize/internal/config/config_test.go b/tooling/templatize/internal/config/config_test.go index 557400a5d..fe98028a3 100644 --- a/tooling/templatize/internal/config/config_test.go +++ b/tooling/templatize/internal/config/config_test.go @@ -19,7 +19,7 @@ func TestConfigProvider(t *testing.T) { assert.NotNil(t, variables) // key is not in the config file - assert.Equal(t, "", variables["svc_resourcegroup"]) + assert.Nil(t, variables["svc_resourcegroup"]) // key is in the config file, region constant value assert.Equal(t, "uksouth", variables["test"]) diff --git a/tooling/templatize/internal/config/types.go b/tooling/templatize/internal/config/types.go index 9eb8c91c1..aafc3e593 100644 --- a/tooling/templatize/internal/config/types.go +++ b/tooling/templatize/internal/config/types.go @@ -8,7 +8,7 @@ type configProviderImpl struct { cxStamp string } -type Variables map[string]string +type Variables map[string]interface{} type VariableOverrides struct { Defaults Variables `yaml:"defaults"` diff --git a/tooling/templatize/internal/naming/azure.go b/tooling/templatize/internal/naming/azure.go index 4cf4976d5..3c99eb933 100644 --- a/tooling/templatize/internal/naming/azure.go +++ b/tooling/templatize/internal/naming/azure.go @@ -11,3 +11,11 @@ func AzurePostgresName(prefix string, suffixLength int, suffixDigestArgs ...stri func AzureKeyVaultName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { return suffixedName(prefix, "-", 24, suffixLength, suffixDigestArgs...) } + +func AzureStorageAccountName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { + return suffixedName(prefix, "", 24, suffixLength, suffixDigestArgs...) +} + +func AzureCosmosDBName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { + return suffixedName(prefix, "-", 44, suffixLength, suffixDigestArgs...) +} diff --git a/tooling/templatize/internal/naming/common.go b/tooling/templatize/internal/naming/common.go index cd43db7b6..7822ebb87 100644 --- a/tooling/templatize/internal/naming/common.go +++ b/tooling/templatize/internal/naming/common.go @@ -33,3 +33,7 @@ func suffixedName(prefix string, suffixDelim string, maxLength int, suffixLength } return name, nil } + +func UniqueString(length int, digestArgs ...string) (string, error) { + return suffixDigest(length, digestArgs...) +} diff --git a/tooling/templatize/testdata/helm.sh b/tooling/templatize/testdata/helm.sh index 4602ad993..cf099fc08 100644 --- a/tooling/templatize/testdata/helm.sh +++ b/tooling/templatize/testdata/helm.sh @@ -1,13 +1,13 @@ # copy from maestro/Makefile#L14 deploy-server: - TENANT_ID="{{index . "tenantId"}}" - REGION_RG="{{index . "region_resourcegroup"}}" - EVENTGRID_NS="{{index . "region_eventgrid_namespace"}}" - MAESTRO_KV="{{index . "region_maestro_keyvault"}}" - SERVICE_RG="{{index . "svc_resourcegroup"}}" - AKS="{{index . "aks_name"}}" - MAESTRO_MI="{{index . "maestro_msi"}}" - HELM_CHART="{{index . "maestro_helm_chart"}}" + TENANT_ID="{{ .tenantId }}" + REGION_RG="{{ .region_resourcegroup }}" + EVENTGRID_NS="{{ .region_eventgrid_namespace }}" + MAESTRO_KV="{{ .region_maestro_keyvault }}" + SERVICE_RG="{{ .svc_resourcegroup }}" + AKS="{{ .aks_name }}" + MAESTRO_MI="{{ .maestro_msi }}" + HELM_CHART="{{ .maestro_helm_chart }}" EVENTGRID_HOSTNAME=$(az event namespace show -g "${REGION_RG}" -n "${EVENTGRID_NS}" --query "properties.topicSpacesConfiguration.hostname") MAESTRO_MI_CLIENT_ID=$(az identity show -g "${SERVICE_RG}" -n "${MAESTRO_MI}" --query "clientId") diff --git a/tooling/templatize/testdata/test.bicepparam b/tooling/templatize/testdata/test.bicepparam index ffae6aad8..8fa281a11 100644 --- a/tooling/templatize/testdata/test.bicepparam +++ b/tooling/templatize/testdata/test.bicepparam @@ -6,8 +6,8 @@ param baseDNSZoneName = 'hcp.osadev.cloud' param baseDNSZoneResourceGroup = 'global' // maestro -param maestroKeyVaultName = '{{index . "region_maestro_keyvault"}}' -param maestroEventGridNamespacesName = '{{index . "region_eventgrid_namespace"}}' +param maestroKeyVaultNamea = '{{ .region_maestro_keyvault }}' +param maestroEventGridNamespacesName = '{{ .region_eventgrid_namespace }}' param maestroEventGridMaxClientSessionsPerAuthName = 4 // These parameters are always overriden in the Makefile diff --git a/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh b/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh index 200d3d0cb..198bb1a47 100644 --- a/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh +++ b/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh @@ -3,11 +3,11 @@ deploy-server: TENANT_ID="72f988bf-86f1-41af-91ab-2d7cd011db47" REGION_RG="hcp-underlay-uksouth-1" EVENTGRID_NS="maestro-eventgrid-uksouth-1" - MAESTRO_KV="" - SERVICE_RG="" + MAESTRO_KV="maestro-kv-uksouth-1" + SERVICE_RG="hcp-underlay-uksouth-svc-1" AKS="aro-hcp-aks" MAESTRO_MI="maestro-server" - HELM_CHART="" + HELM_CHART="../maestro/deploy/helm/server" EVENTGRID_HOSTNAME=$(az event namespace show -g "${REGION_RG}" -n "${EVENTGRID_NS}" --query "properties.topicSpacesConfiguration.hostname") MAESTRO_MI_CLIENT_ID=$(az identity show -g "${SERVICE_RG}" -n "${MAESTRO_MI}" --query "clientId")