Allow creating both mgmt/svc cluster idempotently

Signed-off-by: Michael Shen <[email protected]>

dev-infrastructure/Makefile

@@ -4,9 +4,9 @@ HCPDEVSUBSCRIPTION="ARO Hosted Control Planes (EA Subscription 1)"
 HCPDEVSUBSCRIPTIONID=1d3378d3-5a3f-4712-85a1-2485495dfc4b
 
 CURRENTUSER=$(shell az ad signed-in-user show | jq -r '.id')
-DEPLOYMENTNAME=hcp-$(USER)-dev-infra
 LOCATION?=eastus
 RESOURCEGROUP=aro-hcp-${AKSCONFIG}-$(USER)
+DEPLOYMENTNAME=$(RESOURCEGROUP)
 
 list:
 	@grep '^[^#[:space:]].*:' Makefile
@@ -27,15 +27,6 @@ lint:
 	az bicep lint --file $$file; \
 	done
 
-show:
-	@echo "Resource group      : $(RESOURCEGROUP)"
-	@echo "VPN resource group  : $(VPNRESOURCEGROUP)"
-	@echo "AddressSpace prefix : $(PREFIX)"
-	@echo "AKS version         : $(AKSVERSION)"
-	@echo "Location            : $(LOCATION)"
-	@echo "AKS VNet Name       : $(AKSVNETNAME)"
-	@echo "DNSZone/deployment  : $(DEPLOYMENTNAME)"
-
 setsubscription:
 ifndef AKSCONFIG
 	$(error "Must set AKSCONFIG")
@@ -71,7 +62,8 @@ mgmt-cluster: setsubscription rg
 			currentUserId=$(CURRENTUSER)
 
 aks.kubeconfig: setsubscription
-	az aks get-credentials -n aro-hcp-cluster-001 -g $(RESOURCEGROUP) -a -f aks.kubeconfig
+	AKS_NAME="$$(az aks list --query "[?tags.clusterType == '$(AKSCONFIG)'].name" -ojson | jq -r '.[0]')"; \
+	az aks get-credentials -n "$${AKS_NAME}" -g $(RESOURCEGROUP) -a -f "$(AKSCONFIG).kubeconfig"
 
 secrets-download: setsubscription
 	@[ "${SECRET_SA_ACCOUNT_NAME}" ] || ( echo ">> SECRET_SA_ACCOUNT_NAME is not set"; exit 1 )

dev-infrastructure/configurations/mgmt-cluster.bicepparam

@@ -1,9 +1,10 @@
 using '../templates/mgmt-cluster.bicep'
 
 param kubernetesVersion = '1.29.2'
-param vnetAddressPrefix = enablePrivateCluster ? '10.132.0.0/14' : '10.128.0.0/14'
-param subnetPrefix = enablePrivateCluster ? '10.132.8.0/21' : '10.128.8.0/21'
-param podSubnetPrefix = enablePrivateCluster ? '10.132.64.0/18' : '10.128.64.0/18'
+param istioVersion = 'asm-1-20'
+param vnetAddressPrefix = '10.132.0.0/14'
+param subnetPrefix = '10.132.8.0/21'
+param podSubnetPrefix = '10.132.64.0/18'
 param enablePrivateCluster = false
 param persist = false
 param workloadIdentities = []

dev-infrastructure/configurations/svc-cluster.bicepparam

@@ -1,9 +1,10 @@
 using '../templates/svc-cluster.bicep'
 
 param kubernetesVersion = '1.29.2'
-param vnetAddressPrefix = enablePrivateCluster ? '10.132.0.0/14' : '10.128.0.0/14'
-param subnetPrefix = enablePrivateCluster ? '10.132.8.0/21' : '10.128.8.0/21'
-param podSubnetPrefix = enablePrivateCluster ? '10.132.64.0/18' : '10.128.64.0/18'
+param istioVersion = 'asm-1-20'
+param vnetAddressPrefix = '10.128.0.0/14'
+param subnetPrefix = '10.128.8.0/21'
+param podSubnetPrefix = '10.128.64.0/18'
 param enablePrivateCluster = false
 param persist = false
 param disableLocalAuth = false

dev-infrastructure/modules/aks-cluster-base.bicep

@@ -1,5 +1,5 @@
 // Constants
-param aksClusterName string = 'aro-hcp-cluster-001'
+param aksClusterName string = take('aro-hcp-${clusterType}-${uniqueString(clusterType)}', 63)
 param agentMinCount int = 2
 param agentMaxCount int = 3
 param agentVMSize string = 'Standard_D2s_v3'
@@ -15,6 +15,7 @@ param persist bool = false
 param currentUserId string
 param enablePrivateCluster bool = true
 param kubernetesVersion string
+param istioVersion string
 param vnetAddressPrefix string
 param subnetPrefix string
 param podSubnetPrefix string
@@ -51,7 +52,7 @@ var networkContributorRoleId = subscriptionResourceId(
 
 // Main
 // Tags the subscription
-resource subscriptionTags 'Microsoft.Resources/tags@2023-07-01' = {
+resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = {
   name: 'default'
   properties: {
     tags: {
@@ -61,12 +62,12 @@ resource subscriptionTags 'Microsoft.Resources/tags@2023-07-01' = {
   }
 }
 
-resource aks_nsg 'Microsoft.Network/networkSecurityGroups@2023-09-01' = {
+resource aks_nsg 'Microsoft.Network/networkSecurityGroups@2023-11-01' = {
   name: 'aks-nsg'
   location: location
 }
 
-resource aks_pod_nsg 'Microsoft.Network/networkSecurityGroups@2023-09-01' = {
+resource aks_pod_nsg 'Microsoft.Network/networkSecurityGroups@2023-11-01' = {
   name: 'aks-pod-nsg'
   location: location
 }
@@ -112,6 +113,18 @@ resource aks_etcd_kms 'Microsoft.KeyVault/vaults/keys@2023-07-01' = {
       'decrypt'
     ]
     keySize: 2048
+    rotationPolicy: {
+      lifetimeActions: [
+        {
+          action: {
+            type: 'notify'
+          }
+          trigger: {
+            timeBeforeExpiry: 'P30D'
+          }
+        }
+      ]
+    }
   }
 }
 
@@ -125,7 +138,7 @@ resource aks_keyvault_crypto_user 'Microsoft.Authorization/roleAssignments@2022-
   }
 }
 
-resource vnet 'Microsoft.Network/virtualNetworks@2023-09-01' = {
+resource vnet 'Microsoft.Network/virtualNetworks@2023-11-01' = {
   location: location
   name: 'aks-net'
   properties: {
@@ -137,7 +150,7 @@ resource vnet 'Microsoft.Network/virtualNetworks@2023-09-01' = {
   }
 }
 
-resource aksNodeSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' = {
+resource aksNodeSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' = {
   parent: vnet
   name: 'ClusterSubnet-001'
   properties: {
@@ -163,7 +176,7 @@ resource aksNodeSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' =
   }
 }
 
-resource aksPodSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' = {
+resource aksPodSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' = {
   parent: vnet
   name: 'PodSubnet-001'
   properties: {
@@ -208,6 +221,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = {
   name: aksClusterName
   tags: {
     persist: toLower(string(persist))
+    clusterType: clusterType
   }
   identity: {
     type: 'UserAssigned'
@@ -310,6 +324,9 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = {
             }
           ]
         }
+        revisions: [
+          istioVersion
+        ]
       }
     }
   }

dev-infrastructure/templates/mgmt-cluster.bicep

@@ -22,6 +22,9 @@ param enablePrivateCluster bool
 @description('Kuberentes version to use with AKS')
 param kubernetesVersion string
 
+@description('Istio control plane version to use with AKS')
+param istioVersion string
+
 @description('List of workload identities to create and their required values')
 param workloadIdentities array
 
@@ -33,11 +36,12 @@ module mgmtCluster '../modules/aks-cluster-base.bicep' = {
     persist: persist
     currentUserId: currentUserId
     enablePrivateCluster: enablePrivateCluster
+    istioVersion: istioVersion
     kubernetesVersion: kubernetesVersion
     vnetAddressPrefix: vnetAddressPrefix
     subnetPrefix: subnetPrefix
     podSubnetPrefix: podSubnetPrefix
-    clusterType: 'mgmt'
+    clusterType: 'mgmt-cluster'
     workloadIdentities: workloadIdentities
   }
 }

dev-infrastructure/templates/svc-cluster.bicep

@@ -22,6 +22,9 @@ param enablePrivateCluster bool
 @description('Kuberentes version to use with AKS')
 param kubernetesVersion string
 
+@description('Istio control plane version to use with AKS')
+param istioVersion string
+
 // TODO: When the work around workload identity for the RP is finalized, change this to true
 @description('disableLocalAuth for the ARO HCP RP CosmosDB')
 param disableLocalAuth bool
@@ -41,10 +44,11 @@ module svcCluster '../modules/aks-cluster-base.bicep' = {
     currentUserId: currentUserId
     enablePrivateCluster: enablePrivateCluster
     kubernetesVersion: kubernetesVersion
+    istioVersion: istioVersion
     vnetAddressPrefix: vnetAddressPrefix
     subnetPrefix: subnetPrefix
     podSubnetPrefix: podSubnetPrefix
-    clusterType: 'svc'
+    clusterType: 'svc-cluster'
     workloadIdentities: workloadIdentities
   }
 }