Skip to content

Commit

Permalink
AKS manifest placement
Browse files Browse the repository at this point in the history
add a module aks-manifest.bicep to manage resources on AKS clusters.
this can be used to

* create expected `Namespaces`
* create exüected `ServiceAccounts` with MIWI annotations for cloud resource
  access via Entra
* create expected `ConfigMaps` and `Secrets` holding configuration information
  for cloud resources, e.g. DB hostnames, ...

An example usage can be seen in the aks-cluster-base.bicep template,
where the namespace and serviceaccount are created for each federated
managed identity.

part of
[SD-DDR-0030](https://docs.google.com/document/d/1sxnNGscIuEaLRjbILlQrb3sepa4ZVddvj1-COHwQvSQ/edit#heading=h.bupciudrwmna)

Signed-off-by: Gerd Oberlechner <[email protected]>
  • Loading branch information
geoberle committed May 13, 2024
1 parent 3e49dc2 commit 89f9cd2
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 0 deletions.
35 changes: 35 additions & 0 deletions dev-infrastructure/modules/aks-cluster-base.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,16 @@ resource aksNetworkContributorRoleAssignment 'Microsoft.Authorization/roleAssign
}
}

resource aksClusterAdminRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
scope: aksCluster
name: guid(aksClusterUserDefinedManagedIdentity.id, aksClusterAdminRoleId, aksCluster.id)
properties: {
roleDefinitionId: aksClusterAdminRoleId
principalId: aksClusterUserDefinedManagedIdentity.properties.principalId
principalType: 'ServicePrincipal'
}
}

resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = {
location: location
name: aksClusterName
Expand Down Expand Up @@ -375,6 +385,31 @@ resource uami_fedcred 'Microsoft.ManagedIdentity/userAssignedIdentities/federate
}
]

module serviceAccounts './aks-manifest.bicep' = {
name: '${aksClusterName}-service-accounts'
params: {
aksClusterName: aksClusterName
manifests: [
for i in range(0, length(workloadIdentities)): {
apiVersion: 'v1'
kind: 'ServiceAccount'
metadata: {
name: workloadIdentities[i].value.uamiName
namespace: workloadIdentities[i].value.namespace
annotations: {
'azure.workload.identity/client-id': uami[i].properties.clientId
}
}
}
]
aksManagedIdentityId: aksClusterUserDefinedManagedIdentity.id
location: location
}
dependsOn: [
aksClusterAdminRoleAssignment
]
}

// Outputs
output userAssignedIdentities array = [
for i in range(0, length(workloadIdentities)): {
Expand Down
76 changes: 76 additions & 0 deletions dev-infrastructure/modules/aks-manifest.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
param aksClusterName string
param location string
param aksManagedIdentityId string
param manifests array

var namespaces = [for manifest in manifests: manifest.metadata.namespace]
var uniqueNamespaces = union(namespaces, [])
var namespaceManifests = [
for i in range(0, length(uniqueNamespaces)): {
apiVersion: 'v1'
kind: 'Namespace'
metadata: {
name: uniqueNamespaces[i]
}
}
]
var namespaceManifestList = {
apiVersion: 'v1'
kind: 'List'
items: namespaceManifests
}

var mainfestList = {
apiVersion: 'v1'
kind: 'List'
items: manifests
}

resource deploymentScript 'Microsoft.Resources/deploymentScripts@2023-08-01' = {
name: '${aksClusterName}-manifests'
location: location
kind: 'AzureCLI'
identity: {
type: 'UserAssigned'
userAssignedIdentities: {
'${aksManagedIdentityId}': {}
}
}

properties: {
azCliVersion: '2.30.0'
cleanupPreference: 'OnSuccess'
retentionInterval: 'P1D'
scriptContent: '''
az login --identity
az aks install-cli
az aks get-credentials --resource-group ${AKS_CLUSTER_RG} --name ${AKS_CLUSTER_NAME} --overwrite-existing -a
echo "${NAMESPACE_MANIFESTS}" | base64 -d | kubectl apply -f -
echo "${MANIFESTS}" | base64 -d | kubectl apply -f -
'''
// todo figure out how to leverage az aks command invoke to
// * avoid installing kubectl
// * avoid the need for a network path to the cluster
//
// right now az aks command invoke fails with `MissingAADClusterToken` when run within a deploymentscript
environmentVariables: [
{
name: 'AKS_CLUSTER_RG'
value: resourceGroup().name
}
{
name: 'AKS_CLUSTER_NAME'
value: aksClusterName
}
{
name: 'NAMESPACE_MANIFESTS'
value: base64(string(namespaceManifestList))
}
{
name: 'MANIFESTS'
value: base64(string(mainfestList))
}
]
timeout: 'PT30M'
}
}

0 comments on commit 89f9cd2

Please sign in to comment.