diff --git a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go index 01b82b3182d4..7fcc60e286a6 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go @@ -82,10 +82,10 @@ func (as *AgentPool) initialize() error { ctx, cancel := getContextWithCancel() defer cancel() - template, err := as.manager.azClient.deploymentsClient.ExportTemplate(ctx, as.manager.config.ResourceGroup, as.manager.config.Deployment) + template, err := as.manager.azClient.deploymentClient.ExportTemplate(ctx, as.manager.config.ResourceGroup, as.manager.config.Deployment) if err != nil { - klog.Errorf("deploymentsClient.ExportTemplate(%s, %s) failed: %v", as.manager.config.ResourceGroup, as.manager.config.Deployment, err) - return err + klog.Errorf("deploymentClient.ExportTemplate(%s, %s) failed: %v", as.manager.config.ResourceGroup, as.manager.config.Deployment, err) + return err.Error() } as.template = template.Template.(map[string]interface{}) @@ -211,18 +211,27 @@ func (as *AgentPool) TargetSize() (int, error) { return int(size), nil } -func (as *AgentPool) getAllSucceededAndFailedDeployments() (succeededAndFailedDeployments []resources.DeploymentExtended, err error) { +func (as *AgentPool) getAllSucceededAndFailedDeployments() ([]resources.DeploymentExtended, error) { ctx, cancel := getContextWithCancel() defer cancel() - deploymentsFilter := "provisioningState eq 'Succeeded' or provisioningState eq 'Failed'" - succeededAndFailedDeployments, err = as.manager.azClient.deploymentsClient.List(ctx, as.manager.config.ResourceGroup, deploymentsFilter, nil) - if err != nil { - klog.Errorf("getAllSucceededAndFailedDeployments: failed to list succeeded or failed deployments with error: %v", err) - return nil, err + allDeployments, rerr := as.manager.azClient.deploymentClient.List(ctx, as.manager.config.ResourceGroup) + if rerr != nil { + klog.Errorf("getAllSucceededAndFailedDeployments: failed to list deployments with error: %v", rerr.Error()) + return nil, rerr.Error() + } + + result := make([]resources.DeploymentExtended, 0) + for _, deployment := range allDeployments { + if deployment.Properties == nil || deployment.Properties.ProvisioningState == nil { + continue + } + if *deployment.Properties.ProvisioningState == "Succeeded" || *deployment.Properties.ProvisioningState == "Failed" { + result = append(result, deployment) + } } - return succeededAndFailedDeployments, err + return result, rerr.Error() } // deleteOutdatedDeployments keeps the newest deployments in the resource group and delete others, @@ -258,9 +267,9 @@ func (as *AgentPool) deleteOutdatedDeployments() (err error) { errList := make([]error, 0) for _, deployment := range toBeDeleted { klog.V(4).Infof("deleteOutdatedDeployments: starts deleting outdated deployment (%s)", *deployment.Name) - _, err := as.manager.azClient.deploymentsClient.Delete(ctx, as.manager.config.ResourceGroup, *deployment.Name) - if err != nil { - errList = append(errList, err) + rerr := as.manager.azClient.deploymentClient.Delete(ctx, as.manager.config.ResourceGroup, *deployment.Name) + if rerr != nil { + errList = append(errList, rerr.Error()) } } @@ -317,22 +326,20 @@ func (as *AgentPool) IncreaseSize(delta int) error { } ctx, cancel := getContextWithCancel() defer cancel() - klog.V(3).Infof("Waiting for deploymentsClient.CreateOrUpdate(%s, %s, %v)", as.manager.config.ResourceGroup, newDeploymentName, newDeployment) - resp, err := as.manager.azClient.deploymentsClient.CreateOrUpdate(ctx, as.manager.config.ResourceGroup, newDeploymentName, newDeployment) - isSuccess, realError := isSuccessHTTPResponse(resp, err) - if isSuccess { - klog.V(3).Infof("deploymentsClient.CreateOrUpdate(%s, %s, %v) success", as.manager.config.ResourceGroup, newDeploymentName, newDeployment) - - // Update cache after scale success. - as.curSize = int64(expectedSize) - as.lastRefresh = time.Now() - klog.V(6).Info("IncreaseSize: invalidating cache") - as.manager.invalidateCache() - return nil + klog.V(3).Infof("Waiting for deploymentClient.CreateOrUpdate(%s, %s, %v)", as.manager.config.ResourceGroup, newDeploymentName, newDeployment) + rerr := as.manager.azClient.deploymentClient.CreateOrUpdate(ctx, as.manager.config.ResourceGroup, newDeploymentName, newDeployment, "") + if rerr != nil { + klog.Errorf("deploymentClient.CreateOrUpdate for deployment %q failed: %v", newDeploymentName, rerr.Error()) + return rerr.Error() } + klog.V(3).Infof("deploymentClient.CreateOrUpdate(%s, %s, %v) success", as.manager.config.ResourceGroup, newDeploymentName, newDeployment) - klog.Errorf("deploymentsClient.CreateOrUpdate for deployment %q failed: %v", newDeploymentName, realError) - return realError + // Update cache after scale success. + as.curSize = int64(expectedSize) + as.lastRefresh = time.Now() + klog.V(6).Info("IncreaseSize: invalidating cache") + as.manager.invalidateCache() + return nil } // AtomicIncreaseSize is not implemented. diff --git a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool_test.go b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool_test.go index 444f2f9f235c..a242e0b7b666 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool_test.go @@ -28,6 +28,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/storageaccountclient/mockstorageaccountclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmclient/mockvmclient" + providerazureconsts "sigs.k8s.io/cloud-provider-azure/pkg/consts" "sigs.k8s.io/cloud-provider-azure/pkg/retry" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" @@ -156,13 +157,13 @@ func TestDeleteOutdatedDeployments(t *testing.T) { for _, test := range testCases { testAS := newTestAgentPool(newTestAzureManager(t), "testAS") - testAS.manager.azClient.deploymentsClient = &DeploymentsClientMock{ + testAS.manager.azClient.deploymentClient = &DeploymentClientMock{ FakeStore: test.deployments, } err := testAS.deleteOutdatedDeployments() assert.Equal(t, test.expectedErr, err, test.desc) - existedDeployments, err := testAS.manager.azClient.deploymentsClient.List(context.Background(), "", "", to.Int32Ptr(0)) + existedDeployments, _ := testAS.manager.azClient.deploymentClient.List(context.Background(), "") existedDeploymentsNames := make(map[string]bool) for _, deployment := range existedDeployments { existedDeploymentsNames[*deployment.Name] = true @@ -185,7 +186,7 @@ func TestGetVMsFromCache(t *testing.T) { mockVMClient := mockvmclient.NewMockInterface(ctrl) testAS.manager.azClient.virtualMachinesClient = mockVMClient mockVMClient.EXPECT().List(gomock.Any(), testAS.manager.config.ResourceGroup).Return(expectedVMs, nil) - testAS.manager.config.VMType = vmTypeStandard + testAS.manager.config.VMType = providerazureconsts.VMTypeStandard ac, err := newAzureCache(testAS.manager.azClient, refreshInterval, *testAS.manager.config) assert.NoError(t, err) testAS.manager.azureCache = ac @@ -204,7 +205,7 @@ func TestGetVMIndexes(t *testing.T) { mockVMClient := mockvmclient.NewMockInterface(ctrl) as.manager.azClient.virtualMachinesClient = mockVMClient mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - as.manager.config.VMType = vmTypeStandard + as.manager.config.VMType = providerazureconsts.VMTypeStandard ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -244,7 +245,7 @@ func TestGetCurSize(t *testing.T) { mockVMClient := mockvmclient.NewMockInterface(ctrl) as.manager.azClient.virtualMachinesClient = mockVMClient mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - as.manager.config.VMType = vmTypeStandard + as.manager.config.VMType = providerazureconsts.VMTypeStandard ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -269,7 +270,7 @@ func TestAgentPoolTargetSize(t *testing.T) { as.manager.azClient.virtualMachinesClient = mockVMClient expectedVMs := getExpectedVMs() mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - as.manager.config.VMType = vmTypeStandard + as.manager.config.VMType = providerazureconsts.VMTypeStandard ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -289,7 +290,7 @@ func TestAgentPoolIncreaseSize(t *testing.T) { as.manager.azClient.virtualMachinesClient = mockVMClient expectedVMs := getExpectedVMs() mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil).MaxTimes(2) - as.manager.config.VMType = vmTypeStandard + as.manager.config.VMType = providerazureconsts.VMTypeStandard ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -318,7 +319,7 @@ func TestDecreaseTargetSize(t *testing.T) { as.manager.azClient.virtualMachinesClient = mockVMClient expectedVMs := getExpectedVMs() mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil).MaxTimes(3) - as.manager.config.VMType = vmTypeStandard + as.manager.config.VMType = providerazureconsts.VMTypeStandard ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -437,9 +438,9 @@ func TestAgentPoolDeleteNodes(t *testing.T) { mockSAClient := mockstorageaccountclient.NewMockInterface(ctrl) as.manager.azClient.storageAccountsClient = mockSAClient mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - as.manager.config.VMType = vmTypeStandard + as.manager.config.VMType = providerazureconsts.VMTypeStandard ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) - as.manager.config.VMType = vmTypeVMSS + as.manager.config.VMType = providerazureconsts.VMTypeVMSS assert.NoError(t, err) as.manager.azureCache = ac @@ -505,7 +506,7 @@ func TestAgentPoolNodes(t *testing.T) { mockVMClient := mockvmclient.NewMockInterface(ctrl) as.manager.azClient.virtualMachinesClient = mockVMClient mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - as.manager.config.VMType = vmTypeStandard + as.manager.config.VMType = providerazureconsts.VMTypeStandard ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cache.go b/cluster-autoscaler/cloudprovider/azure/azure_cache.go index bb68567e8f40..c8334ac78771 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_cache.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_cache.go @@ -28,6 +28,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/Azure/skewer" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + providerazureconsts "sigs.k8s.io/cloud-provider-azure/pkg/consts" "k8s.io/klog/v2" ) @@ -436,7 +437,7 @@ func (m *azureCache) FindForInstance(instance *azureRef, vmType string) (cloudpr } // cluster with vmss pool only - if vmType == vmTypeVMSS && len(vmsPoolSet) == 0 { + if vmType == providerazureconsts.VMTypeVMSS && len(vmsPoolSet) == 0 { if m.areAllScaleSetsUniform() { // Omit virtual machines not managed by vmss only in case of uniform scale set. if ok := virtualMachineRE.Match([]byte(inst.Name)); ok { @@ -447,7 +448,7 @@ func (m *azureCache) FindForInstance(instance *azureRef, vmType string) (cloudpr } } - if vmType == vmTypeStandard { + if vmType == providerazureconsts.VMTypeStandard { // Omit virtual machines with providerID not in Azure resource ID format. if ok := virtualMachineRE.Match([]byte(inst.Name)); !ok { klog.V(3).Infof("Instance %q is not in Azure resource ID format, omit it in autoscaler", instance.Name) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cache_test.go b/cluster-autoscaler/cloudprovider/azure/azure_cache_test.go index 2b87ab938486..3fd801afbe8f 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_cache_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_cache_test.go @@ -20,6 +20,7 @@ import ( "testing" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + providerazureconsts "sigs.k8s.io/cloud-provider-azure/pkg/consts" "github.com/stretchr/testify/assert" ) @@ -60,12 +61,12 @@ func TestFindForInstance(t *testing.T) { inst := azureRef{Name: "/subscriptions/sub/resourceGroups/rg/providers/foo"} ac.unownedInstances = make(map[azureRef]bool) ac.unownedInstances[inst] = true - nodeGroup, err := ac.FindForInstance(&inst, vmTypeVMSS) + nodeGroup, err := ac.FindForInstance(&inst, providerazureconsts.VMTypeVMSS) assert.Nil(t, nodeGroup) assert.NoError(t, err) ac.unownedInstances[inst] = false - nodeGroup, err = ac.FindForInstance(&inst, vmTypeStandard) + nodeGroup, err = ac.FindForInstance(&inst, providerazureconsts.VMTypeStandard) assert.Nil(t, nodeGroup) assert.NoError(t, err) assert.True(t, ac.unownedInstances[inst]) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client.go b/cluster-autoscaler/cloudprovider/azure/azure_client.go index 2bf337a4e8d4..fbc39a62ed28 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_client.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_client.go @@ -19,10 +19,6 @@ package azure import ( "context" "fmt" - "io/ioutil" - "net/http" - "os" - "time" _ "go.uber.org/mock/mockgen/model" // for go:generate @@ -35,119 +31,22 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources" - "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-02-01/storage" "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/azure/auth" klog "k8s.io/klog/v2" + "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/deploymentclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/diskclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/interfaceclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/storageaccountclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssvmclient" + providerazureconfig "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" ) -// DeploymentsClient defines needed functions for azure network.DeploymentsClient. -type DeploymentsClient interface { - Get(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExtended, err error) - List(ctx context.Context, resourceGroupName string, filter string, top *int32) (result []resources.DeploymentExtended, err error) - ExportTemplate(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExportResult, err error) - CreateOrUpdate(ctx context.Context, resourceGroupName string, deploymentName string, parameters resources.Deployment) (resp *http.Response, err error) - Delete(ctx context.Context, resourceGroupName string, deploymentName string) (resp *http.Response, err error) -} - -type azDeploymentsClient struct { - client resources.DeploymentsClient -} - -func newAzDeploymentsClient(subscriptionID, endpoint string, authorizer autorest.Authorizer) *azDeploymentsClient { - deploymentsClient := resources.NewDeploymentsClient(subscriptionID) - deploymentsClient.BaseURI = endpoint - deploymentsClient.Authorizer = authorizer - deploymentsClient.PollingDelay = 5 * time.Second - configureUserAgent(&deploymentsClient.Client) - - return &azDeploymentsClient{ - client: deploymentsClient, - } -} - -func (az *azDeploymentsClient) Get(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExtended, err error) { - klog.V(10).Infof("azDeploymentsClient.Get(%q,%q): start", resourceGroupName, deploymentName) - defer func() { - klog.V(10).Infof("azDeploymentsClient.Get(%q,%q): end", resourceGroupName, deploymentName) - }() - - return az.client.Get(ctx, resourceGroupName, deploymentName) -} - -func (az *azDeploymentsClient) ExportTemplate(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExportResult, err error) { - klog.V(10).Infof("azDeploymentsClient.ExportTemplate(%q,%q): start", resourceGroupName, deploymentName) - defer func() { - klog.V(10).Infof("azDeploymentsClient.ExportTemplate(%q,%q): end", resourceGroupName, deploymentName) - }() - - return az.client.ExportTemplate(ctx, resourceGroupName, deploymentName) -} - -func (az *azDeploymentsClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, deploymentName string, parameters resources.Deployment) (resp *http.Response, err error) { - klog.V(10).Infof("azDeploymentsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, deploymentName) - defer func() { - klog.V(10).Infof("azDeploymentsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, deploymentName) - }() - - future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, deploymentName, parameters) - if err != nil { - return future.Response(), err - } - - err = future.WaitForCompletionRef(ctx, az.client.Client) - return future.Response(), err -} - -func (az *azDeploymentsClient) List(ctx context.Context, resourceGroupName, filter string, top *int32) (result []resources.DeploymentExtended, err error) { - klog.V(10).Infof("azDeploymentsClient.List(%q): start", resourceGroupName) - defer func() { - klog.V(10).Infof("azDeploymentsClient.List(%q): end", resourceGroupName) - }() - - iterator, err := az.client.ListByResourceGroupComplete(ctx, resourceGroupName, filter, top) - if err != nil { - return nil, err - } - - result = make([]resources.DeploymentExtended, 0) - for ; iterator.NotDone(); err = iterator.Next() { - if err != nil { - return nil, err - } - - result = append(result, iterator.Value()) - } - - return result, err -} - -func (az *azDeploymentsClient) Delete(ctx context.Context, resourceGroupName, deploymentName string) (resp *http.Response, err error) { - klog.V(10).Infof("azDeploymentsClient.Delete(%q,%q): start", resourceGroupName, deploymentName) - defer func() { - klog.V(10).Infof("azDeploymentsClient.Delete(%q,%q): end", resourceGroupName, deploymentName) - }() - - future, err := az.client.Delete(ctx, resourceGroupName, deploymentName) - if err != nil { - return future.Response(), err - } - - err = future.WaitForCompletionRef(ctx, az.client.Client) - return future.Response(), err -} - //go:generate sh -c "mockgen k8s.io/autoscaler/cluster-autoscaler/cloudprovider/azure AgentPoolsClient >./agentpool_client.go" // AgentPoolsClient interface defines the methods needed for scaling vms pool. @@ -260,15 +159,11 @@ func newAgentpoolClientWithPublicEndpoint(cfg *Config, retryOptions azurecore_po return newAgentpoolClientWithConfig(cfg.SubscriptionID, cred, env.ResourceManagerEndpoint, env.TokenAudience, retryOptions) } -type azAccountsClient struct { - client storage.AccountsClient -} - type azClient struct { virtualMachineScaleSetsClient vmssclient.Interface virtualMachineScaleSetVMsClient vmssvmclient.Interface virtualMachinesClient vmclient.Interface - deploymentsClient DeploymentsClient + deploymentClient deploymentclient.Interface interfacesClient interfaceclient.Interface disksClient diskclient.Interface storageAccountsClient storageaccountclient.Interface @@ -276,80 +171,12 @@ type azClient struct { agentPoolClient AgentPoolsClient } -// newServicePrincipalTokenFromCredentials creates a new ServicePrincipalToken using values of the -// passed credentials map. -func newServicePrincipalTokenFromCredentials(config *Config, env *azure.Environment) (*adal.ServicePrincipalToken, error) { - oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID) - if err != nil { - return nil, fmt.Errorf("creating the OAuth config: %v", err) - } - - if config.UseWorkloadIdentityExtension { - klog.V(2).Infoln("azure: using workload identity extension to retrieve access token") - jwt, err := os.ReadFile(config.AADFederatedTokenFile) - if err != nil { - return nil, fmt.Errorf("failed to read a file with a federated token: %v", err) - } - token, err := adal.NewServicePrincipalTokenFromFederatedToken(*oauthConfig, config.AADClientID, string(jwt), env.ResourceManagerEndpoint) - if err != nil { - return nil, fmt.Errorf("failed to create a workload identity token: %v", err) - } - return token, nil - } - if config.UseManagedIdentityExtension { - klog.V(2).Infoln("azure: using managed identity extension to retrieve access token") - msiEndpoint, err := adal.GetMSIVMEndpoint() - if err != nil { - return nil, fmt.Errorf("getting the managed service identity endpoint: %v", err) - } - if config.UserAssignedIdentityID != "" { - klog.V(4).Info("azure: using User Assigned MSI ID to retrieve access token") - return adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, - env.ServiceManagementEndpoint, - config.UserAssignedIdentityID) - } - klog.V(4).Info("azure: using System Assigned MSI to retrieve access token") - return adal.NewServicePrincipalTokenFromMSI( - msiEndpoint, - env.ServiceManagementEndpoint) - } - - if config.AADClientSecret != "" { - klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token") - return adal.NewServicePrincipalToken( - *oauthConfig, - config.AADClientID, - config.AADClientSecret, - env.ServiceManagementEndpoint) - } - - if config.AADClientCertPath != "" { - klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token") - certData, err := ioutil.ReadFile(config.AADClientCertPath) - if err != nil { - return nil, fmt.Errorf("reading the client certificate from file %s: %v", config.AADClientCertPath, err) - } - certificate, privateKey, err := adal.DecodePfxCertificateData(certData, config.AADClientCertPassword) - if err != nil { - return nil, fmt.Errorf("decoding the client certificate: %v", err) - } - return adal.NewServicePrincipalTokenFromCertificate( - *oauthConfig, - config.AADClientID, - certificate, - privateKey, - env.ServiceManagementEndpoint) - } - - return nil, fmt.Errorf("no credentials provided for AAD application %s", config.AADClientID) -} - func newAuthorizer(config *Config, env *azure.Environment) (autorest.Authorizer, error) { switch config.AuthMethod { case authMethodCLI: return auth.NewAuthorizerFromCLI() case "", authMethodPrincipal: - token, err := newServicePrincipalTokenFromCredentials(config, env) + token, err := providerazureconfig.GetServicePrincipalToken(&config.AzureAuthConfig, env, "") if err != nil { return nil, fmt.Errorf("retrieve service principal token: %v", err) } @@ -380,8 +207,9 @@ func newAzClient(cfg *Config, env *azure.Environment) (*azClient, error) { virtualMachinesClient := vmclient.New(vmClientConfig) klog.V(5).Infof("Created vm client with authorizer: %v", virtualMachinesClient) - deploymentsClient := newAzDeploymentsClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, authorizer) - klog.V(5).Infof("Created deployments client with authorizer: %v", deploymentsClient) + deploymentConfig := azClientConfig.WithRateLimiter(cfg.DeploymentRateLimit) + deploymentClient := deploymentclient.New(deploymentConfig) + klog.V(5).Infof("Created deployments client with authorizer: %v", deploymentClient) interfaceClientConfig := azClientConfig.WithRateLimiter(cfg.InterfaceRateLimit) interfacesClient := interfaceclient.New(interfaceClientConfig) @@ -414,7 +242,7 @@ func newAzClient(cfg *Config, env *azure.Environment) (*azClient, error) { interfacesClient: interfacesClient, virtualMachineScaleSetsClient: scaleSetsClient, virtualMachineScaleSetVMsClient: scaleSetVMsClient, - deploymentsClient: deploymentsClient, + deploymentClient: deploymentClient, virtualMachinesClient: virtualMachinesClient, storageAccountsClient: storageAccountsClient, skuClient: skuClient, diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client_test.go b/cluster-autoscaler/cloudprovider/azure/azure_client_test.go index 7ed0dd4c01f7..a1e94d9a5316 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_client_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_client_test.go @@ -23,17 +23,28 @@ import ( "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" "github.com/stretchr/testify/assert" + azclient "sigs.k8s.io/cloud-provider-azure/pkg/azclient" + providerazure "sigs.k8s.io/cloud-provider-azure/pkg/provider" + providerazureconfig "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" ) func TestGetServicePrincipalTokenFromCertificate(t *testing.T) { config := &Config{ - TenantID: "TenantID", - AADClientID: "AADClientID", - AADClientCertPath: "./testdata/test.pfx", - AADClientCertPassword: "id", + Config: providerazure.Config{ + AzureAuthConfig: providerazureconfig.AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "AADClientID", + AADClientCertPath: "./testdata/test.pfx", + AADClientCertPassword: "id", + }, + }, + }, } env := &azure.PublicCloud - token, err := newServicePrincipalTokenFromCredentials(config, env) + token, err := providerazureconfig.GetServicePrincipalToken(&config.AzureAuthConfig, env, "") assert.NoError(t, err) oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID) @@ -45,17 +56,25 @@ func TestGetServicePrincipalTokenFromCertificate(t *testing.T) { spt, err := adal.NewServicePrincipalTokenFromCertificate( *oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) assert.NoError(t, err) - assert.Equal(t, token, spt) + assert.Equal(t, token.Token(), spt.Token()) } func TestGetServicePrincipalTokenFromCertificateWithoutPassword(t *testing.T) { config := &Config{ - TenantID: "TenantID", - AADClientID: "AADClientID", - AADClientCertPath: "./testdata/testnopassword.pfx", + Config: providerazure.Config{ + AzureAuthConfig: providerazureconfig.AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + TenantID: "TenantID", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "AADClientID", + AADClientCertPath: "./testdata/testnopassword.pfx", + }, + }, + }, } env := &azure.PublicCloud - token, err := newServicePrincipalTokenFromCredentials(config, env) + token, err := providerazureconfig.GetServicePrincipalToken(&config.AzureAuthConfig, env, "") assert.NoError(t, err) oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID) @@ -67,5 +86,5 @@ func TestGetServicePrincipalTokenFromCertificateWithoutPassword(t *testing.T) { spt, err := adal.NewServicePrincipalTokenFromCertificate( *oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) assert.NoError(t, err) - assert.Equal(t, token, spt) + assert.Equal(t, token.Token(), spt.Token()) } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go b/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go index da37ed8492da..cd88602da479 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go @@ -31,6 +31,8 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmclient/mockvmclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssclient/mockvmssclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssvmclient/mockvmssvmclient" + providerazureconsts "sigs.k8s.io/cloud-provider-azure/pkg/consts" + providerazure "sigs.k8s.io/cloud-provider-azure/pkg/provider" "github.com/Azure/go-autorest/autorest/azure" "github.com/stretchr/testify/assert" @@ -55,18 +57,20 @@ func newTestAzureManager(t *testing.T) *AzureManager { env: azure.PublicCloud, explicitlyConfigured: make(map[string]bool), config: &Config{ - ResourceGroup: "rg", - VMType: vmTypeVMSS, + Config: providerazure.Config{ + ResourceGroup: "rg", + VMType: providerazureconsts.VMTypeVMSS, + Location: "eastus", + }, MaxDeploymentsCount: 2, Deployment: "deployment", EnableForceDelete: true, - Location: "eastus", }, azClient: &azClient{ virtualMachineScaleSetsClient: mockVMSSClient, virtualMachineScaleSetVMsClient: mockVMSSVMClient, virtualMachinesClient: mockVMClient, - deploymentsClient: &DeploymentsClientMock{ + deploymentClient: &DeploymentClientMock{ FakeStore: map[string]resources.DeploymentExtended{ "deployment": { Name: to.StringPtr("deployment"), @@ -332,7 +336,7 @@ func TestNodeGroupForNode(t *testing.T) { mockVMSSVMClient.EXPECT().List(gomock.Any(), provider.azureManager.config.ResourceGroup, "test-asg", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() provider.azureManager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient } else { - provider.azureManager.config.EnableVmssFlex = true + provider.azureManager.config.EnableVmssFlexNodes = true mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), "test-asg").Return(expectedVMs, nil).AnyTimes() } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_config.go b/cluster-autoscaler/cloudprovider/azure/azure_config.go index 6c354c2a23e4..49d0bdf32d4d 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_config.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_config.go @@ -18,7 +18,6 @@ package azure import ( "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -32,7 +31,9 @@ import ( "github.com/Azure/go-autorest/autorest/azure" "k8s.io/klog/v2" azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients" + providerazureconsts "sigs.k8s.io/cloud-provider-azure/pkg/consts" providerazure "sigs.k8s.io/cloud-provider-azure/pkg/provider" + providerazureconfig "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" "sigs.k8s.io/cloud-provider-azure/pkg/retry" ) @@ -42,60 +43,23 @@ const ( imdsServerURL = "http://169.254.169.254" - // backoff - backoffRetriesDefault = 6 - backoffExponentDefault = 1.5 - backoffDurationDefault = 5 // in seconds - backoffJitterDefault = 1.0 - - // rate limit - rateLimitQPSDefault float32 = 1.0 - rateLimitBucketDefault = 5 - rateLimitReadQPSEnvVar = "RATE_LIMIT_READ_QPS" - rateLimitReadBucketsEnvVar = "RATE_LIMIT_READ_BUCKETS" - rateLimitWriteQPSEnvVar = "RATE_LIMIT_WRITE_QPS" - rateLimitWriteBucketsEnvVar = "RATE_LIMIT_WRITE_BUCKETS" - - // VmssSizeRefreshPeriodDefault in seconds - VmssSizeRefreshPeriodDefault = 30 - // auth methods authMethodPrincipal = "principal" authMethodCLI = "cli" - - // toggle - dynamicInstanceListDefault = false - enableVmssFlexDefault = false ) -// CloudProviderRateLimitConfig indicates the rate limit config for each clients. -type CloudProviderRateLimitConfig struct { - // The default rate limit config options. - azclients.RateLimitConfig - - // Rate limit config for each clients. Values would override default settings above. - InterfaceRateLimit *azclients.RateLimitConfig `json:"interfaceRateLimit,omitempty" yaml:"interfaceRateLimit,omitempty"` - VirtualMachineRateLimit *azclients.RateLimitConfig `json:"virtualMachineRateLimit,omitempty" yaml:"virtualMachineRateLimit,omitempty"` - StorageAccountRateLimit *azclients.RateLimitConfig `json:"storageAccountRateLimit,omitempty" yaml:"storageAccountRateLimit,omitempty"` - DiskRateLimit *azclients.RateLimitConfig `json:"diskRateLimit,omitempty" yaml:"diskRateLimit,omitempty"` - VirtualMachineScaleSetRateLimit *azclients.RateLimitConfig `json:"virtualMachineScaleSetRateLimit,omitempty" yaml:"virtualMachineScaleSetRateLimit,omitempty"` - KubernetesServiceRateLimit *azclients.RateLimitConfig `json:"kubernetesServiceRateLimit,omitempty" yaml:"kubernetesServiceRateLimit,omitempty"` -} - -// Config holds the configuration parsed from the --cloud-config flag +// Config holds the configuration parsed from the --cloud-config flag or the environment variables. +// Contains both general Azure cloud provider configuration (i.e., in azure.json) and CAS configurations/options specifically for Azure provider. type Config struct { - CloudProviderRateLimitConfig - - Cloud string `json:"cloud" yaml:"cloud"` - Location string `json:"location" yaml:"location"` - TenantID string `json:"tenantId" yaml:"tenantId"` - SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"` - ClusterName string `json:"clusterName" yaml:"clusterName"` - // ResourceGroup is the MC_ resource group where the nodes are located. - ResourceGroup string `json:"resourceGroup" yaml:"resourceGroup"` + // Azure cloud provider configuration, which is generally shared with other Azure components. + providerazure.Config `json:",inline" yaml:",inline"` + + // Legacy fields, which are only here for backward compatibility. To be deprecated. + legacyConfig `json:",inline" yaml:",inline"` + + ClusterName string `json:"clusterName" yaml:"clusterName"` // ClusterResourceGroup is the resource group where the cluster is located. ClusterResourceGroup string `json:"clusterResourceGroup" yaml:"clusterResourceGroup"` - VMType string `json:"vmType" yaml:"vmType"` // ARMBaseURLForAPClient is the URL to use for operations for the VMs pool. // It can override the default public ARM endpoint for VMs pool scale operations. @@ -105,51 +69,26 @@ type Config struct { // cloud. Valid options are "principal" (= the traditional // service principle approach) and "cli" (= load az command line // config file). The default is "principal". + // 08/16/2024: This field is awkward, given the existence of UseManagedIdentityExtension and UseFederatedWorkloadIdentityExtension. + // Ideally, either it should be deprecated, or reworked to be on the same "dimension" as the two above, if not reworking those two. AuthMethod string `json:"authMethod" yaml:"authMethod"` - // Settings for a service principal. - - AADClientID string `json:"aadClientId" yaml:"aadClientId"` - AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` - AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"` - AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"` - AADFederatedTokenFile string `json:"aadFederatedTokenFile" yaml:"aadFederatedTokenFile"` - UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"` - UseWorkloadIdentityExtension bool `json:"useWorkloadIdentityExtension" yaml:"useWorkloadIdentityExtension"` - UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"` - // Configs only for standard vmType (agent pools). Deployment string `json:"deployment" yaml:"deployment"` DeploymentParameters map[string]interface{} `json:"deploymentParameters" yaml:"deploymentParameters"` - // VMSS metadata cache TTL in seconds, only applies for vmss type - VmssCacheTTL int64 `json:"vmssCacheTTL" yaml:"vmssCacheTTL"` - - // VMSS instances cache TTL in seconds, only applies for vmss type - VmssVmsCacheTTL int64 `json:"vmssVmsCacheTTL" yaml:"vmssVmsCacheTTL"` - // Jitter in seconds subtracted from the VMSS cache TTL before the first refresh VmssVmsCacheJitter int `json:"vmssVmsCacheJitter" yaml:"vmssVmsCacheJitter"` // number of latest deployments that will not be deleted MaxDeploymentsCount int64 `json:"maxDeploymentsCount" yaml:"maxDeploymentsCount"` - // Enable exponential backoff to manage resource request retries - CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty" yaml:"cloudProviderBackoff,omitempty"` - CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty" yaml:"cloudProviderBackoffRetries,omitempty"` - CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty" yaml:"cloudProviderBackoffExponent,omitempty"` - CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty" yaml:"cloudProviderBackoffDuration,omitempty"` - CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty" yaml:"cloudProviderBackoffJitter,omitempty"` - // EnableForceDelete defines whether to enable force deletion on the APIs EnableForceDelete bool `json:"enableForceDelete,omitempty" yaml:"enableForceDelete,omitempty"` // EnableDynamicInstanceList defines whether to enable dynamic instance workflow for instance information check EnableDynamicInstanceList bool `json:"enableDynamicInstanceList,omitempty" yaml:"enableDynamicInstanceList,omitempty"` - // EnableVmssFlex defines whether to enable Vmss Flex support or not - EnableVmssFlex bool `json:"enableVmssFlex,omitempty" yaml:"enableVmssFlex,omitempty"` - // (DEPRECATED, DO NOT USE) EnableDetailedCSEMessage defines whether to emit error messages in the CSE error body info EnableDetailedCSEMessage bool `json:"enableDetailedCSEMessage,omitempty" yaml:"enableDetailedCSEMessage,omitempty"` @@ -157,11 +96,34 @@ type Config struct { GetVmssSizeRefreshPeriod int `json:"getVmssSizeRefreshPeriod,omitempty" yaml:"getVmssSizeRefreshPeriod,omitempty"` } +// These are only here for backward compabitility. Their equivalent exists in providerazure.Config with a different name. +type legacyConfig struct { + // Being renamed to UseFederatedWorkloadIdentityExtension + UseWorkloadIdentityExtension *bool `json:"useWorkloadIdentityExtension" yaml:"useWorkloadIdentityExtension"` + // VMSS metadata cache TTL in seconds, only applies for vmss type; being renamed to VmssCacheTTLInSeconds + VmssCacheTTL *int64 `json:"vmssCacheTTL" yaml:"vmssCacheTTL"` + // VMSS instances cache TTL in seconds, only applies for vmss type; being renamed to VmssVirtualMachinesCacheTTLInSeconds + VmssVmsCacheTTL *int64 `json:"vmssVmsCacheTTL" yaml:"vmssVmsCacheTTL"` + // EnableVmssFlex defines whether to enable Vmss Flex support or not; being renamed to EnableVmssFlexNodes + EnableVmssFlex *bool `json:"enableVmssFlex,omitempty" yaml:"enableVmssFlex,omitempty"` +} + // BuildAzureConfig returns a Config object for the Azure clients func BuildAzureConfig(configReader io.Reader) (*Config, error) { var err error cfg := &Config{} + // Static defaults + cfg.EnableDynamicInstanceList = false + cfg.EnableVmssFlexNodes = false + cfg.CloudProviderBackoffRetries = providerazureconsts.BackoffRetriesDefault + cfg.CloudProviderBackoffExponent = providerazureconsts.BackoffExponentDefault + cfg.CloudProviderBackoffDuration = providerazureconsts.BackoffDurationDefault + cfg.CloudProviderBackoffJitter = providerazureconsts.BackoffJitterDefault + cfg.VMType = providerazureconsts.VMTypeVMSS + cfg.MaxDeploymentsCount = int64(defaultMaxDeploymentsCount) + + // Config file overrides defaults if configReader != nil { body, err := ioutil.ReadAll(configReader) if err != nil { @@ -171,192 +133,181 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) { if err != nil { return nil, fmt.Errorf("failed to unmarshal config body: %v", err) } - } else { - cfg.Cloud = os.Getenv("ARM_CLOUD") - cfg.Location = os.Getenv("LOCATION") - cfg.ResourceGroup = os.Getenv("ARM_RESOURCE_GROUP") - cfg.TenantID = os.Getenv("ARM_TENANT_ID") - if tenantId := os.Getenv("AZURE_TENANT_ID"); tenantId != "" { - cfg.TenantID = tenantId - } - cfg.AADClientID = os.Getenv("ARM_CLIENT_ID") - if clientId := os.Getenv("AZURE_CLIENT_ID"); clientId != "" { - cfg.AADClientID = clientId - } - cfg.AADFederatedTokenFile = os.Getenv("AZURE_FEDERATED_TOKEN_FILE") - cfg.AADClientSecret = os.Getenv("ARM_CLIENT_SECRET") - cfg.VMType = strings.ToLower(os.Getenv("ARM_VM_TYPE")) - cfg.AADClientCertPath = os.Getenv("ARM_CLIENT_CERT_PATH") - cfg.AADClientCertPassword = os.Getenv("ARM_CLIENT_CERT_PASSWORD") - cfg.Deployment = os.Getenv("ARM_DEPLOYMENT") - - subscriptionID, err := getSubscriptionIdFromInstanceMetadata() - if err != nil { - return nil, err - } - cfg.SubscriptionID = subscriptionID - - useManagedIdentityExtensionFromEnv := os.Getenv("ARM_USE_MANAGED_IDENTITY_EXTENSION") - if len(useManagedIdentityExtensionFromEnv) > 0 { - cfg.UseManagedIdentityExtension, err = strconv.ParseBool(useManagedIdentityExtensionFromEnv) - if err != nil { - return nil, err - } - } - - useWorkloadIdentityExtensionFromEnv := os.Getenv("ARM_USE_WORKLOAD_IDENTITY_EXTENSION") - if len(useWorkloadIdentityExtensionFromEnv) > 0 { - cfg.UseWorkloadIdentityExtension, err = strconv.ParseBool(useWorkloadIdentityExtensionFromEnv) - if err != nil { - return nil, err - } - } - - if cfg.UseManagedIdentityExtension && cfg.UseWorkloadIdentityExtension { - return nil, errors.New("you can not combine both managed identity and workload identity as an authentication mechanism") - } - - userAssignedIdentityIDFromEnv := os.Getenv("ARM_USER_ASSIGNED_IDENTITY_ID") - if userAssignedIdentityIDFromEnv != "" { - cfg.UserAssignedIdentityID = userAssignedIdentityIDFromEnv - } - - if vmssCacheTTL := os.Getenv("AZURE_VMSS_CACHE_TTL"); vmssCacheTTL != "" { - cfg.VmssCacheTTL, err = strconv.ParseInt(vmssCacheTTL, 10, 0) - if err != nil { - return nil, fmt.Errorf("failed to parse AZURE_VMSS_CACHE_TTL %q: %v", vmssCacheTTL, err) - } - } - - if vmssVmsCacheTTL := os.Getenv("AZURE_VMSS_VMS_CACHE_TTL"); vmssVmsCacheTTL != "" { - cfg.VmssVmsCacheTTL, err = strconv.ParseInt(vmssVmsCacheTTL, 10, 0) - if err != nil { - return nil, fmt.Errorf("failed to parse AZURE_VMSS_VMS_CACHE_TTL %q: %v", vmssVmsCacheTTL, err) - } - } - - if vmssVmsCacheJitter := os.Getenv("AZURE_VMSS_VMS_CACHE_JITTER"); vmssVmsCacheJitter != "" { - cfg.VmssVmsCacheJitter, err = strconv.Atoi(vmssVmsCacheJitter) - if err != nil { - return nil, fmt.Errorf("failed to parse AZURE_VMSS_VMS_CACHE_JITTER %q: %v", vmssVmsCacheJitter, err) - } - } + } - if threshold := os.Getenv("AZURE_MAX_DEPLOYMENT_COUNT"); threshold != "" { - cfg.MaxDeploymentsCount, err = strconv.ParseInt(threshold, 10, 0) - if err != nil { - return nil, fmt.Errorf("failed to parse AZURE_MAX_DEPLOYMENT_COUNT %q: %v", threshold, err) - } + // Legacy config fields, take precedence if provided. + if cfg.UseWorkloadIdentityExtension != nil { + cfg.UseFederatedWorkloadIdentityExtension = *cfg.UseWorkloadIdentityExtension + } + if cfg.VmssCacheTTL != nil { + if *cfg.VmssCacheTTL > int64(^uint32(0)) { + return nil, fmt.Errorf("VmssCacheTTL value %d is too large", *cfg.VmssCacheTTL) } - - if enableBackoff := os.Getenv("ENABLE_BACKOFF"); enableBackoff != "" { - cfg.CloudProviderBackoff, err = strconv.ParseBool(enableBackoff) - if err != nil { - return nil, fmt.Errorf("failed to parse ENABLE_BACKOFF %q: %v", enableBackoff, err) - } + cfg.VmssCacheTTLInSeconds = int(*cfg.VmssCacheTTL) + } + if cfg.VmssVmsCacheTTL != nil { + if *cfg.VmssVmsCacheTTL > int64(^uint32(0)) { + return nil, fmt.Errorf("VmssVmsCacheTTL value %d is too large", *cfg.VmssVmsCacheTTL) } + cfg.VmssVirtualMachinesCacheTTLInSeconds = int(*cfg.VmssVmsCacheTTL) + } + if cfg.EnableVmssFlex != nil { + cfg.EnableVmssFlexNodes = *cfg.EnableVmssFlex + } - if enableDynamicInstanceList := os.Getenv("AZURE_ENABLE_DYNAMIC_INSTANCE_LIST"); enableDynamicInstanceList != "" { - cfg.EnableDynamicInstanceList, err = strconv.ParseBool(enableDynamicInstanceList) - if err != nil { - return nil, fmt.Errorf("failed to parse AZURE_ENABLE_DYNAMIC_INSTANCE_LIST %q: %v", enableDynamicInstanceList, err) - } - } else { - cfg.EnableDynamicInstanceList = dynamicInstanceListDefault + // Each of these environment variables, if provided, will override what's in the config file. + // Note that this "retrieval from env" does not exist in cloud-provider-azure library (at the time of this comment). + if _, err = assignFromEnvIfExists(&cfg.ClusterName, "CLUSTER_NAME"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.ClusterResourceGroup, "ARM_CLUSTER_RESOURCE_GROUP"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.ARMBaseURLForAPClient, "ARM_BASE_URL_FOR_AP_CLIENT"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.Cloud, "ARM_CLOUD"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.Location, "LOCATION"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.ResourceGroup, "ARM_RESOURCE_GROUP"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.TenantID, "ARM_TENANT_ID"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.TenantID, "AZURE_TENANT_ID"); err != nil { // taking precedence + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.AADClientID, "ARM_CLIENT_ID"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.AADClientID, "AZURE_CLIENT_ID"); err != nil { // taking precedence + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.AADFederatedTokenFile, "AZURE_FEDERATED_TOKEN_FILE"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.AADClientSecret, "ARM_CLIENT_SECRET"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.VMType, "ARM_VM_TYPE"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.AADClientCertPath, "ARM_CLIENT_CERT_PATH"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.AADClientCertPassword, "ARM_CLIENT_CERT_PASSWORD"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.Deployment, "ARM_DEPLOYMENT"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.SubscriptionID, "ARM_SUBSCRIPTION_ID"); err != nil { + return nil, err + } + if _, err = assignBoolFromEnvIfExists(&cfg.UseManagedIdentityExtension, "ARM_USE_MANAGED_IDENTITY_EXTENSION"); err != nil { + return nil, err + } + if _, err = assignBoolFromEnvIfExists(&cfg.UseFederatedWorkloadIdentityExtension, "ARM_USE_FEDERATED_WORKLOAD_IDENTITY_EXTENSION"); err != nil { + return nil, err + } + if _, err = assignBoolFromEnvIfExists(&cfg.UseFederatedWorkloadIdentityExtension, "ARM_USE_WORKLOAD_IDENTITY_EXTENSION"); err != nil { // taking precedence + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.UserAssignedIdentityID, "ARM_USER_ASSIGNED_IDENTITY_ID"); err != nil { + return nil, err + } + if _, err = assignIntFromEnvIfExists(&cfg.VmssCacheTTLInSeconds, "AZURE_VMSS_CACHE_TTL_IN_SECONDS"); err != nil { + return nil, err + } + if _, err = assignIntFromEnvIfExists(&cfg.VmssCacheTTLInSeconds, "AZURE_VMSS_CACHE_TTL"); err != nil { // taking precedence + return nil, err + } + if _, err = assignIntFromEnvIfExists(&cfg.VmssVirtualMachinesCacheTTLInSeconds, "AZURE_VMSS_VMS_CACHE_TTL_IN_SECONDS"); err != nil { + return nil, err + } + if _, err = assignIntFromEnvIfExists(&cfg.VmssVirtualMachinesCacheTTLInSeconds, "AZURE_VMSS_VMS_CACHE_TTL"); err != nil { // taking precedence + return nil, err + } + if _, err = assignIntFromEnvIfExists(&cfg.VmssVmsCacheJitter, "AZURE_VMSS_VMS_CACHE_JITTER"); err != nil { + return nil, err + } + if _, err = assignIntFromEnvIfExists(&cfg.GetVmssSizeRefreshPeriod, "AZURE_GET_VMSS_SIZE_REFRESH_PERIOD"); err != nil { + return nil, err + } + if _, err = assignInt64FromEnvIfExists(&cfg.MaxDeploymentsCount, "AZURE_MAX_DEPLOYMENT_COUNT"); err != nil { + return nil, err + } + if _, err = assignBoolFromEnvIfExists(&cfg.CloudProviderBackoff, "ENABLE_BACKOFF"); err != nil { + return nil, err + } + if _, err = assignBoolFromEnvIfExists(&cfg.EnableForceDelete, "AZURE_ENABLE_FORCE_DELETE"); err != nil { + return nil, err + } + if _, err = assignBoolFromEnvIfExists(&cfg.EnableDynamicInstanceList, "AZURE_ENABLE_DYNAMIC_INSTANCE_LIST"); err != nil { + return nil, err + } + if _, err = assignBoolFromEnvIfExists(&cfg.EnableVmssFlexNodes, "AZURE_ENABLE_VMSS_FLEX_NODES"); err != nil { + return nil, err + } + if _, err = assignBoolFromEnvIfExists(&cfg.EnableVmssFlexNodes, "AZURE_ENABLE_VMSS_FLEX"); err != nil { // taking precedence + return nil, err + } + if cfg.CloudProviderBackoff { + if _, err = assignIntFromEnvIfExists(&cfg.CloudProviderBackoffRetries, "BACKOFF_RETRIES"); err != nil { + return nil, err } - - if getVmssSizeRefreshPeriod := os.Getenv("AZURE_GET_VMSS_SIZE_REFRESH_PERIOD"); getVmssSizeRefreshPeriod != "" { - cfg.GetVmssSizeRefreshPeriod, err = strconv.Atoi(getVmssSizeRefreshPeriod) - if err != nil { - return nil, fmt.Errorf("failed to parse AZURE_GET_VMSS_SIZE_REFRESH_PERIOD %q: %v", getVmssSizeRefreshPeriod, err) - } - } else { - cfg.GetVmssSizeRefreshPeriod = VmssSizeRefreshPeriodDefault + if _, err = assignFloat64FromEnvIfExists(&cfg.CloudProviderBackoffExponent, "BACKOFF_EXPONENT"); err != nil { + return nil, err } - - if enableVmssFlex := os.Getenv("AZURE_ENABLE_VMSS_FLEX"); enableVmssFlex != "" { - cfg.EnableVmssFlex, err = strconv.ParseBool(enableVmssFlex) - if err != nil { - return nil, fmt.Errorf("failed to parse AZURE_ENABLE_VMSS_FLEX %q: %v", enableVmssFlex, err) - } - } else { - cfg.EnableVmssFlex = enableVmssFlexDefault + if _, err = assignIntFromEnvIfExists(&cfg.CloudProviderBackoffDuration, "BACKOFF_DURATION"); err != nil { + return nil, err } - - if cfg.CloudProviderBackoff { - if backoffRetries := os.Getenv("BACKOFF_RETRIES"); backoffRetries != "" { - retries, err := strconv.ParseInt(backoffRetries, 10, 0) - if err != nil { - return nil, fmt.Errorf("failed to parse BACKOFF_RETRIES %q: %v", retries, err) - } - cfg.CloudProviderBackoffRetries = int(retries) - } else { - cfg.CloudProviderBackoffRetries = backoffRetriesDefault - } - - if backoffExponent := os.Getenv("BACKOFF_EXPONENT"); backoffExponent != "" { - cfg.CloudProviderBackoffExponent, err = strconv.ParseFloat(backoffExponent, 64) - if err != nil { - return nil, fmt.Errorf("failed to parse BACKOFF_EXPONENT %q: %v", backoffExponent, err) - } - } else { - cfg.CloudProviderBackoffExponent = backoffExponentDefault - } - - if backoffDuration := os.Getenv("BACKOFF_DURATION"); backoffDuration != "" { - duration, err := strconv.ParseInt(backoffDuration, 10, 0) - if err != nil { - return nil, fmt.Errorf("failed to parse BACKOFF_DURATION %q: %v", backoffDuration, err) - } - cfg.CloudProviderBackoffDuration = int(duration) - } else { - cfg.CloudProviderBackoffDuration = backoffDurationDefault - } - - if backoffJitter := os.Getenv("BACKOFF_JITTER"); backoffJitter != "" { - cfg.CloudProviderBackoffJitter, err = strconv.ParseFloat(backoffJitter, 64) - if err != nil { - return nil, fmt.Errorf("failed to parse BACKOFF_JITTER %q: %v", backoffJitter, err) - } - } else { - cfg.CloudProviderBackoffJitter = backoffJitterDefault - } + if _, err = assignFloat64FromEnvIfExists(&cfg.CloudProviderBackoffJitter, "BACKOFF_JITTER"); err != nil { + return nil, err } } + if _, err = assignBoolFromEnvIfExists(&cfg.CloudProviderRateLimit, "CLOUD_PROVIDER_RATE_LIMIT"); err != nil { + return nil, err + } + if _, err = assignFloat32FromEnvIfExists(&cfg.CloudProviderRateLimitQPS, "RATE_LIMIT_READ_QPS"); err != nil { + return nil, err + } + if _, err = assignIntFromEnvIfExists(&cfg.CloudProviderRateLimitBucket, "RATE_LIMIT_READ_BUCKETS"); err != nil { + return nil, err + } + if _, err = assignFloat32FromEnvIfExists(&cfg.CloudProviderRateLimitQPSWrite, "RATE_LIMIT_WRITE_QPS"); err != nil { + return nil, err + } + if _, err = assignIntFromEnvIfExists(&cfg.CloudProviderRateLimitBucketWrite, "RATE_LIMIT_WRITE_BUCKETS"); err != nil { + return nil, err + } - // always read the following from environment variables since azure.json doesn't have these fields - cfg.ClusterName = os.Getenv("CLUSTER_NAME") - cfg.ClusterResourceGroup = os.Getenv("ARM_CLUSTER_RESOURCE_GROUP") - cfg.ARMBaseURLForAPClient = os.Getenv("ARM_BASE_URL_FOR_AP_CLIENT") - - cfg.TrimSpace() - - if cloudProviderRateLimit := os.Getenv("CLOUD_PROVIDER_RATE_LIMIT"); cloudProviderRateLimit != "" { - cfg.CloudProviderRateLimit, err = strconv.ParseBool(cloudProviderRateLimit) + // Nonstatic defaults + cfg.VMType = strings.ToLower(cfg.VMType) + if cfg.MaxDeploymentsCount == 0 { + // 0 means "use default" in this case. + // This means, if it is valued by the config file, but explicitly set to 0 in the env, it will retreat to default. + cfg.MaxDeploymentsCount = int64(defaultMaxDeploymentsCount) + } + if cfg.SubscriptionID == "" { + metadataService, err := providerazure.NewInstanceMetadataService(imdsServerURL) if err != nil { - return nil, fmt.Errorf("failed to parse CLOUD_PROVIDER_RATE_LIMIT: %q, %v", cloudProviderRateLimit, err) + return nil, err } - } - if enableForceDelete := os.Getenv("AZURE_ENABLE_FORCE_DELETE"); enableForceDelete != "" { - cfg.EnableForceDelete, err = strconv.ParseBool(enableForceDelete) + metadata, err := metadataService.GetMetadata(0) if err != nil { - return nil, fmt.Errorf("failed to parse AZURE_ENABLE_FORCE_DELETE: %q, %v", enableForceDelete, err) + return nil, err } - } - err = initializeCloudProviderRateLimitConfig(&cfg.CloudProviderRateLimitConfig) - if err != nil { - return nil, err + cfg.SubscriptionID = metadata.Compute.SubscriptionID } - - // Defaulting vmType to vmss. - if cfg.VMType == "" { - cfg.VMType = vmTypeVMSS - } - - // Read parameters from deploymentParametersPath if it is not set. - if cfg.VMType == vmTypeStandard && len(cfg.DeploymentParameters) == 0 { + if cfg.VMType == providerazureconsts.VMTypeStandard && len(cfg.DeploymentParameters) == 0 { + // Read parameters from deploymentParametersPath if it is not set. parameters, err := readDeploymentParameters(deploymentParametersPath) if err != nil { klog.Errorf("readDeploymentParameters failed with error: %v", err) @@ -365,10 +316,7 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) { cfg.DeploymentParameters = parameters } - - if cfg.MaxDeploymentsCount == 0 { - cfg.MaxDeploymentsCount = int64(defaultMaxDeploymentsCount) - } + providerazureconfig.InitializeCloudProviderRateLimitConfig(&cfg.CloudProviderRateLimitConfig) if err := cfg.validate(); err != nil { return nil, err @@ -376,104 +324,11 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) { return cfg, nil } -// initializeCloudProviderRateLimitConfig initializes rate limit configs. -func initializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig) error { - if config == nil { - return nil - } - - // Assign read rate limit defaults if no configuration was passed in. - if config.CloudProviderRateLimitQPS == 0 { - if rateLimitQPSFromEnv := os.Getenv(rateLimitReadQPSEnvVar); rateLimitQPSFromEnv != "" { - rateLimitQPS, err := strconv.ParseFloat(rateLimitQPSFromEnv, 0) - if err != nil { - return fmt.Errorf("failed to parse %s: %q, %v", rateLimitReadQPSEnvVar, rateLimitQPSFromEnv, err) - } - config.CloudProviderRateLimitQPS = float32(rateLimitQPS) - } else { - config.CloudProviderRateLimitQPS = rateLimitQPSDefault - } - } - - if config.CloudProviderRateLimitBucket == 0 { - if rateLimitBucketFromEnv := os.Getenv(rateLimitReadBucketsEnvVar); rateLimitBucketFromEnv != "" { - rateLimitBucket, err := strconv.ParseInt(rateLimitBucketFromEnv, 10, 0) - if err != nil { - return fmt.Errorf("failed to parse %s: %q, %v", rateLimitReadBucketsEnvVar, rateLimitBucketFromEnv, err) - } - config.CloudProviderRateLimitBucket = int(rateLimitBucket) - } else { - config.CloudProviderRateLimitBucket = rateLimitBucketDefault - } - } - - // Assign write rate limit defaults if no configuration was passed in. - if config.CloudProviderRateLimitQPSWrite == 0 { - if rateLimitQPSWriteFromEnv := os.Getenv(rateLimitWriteQPSEnvVar); rateLimitQPSWriteFromEnv != "" { - rateLimitQPSWrite, err := strconv.ParseFloat(rateLimitQPSWriteFromEnv, 0) - if err != nil { - return fmt.Errorf("failed to parse %s: %q, %v", rateLimitWriteQPSEnvVar, rateLimitQPSWriteFromEnv, err) - } - config.CloudProviderRateLimitQPSWrite = float32(rateLimitQPSWrite) - } else { - config.CloudProviderRateLimitQPSWrite = config.CloudProviderRateLimitQPS - } - } - - if config.CloudProviderRateLimitBucketWrite == 0 { - if rateLimitBucketWriteFromEnv := os.Getenv(rateLimitWriteBucketsEnvVar); rateLimitBucketWriteFromEnv != "" { - rateLimitBucketWrite, err := strconv.ParseInt(rateLimitBucketWriteFromEnv, 10, 0) - if err != nil { - return fmt.Errorf("failed to parse %s: %q, %v", rateLimitWriteBucketsEnvVar, rateLimitBucketWriteFromEnv, err) - } - config.CloudProviderRateLimitBucketWrite = int(rateLimitBucketWrite) - } else { - config.CloudProviderRateLimitBucketWrite = config.CloudProviderRateLimitBucket - } - } - - config.InterfaceRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.InterfaceRateLimit) - config.VirtualMachineRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineRateLimit) - config.StorageAccountRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.StorageAccountRateLimit) - config.DiskRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.DiskRateLimit) - config.VirtualMachineScaleSetRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineScaleSetRateLimit) - config.KubernetesServiceRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.KubernetesServiceRateLimit) - - return nil -} - -// overrideDefaultRateLimitConfig overrides the default CloudProviderRateLimitConfig. -func overrideDefaultRateLimitConfig(defaults, config *azclients.RateLimitConfig) *azclients.RateLimitConfig { - // If config not set, apply defaults. - if config == nil { - return defaults - } - - // Remain disabled if it's set explicitly. - if !config.CloudProviderRateLimit { - return &azclients.RateLimitConfig{CloudProviderRateLimit: false} - } - - // Apply default values. - if config.CloudProviderRateLimitQPS == 0 { - config.CloudProviderRateLimitQPS = defaults.CloudProviderRateLimitQPS - } - if config.CloudProviderRateLimitBucket == 0 { - config.CloudProviderRateLimitBucket = defaults.CloudProviderRateLimitBucket - } - if config.CloudProviderRateLimitQPSWrite == 0 { - config.CloudProviderRateLimitQPSWrite = defaults.CloudProviderRateLimitQPSWrite - } - if config.CloudProviderRateLimitBucketWrite == 0 { - config.CloudProviderRateLimitBucketWrite = defaults.CloudProviderRateLimitBucketWrite - } - - return config -} - +// A "fork" of az.getAzureClientConfig with BYO authorizer (e.g., for CLI auth) and custom polling delay support func (cfg *Config) getAzureClientConfig(authorizer autorest.Authorizer, env *azure.Environment) *azclients.ClientConfig { pollingDelay := 30 * time.Second azClientConfig := &azclients.ClientConfig{ + CloudName: cfg.Cloud, Location: cfg.Location, SubscriptionID: cfg.SubscriptionID, ResourceManagerEndpoint: env.ResourceManagerEndpoint, @@ -482,6 +337,8 @@ func (cfg *Config) getAzureClientConfig(authorizer autorest.Authorizer, env *azu RestClientConfig: azclients.RestClientConfig{ PollingDelay: &pollingDelay, }, + DisableAzureStackCloud: cfg.DisableAzureStackCloud, + UserAgent: cfg.UserAgent, } if cfg.CloudProviderBackoff { @@ -493,24 +350,14 @@ func (cfg *Config) getAzureClientConfig(authorizer autorest.Authorizer, env *azu } } - return azClientConfig -} + if cfg.HasExtendedLocation() { + azClientConfig.ExtendedLocation = &azclients.ExtendedLocation{ + Name: cfg.ExtendedLocationName, + Type: cfg.ExtendedLocationType, + } + } -// TrimSpace removes all leading and trailing white spaces. -func (cfg *Config) TrimSpace() { - cfg.Cloud = strings.TrimSpace(cfg.Cloud) - cfg.Location = strings.TrimSpace(cfg.Location) - cfg.TenantID = strings.TrimSpace(cfg.TenantID) - cfg.SubscriptionID = strings.TrimSpace(cfg.SubscriptionID) - cfg.ClusterName = strings.TrimSpace(cfg.ClusterName) - cfg.ResourceGroup = strings.TrimSpace(cfg.ResourceGroup) - cfg.ClusterResourceGroup = strings.TrimSpace(cfg.ClusterResourceGroup) - cfg.VMType = strings.TrimSpace(cfg.VMType) - cfg.AADClientID = strings.TrimSpace(cfg.AADClientID) - cfg.AADClientSecret = strings.TrimSpace(cfg.AADClientSecret) - cfg.AADClientCertPath = strings.TrimSpace(cfg.AADClientCertPath) - cfg.AADClientCertPassword = strings.TrimSpace(cfg.AADClientCertPassword) - cfg.Deployment = strings.TrimSpace(cfg.Deployment) + return azClientConfig } func (cfg *Config) validate() error { @@ -518,7 +365,7 @@ func (cfg *Config) validate() error { return fmt.Errorf("resource group not set") } - if cfg.VMType == vmTypeStandard { + if cfg.VMType == providerazureconsts.VMTypeStandard { if cfg.Deployment == "" { return fmt.Errorf("deployment not set") } @@ -532,23 +379,29 @@ func (cfg *Config) validate() error { return fmt.Errorf("subscription ID not set") } - if cfg.UseManagedIdentityExtension { - return nil + if cfg.UseManagedIdentityExtension && cfg.UseFederatedWorkloadIdentityExtension { + return fmt.Errorf("you can not combine both managed identity and workload identity as an authentication mechanism") } - if cfg.TenantID == "" { - return fmt.Errorf("tenant ID not set") + if cfg.VMType != providerazureconsts.VMTypeStandard && cfg.VMType != providerazureconsts.VMTypeVMSS { + return fmt.Errorf("unsupported VM type: %s", cfg.VMType) } - switch cfg.AuthMethod { - case "", authMethodPrincipal: - if cfg.AADClientID == "" { - return errors.New("ARM Client ID not set") + if !cfg.UseManagedIdentityExtension && !cfg.UseFederatedWorkloadIdentityExtension { + if cfg.TenantID == "" { + return fmt.Errorf("tenant ID not set") + } + + switch cfg.AuthMethod { + case "", authMethodPrincipal: + if cfg.AADClientID == "" { + return fmt.Errorf("ARM Client ID not set") + } + case authMethodCLI: + // Nothing to check at the moment. + default: + return fmt.Errorf("unsupported authorization method: %s", cfg.AuthMethod) } - case authMethodCLI: - // Nothing to check at the moment. - default: - return fmt.Errorf("unsupported authorization method: %s", cfg.AuthMethod) } if cfg.CloudProviderBackoff && cfg.CloudProviderBackoffRetries == 0 { @@ -558,21 +411,88 @@ func (cfg *Config) validate() error { return nil } -// getSubscriptionId reads the Subscription ID from the instance metadata. -func getSubscriptionIdFromInstanceMetadata() (string, error) { - subscriptionID, present := os.LookupEnv("ARM_SUBSCRIPTION_ID") - if !present { - metadataService, err := providerazure.NewInstanceMetadataService(imdsServerURL) +func assignFromEnvIfExists(assignee *string, name string) (bool, error) { + if assignee == nil { + return false, fmt.Errorf("assignee is nil") + } + if val, present := os.LookupEnv(name); present && strings.TrimSpace(val) != "" { + *assignee = strings.TrimSpace(val) + return true, nil + } + return false, nil +} + +func assignBoolFromEnvIfExists(assignee *bool, name string) (bool, error) { + if assignee == nil { + return false, fmt.Errorf("assignee is nil") + } + var err error + if val, present := os.LookupEnv(name); present && strings.TrimSpace(val) != "" { + *assignee, err = strconv.ParseBool(val) if err != nil { - return "", err + return false, fmt.Errorf("failed to parse %s %q: %v", name, val, err) } + return true, nil + } + return false, nil +} - metadata, err := metadataService.GetMetadata(0) +func assignIntFromEnvIfExists(assignee *int, name string) (bool, error) { + if assignee == nil { + return false, fmt.Errorf("assignee is nil") + } + var err error + if val, present := os.LookupEnv(name); present && strings.TrimSpace(val) != "" { + *assignee, err = parseInt32(val, 10) if err != nil { - return "", err + return false, fmt.Errorf("failed to parse %s %q: %v", name, val, err) } + return true, nil + } + return false, nil +} - return metadata.Compute.SubscriptionID, nil +func assignInt64FromEnvIfExists(assignee *int64, name string) (bool, error) { + if assignee == nil { + return false, fmt.Errorf("assignee is nil") + } + var err error + if val, present := os.LookupEnv(name); present && strings.TrimSpace(val) != "" { + *assignee, err = strconv.ParseInt(val, 10, 0) + if err != nil { + return false, fmt.Errorf("failed to parse %s %q: %v", name, val, err) + } + return true, nil + } + return false, nil +} + +func assignFloat32FromEnvIfExists(assignee *float32, name string) (bool, error) { + if assignee == nil { + return false, fmt.Errorf("assignee is nil") + } + var err error + if val, present := os.LookupEnv(name); present && strings.TrimSpace(val) != "" { + *assignee, err = parseFloat32(val) + if err != nil { + return false, fmt.Errorf("failed to parse %s %q: %v", name, val, err) + } + return true, nil + } + return false, nil +} + +func assignFloat64FromEnvIfExists(assignee *float64, name string) (bool, error) { + if assignee == nil { + return false, fmt.Errorf("assignee is nil") + } + var err error + if val, present := os.LookupEnv(name); present && strings.TrimSpace(val) != "" { + *assignee, err = strconv.ParseFloat(val, 64) + if err != nil { + return false, fmt.Errorf("failed to parse %s %q: %v", name, val, err) + } + return true, nil } - return subscriptionID, nil + return false, nil } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_config_test.go b/cluster-autoscaler/cloudprovider/azure/azure_config_test.go index 65c74a5c90a1..6bfd19ce2319 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_config_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_config_test.go @@ -17,174 +17,74 @@ limitations under the License. package azure import ( - "fmt" "testing" "github.com/stretchr/testify/assert" azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients" + providerazureconsts "sigs.k8s.io/cloud-provider-azure/pkg/consts" + providerazureconfig "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" ) -func TestInitializeCloudProviderRateLimitConfigWithNoConfigReturnsNoError(t *testing.T) { - err := initializeCloudProviderRateLimitConfig(nil) - assert.Nil(t, err, "err should be nil") +func TestCloudProviderAzureConsts(t *testing.T) { + // Just detect user-facing breaking changes from cloud-provider-azure. + // Shouldn't really change a lot, but just in case. + assert.Equal(t, "vmss", providerazureconsts.VMTypeVMSS) + assert.Equal(t, "standard", providerazureconsts.VMTypeStandard) } func TestInitializeCloudProviderRateLimitConfigWithNoRateLimitSettingsReturnsDefaults(t *testing.T) { - emptyConfig := &CloudProviderRateLimitConfig{} - err := initializeCloudProviderRateLimitConfig(emptyConfig) - - assert.NoError(t, err) - assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitQPSDefault) - assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitBucketDefault) - assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitQPSDefault) - assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitBucketDefault) -} - -func TestInitializeCloudProviderRateLimitConfigWithReadRateLimitSettingsFromEnv(t *testing.T) { - emptyConfig := &CloudProviderRateLimitConfig{} - var rateLimitReadQPS float32 = 3.0 - rateLimitReadBuckets := 10 - t.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS)) - t.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets)) - - err := initializeCloudProviderRateLimitConfig(emptyConfig) - assert.NoError(t, err) - assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitReadQPS) - assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitReadBuckets) - assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitReadQPS) - assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets) -} - -func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitSettingsFromEnv(t *testing.T) { - emptyConfig := &CloudProviderRateLimitConfig{} - var rateLimitReadQPS float32 = 3.0 - rateLimitReadBuckets := 10 - var rateLimitWriteQPS float32 = 6.0 - rateLimitWriteBuckets := 20 - - t.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS)) - t.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets)) - t.Setenv(rateLimitWriteQPSEnvVar, fmt.Sprintf("%.1f", rateLimitWriteQPS)) - t.Setenv(rateLimitWriteBucketsEnvVar, fmt.Sprintf("%d", rateLimitWriteBuckets)) + emptyConfig := &providerazureconfig.CloudProviderRateLimitConfig{} + providerazureconfig.InitializeCloudProviderRateLimitConfig(emptyConfig) - err := initializeCloudProviderRateLimitConfig(emptyConfig) - - assert.NoError(t, err) - assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitReadQPS) - assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitReadBuckets) - assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS) - assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets) + assert.InDelta(t, emptyConfig.CloudProviderRateLimitQPS, providerazureconsts.RateLimitQPSDefault, 0.0001) + assert.InDelta(t, emptyConfig.CloudProviderRateLimitBucket, providerazureconsts.RateLimitBucketDefault, 0.0001) + assert.InDelta(t, emptyConfig.CloudProviderRateLimitQPSWrite, providerazureconsts.RateLimitQPSDefault, 0.0001) + assert.InDelta(t, emptyConfig.CloudProviderRateLimitBucketWrite, providerazureconsts.RateLimitBucketDefault, 0.0001) } -func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitAlreadySetInConfig(t *testing.T) { +func TestInitializeCloudProviderRateLimitConfigWithReadRateLimitSettings(t *testing.T) { var rateLimitReadQPS float32 = 3.0 rateLimitReadBuckets := 10 - var rateLimitWriteQPS float32 = 6.0 - rateLimitWriteBuckets := 20 - configWithRateLimits := &CloudProviderRateLimitConfig{ + cfg := &providerazureconfig.CloudProviderRateLimitConfig{ RateLimitConfig: azclients.RateLimitConfig{ - CloudProviderRateLimitBucket: rateLimitReadBuckets, - CloudProviderRateLimitBucketWrite: rateLimitWriteBuckets, - CloudProviderRateLimitQPS: rateLimitReadQPS, - CloudProviderRateLimitQPSWrite: rateLimitWriteQPS, - }, - } - - t.Setenv(rateLimitReadQPSEnvVar, "99") - t.Setenv(rateLimitReadBucketsEnvVar, "99") - t.Setenv(rateLimitWriteQPSEnvVar, "99") - t.Setenv(rateLimitWriteBucketsEnvVar, "99") - - err := initializeCloudProviderRateLimitConfig(configWithRateLimits) - - assert.NoError(t, err) - assert.Equal(t, configWithRateLimits.CloudProviderRateLimitQPS, rateLimitReadQPS) - assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucket, rateLimitReadBuckets) - assert.Equal(t, configWithRateLimits.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS) - assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets) -} - -// nolint: goconst -func TestInitializeCloudProviderRateLimitConfigWithInvalidReadAndWriteRateLimitSettingsFromEnv(t *testing.T) { - emptyConfig := &CloudProviderRateLimitConfig{} - var rateLimitReadQPS float32 = 3.0 - rateLimitReadBuckets := 10 - var rateLimitWriteQPS float32 = 6.0 - rateLimitWriteBuckets := 20 - - invalidSetting := "invalid" - testCases := []struct { - desc string - isInvalidRateLimitReadQPSEnvVar bool - isInvalidRateLimitReadBucketsEnvVar bool - isInvalidRateLimitWriteQPSEnvVar bool - isInvalidRateLimitWriteBucketsEnvVar bool - expectedErr bool - expectedErrMsg error - }{ - { - desc: "an error shall be returned if invalid rateLimitReadQPSEnvVar", - isInvalidRateLimitReadQPSEnvVar: true, - expectedErr: true, - expectedErrMsg: fmt.Errorf("failed to parse %s: %q, strconv.ParseFloat: parsing \"invalid\": invalid syntax", rateLimitReadQPSEnvVar, invalidSetting), - }, - { - desc: "an error shall be returned if invalid rateLimitReadBucketsEnvVar", - isInvalidRateLimitReadBucketsEnvVar: true, - expectedErr: true, - expectedErrMsg: fmt.Errorf("failed to parse %s: %q, strconv.ParseInt: parsing \"invalid\": invalid syntax", rateLimitReadBucketsEnvVar, invalidSetting), - }, - { - desc: "an error shall be returned if invalid rateLimitWriteQPSEnvVar", - isInvalidRateLimitWriteQPSEnvVar: true, - expectedErr: true, - expectedErrMsg: fmt.Errorf("failed to parse %s: %q, strconv.ParseFloat: parsing \"invalid\": invalid syntax", rateLimitWriteQPSEnvVar, invalidSetting), - }, - { - desc: "an error shall be returned if invalid rateLimitWriteBucketsEnvVar", - isInvalidRateLimitWriteBucketsEnvVar: true, - expectedErr: true, - expectedErrMsg: fmt.Errorf("failed to parse %s: %q, strconv.ParseInt: parsing \"invalid\": invalid syntax", rateLimitWriteBucketsEnvVar, invalidSetting), + CloudProviderRateLimitQPS: rateLimitReadQPS, + CloudProviderRateLimitBucket: rateLimitReadBuckets, }, } - - for i, test := range testCases { - if test.isInvalidRateLimitReadQPSEnvVar { - t.Setenv(rateLimitReadQPSEnvVar, invalidSetting) - } else { - t.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS)) - } - if test.isInvalidRateLimitReadBucketsEnvVar { - t.Setenv(rateLimitReadBucketsEnvVar, invalidSetting) - } else { - t.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets)) - } - if test.isInvalidRateLimitWriteQPSEnvVar { - t.Setenv(rateLimitWriteQPSEnvVar, invalidSetting) - } else { - t.Setenv(rateLimitWriteQPSEnvVar, fmt.Sprintf("%.1f", rateLimitWriteQPS)) - } - if test.isInvalidRateLimitWriteBucketsEnvVar { - t.Setenv(rateLimitWriteBucketsEnvVar, invalidSetting) - } else { - t.Setenv(rateLimitWriteBucketsEnvVar, fmt.Sprintf("%d", rateLimitWriteBuckets)) - } - - err := initializeCloudProviderRateLimitConfig(emptyConfig) - - assert.Equal(t, test.expectedErr, err != nil, "TestCase[%d]: %s, return error: %v", i, test.desc, err) - assert.Equal(t, test.expectedErrMsg, err, "TestCase[%d]: %s, expected: %v, return: %v", i, test.desc, test.expectedErrMsg, err) - } + //t.Setenv("RATE_LIMIT_READ_QPS", fmt.Sprintf("%.1f", rateLimitReadQPS) + //t.Setenv("RATE_LIMIT_READ_BUCKETS", fmt.Sprintf("%d", rateLimitReadBuckets) + + providerazureconfig.InitializeCloudProviderRateLimitConfig(cfg) + assert.InDelta(t, cfg.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.CloudProviderRateLimitQPSWrite, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.InterfaceRateLimit.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.InterfaceRateLimit.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.InterfaceRateLimit.CloudProviderRateLimitQPSWrite, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.InterfaceRateLimit.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.VirtualMachineRateLimit.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.VirtualMachineRateLimit.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.VirtualMachineRateLimit.CloudProviderRateLimitQPSWrite, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.VirtualMachineRateLimit.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.StorageAccountRateLimit.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.StorageAccountRateLimit.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.StorageAccountRateLimit.CloudProviderRateLimitQPSWrite, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.StorageAccountRateLimit.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.VirtualMachineScaleSetRateLimit.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.VirtualMachineScaleSetRateLimit.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.VirtualMachineScaleSetRateLimit.CloudProviderRateLimitQPSWrite, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.VirtualMachineScaleSetRateLimit.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets, 0.0001) } -func TestOverrideDefaultRateLimitConfig(t *testing.T) { +func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitSettings(t *testing.T) { var rateLimitReadQPS float32 = 3.0 rateLimitReadBuckets := 10 var rateLimitWriteQPS float32 = 6.0 rateLimitWriteBuckets := 20 - defaultConfigWithRateLimits := &CloudProviderRateLimitConfig{ + cfg := &providerazureconfig.CloudProviderRateLimitConfig{ RateLimitConfig: azclients.RateLimitConfig{ CloudProviderRateLimitBucket: rateLimitReadBuckets, CloudProviderRateLimitBucketWrite: rateLimitWriteBuckets, @@ -193,28 +93,31 @@ func TestOverrideDefaultRateLimitConfig(t *testing.T) { }, } - configWithRateLimits := &CloudProviderRateLimitConfig{ - RateLimitConfig: azclients.RateLimitConfig{ - CloudProviderRateLimit: true, - CloudProviderRateLimitBucket: 0, - CloudProviderRateLimitBucketWrite: 0, - CloudProviderRateLimitQPS: 0, - CloudProviderRateLimitQPSWrite: 0, - }, - } - - newconfig := overrideDefaultRateLimitConfig(&defaultConfigWithRateLimits.RateLimitConfig, &configWithRateLimits.RateLimitConfig) - - assert.Equal(t, defaultConfigWithRateLimits.CloudProviderRateLimitQPS, newconfig.CloudProviderRateLimitQPS) - assert.Equal(t, defaultConfigWithRateLimits.CloudProviderRateLimitBucket, newconfig.CloudProviderRateLimitBucket) - assert.Equal(t, defaultConfigWithRateLimits.CloudProviderRateLimitQPSWrite, newconfig.CloudProviderRateLimitQPSWrite) - assert.Equal(t, defaultConfigWithRateLimits.CloudProviderRateLimitBucketWrite, newconfig.CloudProviderRateLimitBucketWrite) - - falseCloudProviderRateLimit := &CloudProviderRateLimitConfig{ - RateLimitConfig: azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - }, - } - newconfig = overrideDefaultRateLimitConfig(&defaultConfigWithRateLimits.RateLimitConfig, &falseCloudProviderRateLimit.RateLimitConfig) - assert.Equal(t, &falseCloudProviderRateLimit.RateLimitConfig, newconfig) + //t.Setenv("RATE_LIMIT_READ_QPS", fmt.Sprintf("%.1f", rateLimitReadQPS) + //t.Setenv("RATE_LIMIT_READ_BUCKETS", fmt.Sprintf("%d", rateLimitReadBuckets) + //t.Setenv("RATE_LIMIT_WRITE_QPS", fmt.Sprintf("%.1f", rateLimitWriteQPS) + //t.Setenv("RATE_LIMIT_WRITE_BUCKETS", fmt.Sprintf("%d", rateLimitWriteBuckets) + + providerazureconfig.InitializeCloudProviderRateLimitConfig(cfg) + + assert.InDelta(t, cfg.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS, 0.0001) + assert.InDelta(t, cfg.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets, 0.0001) + assert.InDelta(t, cfg.InterfaceRateLimit.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.InterfaceRateLimit.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.InterfaceRateLimit.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS, 0.0001) + assert.InDelta(t, cfg.InterfaceRateLimit.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets, 0.0001) + assert.InDelta(t, cfg.VirtualMachineRateLimit.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.VirtualMachineRateLimit.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.VirtualMachineRateLimit.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS, 0.0001) + assert.InDelta(t, cfg.VirtualMachineRateLimit.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets, 0.0001) + assert.InDelta(t, cfg.StorageAccountRateLimit.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.StorageAccountRateLimit.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.StorageAccountRateLimit.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS, 0.0001) + assert.InDelta(t, cfg.StorageAccountRateLimit.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets, 0.0001) + assert.InDelta(t, cfg.VirtualMachineScaleSetRateLimit.CloudProviderRateLimitQPS, rateLimitReadQPS, 0.0001) + assert.InDelta(t, cfg.VirtualMachineScaleSetRateLimit.CloudProviderRateLimitBucket, rateLimitReadBuckets, 0.0001) + assert.InDelta(t, cfg.VirtualMachineScaleSetRateLimit.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS, 0.0001) + assert.InDelta(t, cfg.VirtualMachineScaleSetRateLimit.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets, 0.0001) } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_fakes.go b/cluster-autoscaler/cloudprovider/azure/azure_fakes.go index 481fcaacf837..adc38e80b39a 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_fakes.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_fakes.go @@ -19,12 +19,12 @@ package azure import ( "context" "fmt" - "net/http" "sync" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources" "github.com/stretchr/testify/mock" + "sigs.k8s.io/cloud-provider-azure/pkg/retry" ) const ( @@ -32,8 +32,8 @@ const ( fakeVirtualMachineVMID = "/subscriptions/test-subscription-id/resourceGroups/test-asg/providers/Microsoft.Compute/virtualMachines/%d" ) -// DeploymentsClientMock mocks for DeploymentsClient. -type DeploymentsClientMock struct { +// DeploymentClientMock mocks for DeploymentsClient. +type DeploymentClientMock struct { mock.Mock mutex sync.Mutex @@ -41,26 +41,26 @@ type DeploymentsClientMock struct { } // Get gets the DeploymentExtended by deploymentName. -func (m *DeploymentsClientMock) Get(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExtended, err error) { +func (m *DeploymentClientMock) Get(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExtended, err *retry.Error) { m.mutex.Lock() defer m.mutex.Unlock() deploy, ok := m.FakeStore[deploymentName] if !ok { - return result, fmt.Errorf("deployment not found") + return result, retry.NewError(false, fmt.Errorf("deployment not found")) } return deploy, nil } // ExportTemplate exports the deployment's template. -func (m *DeploymentsClientMock) ExportTemplate(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExportResult, err error) { +func (m *DeploymentClientMock) ExportTemplate(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExportResult, err *retry.Error) { m.mutex.Lock() defer m.mutex.Unlock() deploy, ok := m.FakeStore[deploymentName] if !ok { - return result, fmt.Errorf("deployment not found") + return result, retry.NewError(false, fmt.Errorf("deployment not found")) } return resources.DeploymentExportResult{ @@ -69,7 +69,7 @@ func (m *DeploymentsClientMock) ExportTemplate(ctx context.Context, resourceGrou } // CreateOrUpdate creates or updates the Deployment. -func (m *DeploymentsClientMock) CreateOrUpdate(ctx context.Context, resourceGroupName string, deploymentName string, parameters resources.Deployment) (resp *http.Response, err error) { +func (m *DeploymentClientMock) CreateOrUpdate(ctx context.Context, resourceGroupName string, deploymentName string, parameters resources.Deployment, etag string) (err *retry.Error) { m.mutex.Lock() defer m.mutex.Unlock() @@ -83,11 +83,11 @@ func (m *DeploymentsClientMock) CreateOrUpdate(ctx context.Context, resourceGrou deploy.Properties.Parameters = parameters.Properties.Parameters deploy.Properties.Template = parameters.Properties.Template - return &http.Response{StatusCode: 200}, nil + return nil } // List gets all the deployments for a resource group. -func (m *DeploymentsClientMock) List(ctx context.Context, resourceGroupName, filter string, top *int32) (result []resources.DeploymentExtended, err error) { +func (m *DeploymentClientMock) List(ctx context.Context, resourceGroupName string) (result []resources.DeploymentExtended, err *retry.Error) { m.mutex.Lock() defer m.mutex.Unlock() @@ -100,12 +100,12 @@ func (m *DeploymentsClientMock) List(ctx context.Context, resourceGroupName, fil } // Delete deletes the given deployment -func (m *DeploymentsClientMock) Delete(ctx context.Context, resourceGroupName, deploymentName string) (resp *http.Response, err error) { +func (m *DeploymentClientMock) Delete(ctx context.Context, resourceGroupName, deploymentName string) (err *retry.Error) { m.mutex.Lock() defer m.mutex.Unlock() if _, ok := m.FakeStore[deploymentName]; !ok { - return nil, fmt.Errorf("there is no such a deployment with name %s", deploymentName) + return retry.NewError(false, fmt.Errorf("there is no such a deployment with name %s", deploymentName)) } delete(m.FakeStore, deploymentName) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager.go b/cluster-autoscaler/cloudprovider/azure/azure_manager.go index 0802d40e085c..6c0f62a099fa 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_manager.go @@ -32,15 +32,13 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/config/dynamic" kretry "k8s.io/client-go/util/retry" klog "k8s.io/klog/v2" + providerazureconsts "sigs.k8s.io/cloud-provider-azure/pkg/consts" "sigs.k8s.io/cloud-provider-azure/pkg/retry" ) const ( azurePrefix = "azure://" - vmTypeVMSS = "vmss" - vmTypeStandard = "standard" - scaleToZeroSupportedStandard = false scaleToZeroSupportedVMSS = true refreshInterval = 1 * time.Minute @@ -103,8 +101,8 @@ func createAzureManagerInternal(configReader io.Reader, discoveryOpts cloudprovi } cacheTTL := refreshInterval - if cfg.VmssCacheTTL != 0 { - cacheTTL = time.Duration(cfg.VmssCacheTTL) * time.Second + if cfg.VmssCacheTTLInSeconds != 0 { + cacheTTL = time.Duration(cfg.VmssCacheTTLInSeconds) * time.Second } cache, err := newAzureCache(azClient, cacheTTL, *cfg) if err != nil { @@ -166,7 +164,7 @@ func (m *AzureManager) fetchExplicitNodeGroups(specs []string) error { func (m *AzureManager) buildNodeGroupFromSpec(spec string) (cloudprovider.NodeGroup, error) { scaleToZeroSupported := scaleToZeroSupportedStandard - if strings.EqualFold(m.config.VMType, vmTypeVMSS) { + if strings.EqualFold(m.config.VMType, providerazureconsts.VMTypeVMSS) { scaleToZeroSupported = scaleToZeroSupportedVMSS } s, err := dynamic.SpecFromString(spec, scaleToZeroSupported) @@ -179,9 +177,9 @@ func (m *AzureManager) buildNodeGroupFromSpec(spec string) (cloudprovider.NodeGr } switch m.config.VMType { - case vmTypeStandard: + case providerazureconsts.VMTypeStandard: return NewAgentPool(s, m) - case vmTypeVMSS: + case providerazureconsts.VMTypeVMSS: return NewScaleSet(s, m, -1, false) default: return nil, fmt.Errorf("vmtype %s not supported", m.config.VMType) @@ -310,7 +308,7 @@ func (m *AzureManager) getFilteredNodeGroups(filter []labelAutoDiscoveryConfig) return nil, nil } - if m.config.VMType == vmTypeVMSS { + if m.config.VMType == providerazureconsts.VMTypeVMSS { return m.getFilteredScaleSets(filter) } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go b/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go index baddccff26b1..70dfe5073b9c 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go @@ -24,6 +24,7 @@ import ( "time" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmclient/mockvmclient" + "sigs.k8s.io/cloud-provider-azure/pkg/retry" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources" @@ -33,9 +34,13 @@ import ( "go.uber.org/mock/gomock" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/config" + azclient "sigs.k8s.io/cloud-provider-azure/pkg/azclient" azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssclient/mockvmssclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssvmclient/mockvmssvmclient" + providerazureconsts "sigs.k8s.io/cloud-provider-azure/pkg/consts" + providerazure "sigs.k8s.io/cloud-provider-azure/pkg/provider" + providerazureconfig "sigs.k8s.io/cloud-provider-azure/pkg/provider/config" ) const validAzureCfg = `{ @@ -51,6 +56,29 @@ const validAzureCfg = `{ "vnetName": "fakeName", "routeTableName": "fakeName", "primaryAvailabilitySetName": "fakeName", + "vmssCacheTTLInSeconds": 60, + "vmssVirtualMachinesCacheTTLInSeconds": 240, + "vmssVmsCacheJitter": 120, + "maxDeploymentsCount": 8, + "cloudProviderRateLimit": false, + "routeRateLimit": { + "cloudProviderRateLimit": true, + "cloudProviderRateLimitQPS": 3 + } +}` + +const validAzureCfgLegacy = `{ + "cloud": "AzurePublicCloud", + "tenantId": "fakeId", + "subscriptionId": "fakeId", + "resourceGroup": "fakeId", + "location": "southeastasia", + "useWorkloadIdentityExtension": true, + "subnetName": "fakeName", + "securityGroupName": "fakeName", + "vnetName": "fakeName", + "routeTableName": "fakeName", + "primaryAvailabilitySetName": "fakeName", "vmssCacheTTL": 60, "vmssVmsCacheTTL": 240, "vmssVmsCacheJitter": 120, @@ -76,8 +104,8 @@ const validAzureCfgForStandardVMType = `{ "vnetName": "fakeName", "routeTableName": "fakeName", "primaryAvailabilitySetName": "fakeName", - "vmssCacheTTL": 60, - "vmssVmsCacheTTL": 240, + "vmssCacheTTLInSeconds": 60, + "vmssVirtualMachinesCacheTTLInSeconds": 240, "vmssVmsCacheJitter": 120, "maxDeploymentsCount": 8, "cloudProviderRateLimit": false, @@ -111,29 +139,57 @@ const validAzureCfgForStandardVMType = `{ }` const validAzureCfgForStandardVMTypeWithoutDeploymentParameters = `{ - "cloud": "AzurePublicCloud", - "tenantId": "fakeId", - "subscriptionId": "fakeId", - "aadClientId": "fakeId", - "aadClientSecret": "fakeId", - "resourceGroup": "fakeId", - "vmType":"standard", - "location": "southeastasia", - "subnetName": "fakeName", - "securityGroupName": "fakeName", - "vnetName": "fakeName", - "routeTableName": "fakeName", - "primaryAvailabilitySetName": "fakeName", - "vmssCacheTTL": 60, - "vmssVmsCacheTTL": 240, + "cloud": "AzurePublicCloud", + "tenantId": "fakeId", + "subscriptionId": "fakeId", + "aadClientId": "fakeId", + "aadClientSecret": "fakeId", + "resourceGroup": "fakeId", + "vmType":"standard", + "location": "southeastasia", + "subnetName": "fakeName", + "securityGroupName": "fakeName", + "vnetName": "fakeName", + "routeTableName": "fakeName", + "primaryAvailabilitySetName": "fakeName", + "vmssCacheTTLInSeconds": 60, + "vmssVirtualMachinesCacheTTLInSeconds": 240, "vmssVmsCacheJitter": 120, - "maxDeploymentsCount": 8, - "cloudProviderRateLimit": false, - "routeRateLimit": { - "cloudProviderRateLimit": true, - "cloudProviderRateLimitQPS": 3 - }, - "deployment":"cluster-autoscaler-0001" + "maxDeploymentsCount": 8, + "cloudProviderRateLimit": false, + "routeRateLimit": { + "cloudProviderRateLimit": true, + "cloudProviderRateLimitQPS": 3 + }, + "deployment":"cluster-autoscaler-0001" +}` + +const validAzureCfgForVMsPool = `{ + "cloud": "AzurePublicCloud", + "tenantId": "fakeId", + "subscriptionId": "fakeId", + "aadClientId": "fakeId", + "aadClientSecret": "fakeId", + "resourceGroup": "fakeId", + "location": "southeastasia", + "subnetName": "fakeName", + "securityGroupName": "fakeName", + "vnetName": "fakeName", + "routeTableName": "fakeName", + "primaryAvailabilitySetName": "fakeName", + "vmssCacheTTLInSeconds": 60, + "vmssVirtualMachinesCacheTTLInSeconds": 240, + "vmssVmsCacheJitter": 120, + "maxDeploymentsCount": 8, + "cloudProviderRateLimit": false, + "routeRateLimit": { + "cloudProviderRateLimit": true, + "cloudProviderRateLimitQPS": 3 + }, + + "clusterName": "mycluster", + "clusterResourceGroup": "myrg", + "armBaseURLForAPClient": "nodeprovisioner-svc.nodeprovisioner.svc.cluster.local" }` const ( @@ -155,73 +211,157 @@ func TestCreateAzureManagerValidConfig(t *testing.T) { manager, err := createAzureManagerInternal(strings.NewReader(validAzureCfg), cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) expectedConfig := &Config{ - Cloud: "AzurePublicCloud", - Location: "southeastasia", - TenantID: "fakeId", - SubscriptionID: "fakeId", - ResourceGroup: "fakeId", - VMType: "vmss", - AADClientID: "fakeId", - AADClientSecret: "fakeId", - VmssCacheTTL: 60, - VmssVmsCacheTTL: 240, - VmssVmsCacheJitter: 120, - MaxDeploymentsCount: 8, - CloudProviderRateLimitConfig: CloudProviderRateLimitConfig{ - RateLimitConfig: azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - InterfaceRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - VirtualMachineRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - StorageAccountRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, + Config: providerazure.Config{ + AzureAuthConfig: providerazureconfig.AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "AzurePublicCloud", + TenantID: "fakeId", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "fakeId", + AADClientSecret: "fakeId", + }, + SubscriptionID: "fakeId", }, - DiskRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, + Location: "southeastasia", + ResourceGroup: "fakeId", + VMType: "vmss", + VmssCacheTTLInSeconds: 60, + VmssVirtualMachinesCacheTTLInSeconds: 240, + CloudProviderRateLimitConfig: providerazureconfig.CloudProviderRateLimitConfig{ + RateLimitConfig: azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + InterfaceRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + StorageAccountRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + DiskRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, }, - VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, + }, + VmssVmsCacheJitter: 120, + MaxDeploymentsCount: 8, + } + + assert.NoError(t, err) + assertStructsMinimallyEqual(t, *expectedConfig, *manager.config) +} + +func TestCreateAzureManagerLegacyConfig(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockVMClient := mockvmclient.NewMockInterface(ctrl) + mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) + mockVMSSClient.EXPECT().List(gomock.Any(), "fakeId").Return([]compute.VirtualMachineScaleSet{}, nil).Times(2) + mockVMClient.EXPECT().List(gomock.Any(), "fakeId").Return([]compute.VirtualMachine{}, nil).Times(2) + mockAzClient := &azClient{ + virtualMachinesClient: mockVMClient, + virtualMachineScaleSetsClient: mockVMSSClient, + } + manager, err := createAzureManagerInternal(strings.NewReader(validAzureCfgLegacy), cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) + + expectedConfig := &Config{ + Config: providerazure.Config{ + AzureAuthConfig: providerazureconfig.AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "AzurePublicCloud", + TenantID: "fakeId", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseFederatedWorkloadIdentityExtension: true, + }, + SubscriptionID: "fakeId", }, - KubernetesServiceRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, + Location: "southeastasia", + ResourceGroup: "fakeId", + VMType: "vmss", + VmssCacheTTLInSeconds: 60, + VmssVirtualMachinesCacheTTLInSeconds: 240, + CloudProviderRateLimitConfig: providerazureconfig.CloudProviderRateLimitConfig{ + RateLimitConfig: azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + InterfaceRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + StorageAccountRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + DiskRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, }, }, + VmssVmsCacheJitter: 120, + MaxDeploymentsCount: 8, } assert.NoError(t, err) - assert.Equal(t, true, reflect.DeepEqual(*expectedConfig, *manager.config), "unexpected azure manager configuration") + assertStructsMinimallyEqual(t, *expectedConfig, *manager.config) } func TestCreateAzureManagerValidConfigForStandardVMType(t *testing.T) { @@ -238,70 +378,71 @@ func TestCreateAzureManagerValidConfigForStandardVMType(t *testing.T) { manager, err := createAzureManagerInternal(strings.NewReader(validAzureCfgForStandardVMType), cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) expectedConfig := &Config{ - Cloud: "AzurePublicCloud", - Location: "southeastasia", - TenantID: "fakeId", - SubscriptionID: "fakeId", - ResourceGroup: "fakeId", - VMType: "standard", - AADClientID: "fakeId", - AADClientSecret: "fakeId", - VmssCacheTTL: 60, - VmssVmsCacheTTL: 240, - VmssVmsCacheJitter: 120, - MaxDeploymentsCount: 8, - CloudProviderRateLimitConfig: CloudProviderRateLimitConfig{ - RateLimitConfig: azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - InterfaceRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - VirtualMachineRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - StorageAccountRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - DiskRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, + Config: providerazure.Config{ + AzureAuthConfig: providerazureconfig.AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "AzurePublicCloud", + TenantID: "fakeId", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "fakeId", + AADClientSecret: "fakeId", + }, + SubscriptionID: "fakeId", }, - KubernetesServiceRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: false, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, + Location: "southeastasia", + ResourceGroup: "fakeId", + VMType: "standard", + VmssCacheTTLInSeconds: 60, + VmssVirtualMachinesCacheTTLInSeconds: 240, + CloudProviderRateLimitConfig: providerazureconfig.CloudProviderRateLimitConfig{ + RateLimitConfig: azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + InterfaceRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + StorageAccountRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + DiskRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, }, }, - Deployment: "cluster-autoscaler-0001", + VmssVmsCacheJitter: 120, + MaxDeploymentsCount: 8, + Deployment: "cluster-autoscaler-0001", DeploymentParameters: map[string]interface{}{ "Name": "cluster-autoscaler-0001", "Properties": map[string]interface{}{ @@ -327,7 +468,7 @@ func TestCreateAzureManagerValidConfigForStandardVMType(t *testing.T) { } assert.NoError(t, err) - assert.Equal(t, *expectedConfig, *manager.config, "unexpected azure manager configuration, expected: %v, actual: %v", *expectedConfig, *manager.config) + assertStructsMinimallyEqual(t, *expectedConfig, *manager.config) } func TestCreateAzureManagerValidConfigForStandardVMTypeWithoutDeploymentParameters(t *testing.T) { @@ -336,6 +477,92 @@ func TestCreateAzureManagerValidConfigForStandardVMTypeWithoutDeploymentParamete assert.Nil(t, manager) assert.Equal(t, expectedErr, err.Error(), "return error does not match, expected: %v, actual: %v", expectedErr, err.Error()) } +func TestCreateAzureManagerValidConfigForVMsPool(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockVMClient := mockvmclient.NewMockInterface(ctrl) + mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) + mockVMSSClient.EXPECT().List(gomock.Any(), "fakeId").Return([]compute.VirtualMachineScaleSet{}, nil).Times(2) + mockVMClient.EXPECT().List(gomock.Any(), "fakeId").Return([]compute.VirtualMachine{}, nil).Times(2) + mockAzClient := &azClient{ + virtualMachinesClient: mockVMClient, + virtualMachineScaleSetsClient: mockVMSSClient, + } + manager, err := createAzureManagerInternal(strings.NewReader(validAzureCfgForVMsPool), cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) + + expectedConfig := &Config{ + Config: providerazure.Config{ + AzureAuthConfig: providerazureconfig.AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "AzurePublicCloud", + TenantID: "fakeId", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "fakeId", + AADClientSecret: "fakeId", + }, + SubscriptionID: "fakeId", + }, + Location: "southeastasia", + ResourceGroup: "fakeId", + VMType: "vmss", + VmssCacheTTLInSeconds: 60, + VmssVirtualMachinesCacheTTLInSeconds: 240, + CloudProviderRateLimitConfig: providerazureconfig.CloudProviderRateLimitConfig{ + RateLimitConfig: azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + InterfaceRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + StorageAccountRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + DiskRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: false, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + }, + }, + VmssVmsCacheJitter: 120, + MaxDeploymentsCount: 8, + ClusterName: "mycluster", + ClusterResourceGroup: "myrg", + ARMBaseURLForAPClient: "nodeprovisioner-svc.nodeprovisioner.svc.cluster.local", + } + + assert.NoError(t, err) + assertStructsMinimallyEqual(t, *expectedConfig, *manager.config) +} func TestCreateAzureManagerWithNilConfig(t *testing.T) { ctrl := gomock.NewController(t) @@ -350,83 +577,83 @@ func TestCreateAzureManagerWithNilConfig(t *testing.T) { } expectedConfig := &Config{ - Cloud: "AzurePublicCloud", - Location: "southeastasia", - TenantID: "tenantId", - SubscriptionID: "subscriptionId", - ResourceGroup: "resourceGroup", - ClusterName: "mycluster", - ClusterResourceGroup: "myrg", - ARMBaseURLForAPClient: "nodeprovisioner-svc.nodeprovisioner.svc.cluster.local", - VMType: "vmss", - AADClientID: "aadClientId", - AADClientSecret: "aadClientSecret", - AADClientCertPath: "aadClientCertPath", - AADClientCertPassword: "aadClientCertPassword", - Deployment: "deployment", - UseManagedIdentityExtension: true, - UserAssignedIdentityID: "UserAssignedIdentityID", - VmssCacheTTL: 100, - VmssVmsCacheTTL: 110, - VmssVmsCacheJitter: 90, - GetVmssSizeRefreshPeriod: 30, - MaxDeploymentsCount: 8, - CloudProviderBackoff: true, - CloudProviderBackoffRetries: 1, - CloudProviderBackoffExponent: 1, - CloudProviderBackoffDuration: 1, - CloudProviderBackoffJitter: 1, - CloudProviderRateLimitConfig: CloudProviderRateLimitConfig{ - RateLimitConfig: azclients.RateLimitConfig{ - CloudProviderRateLimit: true, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - InterfaceRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: true, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - VirtualMachineRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: true, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - StorageAccountRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: true, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - DiskRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: true, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, - }, - VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: true, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, + Config: providerazure.Config{ + AzureAuthConfig: providerazureconfig.AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "AzurePublicCloud", + TenantID: "tenantId", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "aadClientId", + AADClientSecret: "aadClientSecret", + AADClientCertPath: "aadClientCertPath", + AADClientCertPassword: "aadClientCertPassword", + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "UserAssignedIdentityID", + }, + SubscriptionID: "subscriptionId", }, - KubernetesServiceRateLimit: &azclients.RateLimitConfig{ - CloudProviderRateLimit: true, - CloudProviderRateLimitBucket: 5, - CloudProviderRateLimitBucketWrite: 5, - CloudProviderRateLimitQPS: 1, - CloudProviderRateLimitQPSWrite: 1, + Location: "southeastasia", + ResourceGroup: "resourceGroup", + VMType: "vmss", + VmssCacheTTLInSeconds: 100, + VmssVirtualMachinesCacheTTLInSeconds: 110, + CloudProviderBackoff: true, + CloudProviderBackoffRetries: 1, + CloudProviderBackoffExponent: 1, + CloudProviderBackoffDuration: 1, + CloudProviderBackoffJitter: 1, + CloudProviderRateLimitConfig: providerazureconfig.CloudProviderRateLimitConfig{ + RateLimitConfig: azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + InterfaceRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + StorageAccountRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + DiskRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, }, }, + ClusterName: "mycluster", + ClusterResourceGroup: "myrg", + ARMBaseURLForAPClient: "nodeprovisioner-svc.nodeprovisioner.svc.cluster.local", + Deployment: "deployment", + VmssVmsCacheJitter: 90, + MaxDeploymentsCount: 8, } t.Setenv("ARM_CLOUD", "AzurePublicCloud") @@ -463,15 +690,13 @@ func TestCreateAzureManagerWithNilConfig(t *testing.T) { t.Run("environment variables correctly set", func(t *testing.T) { manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) assert.NoError(t, err) - manager.config.TenantID = "tenantId" - manager.config.AADClientID = "aadClientId" - assert.Equal(t, true, reflect.DeepEqual(*expectedConfig, *manager.config), "unexpected azure manager configuration") + assertStructsMinimallyEqual(t, *expectedConfig, *manager.config) }) t.Run("invalid bool for ARM_USE_MANAGED_IDENTITY_EXTENSION", func(t *testing.T) { t.Setenv("ARM_USE_MANAGED_IDENTITY_EXTENSION", "invalidbool") manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) - expectedErr0 := "strconv.ParseBool: parsing \"invalidbool\": invalid syntax" + expectedErr0 := "failed to parse ARM_USE_MANAGED_IDENTITY_EXTENSION \"invalidbool\": strconv.ParseBool: parsing \"invalidbool\": invalid syntax" assert.Nil(t, manager) assert.Equal(t, expectedErr0, err.Error(), "Return err does not match, expected: %v, actual: %v", expectedErr0, err.Error()) }) @@ -487,7 +712,7 @@ func TestCreateAzureManagerWithNilConfig(t *testing.T) { t.Run("invalid int for AZURE_GET_VMSS_SIZE_REFRESH_PERIOD", func(t *testing.T) { t.Setenv("AZURE_GET_VMSS_SIZE_REFRESH_PERIOD", "invalidint") manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) - expectedErr := fmt.Errorf("failed to parse AZURE_GET_VMSS_SIZE_REFRESH_PERIOD \"invalidint\": strconv.Atoi: parsing \"invalidint\": invalid syntax") + expectedErr := fmt.Errorf("failed to parse AZURE_GET_VMSS_SIZE_REFRESH_PERIOD \"invalidint\": strconv.ParseInt: parsing \"invalidint\": invalid syntax") assert.Nil(t, manager) assert.Equal(t, expectedErr, err, "Return err does not match, expected: %v, actual: %v", expectedErr, err) }) @@ -518,7 +743,7 @@ func TestCreateAzureManagerWithNilConfig(t *testing.T) { t.Run("invalid int for BACKOFF_RETRIES", func(t *testing.T) { t.Setenv("BACKOFF_RETRIES", "invalidint") manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) - expectedErr := fmt.Errorf("failed to parse BACKOFF_RETRIES '\\x00': strconv.ParseInt: parsing \"invalidint\": invalid syntax") + expectedErr := fmt.Errorf("failed to parse BACKOFF_RETRIES \"invalidint\": strconv.ParseInt: parsing \"invalidint\": invalid syntax") assert.Nil(t, manager) assert.Equal(t, expectedErr, err, "Return err does not match, expected: %v, actual: %v", expectedErr, err) }) @@ -527,7 +752,7 @@ func TestCreateAzureManagerWithNilConfig(t *testing.T) { t.Setenv("BACKOFF_RETRIES", "") manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) assert.NoError(t, err) - assert.Equal(t, backoffRetriesDefault, (*manager.config).CloudProviderBackoffRetries, "CloudProviderBackoffRetries does not match.") + assert.Equal(t, providerazureconsts.BackoffRetriesDefault, (*manager.config).CloudProviderBackoffRetries, "CloudProviderBackoffRetries does not match.") }) t.Run("invalid float for BACKOFF_EXPONENT", func(t *testing.T) { @@ -542,7 +767,7 @@ func TestCreateAzureManagerWithNilConfig(t *testing.T) { t.Setenv("BACKOFF_EXPONENT", "") manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) assert.NoError(t, err) - assert.Equal(t, backoffExponentDefault, (*manager.config).CloudProviderBackoffExponent, "CloudProviderBackoffExponent does not match.") + assert.Equal(t, providerazureconsts.BackoffExponentDefault, (*manager.config).CloudProviderBackoffExponent, "CloudProviderBackoffExponent does not match.") }) t.Run("invalid int for BACKOFF_DURATION", func(t *testing.T) { @@ -557,7 +782,7 @@ func TestCreateAzureManagerWithNilConfig(t *testing.T) { t.Setenv("BACKOFF_DURATION", "") manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) assert.NoError(t, err) - assert.Equal(t, backoffDurationDefault, (*manager.config).CloudProviderBackoffDuration, "CloudProviderBackoffDuration does not match.") + assert.Equal(t, providerazureconsts.BackoffDurationDefault, (*manager.config).CloudProviderBackoffDuration, "CloudProviderBackoffDuration does not match.") }) t.Run("invalid float for BACKOFF_JITTER", func(t *testing.T) { @@ -572,18 +797,148 @@ func TestCreateAzureManagerWithNilConfig(t *testing.T) { t.Setenv("BACKOFF_JITTER", "") manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) assert.NoError(t, err) - assert.Equal(t, backoffJitterDefault, (*manager.config).CloudProviderBackoffJitter, "CloudProviderBackoffJitter does not match.") + assert.Equal(t, providerazureconsts.BackoffJitterDefault, (*manager.config).CloudProviderBackoffJitter, "CloudProviderBackoffJitter does not match.") }) t.Run("invalid bool for CLOUD_PROVIDER_RATE_LIMIT", func(t *testing.T) { t.Setenv("CLOUD_PROVIDER_RATE_LIMIT", "invalidbool") manager, err := createAzureManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) - expectedErr := fmt.Errorf("failed to parse CLOUD_PROVIDER_RATE_LIMIT: \"invalidbool\", strconv.ParseBool: parsing \"invalidbool\": invalid syntax") + expectedErr := fmt.Errorf("failed to parse CLOUD_PROVIDER_RATE_LIMIT \"invalidbool\": strconv.ParseBool: parsing \"invalidbool\": invalid syntax") assert.Nil(t, manager) assert.Equal(t, expectedErr, err, "Return err does not match, expected: %v, actual: %v", expectedErr, err) }) } +func TestCreateAzureManagerWithEnvOverridingConfig(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockVMClient := mockvmclient.NewMockInterface(ctrl) + mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) + mockVMSSClient.EXPECT().List(gomock.Any(), "resourceGroup").Return([]compute.VirtualMachineScaleSet{}, nil).AnyTimes() + mockVMClient.EXPECT().List(gomock.Any(), "resourceGroup").Return([]compute.VirtualMachine{}, nil).AnyTimes() + mockAzClient := &azClient{ + virtualMachinesClient: mockVMClient, + virtualMachineScaleSetsClient: mockVMSSClient, + } + + expectedConfig := &Config{ + Config: providerazure.Config{ + AzureAuthConfig: providerazureconfig.AzureAuthConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "AzurePublicCloud", + TenantID: "tenantId", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + AADClientID: "aadClientId", + AADClientSecret: "aadClientSecret", + AADClientCertPath: "aadClientCertPath", + AADClientCertPassword: "aadClientCertPassword", + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "UserAssignedIdentityID", + }, + SubscriptionID: "subscriptionId", + }, + Location: "southeastasia", + ResourceGroup: "resourceGroup", + VMType: "vmss", + VmssCacheTTLInSeconds: 100, + VmssVirtualMachinesCacheTTLInSeconds: 110, + CloudProviderBackoff: true, + CloudProviderBackoffRetries: 1, + CloudProviderBackoffExponent: 1, + CloudProviderBackoffDuration: 1, + CloudProviderBackoffJitter: 1, + CloudProviderRateLimitConfig: providerazureconfig.CloudProviderRateLimitConfig{ + RateLimitConfig: azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + InterfaceRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + StorageAccountRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + DiskRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + VirtualMachineScaleSetRateLimit: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitBucketWrite: 5, + CloudProviderRateLimitQPS: 1, + CloudProviderRateLimitQPSWrite: 1, + }, + }, + }, + ClusterName: "mycluster", + ClusterResourceGroup: "myrg", + ARMBaseURLForAPClient: "nodeprovisioner-svc.nodeprovisioner.svc.cluster.local", + Deployment: "deployment", + VmssVmsCacheJitter: 90, + MaxDeploymentsCount: 8, + } + + t.Setenv("ARM_CLOUD", "AzurePublicCloud") + // LOCATION is not set from env to test getting it from config file + t.Setenv("AZURE_TENANT_ID", "tenantId") + t.Setenv("AZURE_CLIENT_ID", "aadClientId") + t.Setenv("ARM_SUBSCRIPTION_ID", "subscriptionId") + t.Setenv("ARM_RESOURCE_GROUP", "resourceGroup") + t.Setenv("AZURE_TENANT_ID", "tenantId") + t.Setenv("ARM_TENANT_ID", "tenantId") + t.Setenv("AZURE_CLIENT_ID", "aadClientId") + t.Setenv("ARM_CLIENT_ID", "aadClientId") + t.Setenv("ARM_CLIENT_SECRET", "aadClientSecret") + t.Setenv("ARM_VM_TYPE", "vmss") // this is one of the differences with the config file, expect this to take precedence + t.Setenv("ARM_CLIENT_CERT_PATH", "aadClientCertPath") + t.Setenv("ARM_CLIENT_CERT_PASSWORD", "aadClientCertPassword") + t.Setenv("ARM_DEPLOYMENT", "deployment") + t.Setenv("ARM_USE_MANAGED_IDENTITY_EXTENSION", "true") + t.Setenv("ARM_USER_ASSIGNED_IDENTITY_ID", "UserAssignedIdentityID") + t.Setenv("AZURE_VMSS_CACHE_TTL", "100") + t.Setenv("AZURE_VMSS_VMS_CACHE_TTL", "110") + t.Setenv("AZURE_VMSS_VMS_CACHE_JITTER", "90") + t.Setenv("AZURE_MAX_DEPLOYMENT_COUNT", "8") + t.Setenv("ENABLE_BACKOFF", "true") + t.Setenv("BACKOFF_RETRIES", "1") + t.Setenv("BACKOFF_EXPONENT", "1") + t.Setenv("BACKOFF_DURATION", "1") + t.Setenv("BACKOFF_JITTER", "1") + t.Setenv("CLOUD_PROVIDER_RATE_LIMIT", "true") + t.Setenv("CLUSTER_NAME", "mycluster") + t.Setenv("ARM_CLUSTER_RESOURCE_GROUP", "myrg") + t.Setenv("ARM_BASE_URL_FOR_AP_CLIENT", "nodeprovisioner-svc.nodeprovisioner.svc.cluster.local") + + t.Run("environment variables correctly set", func(t *testing.T) { + manager, err := createAzureManagerInternal(strings.NewReader(validAzureCfgForStandardVMType), cloudprovider.NodeGroupDiscoveryOptions{}, mockAzClient) + assert.NoError(t, err) + assertStructsMinimallyEqual(t, *expectedConfig, *manager.config) + }) +} + func TestCreateAzureManagerInvalidConfig(t *testing.T) { _, err := createAzureManagerInternal(strings.NewReader(invalidAzureCfg), cloudprovider.NodeGroupDiscoveryOptions{}, &azClient{}) assert.Error(t, err, "failed to unmarshal config body") @@ -620,7 +975,7 @@ func TestFetchExplicitNodeGroups(t *testing.T) { } else { mockVMClient := mockvmclient.NewMockInterface(ctrl) - manager.config.EnableVmssFlex = true + manager.config.EnableVmssFlexNodes = true mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), "test-asg").Return(expectedVMs, nil).AnyTimes() manager.azClient.virtualMachinesClient = mockVMClient } @@ -638,7 +993,7 @@ func TestFetchExplicitNodeGroups(t *testing.T) { testAS := newTestAgentPool(newTestAzureManager(t), "testAS") timeLayout := "2006-01-02 15:04:05" timeBenchMark, _ := time.Parse(timeLayout, "2000-01-01 00:00:00") - testAS.manager.azClient.deploymentsClient = &DeploymentsClientMock{ + testAS.manager.azClient.deploymentClient = &DeploymentClientMock{ FakeStore: map[string]resources.DeploymentExtended{ "cluster-autoscaler-0001": { Name: to.StringPtr("cluster-autoscaler-0001"), @@ -649,9 +1004,9 @@ func TestFetchExplicitNodeGroups(t *testing.T) { }, }, } - testAS.manager.config.VMType = vmTypeStandard + testAS.manager.config.VMType = providerazureconsts.VMTypeStandard err := testAS.manager.fetchExplicitNodeGroups([]string{"1:5:testAS"}) - expectedErr := fmt.Errorf("failed to parse node group spec: deployment not found") + expectedErr := fmt.Errorf("failed to parse node group spec: %v", retry.NewError(false, fmt.Errorf("deployment not found")).Error()) assert.Equal(t, expectedErr, err, "testAS.manager.fetchExplicitNodeGroups return error does not match, expected: %v, actual: %v", expectedErr, err) err = testAS.manager.fetchExplicitNodeGroups(nil) assert.NoError(t, err) @@ -888,3 +1243,36 @@ func TestGetScaleSetOptions(t *testing.T) { opts = manager.GetScaleSetOptions("test3", defaultOptions) assert.Equal(t, *opts, defaultOptions) } + +func assertStructsMinimallyEqual(t *testing.T, struct1, struct2 interface{}) bool { + return compareStructFields(t, reflect.ValueOf(struct1), reflect.ValueOf(struct2)) +} + +func compareStructFields(t *testing.T, v1, v2 reflect.Value) bool { + if v1.Type() != v2.Type() { + return assert.Fail(t, "different types", "v1 type: %v, v2 type: %v", v1.Type(), v2.Type()) + } + + for i := 0; i < v1.NumField(); i++ { + field1 := v1.Field(i) + field2 := v2.Field(i) + fieldType := v1.Type().Field(i) + + if field1.IsZero() || reflect.DeepEqual(field1.Interface(), reflect.Zero(field1.Type()).Interface()) { + continue // Skip zero value fields in struct1 + } + + if field1.Kind() == reflect.Struct { + // Recursively compare nested structs + if !compareStructFields(t, field1, field2) { + return false + } + } else { + if !assert.Equal(t, field1.Interface(), field2.Interface(), "field %s", fieldType.Name) { + return false + } + } + } + + return true +} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go b/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go index 3d03591f5e0c..05cf4301080f 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go @@ -112,8 +112,8 @@ func NewScaleSet(spec *dynamic.NodeGroupSpec, az *AzureManager, curSize int64, d dedicatedHost: dedicatedHost, } - if az.config.VmssVmsCacheTTL != 0 { - scaleSet.instancesRefreshPeriod = time.Duration(az.config.VmssVmsCacheTTL) * time.Second + if az.config.VmssVirtualMachinesCacheTTLInSeconds != 0 { + scaleSet.instancesRefreshPeriod = time.Duration(az.config.VmssVirtualMachinesCacheTTLInSeconds) * time.Second } else { scaleSet.instancesRefreshPeriod = defaultVmssInstancesRefreshPeriod } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_instance_cache.go b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_instance_cache.go index 5b6843caf412..bc6bda019f3d 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_instance_cache.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_instance_cache.go @@ -106,10 +106,10 @@ func (scaleSet *ScaleSet) updateInstanceCache() error { } if orchestrationMode == compute.Flexible { - if scaleSet.manager.config.EnableVmssFlex { + if scaleSet.manager.config.EnableVmssFlexNodes { return scaleSet.buildScaleSetCacheForFlex() } - return fmt.Errorf("vmss - %q with Flexible orchestration detected but 'enableVmssFlex' feature flag is turned off", scaleSet.Name) + return fmt.Errorf("vmss - %q with Flexible orchestration detected but 'enableVmssFlexNodes' feature flag is turned off", scaleSet.Name) } else if orchestrationMode == compute.Uniform { return scaleSet.buildScaleSetCacheForUniform() } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go index f3aa0dc356d8..b3bd5fca5b5c 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go @@ -203,7 +203,7 @@ func TestTargetSize(t *testing.T) { mockVMSSVMClient.EXPECT().List(gomock.Any(), provider.azureManager.config.ResourceGroup, "test-asg", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() provider.azureManager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient } else { - provider.azureManager.config.EnableVmssFlex = true + provider.azureManager.config.EnableVmssFlexNodes = true mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), "test-asg").Return(expectedVMs, nil).AnyTimes() } @@ -278,7 +278,7 @@ func TestIncreaseSize(t *testing.T) { mockVMSSVMClient.EXPECT().List(gomock.Any(), provider.azureManager.config.ResourceGroup, "test-asg", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() provider.azureManager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient } else { - provider.azureManager.config.EnableVmssFlex = true + provider.azureManager.config.EnableVmssFlexNodes = true mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), "test-asg").Return(expectedVMs, nil).AnyTimes() } err := provider.azureManager.forceRefresh() @@ -514,7 +514,7 @@ func TestBelongs(t *testing.T) { mockVMSSVMClient.EXPECT().List(gomock.Any(), provider.azureManager.config.ResourceGroup, "test-asg", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() provider.azureManager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient } else { - provider.azureManager.config.EnableVmssFlex = true + provider.azureManager.config.EnableVmssFlexNodes = true mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), "test-asg").Return(expectedVMs, nil).AnyTimes() } @@ -603,7 +603,7 @@ func TestDeleteNodes(t *testing.T) { mockVMSSVMClient.EXPECT().List(gomock.Any(), manager.config.ResourceGroup, "test-asg", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() manager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient } else { - manager.config.EnableVmssFlex = true + manager.config.EnableVmssFlexNodes = true mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), "test-asg").Return(expectedVMs, nil).AnyTimes() manager.azClient.virtualMachinesClient = mockVMClient } @@ -739,7 +739,7 @@ func TestDeleteNodeUnregistered(t *testing.T) { mockVMSSVMClient.EXPECT().List(gomock.Any(), manager.config.ResourceGroup, "test-asg", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() manager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient } else { - manager.config.EnableVmssFlex = true + manager.config.EnableVmssFlexNodes = true mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), "test-asg").Return(expectedVMs, nil).AnyTimes() } err := manager.forceRefresh() @@ -1013,7 +1013,7 @@ func TestScaleSetNodes(t *testing.T) { provider.azureManager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient } else { - provider.azureManager.config.EnableVmssFlex = true + provider.azureManager.config.EnableVmssFlexNodes = true mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), "test-asg").Return(expectedVMs, nil).AnyTimes() } @@ -1057,7 +1057,7 @@ func TestScaleSetNodes(t *testing.T) { } -func TestEnableVmssFlexFlag(t *testing.T) { +func TestEnableVmssFlexNodesFlag(t *testing.T) { // flag set to false ctrl := gomock.NewController(t) @@ -1069,7 +1069,7 @@ func TestEnableVmssFlexFlag(t *testing.T) { provider := newTestProvider(t) mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) mockVMSSClient.EXPECT().List(gomock.Any(), provider.azureManager.config.ResourceGroup).Return(expectedScaleSets, nil).AnyTimes() - provider.azureManager.config.EnableVmssFlex = false + provider.azureManager.config.EnableVmssFlexNodes = false provider.azureManager.azClient.virtualMachineScaleSetsClient = mockVMSSClient mockVMClient := mockvmclient.NewMockInterface(ctrl) @@ -1084,7 +1084,7 @@ func TestEnableVmssFlexFlag(t *testing.T) { assert.Error(t, err, "vmss - \"test-asg\" with Flexible orchestration detected but 'enbaleVmssFlex' feature flag is turned off") // flag set to true - provider.azureManager.config.EnableVmssFlex = true + provider.azureManager.config.EnableVmssFlexNodes = true err = provider.azureManager.Refresh() assert.NoError(t, err) } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_util.go b/cluster-autoscaler/cloudprovider/azure/azure_util.go index 9f4a44284ed2..44a87980571c 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_util.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_util.go @@ -636,3 +636,21 @@ func vmPowerStateFromStatuses(statuses []compute.InstanceViewStatus) string { // PowerState is not set if the VM is still creating (or has failed creation) return vmPowerStateUnknown } + +// strconv.ParseInt, but for int +func parseInt32(s string, base int) (int, error) { + val, err := strconv.ParseInt(s, base, 32) + if err != nil { + return 0, err + } + return int(val), nil +} + +// strconv.ParseFloat, but for float32 +func parseFloat32(s string) (float32, error) { + val, err := strconv.ParseFloat(s, 32) + if err != nil { + return 0, err + } + return float32(val), nil +} diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index 7341cbd08217..d308f0db5e26 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -2,7 +2,7 @@ module k8s.io/autoscaler/cluster-autoscaler go 1.22.0 -toolchain go1.22.2 +toolchain go1.22.4 require ( cloud.google.com/go/compute/metadata v0.3.0 @@ -12,8 +12,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.9.0-beta.1 github.com/Azure/go-autorest/autorest v0.11.29 - github.com/Azure/go-autorest/autorest/adal v0.9.23 - github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 + github.com/Azure/go-autorest/autorest/adal v0.9.24 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 github.com/Azure/go-autorest/autorest/date v0.3.0 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/Azure/skewer v0.0.14 @@ -47,7 +47,7 @@ require ( k8s.io/apiserver v0.31.0-beta.0 k8s.io/autoscaler/cluster-autoscaler/apis v0.0.0-20240627115740-d52e4b9665d7 k8s.io/client-go v0.31.0-beta.0 - k8s.io/cloud-provider v0.30.0-alpha.3 + k8s.io/cloud-provider v0.30.1 k8s.io/cloud-provider-aws v1.27.0 k8s.io/cloud-provider-gcp/providers v0.28.2 k8s.io/component-base v0.31.0-beta.0 @@ -57,6 +57,7 @@ require ( k8s.io/kubernetes v1.31.0-beta.0 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/cloud-provider-azure v1.29.4 + sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.13 sigs.k8s.io/yaml v1.4.0 ) @@ -73,7 +74,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 // indirect github.com/Azure/go-armbalancer v0.0.2 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect @@ -216,7 +217,6 @@ require ( k8s.io/kubectl v0.28.0 // indirect k8s.io/mount-utils v0.26.0-alpha.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect - sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.13 // indirect sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index d00b52672e1d..224ac9d942f8 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -45,20 +45,19 @@ github.com/Azure/go-armbalancer v0.0.2/go.mod h1:yTg7MA/8YnfKQc9o97tzAJ7fbdVkod1 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.4/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= -github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= -github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= +github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 h1:Ov8avRZi2vmrE2JcXw+tu5K/yB41r7xK9GZDiBF7NdM= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13/go.mod h1:5BAVfWLWXihP47vYrPuBKKf4cS0bXI+KM9Qx6ETDJYo= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= @@ -169,7 +168,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.27.0 h1:78iE9oVvTnAEqhMip2UHFvL01b8LJcydbNUpr0cAmN4= github.com/digitalocean/godo v1.27.0/go.mod h1:iJnN9rVu6K5LioLxLimlq0uRI+y/eAQjROUmeU/r0hY= -github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= @@ -198,7 +196,6 @@ github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esu github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= @@ -239,6 +236,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= @@ -560,11 +558,10 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -577,6 +574,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -598,6 +596,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -614,6 +613,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -639,12 +639,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -654,6 +658,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -671,6 +677,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=