Skip to content

Commit

Permalink
Sync eng/common directory with azure-sdk-tools repository (#768)
Browse files Browse the repository at this point in the history
  • Loading branch information
azure-sdk authored May 16, 2020
1 parent 24ea81b commit 9bd3125
Show file tree
Hide file tree
Showing 8 changed files with 71 additions and 144 deletions.
2 changes: 1 addition & 1 deletion eng/common/TestResources/New-TestResources.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ if ($ProvisionerApplicationId) {
}

$provisionerAccount = Retry {
Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs
Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs
}

$exitActions += {
Expand Down
26 changes: 7 additions & 19 deletions eng/common/TestResources/Remove-TestResources.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,11 @@ param (
[string] $Environment = 'AzureCloud',

[Parameter()]
[switch] $Force
[switch] $Force,

# Captures any arguments not declared here (no parameter errors)
[Parameter(ValueFromRemainingArguments = $true)]
$RemoveTestResourcesRemainingArguments
)

# By default stop for any error.
Expand Down Expand Up @@ -105,7 +109,7 @@ if ($ProvisionerApplicationId) {
}

$provisionerAccount = Retry {
Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs
Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs
}

$exitActions += {
Expand Down Expand Up @@ -143,54 +147,40 @@ $exitActions.Invoke()
<#
.SYNOPSIS
Deletes the resource group deployed for a service directory from Azure.
.DESCRIPTION
Removes a resource group and all its resources previously deployed using
New-TestResources.ps1.
If you are not currently logged into an account in the Az PowerShell module,
you will be asked to log in with Connect-AzAccount. Alternatively, you (or a
build pipeline) can pass $ProvisionerApplicationId and
$ProvisionerApplicationSecret to authenticate a service principal with access to
create resources.
.PARAMETER BaseName
A name to use in the resource group and passed to the ARM template as 'baseName'.
This will delete the resource group named 'rg-<baseName>'
.PARAMETER ResourceGroupName
The name of the resource group to delete.
.PARAMETER TenantId
The tenant ID of a service principal when a provisioner is specified.
.PARAMETER SubscriptionId
Optional subscription ID to use for new resources when logging in as a
provisioner. You can also use Set-AzContext if not provisioning.
.PARAMETER ProvisionerApplicationId
A service principal ID to provision test resources when a provisioner is specified.
.PARAMETER ProvisionerApplicationSecret
A service principal secret (password) to provision test resources when a provisioner is specified.
.PARAMETER ServiceDirectory
A directory under 'sdk' in the repository root - optionally with subdirectories
specified - in which to discover pre removal script named 'remove-test-resources-pre.json'.
.PARAMETER Environment
Name of the cloud environment. The default is the Azure Public Cloud
('PublicCloud')
.PARAMETER Force
Force removal of resource group without asking for user confirmation
.EXAMPLE
Remove-TestResources.ps1 -BaseName 'uuid123' -Force
Use the currently logged-in account to delete the resource group by the name of
'rg-uuid123'
.EXAMPLE
Remove-TestResources.ps1 `
-ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" `
Expand All @@ -199,11 +189,9 @@ Remove-TestResources.ps1 `
-ProvisionerApplicationSecret '$(AppSecret)' `
-Force `
-Verbose `
When run in the context of an Azure DevOps pipeline, this script removes the
resource group whose name is stored in the environment variable
AZURE_RESOURCEGROUP_NAME.
.LINK
New-TestResources.ps1
#>
#>
92 changes: 28 additions & 64 deletions eng/common/TestResources/deploy-test-resources.yml
Original file line number Diff line number Diff line change
@@ -1,76 +1,40 @@
# Deploys resources to a cloud type specified by the variable (not parameter)
# 'CloudType'. Use this as part of a matrix to deploy resources to a particular
# cloud instance. Normally we would use template parameters instead of variables
# but matrix variables are not available during template expansion so any
# benefits of parameters are lost.

parameters:
ServiceDirectory: not-set
ArmTemplateParameters: '@{}'
DeleteAfterHours: 24
Location: ''
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)

# SubscriptionConfiguration will be splat into the parameters of the test
# resources script. It should be JSON in the form:
# {
# "SubscriptionId": "<subscription id>",
# "TenantId": "<tenant id>",
# "TestApplicationId": "<test app id>",
# "TestApplicationSecret": "<test app secret>",
# "ProvisionerApplicationId": "<provisoner app id>",
# "ProvisoinerApplicationSecret": "<provisoner app secert>",
# "Environment": "AzureCloud | AzureGov | AzureChina | <other environment>"
# }

steps:
# New-TestResources command requires Az module
- pwsh: Install-Module -Name Az -Scope CurrentUser -AllowClobber -Force -Verbose
displayName: Install Azure PowerShell module

- pwsh: >
eng/common/TestResources/New-TestResources.ps1
-BaseName 'Generated'
-ServiceDirectory '${{ parameters.ServiceDirectory }}'
-TenantId '$(aad-azure-sdk-test-tenant-id)'
-SubscriptionId '$(azure-subscription-id)'
-TestApplicationId '$(aad-azure-sdk-test-client-id)'
-TestApplicationSecret '$(aad-azure-sdk-test-client-secret)'
-ProvisionerApplicationId '$(aad-azure-sdk-test-client-id)'
-ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret)'
-AdditionalParameters ${{ parameters.ArmTemplateParameters }}
-DeleteAfterHours ${{ parameters.DeleteAfterHours }}
-Location '${{ parameters.Location }}'
-Environment 'AzureCloud'
-CI
-Force
-Verbose
displayName: Deploy test resources (AzureCloud)
condition: and(succeeded(), eq(variables['CloudType'], 'AzureCloud'))
- pwsh: >
eng/common/TestResources/New-TestResources.ps1
-BaseName 'Generated'
-ServiceDirectory '${{ parameters.ServiceDirectory }}'
-TenantId '$(aad-azure-sdk-test-tenant-id-gov)'
-SubscriptionId '$(azure-subscription-id-gov)'
-TestApplicationId '$(aad-azure-sdk-test-client-id-gov)'
-TestApplicationSecret '$(aad-azure-sdk-test-client-secret-gov)'
-ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-gov)'
-ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-gov)'
-AdditionalParameters ${{ parameters.ArmTemplateParameters }}
-DeleteAfterHours ${{ parameters.DeleteAfterHours }}
-Location '${{ parameters.Location }}'
-Environment 'AzureUSGovernment'
-CI
-Force
-Verbose
displayName: Deploy test resources (AzureUSGovernment)
condition: and(succeeded(), eq(variables['CloudType'], 'AzureUSGovernment'))
- pwsh: |
$subscriptionConfiguration = @"
${{ parameters.SubscriptionConfiguration }}
"@ | ConvertFrom-Json -AsHashtable;
- pwsh: >
eng/common/TestResources/New-TestResources.ps1
-BaseName 'Generated'
-ServiceDirectory '${{ parameters.ServiceDirectory }}'
-TenantId '$(aad-azure-sdk-test-tenant-id-cn)'
-SubscriptionId '$(azure-subscription-id-cn)'
-TestApplicationId '$(aad-azure-sdk-test-client-id-cn)'
-TestApplicationSecret '$(aad-azure-sdk-test-client-secret-cn)'
-ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-cn)'
-ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-cn)'
-AdditionalParameters ${{ parameters.ArmTemplateParameters }}
-DeleteAfterHours ${{ parameters.DeleteAfterHours }}
-Location '${{ parameters.Location }}'
-Environment 'AzureChinaCloud'
-CI
-Force
-Verbose
displayName: Deploy test resources (AzureChinaCloud)
condition: and(succeeded(), eq(variables['CloudType'], 'AzureChinaCloud'))
eng/common/TestResources/New-TestResources.ps1 `
-BaseName 'Generated' `
-ServiceDirectory ${{ parameters.ServiceDirectory }} `
-Location '${{ parameters.Location }}' `
-DeleteAfterHours ${{ parameters.DeleteAfterHours }} `
-AdditionalParameters ${{ parameters.ArmTemplateParameters }} `
@subscriptionConfiguration `
-CI `
-Force `
-Verbose
displayName: Deploy test resources
74 changes: 26 additions & 48 deletions eng/common/TestResources/remove-test-resources.yml
Original file line number Diff line number Diff line change
@@ -1,57 +1,35 @@
# Removes resources from a cloud type specified by the variable (not parameter)
# 'CloudType'. Use this as part of a matrix to remove resources from a
# particular cloud instance. Normally we would use template variables instead of
# parameters but matrix variables are not available during template expansion
# so any benefits of parameters are lost.

# Assumes steps in deploy-test-resources.yml was run previously. Requires
# environment variable: AZURE_RESOURCEGROUP_NAME and Az PowerShell module

parameters:
ServiceDirectory: ''
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)

steps:
- pwsh: >
eng/common/TestResources/Remove-TestResources.ps1
-ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}"
-TenantId '$(aad-azure-sdk-test-tenant-id)'
-SubscriptionId '$(azure-subscription-id)'
-ProvisionerApplicationId '$(aad-azure-sdk-test-client-id)'
-ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret)'
-ServiceDirectory '${{ parameters.ServiceDirectory }}'
-Environment 'AzureCloud'
-Force
-Verbose
displayName: Remove test resources (AzureCloud)
condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), eq(variables['CloudType'], 'AzureCloud'))
continueOnError: true
# SubscriptionConfiguration will be splat into the parameters of the test
# resources script. It should be JSON in the form:
# {
# "SubscriptionId": "<subscription id>",
# "TenantId": "<tenant id>",
# "TestApplicationId": "<test app id>",
# "TestApplicationSecret": "<test app secret>",
# "ProvisionerApplicationId": "<provisoner app id>",
# "ProvisoinerApplicationSecret": "<provisoner app secert>",
# "Environment": "AzureCloud | AzureGov | AzureChina | <other environment>"
# }
# The Remove-TestResources.ps1 script accommodates extra parameters so it will
# not error when parameters are provided which the script doesn't use.

- pwsh: >
eng/common/TestResources/Remove-TestResources.ps1
-ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}"
-TenantId '$(aad-azure-sdk-test-tenant-id-gov)'
-SubscriptionId '$(azure-subscription-id-gov)'
-ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-gov)'
-ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-gov)'
-ServiceDirectory '${{ parameters.ServiceDirectory }}'
-Environment 'AzureUSGovernment'
-Force
-Verbose
displayName: Remove test resources (AzureUSGovernment)
condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), eq(variables['CloudType'], 'AzureUSGovernment'))
continueOnError: true
steps:
- pwsh: |
$subscriptionConfiguration = @"
${{ parameters.SubscriptionConfiguration }}
"@ | ConvertFrom-Json -AsHashtable;
- pwsh: >
eng/common/TestResources/Remove-TestResources.ps1
-ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}"
-TenantId '$(aad-azure-sdk-test-tenant-id-cn)'
-SubscriptionId '$(azure-subscription-id-cn)'
-ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-cn)'
-ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-cn)'
-ServiceDirectory '${{ parameters.ServiceDirectory }}'
-Environment 'AzureChinaCloud'
-Force
-Verbose
displayName: Remove test resources (AzureChinaCloud)
condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), eq(variables['CloudType'], 'AzureChinaCloud'))
eng/common/TestResources/Remove-TestResources.ps1 `
-ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" `
-ServiceDirectory ${{ parameters.ServiceDirectory }} `
@subscriptionConfiguration `
-Force `
-Verbose
displayName: Remove test resources
continueOnError: true
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,3 @@ steps:
BaseBranchName: smoke-test
WorkingDirectory: ${{parameters.WorkingDirectory}}/repo
ScriptDirectory: ${{parameters.WorkingDirectory}}/${{parameters.ScriptDirectory}}



12 changes: 6 additions & 6 deletions eng/common/scripts/artifact-metadata-parsing.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,7 @@ function GetExistingTags($apiUrl) {
}

# Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases
function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $exitOnError = $True) {
function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $continueOnError = $false) {
$pkgList = [array]@()
$ParsePkgInfoFn = ""
$packagePattern = ""
Expand Down Expand Up @@ -404,7 +404,7 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
continue
}

if ($parsedPackage.Deployable -ne $True -and $exitOnError) {
if ($parsedPackage.Deployable -ne $True -and !$continueOnError) {
Write-Host "Package $($parsedPackage.PackageId) is marked with version $($parsedPackage.PackageVersion), the version $($parsedPackage.PackageVersion) has already been deployed to the target repository."
Write-Host "Maybe a pkg version wasn't updated properly?"
exit(1)
Expand All @@ -430,8 +430,8 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a

$intersect = $results | % { $_.Tag } | ? { $existingTags -contains $_ }

if ($intersect.Length -gt 0 -and $exitOnError) {
CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl -exitOnError $exitOnError
if ($intersect.Length -gt 0 -and !$continueOnError) {
CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl -continueOnError $continueOnError

# all the tags are clean. remove them from the list of releases we will publish.
$results = $results | ? { -not ($intersect -contains $_.Tag ) }
Expand All @@ -443,7 +443,7 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
# given a set of tags that we want to release, we need to ensure that if they already DO exist.
# if they DO exist, quietly exit if the commit sha of the artifact matches that of the tag
# if the commit sha does not match, exit with error and report both problem shas
function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl, $exitOnError) {
function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl, $continueOnError) {
$headers = @{
"Content-Type" = "application/json"
"Authorization" = "token $($env:GH_TOKEN)"
Expand All @@ -465,7 +465,7 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap
}
}

if ($unmatchedTags.Length -gt 0 -and $exitOnError) {
if ($unmatchedTags.Length -gt 0 -and !$continueOnError) {
Write-Host "Tags already existing with different SHA versions. Exiting."
exit(1)
}
Expand Down
4 changes: 2 additions & 2 deletions eng/common/scripts/create-tags-and-git-release.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ param (
$repoOwner = "", # the owning organization of the repository. EG "Azure"
$repoName = "", # the name of the repository. EG "azure-sdk-for-java"
$repoId = "$repoOwner/$repoName", # full repo id. EG azure/azure-sdk-for-net DevOps: $(Build.Repository.Id),
[switch]$forceCreate = $false
[switch]$continueOnError = $false
)

Write-Host "> $PSCommandPath $args"
Expand All @@ -26,7 +26,7 @@ $apiUrl = "https://api.github.com/repos/$repoId"
Write-Host "Using API URL $apiUrl"

# VERIFY PACKAGES
$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha
$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError

if ($pkgList) {
Write-Host "Given the visible artifacts, github releases will be created for the following:"
Expand Down
2 changes: 1 addition & 1 deletion eng/common/scripts/update-docs-metadata.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ $pkgs = VerifyPackages -pkgRepository $Repository `
-workingDirectory $WorkDirectory `
-apiUrl $apiUrl `
-releaseSha $ReleaseSHA `
-exitOnError $False
-continueOnError $True

if ($pkgs) {
Write-Host "Given the visible artifacts, readmes will be copied for the following packages"
Expand Down

0 comments on commit 9bd3125

Please sign in to comment.