Skip to content

Commit

Permalink
Merge pull request #26 from lorengordon/archive-race
Browse files Browse the repository at this point in the history
  • Loading branch information
lorengordon authored Jul 20, 2020
2 parents 6841c3b + a25d114 commit 1ebb082
Show file tree
Hide file tree
Showing 14 changed files with 170 additions and 136 deletions.
3 changes: 1 addition & 2 deletions .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
[bumpversion]
current_version = 4.0.1
current_version = 5.0.0
commit = True
message = Bumps version to {new_version}
tag = False
tag_name = {new_version}

5 changes: 1 addition & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,7 @@ No provider.
|------|-------------|------|---------|:--------:|
| bucket\_name | S3 bucket where salt repo will be mirrored | `string` | n/a | yes |
| repo\_endpoint | HTTP/S endpoint URL that hosts the yum repos; used with the baseurl in the yum .repo definitions | `string` | n/a | yes |
| repo\_prefix | S3 key prefix where the repos will be mirrored | `string` | `""` | no |
| salt\_s3\_endpoint | S3 endpoint for the upstream salt repo | `string` | `"https://s3.repo.saltstack.com"` | no |
| salt\_versions | List of salt versions to mirror; will also generate version-specific yum .repo definition files | `list(string)` | `[]` | no |
| yum\_prefix | S3 key where the yum repo definitions will be placed | `string` | `""` | no |
| repos | Schema list of repo objects. `repo_prefix` is the S3 key prefix where the repo will be mirrored. `salt_s3_endpoint` is the upstream s3 endpoint hosting the repos. `salt_versions` is the list of salt versions to mirror. `yum_prefix` is the S3 key prefix for the yum repo definition files. | <pre>list(object({<br> repo_prefix = string<br> salt_s3_endpoint = string<br> salt_versions = list(string)<br> yum_prefix = string<br> }))</pre> | n/a | yes |

## Outputs

Expand Down
10 changes: 3 additions & 7 deletions main.tf
Original file line number Diff line number Diff line change
@@ -1,18 +1,14 @@
module "sync_repo" {
source = "./modules/repo"

bucket_name = var.bucket_name
salt_versions = var.salt_versions
repo_prefix = var.repo_prefix
salt_s3_endpoint = var.salt_s3_endpoint
bucket_name = var.bucket_name
repos = var.repos
}

module "yum_defs" {
source = "./modules/defs"

bucket_name = var.bucket_name
salt_versions = var.salt_versions
repo_endpoint = var.repo_endpoint
repo_prefix = var.repo_prefix
yum_prefix = var.yum_prefix
repos = var.repos
}
4 changes: 1 addition & 3 deletions modules/defs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@ No requirements.
|------|-------------|------|---------|:--------:|
| bucket\_name | n/a | `string` | n/a | yes |
| repo\_endpoint | n/a | `string` | n/a | yes |
| repo\_prefix | n/a | `string` | `""` | no |
| salt\_versions | n/a | `list(string)` | `[]` | no |
| yum\_prefix | n/a | `string` | `""` | no |
| repos | Schema list of repo objects. `repo_prefix` is the S3 key prefix where the repo is located. `salt_versions` is the list of salt versions for which repo definitions will be generated. `yum_prefix` is the S3 key prefix for the yum repo definition files. | <pre>list(object({<br> repo_prefix = string<br> salt_versions = list(string)<br> yum_prefix = string<br> }))</pre> | n/a | yes |

## Outputs

Expand Down
55 changes: 30 additions & 25 deletions modules/defs/main.tf
Original file line number Diff line number Diff line change
@@ -1,35 +1,40 @@
locals {
salt_versions = sort(var.salt_versions)
repo_endpoint = "${trimsuffix(var.repo_endpoint, "/")}/${trim(var.repo_prefix, "/")}"
yum_prefix = trim(var.yum_prefix, "/")
repos = [
for repo in var.repos : {
salt_versions = sort(repo.salt_versions)
repo_endpoint = "${trimsuffix(var.repo_endpoint, "/")}/${trim(repo.repo_prefix, "/")}"
yum_prefix = trim(repo.yum_prefix, "/")
}
]
}

locals {

repo_endpoints = {
amzn = "${local.repo_endpoint}/python2/amazon/latest/$basearch/archive"
amzn2 = "${local.repo_endpoint}/python2/amazon/2/$basearch/archive"
amzn2-python3 = "${local.repo_endpoint}/python3/amazon/2/$basearch/archive"
el6 = "${local.repo_endpoint}/python2/redhat/6/$basearch/archive"
el7 = "${local.repo_endpoint}/python2/redhat/7/$basearch/archive"
el7-python3 = "${local.repo_endpoint}/python3/redhat/7/$basearch/archive"
el8 = "${local.repo_endpoint}/python2/redhat/8/$basearch/archive"
el8-python3 = "${local.repo_endpoint}/python3/redhat/8/$basearch/archive"
repo_paths = {
amzn = "python2/amazon/latest/$basearch/archive"
amzn2 = "python2/amazon/2/$basearch/archive"
amzn2-python3 = "python3/amazon/2/$basearch/archive"
el6 = "python2/redhat/6/$basearch/archive"
el7 = "python2/redhat/7/$basearch/archive"
el7-python3 = "python3/redhat/7/$basearch/archive"
el8 = "python2/redhat/8/$basearch/archive"
el8-python3 = "python3/redhat/8/$basearch/archive"
}

repo_defs = flatten([
for version in local.salt_versions : [
for repo, endpoint in local.repo_endpoints : {
key = "${local.yum_prefix}/${version}/salt-reposync-${repo}.repo"
content = templatefile(
"${path.module}/yum.repo",
{
name = "salt-reposync-${repo}"
baseurl = "${endpoint}/${version}"
gpgkey = "${endpoint}/${version}/SALTSTACK-GPG-KEY.pub"
}
)
}
for repo in local.repos : [
for version in repo.salt_versions : [
for platform, repo_path in local.repo_paths : {
key = "${repo.yum_prefix}/${version}/salt-reposync-${platform}.repo"
content = templatefile(
"${path.module}/yum.repo",
{
name = "salt-reposync-${platform}"
baseurl = "${repo.repo_endpoint}/${repo_path}/${version}"
gpgkey = "${repo.repo_endpoint}/${repo_path}/${version}/SALTSTACK-GPG-KEY.pub"
}
)
}
]
]
])
}
Expand Down
20 changes: 7 additions & 13 deletions modules/defs/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,15 @@ variable "bucket_name" {
type = string
}

variable "salt_versions" {
type = list(string)
default = []
}

variable "repo_endpoint" {
type = string
}

variable "repo_prefix" {
type = string
default = ""
}

variable "yum_prefix" {
type = string
default = ""
variable "repos" {
description = "Schema list of repo objects. `repo_prefix` is the S3 key prefix where the repo is located. `salt_versions` is the list of salt versions for which repo definitions will be generated. `yum_prefix` is the S3 key prefix for the yum repo definition files."
type = list(object({
repo_prefix = string
salt_versions = list(string)
yum_prefix = string
}))
}
4 changes: 1 addition & 3 deletions modules/repo/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@ No requirements.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| bucket\_name | n/a | `string` | n/a | yes |
| repo\_prefix | n/a | `string` | `"/"` | no |
| salt\_s3\_endpoint | n/a | `string` | `"https://s3.repo.saltstack.com"` | no |
| salt\_versions | n/a | `list(string)` | `[]` | no |
| repos | Schema list of repo objects. `repo_prefix` is the S3 key prefix where the repo will be mirrored. `salt_s3_endpoint` is the upstream s3 endpoint hosting the repos. `salt_versions` is the list of salt versions to mirror. | <pre>list(object({<br> repo_prefix = string<br> salt_s3_endpoint = string<br> salt_versions = list(string)<br> }))</pre> | n/a | yes |

## Outputs

Expand Down
59 changes: 41 additions & 18 deletions modules/repo/main.tf
Original file line number Diff line number Diff line change
@@ -1,62 +1,85 @@
locals {
disable = length(var.salt_versions) == 0
salt_versions = formatlist("--filter '+ {amazon,redhat}/{latest,?}/**/archive/%s/**'", sort(var.salt_versions))
repo_prefix_python2 = "${trim(var.repo_prefix, "/")}/python2"
repo_prefix_python3 = "${trim(var.repo_prefix, "/")}/python3"
}
repos = [
for repo in var.repos : {
id = "${repo.salt_s3_endpoint}_${repo.repo_prefix}"
repo_prefix_python2 = "${trim(repo.repo_prefix, "/")}/python2"
repo_prefix_python3 = "${trim(repo.repo_prefix, "/")}/python3"
salt_s3_endpoint = repo.salt_s3_endpoint
salt_versions = formatlist("--filter '+ {amazon,redhat}/{latest,?}/**/archive/%s/**'", sort(repo.salt_versions))
}
]

locals {
rclone_base = [
"RCLONE_CONFIG_SALT_TYPE=s3",
"RCLONE_CONFIG_SALT_PROVIDER=Other",
"RCLONE_CONFIG_SALT_ENV_AUTH=false",
"RCLONE_CONFIG_SALT_ENDPOINT=${var.salt_s3_endpoint}",
"RCLONE_CONFIG_SALT_ENDPOINT=%s", # %s is repo.salt_s3_endpoint
"RCLONE_CONFIG_S3_TYPE=s3",
"RCLONE_CONFIG_S3_ENV_AUTH=true",
"rclone sync",
"--delete-excluded --use-server-modtime --update --fast-list -v",
"--filter '- **/{i386,i686,SRPMS}/**'",
join(" ", local.salt_versions),
"%s", # %s is repo.salt_versions
"--filter '- *'",
]

rclone_python2 = concat(
local.rclone_base,
list(
"salt:s3/yum", # rclone source
"s3:${var.bucket_name}/${local.repo_prefix_python2}" # rclone target
"salt:s3/yum", # rclone source
"s3:${var.bucket_name}/%s" # rclone target, %s is repo.repo_prefix_python2
)
)

rclone_python3 = concat(
local.rclone_base,
list(
"salt:s3/py3", # rclone source
"s3:${var.bucket_name}/${local.repo_prefix_python3}" # rclone target
"salt:s3/py3", # rclone source
"s3:${var.bucket_name}/%s" # rclone target, %s is repo.repo_prefix_python3
)
)
}

resource "null_resource" "sync_python2" {
count = local.disable ? 0 : 1
for_each = { for repo in local.repos : repo.id => repo }

provisioner "local-exec" {
command = join(" ", local.rclone_python2)
command = format(
join(" ", local.rclone_python2),
each.value.salt_s3_endpoint,
join(" ", each.value.salt_versions),
each.value.repo_prefix_python2,
)
}

triggers = {
rclone = join(" ", local.rclone_python2)
rclone = format(
join(" ", local.rclone_python2),
each.value.salt_s3_endpoint,
join(" ", each.value.salt_versions),
each.value.repo_prefix_python2,
)
}
}

resource "null_resource" "sync_python3" {
count = local.disable ? 0 : 1
for_each = { for repo in local.repos : repo.id => repo }

provisioner "local-exec" {
command = join(" ", local.rclone_python3)
command = format(
join(" ", local.rclone_python3),
each.value.salt_s3_endpoint,
join(" ", each.value.salt_versions),
each.value.repo_prefix_python3,
)
}

triggers = {
rclone = join(" ", local.rclone_python3)
rclone = format(
join(" ", local.rclone_python3),
each.value.salt_s3_endpoint,
join(" ", each.value.salt_versions),
each.value.repo_prefix_python3,
)
}
}
20 changes: 7 additions & 13 deletions modules/repo/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,11 @@ variable "bucket_name" {
type = string
}

variable "salt_versions" {
type = list(string)
default = []
}

variable "repo_prefix" {
type = string
default = "/"
}

variable "salt_s3_endpoint" {
type = string
default = "https://s3.repo.saltstack.com"
variable "repos" {
description = "Schema list of repo objects. `repo_prefix` is the S3 key prefix where the repo will be mirrored. `salt_s3_endpoint` is the upstream s3 endpoint hosting the repos. `salt_versions` is the list of salt versions to mirror."
type = list(object({
repo_prefix = string
salt_s3_endpoint = string
salt_versions = list(string)
}))
}
27 changes: 20 additions & 7 deletions tests/defs/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,19 @@ module "defs" {
source = "../../modules/defs"

bucket_name = local.bucket_name
salt_versions = local.salt_versions
repo_endpoint = local.repo_endpoint
repo_prefix = local.repo_prefix
yum_prefix = local.yum_prefix
repos = [
{
salt_versions = local.salt_versions
repo_prefix = local.repo_prefix
yum_prefix = local.yum_prefix
},
{
salt_versions = local.salt_versions_archive
repo_prefix = local.repo_prefix_archive
yum_prefix = local.yum_prefix
},
]
}

resource "aws_s3_bucket" "this" {
Expand All @@ -19,13 +28,17 @@ resource "aws_s3_bucket" "this" {
locals {
bucket_name = aws_s3_bucket.this.id

repo_endpoint = "https://${aws_s3_bucket.this.bucket_regional_domain_name}"
repo_prefix = "repo/"
repo_prefix_archive = "repo/archive/"
yum_prefix = "defs/"

salt_versions = [
"3000",
"2019.2.3",
"2018.3.4",
]

repo_endpoint = "https://${aws_s3_bucket.this.bucket_regional_domain_name}"
repo_prefix = "repo/"
yum_prefix = "defs/"
salt_versions_archive = [
"2018.3.4",
]
}
36 changes: 26 additions & 10 deletions tests/main/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,21 @@ module "main" {
source = "../.."

bucket_name = local.bucket_name
salt_versions = local.salt_versions
repo_endpoint = local.repo_endpoint
repo_prefix = local.repo_prefix
yum_prefix = local.yum_prefix
repos = [
{
yum_prefix = local.yum_prefix
salt_s3_endpoint = local.salt_s3_endpoint
salt_versions = local.salt_versions
repo_prefix = local.repo_prefix
},
{
repo_prefix = local.repo_prefix_archive
salt_s3_endpoint = local.salt_s3_endpoint_archive
salt_versions = local.salt_versions_archive
yum_prefix = local.yum_prefix
},
]
}

resource "aws_s3_bucket" "this" {
Expand All @@ -18,15 +29,20 @@ resource "aws_s3_bucket" "this" {
}

locals {
bucket_name = aws_s3_bucket.this.id
bucket_name = aws_s3_bucket.this.id
repo_endpoint = "https://${aws_s3_bucket.this.bucket_regional_domain_name}"
repo_prefix = "repo/main/"
repo_prefix_archive = "repo/archive/"
salt_s3_endpoint = "https://s3.repo.saltstack.com"
salt_s3_endpoint_archive = "https://s3.archive.repo.saltstack.com"
yum_prefix = "defs/"

salt_versions = [
"3000",
"2019.2.3",
"2018.3.4",
"3000.3",
"2019.2.5",
]

repo_endpoint = "https://${aws_s3_bucket.this.bucket_regional_domain_name}"
repo_prefix = "repo/"
yum_prefix = "defs/"
salt_versions_archive = [
"2018.3.4",
]
}
2 changes: 1 addition & 1 deletion tests/noop/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ module "main" {
source = "../.."

bucket_name = "foo"
salt_versions = []
repo_endpoint = "bar"
repos = []
}
Loading

0 comments on commit 1ebb082

Please sign in to comment.