Skip to content

Commit

Permalink
Update CI files
Browse files Browse the repository at this point in the history
  • Loading branch information
pulpbot committed Oct 6, 2024
1 parent 2a8abba commit 38592cd
Show file tree
Hide file tree
Showing 19 changed files with 186 additions and 104 deletions.
4 changes: 3 additions & 1 deletion .ci/ansible/Containerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ ADD ./{{ item.name }} ./{{ item.name }}
# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
# Hacking botocore (https://github.com/boto/botocore/pull/1990)

# This MUST be the ONLY call to pip install in inside the container.
RUN pip3 install --upgrade pip setuptools wheel && \
rm -rf /root/.cache/pip && \
pip3 install
Expand All @@ -27,7 +28,8 @@ RUN pip3 install --upgrade pip setuptools wheel && \
{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
{%- endif -%}
{%- endfor %}
{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \
pipdeptree && \
rm -rf /root/.cache/pip

{% if pulp_env is defined and pulp_env %}
Expand Down
5 changes: 5 additions & 0 deletions .ci/assets/ci_constraints.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,8 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*

tablib!=3.6.0
# 3.6.0: This release introduced a regression removing the "html" optional dependency.



# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue.
pipdeptree<=3.23.1
11 changes: 8 additions & 3 deletions .ci/scripts/check_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def main():
for branch in branches:
if branch != DEFAULT_BRANCH:
# Check if a Z release is needed
reasons = []
changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
z_changelog = False
for change in changes.split("\n"):
Expand All @@ -76,23 +77,27 @@ def main():
)
elif ext in Z_CHANGELOG_EXTS:
z_changelog = True
if z_changelog:
reasons.append("Backports")

last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}")
req_txt_diff = repo.git.diff(
f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt"
)
if z_changelog or req_txt_diff:
if req_txt_diff:
reasons.append("requirements.txt")

if reasons:
curr_version = Version(last_tag)
assert curr_version.base_version.startswith(
branch
), "Current-version has to belong to the current branch!"
next_version = Version(f"{branch}.{curr_version.micro + 1}")
reason = "CHANGES" if z_changelog else "requirements.txt"
print(
f"A Z-release is needed for {branch}, "
f"Prev: {last_tag}, "
f"Next: {next_version.base_version}, "
f"Reason: {reason}"
f"Reason: {','.join(reasons)}"
)
releases.append(next_version)
else:
Expand Down
2 changes: 1 addition & 1 deletion .ci/scripts/collect_changes.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def main():
for change in main_changes:
fp.write(change[1])

repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE)


if __name__ == "__main__":
Expand Down
60 changes: 60 additions & 0 deletions .ci/scripts/pr_labels.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/bin/env python3

# This script is running with elevated privileges from the main branch against pull requests.

import re
import sys
import tomllib
from pathlib import Path

from git import Repo


def main():
assert len(sys.argv) == 3

with open("pyproject.toml", "rb") as fp:
PYPROJECT_TOML = tomllib.load(fp)
BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")
CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$")
try:
CHANGELOG_EXTS = {
f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
}
except KeyError:
CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"}

repo = Repo(".")

base_commit = repo.commit(sys.argv[1])
head_commit = repo.commit(sys.argv[2])

pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}"))

labels = {
"multi-commit": len(pr_commits) > 1,
"cherry-pick": False,
"no-issue": False,
"no-changelog": False,
"wip": False,
}
for commit in pr_commits:
labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None
no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None
labels["no-issue"] |= no_issue
cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None
labels["cherry-pick"] |= cherry_pick
changelog_snippets = [
k
for k in commit.stats.files
if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS
]
labels["no-changelog"] |= not changelog_snippets

print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v)))
print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v)))


if __name__ == "__main__":
main()
19 changes: 2 additions & 17 deletions .ci/scripts/validate_commit_message.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,16 @@
import sys
from pathlib import Path
import subprocess


import os
import warnings
from github import Github


NO_ISSUE = "[noissue]"
CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
KEYWORDS = ["fixes", "closes"]

sha = sys.argv[1]
message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")


KEYWORDS = ["fixes", "closes"]

g = Github(os.environ.get("GITHUB_TOKEN"))
repo = g.get_repo("pulp/pulp_file")

Expand Down Expand Up @@ -64,15 +59,5 @@ def __check_changelog(issue):
for issue in pattern.findall(message):
__check_status(issue)
__check_changelog(issue)
else:
if NO_ISSUE in message:
print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
elif "Merge" in message and "cherry picked from commit" in message:
pass
else:
sys.exit(
"Error: no attached issues found for {sha}. If this was intentional, add "
" '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
)

print("Commit message for {sha} passed.".format(sha=sha[0:7]))
2 changes: 1 addition & 1 deletion .github/template_gitref
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2021.08.26-364-g6f9579c
2021.08.26-385-g17472a1
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ jobs:
run: |
.github/workflows/scripts/check_commit.sh
docs:
uses: "./.github/workflows/docs.yml"

lint:
uses: "./.github/workflows/lint.yml"

Expand All @@ -61,6 +64,7 @@ jobs:
- "check-commits"
- "lint"
- "test"
- "docs"
if: "always()"
steps:
- name: "Collect needed jobs results"
Expand Down
17 changes: 7 additions & 10 deletions .github/workflows/create-branch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,20 @@ jobs:
fetch-depth: 0
path: "pulp_file"

- uses: "actions/checkout@v4"
with:
fetch-depth: 1
repository: "pulp/plugin_template"
path: "plugin_template"

- uses: "actions/setup-python@v5"
with:
python-version: "3.11"

- name: "Install python dependencies"
run: |
echo ::group::PYDEPS
pip install bump2version jinja2 pyyaml packaging
pip install bump2version packaging -r plugin_template/requirements.txt
echo ::endgroup::
- name: "Setting secrets"
Expand Down Expand Up @@ -71,13 +77,6 @@ jobs:
run: |
find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} +
- name: Checkout plugin template
uses: actions/checkout@v4
with:
repository: pulp/plugin_template
path: plugin_template
fetch-depth: 0

- name: Update CI branches in template_config
working-directory: plugin_template
run: |
Expand All @@ -94,10 +93,8 @@ jobs:
branch: minor-version-bump
base: main
title: Bump minor version
body: '[noissue]'
commit-message: |
Bump minor version
[noissue]
delete-branch: true

- name: Push release branch
Expand Down
54 changes: 54 additions & 0 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# WARNING: DO NOT EDIT!
#
# This file was generated by plugin_template, and is managed by it. Please use
# './plugin-template --github pulp_file' to update this file.
#
# For more info visit https://github.com/pulp/plugin_template

---
name: "Docs"
on:
workflow_call:

jobs:
test:
if: "endsWith(github.base_ref, 'main')"
runs-on: "ubuntu-20.04"
defaults:
run:
working-directory: "pulp_file"
steps:
- uses: "actions/checkout@v4"
with:
fetch-depth: 1
path: "pulp_file"
- uses: "actions/setup-python@v5"
with:
python-version: "3.11"
- name: "Setup cache key"
run: |
git ls-remote https://github.com/pulp/pulp-docs main | tee pulp-docs-main-sha
- uses: "actions/cache@v4"
with:
path: "~/.cache/pip"
key: ${{ runner.os }}-pip-${{ hashFiles('pulp-docs-main-sha') }}
restore-keys: |
${{ runner.os }}-pip-
- name: "Install python dependencies"
run: |
echo ::group::PYDEPS
pip install -r doc_requirements.txt
echo ::endgroup::
- name: "Build changelog"
run: |
towncrier build --yes --version 4.0.0.ci
- name: "Build docs"
run: |
pulp-docs build
no-test:
if: "!endsWith(github.base_ref, 'main')"
runs-on: "ubuntu-20.04"
steps:
- run: |
echo "Skip docs testing on non-main branches."
55 changes: 30 additions & 25 deletions .github/workflows/pr_checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,57 +6,62 @@
# For more info visit https://github.com/pulp/plugin_template

---
name: File PR static checks
name: "File PR static checks"
on:
pull_request_target:
types: [opened, synchronize, reopened]
types: ["opened", "synchronize", "reopened"]

# This workflow runs with elevated permissions.
# Do not even think about running a single bit of code from the PR.
# Static analysis should be fine however.

concurrency:
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
group: "${{ github.event.pull_request.number }}-${{ github.workflow }}"
cancel-in-progress: true

jobs:
single_commit:
runs-on: ubuntu-latest
name: Label multiple commit PR
apply_labels:
runs-on: "ubuntu-latest"
name: "Label PR"
permissions:
pull-requests: write
pull-requests: "write"
steps:
- uses: "actions/checkout@v4"
with:
fetch-depth: 0
- name: Commit Count Check
- uses: "actions/setup-python@v5"
with:
python-version: "3.11"
- name: "Determine PR labels"
run: |
pip install GitPython==3.1.42
git fetch origin ${{ github.event.pull_request.head.sha }}
echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV"
- uses: actions/github-script@v7
python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV"
- uses: "actions/github-script@v7"
name: "Apply PR Labels"
with:
script: |
const labelName = "multi-commit";
const { COMMIT_COUNT } = process.env;
const { ADD_LABELS, REMOVE_LABELS } = process.env;
if (COMMIT_COUNT == 1)
{
try {
await github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
});
} catch(err) {
if (REMOVE_LABELS.length) {
for await (const labelName of REMOVE_LABELS.split(",")) {
try {
await github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
});
} catch(err) {
}
}
}
else
{
if (ADD_LABELS.length) {
await github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: [labelName],
labels: ADD_LABELS.split(","),
});
}
...
2 changes: 1 addition & 1 deletion .github/workflows/scripts/before_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ fi

for i in {1..3}
do
ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3
ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
done
if [[ $s -gt 0 ]]
then
Expand Down
Loading

0 comments on commit 38592cd

Please sign in to comment.