From a3609da1e555163ab8edf8c32c60d5075c83058c Mon Sep 17 00:00:00 2001 From: Haroon Khel <34969545+Haroon-Khel@users.noreply.github.com> Date: Thu, 4 Apr 2024 17:45:44 +0800 Subject: [PATCH 01/10] Remove equinix x64 dockerhost machines from inventory (#3497) --- ansible/inventory.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 0b2fcc92c3..b20812bebd 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -89,8 +89,6 @@ hosts: ubuntu2204-x64-2: {ip: 20.83.24.86, description: 16 cores, 64GB} - equinix: - ubuntu2204-x64-1: {ip: 145.40.113.173, description: Intel Xeon Gold 40 core} - ubuntu2004-x64-1: {ip: 145.40.114.58, description: AMD EPYC 7401P 24 core} ubuntu2004-armv8-1: {ip: 147.75.35.203, description: Ampere Altra 160 core, 512Gb} ubuntu2204-armv8-1: {ip: 139.178.86.243, description: Ampere Altra 160 cores, 512Gb} From b728c86a1b2fe798c29cae85f7b23e50ff9686fa Mon Sep 17 00:00:00 2001 From: Stewart X Addison <6487691+sxa@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:25:35 +0100 Subject: [PATCH 02/10] Changes to support RHEL7/s390x image creation and devkit inclusion (#3492) --------- Signed-off-by: Stewart X Addison --- FAQ.md | 22 +++++++++++++++---- ansible/docker/Dockerfile.RHEL7 | 15 ++++++------- .../AdoptOpenJDK_Unix_Playbook/main.yml | 2 ++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/FAQ.md b/FAQ.md index cc850474fb..45c24e5fae 100644 --- a/FAQ.md +++ b/FAQ.md @@ -85,16 +85,30 @@ have at the moment: | Dockerfile | Image | Platforms | Where is this built? | In use? |---|---|---|---|---| | [Centos7](./ansible/docker/Dockerfile.CentOS7) | [`adoptopenjdk/centos7_build_image`](https://hub.docker.com/r/adoptopenjdk/centos7_build_image) | linux on amd64, arm64, ppc64le | [Jenkins](https://ci.adoptium.net/job/centos7_docker_image_updater/) | Yes +| [RHEL7](./ansible/docker/Dockerfile.RHEL7) | n/a - restricted (*) | s390x | [Jenkins](https://ci.adoptium.net/job/rhel7_docker_image_updater/) | Yes | [Centos6](./ansible/docker/Dockerfile.CentOS6) | [`adoptopenjdk/centos6_build_image`](https://hub.docker.com/r/adoptopenjdk/centos6_build_image)| linux/amd64 | [GH Actions](.github/workflows/build.yml) | Yes | [Alpine3](./ansible/docker/Dockerfile.Alpine3) | [`adoptopenjdk/alpine3_build_image`](https://hub.docker.com/r/adoptopenjdk/alpine3_build_image) | linux/x64 & linux/arm64 | [Jenkins](https://ci.adoptium.net/job/centos7_docker_image_updater/) | Yes | [Ubuntu 20.04 (riscv64 only)](./ansible/docker/Dockerfile.Ubuntu2004-riscv64) | [`adoptopenjdk/ubuntu2004_build_image:linux-riscv64`](https://hub.docker.com/r/adoptopenjdk/ubuntu2004_build_image) | linux/riscv64 | [Jenkins](https://ci.adoptium.net/job/centos7_docker_image_updater/) | Yes +
+(*) - Caveats: + +The RHEL7 image creation for s390x has to be run on a RHEL host using a +container implementation supplied by Red Hat, and we are using RHEL8 for +this as it has a stable implemention. The image creation requires the +following: + +1. The host needs to have an active RHEL subscription +2. The RHEL7 devkit (which cannot be made public) to be available in a tar file under /usr/local on the host as per the name in the Dockerfile +
+ When a change lands into master, the relevant dockerfiles are built using the appropriate CI system listed in the table above by configuring them with -the ansible playbooks and pushing them up to Docker Hub where they can be -consumed by our jenkins build agents when the `DOCKER_IMAGE` value is -defined on the jenkins build pipelines as configured in the -[pipeline_config files](https://github.com/AdoptOpenJDK/ci-jenkins-pipelines/tree/master/pipelines/jobs/configurations). +the ansible playbooks and - with the exception of the RHEL7 image for s390x - +pushing them up to Docker Hub where they can be consumed by our jenkins +build agents when the `DOCKER_IMAGE` value is defined on the jenkins build +pipelines as configured in the [pipeline_config +files](https://github.com/AdoptOpenJDK/ci-jenkins-pipelines/tree/master/pipelines/jobs/configurations). ### Adding a new dockerBuild dockerhub repository diff --git a/ansible/docker/Dockerfile.RHEL7 b/ansible/docker/Dockerfile.RHEL7 index e9e0f96bb6..dc0b773fae 100644 --- a/ansible/docker/Dockerfile.RHEL7 +++ b/ansible/docker/Dockerfile.RHEL7 @@ -1,10 +1,5 @@ FROM registry.access.redhat.com/rhel7 -# This dockerfile should be built using this from the top level of the repository: -# ROSIPW=******* docker build --no-cache -t rhel7_build_image -f ansible/docker/Dockerfile.RHEL7 --build-arg ROSIUSER=******* --secret id=ROSIPW --build-arg git_sha="$(git rev-parse --short HEAD)" `pwd` -ARG ROSIUSER -RUN sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py -RUN --mount=type=secret,id=ROSIPW,required=true subscription-manager register --username=${ROSIUSER} --password="$(cat /run/secrets/ROSIPW)" --auto-attach -RUN subscription-manager repos --enable rhel-7-for-system-z-optional-rpms +RUN yum-config-manager --enable rhel-7-for-system-z-optional-rpms # ^^ Optional repo needed for Xvfb ARG git_sha @@ -15,6 +10,7 @@ RUN yum --enablerepo=rhel-7-server-ansible-2-for-system-z-rpms install -y ansibl RUN yum clean all COPY . /ansible +COPY devkit /usr/local/devkit RUN echo "localhost ansible_connection=local" > /ansible/hosts @@ -31,7 +27,10 @@ RUN useradd -c "Jenkins user" -d /home/${user} -u 1002 -g 1003 -m ${user} ENV \ JDK7_BOOT_DIR="/usr/lib/jvm/java-1.7.0-openjdk" \ JDK8_BOOT_DIR="/usr/lib/jvm/java-1.8.0-openjdk" \ - JDK10_BOOT_DIR="/usr/lib/jvm/jdk-10" \ JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk" -RUN subscription-manager unregister +# While this does bloat the image it is required for building the +# devkit, and the process for that runs as non-root ... +# Disabled for now as we're going to copy from /usr/local/devkit on the host +RUN yum clean all +RUN yum reinstall --downloadonly glibc glibc-headers glibc-devel cups-libs cups-devel libX11 libX11-devel xorg-x11-proto-devel alsa-lib alsa-lib-devel libXext libXext-devel libXtst libXtst-devel libXrender libXrender-devel libXrandr libXrandr-devel freetype freetype-devel libXt libXt-devel libSM libSM-devel libICE libICE-devel libXi libXi-devel libXdmcp libXdmcp-devel libXau libXau-devel libgcc libxcrypt zlib zlib-devel libffi libffi-devel fontconfig fontconfig-devel kernel-headers diff --git a/ansible/playbooks/AdoptOpenJDK_Unix_Playbook/main.yml b/ansible/playbooks/AdoptOpenJDK_Unix_Playbook/main.yml index e43cf47b4e..059989e1f4 100644 --- a/ansible/playbooks/AdoptOpenJDK_Unix_Playbook/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Unix_Playbook/main.yml @@ -94,6 +94,7 @@ when: - (ansible_distribution != "Alpine" or ansible_architecture != "aarch64") - ansible_architecture != "riscv64" + - ansible_architecture != "s390x" tags: build_tools - role: adoptopenjdk_install # JDK11 Build Bootstrap jdk_version: 10 @@ -120,6 +121,7 @@ - ansible_distribution != "Solaris" - ansible_architecture != "riscv64" - ansible_architecture != "armv7l" + - ansible_architecture != "s390x" tags: build_tools - role: adoptopenjdk_install # Current LTS jdk_version: 21 From f20fe120c4cfda8c71c4f5881094e32d3d857dbc Mon Sep 17 00:00:00 2001 From: Scott Fryer <60462088+steelhead31@users.noreply.github.com> Date: Mon, 8 Apr 2024 12:06:44 +0100 Subject: [PATCH 03/10] inventory: Update public IP for win2019 test machine (#3509) --- ansible/inventory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/inventory.yml b/ansible/inventory.yml index b20812bebd..5d15487bca 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -111,7 +111,7 @@ hosts: - azure: win2016-x64-1: {ip: 52.149.211.210, user: adoptopenjdk} - win2019-x64-1: {ip: 20.185.182.137, user: adoptopenjdk} + win2019-x64-1: {ip: 13.92.177.186, user: adoptopenjdk} win2022-x64-1: {ip: 51.132.234.42, user: adoptopenjdk} win2022-x64-2: {ip: 20.26.116.218, user: adoptopenjdk} win11-aarch64-1: {ip: 20.4.31.184, user: adoptopenjdk} From 2d54a9050c7ee470c01ce12a534c3f5ffe79af0f Mon Sep 17 00:00:00 2001 From: Haroon Khel <34969545+Haroon-Khel@users.noreply.github.com> Date: Mon, 8 Apr 2024 20:53:34 +0800 Subject: [PATCH 04/10] Add windows packages to AWX collections (#3510) --- collections/requirements.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/collections/requirements.yml b/collections/requirements.yml index 9ee87128fb..13a584e939 100644 --- a/collections/requirements.yml +++ b/collections/requirements.yml @@ -4,3 +4,6 @@ collections: - name: community.general source: https://galaxy.ansible.com + + - name: community.windows + source: https://galaxy.ansible.com From 79db1c04a42499ebfa44836f880689a9e54dc359 Mon Sep 17 00:00:00 2001 From: Stewart X Addison <6487691+sxa@users.noreply.github.com> Date: Tue, 9 Apr 2024 09:23:52 +0100 Subject: [PATCH 05/10] inventory: remove spearhead freebsd system (#3504) --- ansible/inventory.yml | 3 --- .../nagios/roles/Nagios_Config/files/Nagios_Server_Config.py | 3 +-- ansible/plugins/inventory/adoptopenjdk_yaml.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 5d15487bca..5994ab1f55 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -74,9 +74,6 @@ hosts: win2012r2-x64-2: {ip: 169.48.4.142, user: Administrator} win2022-x64-1: {ip: 52.118.206.11, user: Administrator} - - spearhead: - freebsd12-x64-1: {ip: 185.131.222.224} - - docker: - skytap: diff --git a/ansible/playbooks/nagios/roles/Nagios_Config/files/Nagios_Server_Config.py b/ansible/playbooks/nagios/roles/Nagios_Config/files/Nagios_Server_Config.py index e00f3d27bf..70e2e050ff 100644 --- a/ansible/playbooks/nagios/roles/Nagios_Config/files/Nagios_Server_Config.py +++ b/ansible/playbooks/nagios/roles/Nagios_Config/files/Nagios_Server_Config.py @@ -29,8 +29,7 @@ ## Define Any Hosts That Should Be Excluded -excluded_hosts = {'build-spearhead-freebsd12-x64-1', - 'test-inspira-solaris10u11-sparcv9-1', +excluded_hosts = {'test-inspira-solaris10u11-sparcv9-1', 'build-inspira-solaris10u11-sparcv9-1', 'build-inspira-solaris10u11-sparcv9-2', 'infrastructure-aws-ubuntu1804-x64-1', diff --git a/ansible/plugins/inventory/adoptopenjdk_yaml.py b/ansible/plugins/inventory/adoptopenjdk_yaml.py index f88ba3ba5c..3fe8ef8013 100755 --- a/ansible/plugins/inventory/adoptopenjdk_yaml.py +++ b/ansible/plugins/inventory/adoptopenjdk_yaml.py @@ -47,7 +47,7 @@ # providers - validated for consistency 'provider': ('alibaba', 'azure', 'marist', 'osuosl', - 'macstadium', 'macincloud', 'ibmcloud', 'spearhead', 'siteox', + 'macstadium', 'macincloud', 'ibmcloud', 'siteox', 'equinix', 'linaro','digitalocean', 'ibm', 'godaddy', 'aws', 'inspira', 'equinix_esxi', 'nine', 'scaleway', 'skytap', 'hetzner') From 9892a79e97b6e61cb05bc7abad0701cd5753b5a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 14:45:11 +1200 Subject: [PATCH 06/10] build(deps): bump docker/setup-buildx-action from 3.1.0 to 3.3.0 (#3512) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.1.0 to 3.3.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/0d103c3126aa41d772a8362f6aa67afac040f80c...d70bba72b1f3fd22344832f00baa16ece964efeb) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Martijn Verburg --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9238511028..5f5b1044b6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Set up Docker Buildx to use cache feature - uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 - name: Login to Docker Hub uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3 @@ -68,7 +68,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Set up Docker Buildx to use cache feature - uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 - name: Docker Build Alpine3 Image uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0 From 58d8783ff91fece6e09de91972b4889e32d99cbf Mon Sep 17 00:00:00 2001 From: Haroon Khel <34969545+Haroon-Khel@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:55:34 +0800 Subject: [PATCH 07/10] docker: Assign port number to static docker container during automated deployment (#3511) * Assign port number to static docker container * satisfy linter * modify docker_port variable --------- Co-authored-by: Martijn Verburg --- .../roles/deploy_container/tasks/deploy.yml | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ansible/playbooks/AdoptOpenJDK_Unix_Playbook/roles/deploy_container/tasks/deploy.yml b/ansible/playbooks/AdoptOpenJDK_Unix_Playbook/roles/deploy_container/tasks/deploy.yml index f6382e7038..e12470d7cf 100644 --- a/ansible/playbooks/AdoptOpenJDK_Unix_Playbook/roles/deploy_container/tasks/deploy.yml +++ b/ansible/playbooks/AdoptOpenJDK_Unix_Playbook/roles/deploy_container/tasks/deploy.yml @@ -22,14 +22,20 @@ - name: Build {{ docker_image }} docker images command: docker build --cpu-period=100000 --cpu-quota=800000 -t aqa_{{ docker_image }} --memory=6G -f /tmp/Dockerfile.{{ docker_image }} /tmp/ -# Without specifying a port here, docker will give the container a random unused port -- name: Run {{ docker_image }} docker container - command: docker run --restart unless-stopped -p 22 --cpuset-cpus="0-3" --memory=6G --detach --name {{ docker_image | upper }}.PORT aqa_{{ docker_image }} +# Finds the highest port number already assigned and +1 +- name: Find available port + shell: docker ps --format \"\{\{\.Ports\}\}\" | awk -F[:-] '{print $2}' | sort | tail -n 1 + register: docker_port_output + +- name: Set docker_port variable if empty + set_fact: + docker_port: 32000 + when: docker_port_output.stdout == "" -# Now we want to rename the running container with the port name -- name: Find assigned port of {{ docker_image }} container - shell: docker port {{ docker_image | upper }}.PORT | head -n 1 | cut -d ':' -f 2 - register: docker_port +- name: Set docker_port variable when non empty + set_fact: + docker_port: "{{ docker_port_output.stdout | int + 1 }}" + when: not (docker_port_output.stdout == "") -- name: Rename {{ docker_image }} container to {{ docker_image | upper }}.{{ docker_port.stdout_lines[0] }} - command: docker rename {{ docker_image | upper }}.PORT {{ docker_image | upper }}.{{ docker_port.stdout_lines[0] }} +- name: Run {{ docker_image }} docker container + command: docker run --restart unless-stopped -p {{ docker_port }}:22 --cpuset-cpus="0-3" --memory=6G --detach --name {{ docker_image | upper }}.{{ docker_port }} aqa_{{ docker_image }} From 31214c7708c6c072efdd43063abef686b6258792 Mon Sep 17 00:00:00 2001 From: Scott Fryer <60462088+steelhead31@users.noreply.github.com> Date: Thu, 11 Apr 2024 11:42:07 +0100 Subject: [PATCH 08/10] Inventory: Remove IBMCloud Win2012 Machines (#3516) --- ansible/inventory.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 5994ab1f55..bac7f050ce 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -70,8 +70,6 @@ hosts: solaris10u11-sparcv9-1: {ip: cloud.siteox.com, port: 24322} - ibmcloud: - win2012r2-x64-1: {ip: 169.48.4.138, user: Administrator} - win2012r2-x64-2: {ip: 169.48.4.142, user: Administrator} win2022-x64-1: {ip: 52.118.206.11, user: Administrator} - docker: @@ -173,5 +171,3 @@ hosts: rhel6-x64-1: {ip: 169.48.4.140} rhel7-x64-1: {ip: 169.48.4.136} ubuntu1604-x64-1: {ip: 169.48.4.141} - win2012r2-x64-1: {ip: 169.48.4.131, user: Administrator} - win2012r2-x64-2: {ip: 169.48.4.139, user: Administrator} From 267b00809642c35485fdbbf868ec9e020c6b01fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:46:30 +0200 Subject: [PATCH 09/10] build(deps): bump actions/setup-python from 5.0.0 to 5.1.0 (#3493) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/0a5c61591373683505ea898e09a3ea4f39ef2b9c...82c7e631bb3cdc910f68e0081d67478d79c6982d) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Martijn Verburg --- .github/workflows/linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 4f7dbebf30..f1b487954a 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -42,7 +42,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Set up Python 3.x - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: '3.x' From f643812be18efaae31b931349ace3ab9e136cb73 Mon Sep 17 00:00:00 2001 From: Scott Fryer <60462088+steelhead31@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:50:51 +0100 Subject: [PATCH 10/10] GitHubAction : Update Solaris Build Action (#3527) --- .github/workflows/build_vagrant.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_vagrant.yml b/.github/workflows/build_vagrant.yml index d92af98739..cecfd3789b 100644 --- a/.github/workflows/build_vagrant.yml +++ b/.github/workflows/build_vagrant.yml @@ -24,6 +24,9 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Update Repos + run: sudo apt-get update + - name: Install Ansible run: sudo apt-get install ansible