Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

update packaging #63

Open
wants to merge 1 commit into
base: daos_adio-rpm
Choose a base branch
from
Open

update packaging #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packaging/Dockerfile.centos.7
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
#
# Copyright 2018-2021, Intel Corporation
# Copyright 2018-2022, Intel Corporation
#
# 'recipe' for Docker to build an RPM
#

# Pull base image
FROM centos:7
MAINTAINER daos-stack <daos@daos.groups.io>
LABEL maintainer="[email protected]"

# use same UID as host and default value of 1000 if not specified
ARG UID=1000
Expand Down
32 changes: 23 additions & 9 deletions packaging/Dockerfile.mockbuild
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright 2018-2022, Intel Corporation
# Copyright 2018-2022 Intel Corporation
#
# 'recipe' for Docker to build an RPM
#
Expand All @@ -8,14 +8,24 @@
FROM fedora:latest
LABEL maintainer="[email protected]"

# use same UID as host and default value of 1000 if not specified
ARG UID=1000
# Use local repo server if present
ARG REPO_FILE_URL
RUN if [ -n "$REPO_FILE_URL" ]; then \
cd /etc/yum.repos.d/ && \
curl -f -o daos_ci-fedora-artifactory.repo.tmp \
"$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \
rm -f *.repo && \
mv daos_ci-fedora-artifactory.repo{.tmp,}; \
fi

# Install basic tools
RUN dnf -y install mock make \
rpm-build curl createrepo rpmlint redhat-lsb-core git \
RUN dnf -y install mock make \
rpm-build createrepo rpmlint redhat-lsb-core git \
python-srpm-macros rpmdevtools

# use same UID as host and default value of 1000 if not specified
ARG UID=1000

# Add build user (to keep rpmbuild happy)
ENV USER build
ENV PASSWD build
Expand All @@ -24,10 +34,14 @@ RUN echo "$USER:$PASSWD" | chpasswd
# add the user to the mock group so it can run mock
RUN usermod -a -G mock $USER

# mock in Docker needs to use the old-chroot option
RUN grep use_nspawn /etc/mock/site-defaults.cfg || \
echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg

ARG CACHEBUST
RUN dnf -y upgrade && \
dnf clean all

# Monkey-patch rpmlint until a new release is made with
# https://github.com/rpm-software-management/rpmlint/pull/795 in it
COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch .
RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \
patch -p1 && \
rm -f rpmlint/__pycache__/{cli,lint}.*.pyc) < rpmlint--ignore-unused-rpmlintrc.patch && \
rm -f rpmlint--ignore-unused-rpmlintrc.patch
2 changes: 1 addition & 1 deletion packaging/Dockerfile.ubuntu.20.04
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
autoconf bash ca-certificates curl debhelper dh-make \
dpkg-dev dh-python doxygen gcc git git-buildpackage locales \
make patch pbuilder pkg-config python3-dev python3-distro \
python3-distutils rpm scons wget
python3-distutils rpm scons wget cmake valgrind

# rpmdevtools
RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list
Expand Down
17 changes: 10 additions & 7 deletions packaging/Makefile_distro_vars.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,16 @@ ORIG_TARGET_VER := 7
SED_EXPR := 1s/$(DIST)//p
endif
ifeq ($(patsubst %epel-8-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),)
DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist})
VERSION_ID := 8
DISTRO_ID := el8
DISTRO_BASE := EL_8
DISTRO_VERSION ?= $(VERSION_ID)
ORIG_TARGET_VER := 8
SED_EXPR := 1s/$(DIST)//p
DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist})
VERSION_ID := 8.5
DISTRO_ID := el8
DISTRO_BASE := EL_8
ifneq ($(DISTRO_VERSION_EL8),)
override DISTRO_VERSION := $(DISTRO_VERSION_EL8)
endif
DISTRO_VERSION ?= $(VERSION_ID)
ORIG_TARGET_VER := 8.5
SED_EXPR := 1s/$(DIST)//p
endif
ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64)
VERSION_ID := 15.2
Expand Down
166 changes: 89 additions & 77 deletions packaging/Makefile_packaging.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ endif

CALLING_MAKEFILE := $(word 1, $(MAKEFILE_LIST))

TOPDIR ?= $(CURDIR)
# this Makefile should always be executed from it's own dir
TOPDIR ?= $(abspath $(dir $(firstword $(MAKEFILE_LIST))))

BUILD_PREFIX ?= .

DOT := .
Expand All @@ -28,11 +30,6 @@ RPM_BUILD_OPTIONS += $(EXTERNAL_RPM_BUILD_OPTIONS)
# some defaults the caller can override
PACKAGING_CHECK_DIR ?= ../packaging
LOCAL_REPOS ?= true
ifeq ($(ID_LIKE),debian)
DAOS_REPO_TYPE ?= LOCAL
else
DAOS_REPO_TYPE ?= STABLE
endif
TEST_PACKAGES ?= ${NAME}

# unfortunately we cannot always name the repo the same as the project
Expand All @@ -43,6 +40,7 @@ LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep
EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p')
EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p')
UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p')
REPO_FILES_PR ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p')

ifneq ($(PKG_GIT_COMMIT),)
ifeq ($(GITHUB_PROJECT),)
Expand Down Expand Up @@ -100,27 +98,29 @@ define distro_map
endef

define install_repos
IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \
for baseurl in "$${BASES[@]}"; do \
baseurl="$${baseurl# *}"; \
$(call install_repo,$$baseurl); \
done
for repo in $($(DISTRO_BASE)_PR_REPOS) \
$(PR_REPOS) $(1); do \
branch="master"; \
build_number="lastSuccessfulBuild"; \
if [[ $$repo = *@* ]]; then \
branch="$${repo#*@}"; \
repo="$${repo%@*}"; \
if [[ $$branch = *:* ]]; then \
build_number="$${branch#*:}"; \
branch="$${branch%:*}"; \
fi; \
fi; \
$(call distro_map) \
if [ "$(ID_LIKE)" = "debian" ]; then \
IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \
for baseurl in "$${BASES[@]}"; do \
baseurl="$${baseurl# *}"; \
$(call install_repo,$$baseurl) \
done; \
fi
for repo in $($(DISTRO_BASE)_PR_REPOS) \
$(PR_REPOS) $(1); do \
branch="master"; \
build_number="lastSuccessfulBuild"; \
if [[ $$repo = *@* ]]; then \
branch="$${repo#*@}"; \
repo="$${repo%@*}"; \
if [[ $$branch = *:* ]]; then \
build_number="$${branch#*:}"; \
branch="$${branch%:*}"; \
fi; \
fi; \
$(call distro_map) \
baseurl=$${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$$repo/job/$$branch/; \
baseurl+=$$build_number/artifact/artifacts/$$distro/; \
$(call install_repo,$$baseurl); \
baseurl+=$$build_number/artifact/artifacts/$$distro/; \
$(call install_repo,$$baseurl) \
done
endef

Expand Down Expand Up @@ -154,6 +154,11 @@ ifeq ($(DL_NAME),)
DL_NAME = $(NAME)
endif

# this actually should replace all of the downloaders below
$(notdir $(SOURCE)): $(SPEC) $(CALLING_MAKEFILE)
# TODO: need to clean up old ones
$(SPECTOOL) -g $(SPEC)

$(DL_NAME)$(DL_VERSION).linux-amd64.tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE)
rm -f ./$(DL_NAME)*.tar{gz,bz*,xz}
$(SPECTOOL) -g $(SPEC)
Expand Down Expand Up @@ -334,60 +339,29 @@ patch:
endif

# *_LOCAL_* repos are locally built packages.
# *_GROUP_* repos are a local mirror of a group of upstream repos.
# *_GROUP_* repos may not supply a repomd.xml.key.
ifeq ($(LOCAL_REPOS),true)
ifneq ($(REPOSITORY_URL),)
# group repos are not working in Nexus so we hack in the group members directly below
#ifneq ($(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO),)
#DISTRO_REPOS = $(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO)
#$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO)/
#endif
ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),)
ifeq ($(ID_LIKE),debian)
# $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists
# of values with spaces as environment variables
$(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes]
else
$(DISTRO_BASE)_LOCAL_REPOS := $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)
DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not
endif # ifeq ($(ID_LIKE),debian)
ifeq ($(DISTRO_BASE), EL_8)
# hack to use 8.3 non-group repos on EL_8
$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/rocky-8.5-base-x86_64-proxy|$(REPOSITORY_URL)repository/rocky-8.5-extras-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-8-x86_64-proxy)
else ifeq ($(DISTRO_BASE), EL_7)
# hack to use 7.9 non-group repos on EL_7
$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-7.9-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-extras-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-updates-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-7-x86_64-proxy)
else ifeq ($(DISTRO_BASE), LEAP_15)
# hack to use 15 non-group repos on LEAP_15
$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/opensuse-15.2-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-oss-x86_64-provo-mirror-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-sle-update-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-backports-update-proxy)
else
# debian
$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO)
endif # ifeq ($(DISTRO_BASE), *)
endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),)
ifneq ($(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO),)
$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst centos-8.3,rocky-8.5,$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO))
endif
# group repos are not working in Nexus so we hack in the group members directly above
ifneq ($(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO),)
$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst centos-8.3,rocky-8.5,$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO))
endif
ifneq ($(ID_LIKE),debian)
ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),)
$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_INTEL_ONEAPI_REPO)
endif # ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),)
endif # ifneq ($(ID_LIKE),debian)
endif # ifneq ($(REPOSITORY_URL),)
ifneq ($(ARTIFACTORY_URL),)
ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),)
DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not
# convert to artifactory url
DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
# $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists
# of values with spaces as environment variables
$(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
$(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),)
endif # ifneq ($(ARTIFACTORY_URL),)
endif # ifeq ($(LOCAL_REPOS),true)
ifeq ($(ID_LIKE),debian)
chrootbuild: $(DEB_TOP)/$(DEB_DSC)
$(call distro_map) \
DISTRO="$$distro" \
PR_REPOS="$(PR_REPOS)" \
REPO_FILES_PR="$(REPO_FILES_PR)" \
DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \
JENKINS_URL="$${JENKINS_URL}" \
JOB_REPOS="$(JOB_REPOS)" \
REPO_FILE_URL="$(REPO_FILE_URL)" \
DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \
VERSION_CODENAME="$(VERSION_CODENAME)" \
DEB_TOP="$(DEB_TOP)" \
Expand All @@ -400,26 +374,61 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE)
DISTRO="$$distro" \
CHROOT_NAME="$(CHROOT_NAME)" \
PR_REPOS="$(PR_REPOS)" \
REPO_FILES_PR="$(REPO_FILES_PR)" \
DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \
JENKINS_URL="$${JENKINS_URL}" \
JOB_REPOS="$(JOB_REPOS)" \
DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \
REPO_FILE_URL="$(REPO_FILE_URL)" \
MOCK_OPTIONS="$(MOCK_OPTIONS)" \
RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \
DISTRO_REPOS='$(DISTRO_REPOS)' \
ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \
REPOSITORY_URL="$(REPOSITORY_URL)" \
DISTRO_VERSION="$(DISTRO_VERSION)" \
TARGET="$<" \
packaging/rpm_chrootbuild
endif

podman_chrootbuild:
if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \
-t $(subst +,-,$(CHROOT_NAME))-chrootbuild \
-f packaging/Dockerfile.mockbuild .; then \
echo "Container build failed"; \
exit 1; \
fi
rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log
podman run --rm --privileged -w $(TOPDIR) -v=$(TOPDIR)/..:$(TOPDIR)/.. \
-it $(subst +,-,$(CHROOT_NAME))-chrootbuild \
bash -c 'if ! DISTRO_REPOS=false \
REPO_FILE_URL=$(REPO_FILE_URL) \
REPOSITORY_URL=$(REPOSITORY_URL) \
make REPO_FILES_PR=$(REPO_FILES_PR) \
MOCK_OPTIONS=$(MOCK_OPTIONS) \
CHROOT_NAME=$(CHROOT_NAME) -C $(CURDIR) chrootbuild; then \
cat /var/lib/mock/$(CHROOT_NAME)/{result/{root,build},root/builddir/build/BUILD/*/config}.log; \
exit 1; \
fi; \
rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \
grep -v -e debuginfo -e debugsource -e src.rpm)'

docker_chrootbuild:
$(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \
-f packaging/Dockerfile.mockbuild .
$(DOCKER) run --privileged=true -w $(TOPDIR) -v=$(TOPDIR):$(TOPDIR) \
-it chrootbuild bash -c "make -C $(CURDIR) \
CHROOT_NAME=$(CHROOT_NAME) chrootbuild"
if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \
--build-arg REPO_FILE_URL=$(REPO_FILE_URL) \
-f packaging/Dockerfile.mockbuild .; then \
echo "Container build failed"; \
exit 1; \
fi
rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log
if ! $(DOCKER) run --user=$$(id -u) --privileged=true -w $(TOPDIR) \
-v=$(TOPDIR):$(TOPDIR) \
-it chrootbuild bash -c "make -C $(CURDIR) \
CHROOT_NAME=$(CHROOT_NAME) chrootbuild"; then \
cat /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log; \
exit 1; \
fi

rpmlint: $(SPEC)
rpmlint $<
rpmlint --ignore-unused-rpmlintrc $<

packaging_check:
if grep -e --repo $(CALLING_MAKEFILE); then \
Expand Down Expand Up @@ -456,6 +465,9 @@ test:
$(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER))
dnf -y install $(TEST_PACKAGES)

show_NAME:
@echo '$(NAME)'

show_DISTRO_ID:
@echo '$(DISTRO_ID)'

Expand Down
Loading