Skip to content

Commit

Permalink
Update CI tooling
Browse files Browse the repository at this point in the history
  • Loading branch information
Tyil committed Nov 22, 2018
1 parent 15b8fe3 commit db50125
Show file tree
Hide file tree
Showing 14 changed files with 275 additions and 32 deletions.
13 changes: 13 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
image: docker:latest
services:
- docker:dind
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- /usr/portage
- /var/portage
- /var/lib/docker

build-ebuilds:
script:
- ./.tools/bin/ci-run-mr
23 changes: 23 additions & 0 deletions .tools/bin/ci-mkdockercmds
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#! /usr/bin/env sh

set -e

export PATH=$PATH:$(pwd)/.tools/bin
export DEBIAN_FRONTEND=noninteractive

hash -r

{
# We need git for consecutive commands
apt update -qq
apt install -qqy git

# Make sure the latest `master` is also available to diff against
git fetch -a origin
git checkout master
git pull origin master
git checkout -
} > /dev/null

# Generate the testing commands
updated-ebuilds | ebuild2atom | make-docker-commands --root="$1"
31 changes: 31 additions & 0 deletions .tools/bin/ci-run-mr
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#! /usr/bin/env sh

set -e

main()
{
cmds=$(mktemp)

mkdir -p /usr/portage
mkdir -p /var/portage

# Sync latest portage tree
do_cmd docker run -v /usr/portage:/usr/portage gentoo/stage3-amd64 emerge -q --sync

# Run the script to generate the testing commands
do_cmd docker run -w /app -v "$(pwd):/app" rakudo-star sh .tools/bin/ci-mkdockercmds "$(pwd)" > "$cmds"

# Run the testing commands
while read -r cmd
do
do_cmd $cmd
done < "$cmds"
}

do_cmd()
{
printf "> %s\n" "$*" >&2
$@
}

main "$@"
14 changes: 14 additions & 0 deletions .tools/bin/ebuild2atom
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#! /usr/bin/env perl6

#| Convert a path to an ebuild into an atom that can be passed to emerge.
sub MAIN (*@ebuilds)
{
@ebuilds = $*IN.lines unless @ebuilds;

for @ebuilds {
my $category = $_.split("/").head;
my $package = $_.split("/").tail.IO.extension("").Str;

say "=$category/$package"
}
}
58 changes: 58 additions & 0 deletions .tools/bin/make-docker-commands
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#! /usr/bin/env perl6

#| Build packages
sub MAIN (
#| Host's path to the Portage tree.
Str:D :$tree = "/usr/portage",

#| Host's path to Portage's var dir. This holds binpkgs and distfiles.
Str:D :$var = "/var/portage",

#| Host's path to the root of the overlay.
Str:D :$root is copy = "",

#| A list of packages to build
*@packages,
) {
@packages = $*IN.lines unless @packages;
$root = shell("git rev-parse --show-toplevel", :out).out.slurp.trim unless $root;

build-pkg($_, :$tree, :$var, root => $root.IO) for @packages;
}

#| Build a given package in a clean Docker container
sub build-pkg (
#| The name of the package to build
Str:D $package,

#| Host's path to the Portage tree.
Str:D :$tree = "/usr/portage",

#| Host's path to Portage's var dir. This holds binpkgs and distfiles.
Str:D :$var = "/var/portage",

#| Path to overlay's root
IO::Path:D :$root,
) {
my Pair @volumes = (
$tree => "/usr/portage",
$var => "/var/portage",
$root.add(".tools").add("etc").absolute => "/etc/portage",
$root.absolute => "/app",
);

my Str @cmd = <
docker
run
>;

for @volumes {
@cmd.push: "-v $_.key():$_.value()";
}

@cmd.push: "gentoo/stage3-amd64";
@cmd.push: "emerge";
@cmd.push: $package;

@cmd.join(" ").say;
}
18 changes: 18 additions & 0 deletions .tools/bin/mkcommit
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#! /usr/bin/env perl6

sub MAIN (
*@words,
) {
my @ebuilds = (run « git diff --name-status master », :out).out.lines.grep(*.ends-with(".ebuild"));

if (@ebuilds.elems != 1) {
exit note q:to/EOF/;
Committed ebuilds does not equal to 1. Each commit should only
contain a single ebuild update.
EOF
}

my $ebuild = @ebuilds.head.split("\t").tail.split("/").head(2).join("/");

run « git commit -S -m "$ebuild: @words.join(" ")" »;
}
11 changes: 11 additions & 0 deletions .tools/bin/updated-ebuilds
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#! /usr/bin/env perl6

#| Get a list of all changed package atoms.
sub MAIN ()
{
(run « git diff --name-status master », :out).out.lines
.grep(*.ends-with(".ebuild"))
.map(*.split("\t").tail)
.map(*.say)
;
}
33 changes: 33 additions & 0 deletions .tools/etc/make.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
COMMON_FLAGS="-O2 -pipe"
CFLAGS="${COMMON_FLAGS}"
CXXFLAGS="${COMMON_FLAGS}"
FCFLAGS="${COMMON_FLAGS}"
FFLAGS="${COMMON_FLAGS}"

FEATURES="
$FEATURES
buildpkg
network-sandbox
parallel-fetch
parallel-install
sandbox
userfetch
userpriv
usersandbox
usersync
"

EMERGE_DEFAULT_OPTS="
$EMERGE_DEFAULT_OPTS
--binpkg-respect-use=y
--binpkg-changed-deps=y
--tree
--usepkg
--verbose
"

PKGDIR=/var/portage/packages
DISTDIR=/var/portage/distfiles

LC_MESSAGES=C
L10N="en"
1 change: 1 addition & 0 deletions .tools/etc/make.profile
1 change: 1 addition & 0 deletions .tools/etc/package.accept_keywords
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*/*::sk-overlay ~*
51 changes: 51 additions & 0 deletions .tools/etc/repo.postsync.d/example
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/sh
# Example /etc/portage/repo.postsync.d script. Make it executable (chmod +x) for
# Portage to process it.
#
# With portage-2.2.16 and newer, all repo.postsync.d hooks will be called multiple
# times after syncing each repository.
#
# Older versions of Portage support syncing only one repository.
# In those versions, the postsync.d hooks will be called only once,
# and they will not be passed any parameters.

# On a repo.postsync.d hook call, positional parameters contain
# information about the just-synced repository.

# Your hook can control it's actions depending on any of the three
# parameters passed in to it.
#
# They are as follows:
#
# The repository name.
repository_name=${1}
# The URI to which the repository was synced.
sync_uri=${2}
# The path to the repository.
repository_path=${3}

# Portage assumes that a hook succeeded if it exits with 0 code. If no
# explicit exit is done, the exit code is the exit code of last spawned
# command. Since our script is a bit more complex, we want to control
# the exit code explicitly.
ret=0

if [ -n "${repository_name}" ]; then
# Repository name was provided, so we're in a post-repository hook.
echo "* In post-repository hook for ${repository_name}"
echo "** synced from remote repository ${sync_uri}"
echo "** synced into ${repository_path}"

# Gentoo comes with pregenerated cache but the other repositories
# usually don't. Generate them to improve performance.
if [ "${repository_name}" != "gentoo" ]; then
if ! egencache --update --repo="${repository_name}" --jobs=4
then
echo "!!! egencache failed!"
ret=1
fi
fi
fi

# Return explicit status.
exit "${ret}"
10 changes: 10 additions & 0 deletions .tools/etc/repo.postsync.d/q-reinit
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/sh

repository_name=$1
repository_path=$3

if [ -n "${repository_name}" ]; then
q ${PORTAGE_QUIET:+-q} --reinitialize="${repository_path}"
fi

:
5 changes: 5 additions & 0 deletions .tools/etc/repos.conf/sk-overlay.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[sk-overlay]
priority = 50
location = /app
layman-type = git
auto-sync = No
38 changes: 6 additions & 32 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,33 +1,7 @@
#
# Run repoman via travis
# See https://github.com/mrueg/repoman-travis
#
language: python
python:
- pypy
env:
- PORTAGE_VER="2.3.51"
before_install:
- sudo apt-get -qq update
- pip install lxml pyyaml
before_script:
- sudo chmod a+rwX /etc/passwd /etc/group /etc /usr
- mkdir -p travis-overlay /etc/portage /usr/portage/distfiles
- mv !(travis-overlay) travis-overlay/
- mv .git travis-overlay/
- wget "https://raw.githubusercontent.com/mrueg/repoman-travis/master/.travis.yml" -O .travis.yml.upstream
- wget "https://raw.githubusercontent.com/mrueg/repoman-travis/master/spinner.sh"
- wget -qO - "https://github.com/gentoo/portage/archive/portage-${PORTAGE_VER}.tar.gz" | tar xz
- wget -qO - "https://github.com/gentoo-mirror/gentoo/archive/master.tar.gz" | tar xz -C /usr/portage --strip-components=1
- chmod a+rwx spinner.sh
- echo "portage:x:250:250:portage:/var/tmp/portage:/bin/false" >> /etc/passwd
- echo "portage::250:portage,travis" >> /etc/group
- wget "https://www.gentoo.org/dtd/metadata.dtd" -O /usr/portage/distfiles/metadata.dtd
- ln -s $TRAVIS_BUILD_DIR/portage-portage-${PORTAGE_VER}/cnf/repos.conf /etc/portage/repos.conf
- ln -s /usr/portage/profiles/default/linux/amd64/17.0 /etc/portage/make.profile
- SIZE=$(stat -c %s .travis.yml.upstream)
- if ! cmp -n $SIZE -s .travis.yml .travis.yml.upstream; then echo -e "\e[31m !!! .travis.yml outdated! Update available https://github.com/mrueg/repoman-travis \e[0m" > /tmp/update ; fi
- cd travis-overlay
sudo: required
language: generic
services:
- docker

script:
- ./../spinner.sh "python ../portage-portage-${PORTAGE_VER}/repoman/bin/repoman full -d"
# You can append own scripts after this line
- sudo ./.tools/bin/ci-run-mr

0 comments on commit db50125

Please sign in to comment.