From a0e2e9b9fa900fb22878eae2229ec4eb1d779e13 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Sat, 16 Dec 2023 12:04:11 -0600 Subject: [PATCH] Update and optimize meta tester and inflator images --- .github/workflows/build.yml | 3 +- Dockerfile.metadata | 58 ++++++++++++++++++++++++++++++++----- Dockerfile.netkan | 25 +++++++++++++--- Netkan/Program.cs | 25 +++++++++------- doc/building.md | 2 +- 5 files changed, 88 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec82ae1cb4..c115693af7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,8 +76,7 @@ jobs: sh get-docker.sh docker build --tag inflator --file ../Dockerfile.netkan . docker run --rm --name inflator --entrypoint /bin/bash inflator -c " - curl -O https://raw.githubusercontent.com/KSP-CKAN/NetKAN/master/NetKAN/ZeroMiniAVC.netkan && \ - mono netkan.exe ZeroMiniAVC.netkan + mono netkan.exe https://raw.githubusercontent.com/KSP-CKAN/NetKAN/master/NetKAN/ZeroMiniAVC.netkan " if: matrix.configuration == 'release' diff --git a/Dockerfile.metadata b/Dockerfile.metadata index 05b3dac4ff..8e86963f90 100644 --- a/Dockerfile.metadata +++ b/Dockerfile.metadata @@ -1,15 +1,57 @@ -FROM mono:latest +# Everything we need in both the build and prod images +FROM ubuntu:latest as base -RUN /bin/sed -i 's/^mozilla\/DST_Root_CA_X3.crt$/!mozilla\/DST_Root_CA_X3.crt/' /etc/ca-certificates.conf && \ - /usr/sbin/update-ca-certificates -RUN apt-get update && \ - apt-get install -y --no-install-recommends python3 python3-pip python3-setuptools git build-essential python3-dev libffi-dev && \ - apt-get clean +# Don't prompt for time zone +ENV DEBIAN_FRONTEND=noninteractive + +# Put user-installed Python code in path +ENV PATH "$PATH:/root/.local/bin" + +# Install Git and Python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates gnupg git libffi-dev \ + python3 python-is-python3 + +# Trust all git repos RUN git config --global --add safe.directory '*' + +# Set up Mono's APT repo +RUN gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \ + && echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | tee /etc/apt/sources.list.d/mono-official-stable.list \ + && apt-get update + +# Install the necessary pieces of Mono +RUN apt-get install -y --no-install-recommends \ + mono-runtime ca-certificates-mono libmono-microsoft-csharp4.0-cil libmono-system-data4.0-cil libmono-system-runtime-serialization4.0-cil libmono-system-transactions4.0-cil libmono-system-net-http-webrequest4.0-cil + +# Isolate Python build stuff in a separate container +FROM base as build + +# Install Python build deps +RUN apt-get install -y --no-install-recommends \ + python3-pip python3-setuptools python3-dev + +# Install the meta tester's Python code and its Infra dep +ENV PIP_ROOT_USER_ACTION=ignore RUN pip3 install --upgrade pip -RUN pip3 install 'git+https://github.com/KSP-CKAN/NetKAN-Infra#subdirectory=netkan' -RUN pip3 install 'git+https://github.com/KSP-CKAN/xKAN-meta_testing' +RUN pip3 install --user 'git+https://github.com/KSP-CKAN/NetKAN-Infra#subdirectory=netkan' +RUN pip3 install --user 'git+https://github.com/KSP-CKAN/xKAN-meta_testing' + +# Prune unused deps (`--user` is implicit for uninstall) +RUN pip3 --no-input uninstall -y flask gunicorn werkzeug + +# The image we'll actually use +FROM base as prod + +# Purge APT download cache, package lists, and logs +RUN apt-get clean \ + && rm -r /var/lib/apt/lists /var/log/dpkg.log /var/log/apt + +# Extract built Python packages from the build image +COPY --from=build /root/.local /root/.local +# Install the .NET assemblies the meta tester uses ADD netkan.exe /usr/local/bin/. ADD ckan.exe /usr/local/bin/. diff --git a/Dockerfile.netkan b/Dockerfile.netkan index 2cc12656ae..9b55858f90 100644 --- a/Dockerfile.netkan +++ b/Dockerfile.netkan @@ -1,10 +1,27 @@ -FROM mono:latest -RUN /bin/sed -i 's/^mozilla\/DST_Root_CA_X3.crt$/!mozilla\/DST_Root_CA_X3.crt/' /etc/ca-certificates.conf && \ - /usr/sbin/update-ca-certificates +FROM ubuntu:latest + +# Don't prompt for time zone +ENV DEBIAN_FRONTEND=noninteractive + +# Set up Mono's APT repo +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates gnupg \ + && gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \ + && echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | tee /etc/apt/sources.list.d/mono-official-stable.list \ + && apt-get update + +# Install the necessary pieces of Mono +RUN apt-get install -y --no-install-recommends \ + mono-runtime ca-certificates-mono libmono-microsoft-csharp4.0-cil libmono-system-data4.0-cil libmono-system-runtime-serialization4.0-cil libmono-system-transactions4.0-cil libmono-system-net-http-webrequest4.0-cil + +# Purge APT download cache, package lists, and logs +RUN apt-get clean \ + && rm -r /var/lib/apt/lists /var/log/dpkg.log /var/log/apt + RUN useradd -ms /bin/bash netkan USER netkan WORKDIR /home/netkan -ADD netkan.exe . +ADD --chown=netkan netkan.exe . ENTRYPOINT /usr/bin/mono netkan.exe --game ${GAME:-KSP} --queues $QUEUES \ --net-useragent 'Mozilla/5.0 (compatible; Netkanbot/1.0; CKAN; +https://github.com/KSP-CKAN/NetKAN-Infra)' \ --github-token $GH_Token --gitlab-token "$GL_Token" --cachedir ckan_cache -v diff --git a/Netkan/Program.cs b/Netkan/Program.cs index e0d068f203..84144c1ab1 100644 --- a/Netkan/Program.cs +++ b/Netkan/Program.cs @@ -16,6 +16,7 @@ using CKAN.NetKAN.Processors; using CKAN.NetKAN.Transformers; using CKAN.NetKAN.Extensions; +using YamlDotNet.RepresentationModel; namespace CKAN.NetKAN { @@ -116,7 +117,7 @@ public static int Main(string[] args) else { Log.Fatal( - "Usage: netkan [--verbose|--debug] [--debugger] [--prerelease] [--outputdir=...] " + "Usage: netkan [--verbose|--debug] [--debugger] [--prerelease] [--outputdir=...] " ); return ExitBadOpt; } @@ -183,18 +184,22 @@ private static Metadata[] ReadNetkans() Log.WarnFormat("Input is not a .netkan file"); } - return YamlExtensions.Parse(File.OpenText(Options.File)) - .Select(ymap => new Metadata(ymap)) - .ToArray(); + return ArgContents(Options.File).Select(ymap => new Metadata(ymap)) + .ToArray(); } + private static YamlMappingNode[] ArgContents(string arg) + => Uri.IsWellFormedUriString(arg, UriKind.Absolute) + ? YamlExtensions.Parse(Net.DownloadText(new Uri(arg))) + : YamlExtensions.Parse(File.OpenText(arg)); + internal static string CkanFileName(JObject json) - => Path.Combine( - Options.OutputDir, - string.Format( - "{0}-{1}.ckan", - (string)json["identifier"], - ((string)json["version"]).Replace(':', '-'))); + => Path.Combine( + Options.OutputDir, + string.Format( + "{0}-{1}.ckan", + (string)json["identifier"], + ((string)json["version"]).Replace(':', '-'))); private static void WriteCkan(JObject json) { diff --git a/doc/building.md b/doc/building.md index d5265ba034..f2c76a7044 100644 --- a/doc/building.md +++ b/doc/building.md @@ -199,7 +199,7 @@ The basic operation of the actual build process is as follows: These are the "final" output of the build process and are stored in: `_build/repack/$CONFIGURATION`. - Unit tests are executed - The NUnit unit tests in `CKAN.Tests.dll` are executed. -- Smoke tets are executed +- Smoke tests are executed - These are simple tests designed to make sure there isn't anything grossly wrong with the build. All they do is execute the repacked `ckan.exe` and `netkan.exe` and make sure their version output matches the expected output.