From 71681cd2e870458da1551c57432aa6ad570f9793 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 11 Jan 2024 14:48:15 -0800 Subject: [PATCH] Implement optional persistent registry lookup cache and better 429 handling This adds a `--cache` flag to builds which will read from a cache file and write back out to it a set of images that are ~safe to cache (and because it's just a file, we can clear cache entries trivially by deleting the relevant entries from the file). This also adds better 429 behavior that's twofold: 1. cap our maximum registry requests at ~500/min with an initial burst of 100 unless we hit a 429 (which should, in theory, keep us closer to staying under the Hub abuse rate limits) 2. actually retry requests when they give us a 429 response (capping both new requests and retries at the same ~500/min which is ~8/sec) As a final bonus, I added/generated an appropriate cache file for our local test suite which brings the total number of actual Hub/registry requests our test suite makes down to one single image lookup. --- .test/cache-builds.json | 970 ++++++++++++++++++++++++++++++++++++++++ .test/test.sh | 2 +- builds.go | 202 ++++++++- go.mod | 1 + go.sum | 2 + 5 files changed, 1156 insertions(+), 21 deletions(-) create mode 100644 .test/cache-builds.json diff --git a/.test/cache-builds.json b/.test/cache-builds.json new file mode 100644 index 0000000..8e791e3 --- /dev/null +++ b/.test/cache-builds.json @@ -0,0 +1,970 @@ +{ + "resolve": { + "alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "digest": "sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b", + "size": 1638 + }, + "ImageRef": "docker.io/library/alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b" + }, + "arches": { + "amd64": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:ca452b8ab373e6de9c39d030870a52b8f0d3a9cf998c43183fd114660ae96330", + "size": 528, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "ImageRef": "docker.io/library/alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b" + } + ], + "arm32v6": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:50b367e964ca2624aaa480a693df28fdbc240e0c888c13f978e30f4efc2e7ae6", + "size": 528, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v6" + } + }, + "ImageRef": "docker.io/library/alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b" + } + ], + "arm32v7": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:b5a8cebd2749e0b4601d6c0f0bbf9734d87a64f80f1c20100070756531925069", + "size": 528, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v7" + } + }, + "ImageRef": "docker.io/library/alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b" + } + ], + "arm64v8": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:9796bcac681e439af8e4276c7849dcf38612bff029b3b66e4999d5625b6aef85", + "size": 528, + "platform": { + "architecture": "arm64", + "os": "linux", + "variant": "v8" + } + }, + "ImageRef": "docker.io/library/alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b" + } + ], + "i386": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:69d5211febf80ceb758b09bed56969ca0b00560f812d58406dd8a31b65942b2b", + "size": 528, + "platform": { + "architecture": "386", + "os": "linux" + } + }, + "ImageRef": "docker.io/library/alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b" + } + ], + "ppc64le": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:b1da5b7282cab6b45d04b6da5185060bb7e17b3b70a48994c87f2c4cc97141ff", + "size": 528, + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + "ImageRef": "docker.io/library/alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b" + } + ], + "s390x": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:4fac498c31f00cb2289eae7cca6cfbefb923927d9b0fa9209aec4731c8bf1ea5", + "size": 528, + "platform": { + "architecture": "s390x", + "os": "linux" + } + }, + "ImageRef": "docker.io/library/alpine:3.16@sha256:e4cdb7d47b06ba0a062ad2a97a7d154967c8f83934594d9f2bd3efa89292996b" + } + ] + } + }, + "alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "digest": "sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0", + "size": 1638 + }, + "ImageRef": "docker.io/library/alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0" + }, + "arches": { + "amd64": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389", + "size": 528, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "ImageRef": "docker.io/library/alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0" + } + ], + "arm32v6": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:1832ef473ede9a923cc6affdf13b54a1be6561ad2ce3c3684910260a7582d36b", + "size": 528, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v6" + } + }, + "ImageRef": "docker.io/library/alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0" + } + ], + "arm32v7": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:211fe64069acea47ea680c0943b5a77be1819d0e85365011595391f7562caf27", + "size": 528, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v7" + } + }, + "ImageRef": "docker.io/library/alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0" + } + ], + "arm64v8": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:d4ade3639c27579321046d78cc44ec978cea8357d56932611984f601d27e30ac", + "size": 528, + "platform": { + "architecture": "arm64", + "os": "linux", + "variant": "v8" + } + }, + "ImageRef": "docker.io/library/alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0" + } + ], + "i386": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:5ece42cd6ca30ec1a4cc5e1e10a260ad4906e1d4588ae0ef486874d72b3857ad", + "size": 528, + "platform": { + "architecture": "386", + "os": "linux" + } + }, + "ImageRef": "docker.io/library/alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0" + } + ], + "ppc64le": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:1698bcd6bf339e1578dfb9f0034dff615e3eec8404517045046ecbeb84ad01d6", + "size": 528, + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + "ImageRef": "docker.io/library/alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0" + } + ], + "s390x": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:5c63479aeed37522de78284d99dcd32f9ad288b04a56236f44e78b3b3f62ebd2", + "size": 528, + "platform": { + "architecture": "s390x", + "os": "linux" + } + }, + "ImageRef": "docker.io/library/alpine:3.18@sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0" + } + ] + } + }, + "mcr.microsoft.com/windows/servercore:1809@sha256:4fe58f25a157ea749c7b770acebfdbd70c3cb2088c446943e90fe89ea059558b": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "digest": "sha256:4fe58f25a157ea749c7b770acebfdbd70c3cb2088c446943e90fe89ea059558b", + "size": 375 + }, + "ImageRef": "mcr.microsoft.com/windows/servercore:1809@sha256:4fe58f25a157ea749c7b770acebfdbd70c3cb2088c446943e90fe89ea059558b" + }, + "arches": { + "windows-amd64": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:603e3b4f1d0d2565ad4945575a9fb5cd6b90cb6b4981fe551de509f09853ffc5", + "size": 596, + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.17763.5122", + "os.features": [ + "win32k" + ] + } + }, + "ImageRef": "mcr.microsoft.com/windows/servercore:1809@sha256:4fe58f25a157ea749c7b770acebfdbd70c3cb2088c446943e90fe89ea059558b" + } + ] + } + }, + "mcr.microsoft.com/windows/servercore:ltsc2022@sha256:308ef3f8ee3e9c9a1bdec460009c1e6394b329db13eb3149461f8841be5b538a": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "digest": "sha256:308ef3f8ee3e9c9a1bdec460009c1e6394b329db13eb3149461f8841be5b538a", + "size": 375 + }, + "ImageRef": "mcr.microsoft.com/windows/servercore:ltsc2022@sha256:308ef3f8ee3e9c9a1bdec460009c1e6394b329db13eb3149461f8841be5b538a" + }, + "arches": { + "windows-amd64": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:d4ab2dd7d3d0fce6edc5df459565a4c96bbb1d0148065b215ab5ddcab1e42eb4", + "size": 596, + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.20348.2113", + "os.features": [ + "win32k" + ] + } + }, + "ImageRef": "mcr.microsoft.com/windows/servercore:ltsc2022@sha256:308ef3f8ee3e9c9a1bdec460009c1e6394b329db13eb3149461f8841be5b538a" + } + ] + } + }, + "oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:153793dfbac130679ad1eebd9e88b3772c47d3903a3f299c49d5c3f23a6e35d2", + "size": 1248 + }, + "ImageRef": "docker.io/oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43" + }, + "arches": { + "amd64": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:0432a4d379794811b4a2e01d0d3e67a9bcf95d6c2bf71545f03bce3f1d60f401", + "size": 2372, + "annotations": { + "org.opencontainers.image.revision": "6d541d27b5dd12639e5a33a675ebca04d3837d74", + "org.opencontainers.image.source": "https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli", + "org.opencontainers.image.url": "https://hub.docker.com/_/docker", + "org.opencontainers.image.version": "24.0.7-cli" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43" + } + ] + } + }, + "oisupport/staging-amd64:52e3bf2e5ae5606b777f60a7205b338af7ecf70bfebf714e52979dbf9a055621": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:b41b0f3fcaf8866e4907a9104ece79aef61328789f3a3b4b9bf92eea5124a728", + "size": 1250 + }, + "ImageRef": "docker.io/oisupport/staging-amd64:52e3bf2e5ae5606b777f60a7205b338af7ecf70bfebf714e52979dbf9a055621" + }, + "arches": { + "amd64": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:4c92bd9328191f76e8eec6592ceb2e248aa7406dfc9505870812cf8ebee9326a", + "size": 3327, + "annotations": { + "org.opencontainers.image.revision": "99073a3b6be3aa7e6b5af1e69509e8c532254500", + "org.opencontainers.image.source": "https://github.com/docker-library/docker.git#99073a3b6be3aa7e6b5af1e69509e8c532254500:24/dind", + "org.opencontainers.image.url": "https://hub.docker.com/_/docker", + "org.opencontainers.image.version": "24.0.7-dind" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-amd64:52e3bf2e5ae5606b777f60a7205b338af7ecf70bfebf714e52979dbf9a055621" + } + ] + } + }, + "oisupport/staging-amd64:57c2ee0d050ffb54c7f2b50c57b807cce8a8c478648c2eb6bbdf1604b34dd1b9": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:0c0a250eb8a93d1d576e112f03c43c475748df50c007685863c5bf04d45ca3f3", + "size": 1265 + }, + "ImageRef": "docker.io/oisupport/staging-amd64:57c2ee0d050ffb54c7f2b50c57b807cce8a8c478648c2eb6bbdf1604b34dd1b9" + }, + "arches": { + "amd64": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:a5f3cf14ec1f9dbe64f5038168764468bf8cf36023f8c1d763abd3bcbe2a5952", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-signer", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "signer-0.7.0" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-amd64:57c2ee0d050ffb54c7f2b50c57b807cce8a8c478648c2eb6bbdf1604b34dd1b9" + } + ] + } + }, + "oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:09dd1c0183f992a4507d6e562a8e079b8583d19aaf8d991b0d22711c6b4525d7", + "size": 1265 + }, + "ImageRef": "docker.io/oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694" + }, + "arches": { + "amd64": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:4c3d07b2fed560ab0012452aa8a6f58533ddf2d4a3845fa89b74d9455816b454", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-server", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "server-0.7.0" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694" + } + ] + } + }, + "oisupport/staging-arm32v6:2fa4ebddfdbb2318be99958b867533429ec9fbff92c1e3ee53658609e5490a89": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:8afe342ba4d38a2924c9853d4fc812cebbfe56ad8509b80f752e04c4d0ec3981", + "size": 2203 + }, + "ImageRef": "docker.io/oisupport/staging-arm32v6:2fa4ebddfdbb2318be99958b867533429ec9fbff92c1e3ee53658609e5490a89" + }, + "arches": { + "arm32v6": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:8afe342ba4d38a2924c9853d4fc812cebbfe56ad8509b80f752e04c4d0ec3981", + "size": 2203, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v6" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm32v6:2fa4ebddfdbb2318be99958b867533429ec9fbff92c1e3ee53658609e5490a89" + } + ] + } + }, + "oisupport/staging-arm32v6:4e98c0f9e21663851c8cf186fb114fd2440f3de0b0a8d52f6154e23730d0ca04": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:db900d48c4071b64779ab61dbc65f20a01dfc9955eec6013c575c6df2c178f38", + "size": 3251 + }, + "ImageRef": "docker.io/oisupport/staging-arm32v6:4e98c0f9e21663851c8cf186fb114fd2440f3de0b0a8d52f6154e23730d0ca04" + }, + "arches": { + "arm32v6": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:db900d48c4071b64779ab61dbc65f20a01dfc9955eec6013c575c6df2c178f38", + "size": 3251, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v6" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm32v6:4e98c0f9e21663851c8cf186fb114fd2440f3de0b0a8d52f6154e23730d0ca04" + } + ] + } + }, + "oisupport/staging-arm32v6:d5ab9c988abb75a0da68a726aefdade29331f16f154bd15d41942f8d3476d3d1": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:9f245d4b08cfa9f6934e315f60699db5caacbf638d012c4c6f16dfa10066ae4e", + "size": 1774 + }, + "ImageRef": "docker.io/oisupport/staging-arm32v6:d5ab9c988abb75a0da68a726aefdade29331f16f154bd15d41942f8d3476d3d1" + }, + "arches": { + "arm32v6": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:9f245d4b08cfa9f6934e315f60699db5caacbf638d012c4c6f16dfa10066ae4e", + "size": 1774, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v6" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm32v6:d5ab9c988abb75a0da68a726aefdade29331f16f154bd15d41942f8d3476d3d1" + } + ] + } + }, + "oisupport/staging-arm32v6:fe9c7d0064f7451bf4c72ab5a82845860a0a49af27fb3ffd0a00a1c20eeccc5f": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:65861e496c2a89861f24aae1e230413422204f5d85529405e213e137c03ddf53", + "size": 1774 + }, + "ImageRef": "docker.io/oisupport/staging-arm32v6:fe9c7d0064f7451bf4c72ab5a82845860a0a49af27fb3ffd0a00a1c20eeccc5f" + }, + "arches": { + "arm32v6": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:65861e496c2a89861f24aae1e230413422204f5d85529405e213e137c03ddf53", + "size": 1774, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v6" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm32v6:fe9c7d0064f7451bf4c72ab5a82845860a0a49af27fb3ffd0a00a1c20eeccc5f" + } + ] + } + }, + "oisupport/staging-arm32v7:c87eaae1c3107315f6579700fcecec376d90c8d76e3fa59d5f3d2a18b81c44ff": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:c36ec6ddcbe0e9b0b9a2d957fc0521b01fbeca74d74aea790d3ca7a7dea690a3", + "size": 1273 + }, + "ImageRef": "docker.io/oisupport/staging-arm32v7:c87eaae1c3107315f6579700fcecec376d90c8d76e3fa59d5f3d2a18b81c44ff" + }, + "arches": { + "arm32v7": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:dff21d22d9a1e8a1d927234ae319bbeb724737a921dc1a843fd218012b96e993", + "size": 3327, + "annotations": { + "org.opencontainers.image.revision": "99073a3b6be3aa7e6b5af1e69509e8c532254500", + "org.opencontainers.image.source": "https://github.com/docker-library/docker.git#99073a3b6be3aa7e6b5af1e69509e8c532254500:24/dind", + "org.opencontainers.image.url": "https://hub.docker.com/_/docker", + "org.opencontainers.image.version": "24.0.7-dind" + }, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v7" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm32v7:c87eaae1c3107315f6579700fcecec376d90c8d76e3fa59d5f3d2a18b81c44ff" + } + ] + } + }, + "oisupport/staging-arm32v7:f21b47c585fce7db3ff4f646bc87f483fecb3c920d6ec5fbf8a2d76a03ef7ae9": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:b6ebc3e33976b79e335395f06a860b09bb9482165ca7869a46f8411957eacf24", + "size": 1271 + }, + "ImageRef": "docker.io/oisupport/staging-arm32v7:f21b47c585fce7db3ff4f646bc87f483fecb3c920d6ec5fbf8a2d76a03ef7ae9" + }, + "arches": { + "arm32v7": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:8a0f2a979b051d037be8ada354394105ee3dbfe4bad4e5cf101047cdc2da3abf", + "size": 2372, + "annotations": { + "org.opencontainers.image.revision": "6d541d27b5dd12639e5a33a675ebca04d3837d74", + "org.opencontainers.image.source": "https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli", + "org.opencontainers.image.url": "https://hub.docker.com/_/docker", + "org.opencontainers.image.version": "24.0.7-cli" + }, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v7" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm32v7:f21b47c585fce7db3ff4f646bc87f483fecb3c920d6ec5fbf8a2d76a03ef7ae9" + } + ] + } + }, + "oisupport/staging-arm64v8:535b37066c5ad0d2d9684589705db75f5f33494fbae845627bc8d77f636769e3": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:8a65a848a3d45898b726849eb47ddaa199e71d3962133f107a45f51fb74fd73a", + "size": 1265 + }, + "ImageRef": "docker.io/oisupport/staging-arm64v8:535b37066c5ad0d2d9684589705db75f5f33494fbae845627bc8d77f636769e3" + }, + "arches": { + "arm64v8": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:7cb971c2293a1784f771970101a6f7a684bee811b6ebd0cd9587fecf90d7ee8a", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-server", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "server-0.7.0" + }, + "platform": { + "architecture": "arm64", + "os": "linux", + "variant": "v8" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm64v8:535b37066c5ad0d2d9684589705db75f5f33494fbae845627bc8d77f636769e3" + } + ] + } + }, + "oisupport/staging-arm64v8:bf4deeb14c5a100aed6a596cc974bb4db704d8189d9a09fb433e12644e3274ea": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:7f3e5197835a12756f5233447dbab9a5d67ab5e094a939308299a15edf5eefca", + "size": 1248 + }, + "ImageRef": "docker.io/oisupport/staging-arm64v8:bf4deeb14c5a100aed6a596cc974bb4db704d8189d9a09fb433e12644e3274ea" + }, + "arches": { + "arm64v8": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:bd9404aaf047e27974c67bac58a4b7c2975235056a67c4959d4565f8e2389b4b", + "size": 2372, + "annotations": { + "org.opencontainers.image.revision": "6d541d27b5dd12639e5a33a675ebca04d3837d74", + "org.opencontainers.image.source": "https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli", + "org.opencontainers.image.url": "https://hub.docker.com/_/docker", + "org.opencontainers.image.version": "24.0.7-cli" + }, + "platform": { + "architecture": "arm64", + "os": "linux", + "variant": "v8" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm64v8:bf4deeb14c5a100aed6a596cc974bb4db704d8189d9a09fb433e12644e3274ea" + } + ] + } + }, + "oisupport/staging-arm64v8:d20c425b32e42806dff6d00acb99a8c95e14b773b978f44587f472aa38336691": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:29e6cf08f34de4d30d19bcb21000596af6c215da8fbba5d04ae3d98b0bb665eb", + "size": 1265 + }, + "ImageRef": "docker.io/oisupport/staging-arm64v8:d20c425b32e42806dff6d00acb99a8c95e14b773b978f44587f472aa38336691" + }, + "arches": { + "arm64v8": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:974493bb14128e55fa9a95615788a430df892ca0dcdba0141c81a543f7597ecb", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-signer", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "signer-0.7.0" + }, + "platform": { + "architecture": "arm64", + "os": "linux", + "variant": "v8" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm64v8:d20c425b32e42806dff6d00acb99a8c95e14b773b978f44587f472aa38336691" + } + ] + } + }, + "oisupport/staging-arm64v8:e8e572b413236b909b53fe962f2b91a981cfe91514f0f81b251b65e14a06e60a": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:8cbf87eb14de4783e139d5cc69578f424754a0d7479175997aa30b0b49d436e3", + "size": 1250 + }, + "ImageRef": "docker.io/oisupport/staging-arm64v8:e8e572b413236b909b53fe962f2b91a981cfe91514f0f81b251b65e14a06e60a" + }, + "arches": { + "arm64v8": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:2afeb0c1357ac43eb81976a3a84995645872ab18644f6799677cd786de54cf40", + "size": 3327, + "annotations": { + "org.opencontainers.image.revision": "99073a3b6be3aa7e6b5af1e69509e8c532254500", + "org.opencontainers.image.source": "https://github.com/docker-library/docker.git#99073a3b6be3aa7e6b5af1e69509e8c532254500:24/dind", + "org.opencontainers.image.url": "https://hub.docker.com/_/docker", + "org.opencontainers.image.version": "24.0.7-dind" + }, + "platform": { + "architecture": "arm64", + "os": "linux", + "variant": "v8" + } + }, + "ImageRef": "docker.io/oisupport/staging-arm64v8:e8e572b413236b909b53fe962f2b91a981cfe91514f0f81b251b65e14a06e60a" + } + ] + } + }, + "oisupport/staging-i386:7d563d914d72c37cc7a59c0ef9d5ede304412f9c5417b72e82d1963c70e8cbfb": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:16f71e74b992a74d7c8e9dff93036f10aaf29f0e44a9ceb5b739d8113e82fcf7", + "size": 1263 + }, + "ImageRef": "docker.io/oisupport/staging-i386:7d563d914d72c37cc7a59c0ef9d5ede304412f9c5417b72e82d1963c70e8cbfb" + }, + "arches": { + "i386": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:e6d16088d6a27939ccbde71947f57ef7db02215049c18b82ccbc58ffc19875ab", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-server", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "server-0.7.0" + }, + "platform": { + "architecture": "386", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-i386:7d563d914d72c37cc7a59c0ef9d5ede304412f9c5417b72e82d1963c70e8cbfb" + } + ] + } + }, + "oisupport/staging-i386:8a39d1f803b6bd6db6a5a231b5a4e2fc1d7fc13a10e22dd28cb25e16429b16fd": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:e56f38e952dd40f4f3e5d6f970e8868239e199efba77c43c8ad5f387f24de2fa", + "size": 1263 + }, + "ImageRef": "docker.io/oisupport/staging-i386:8a39d1f803b6bd6db6a5a231b5a4e2fc1d7fc13a10e22dd28cb25e16429b16fd" + }, + "arches": { + "i386": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:b4dab61b2d4468f7afa31ff7fa2156ae7f053044df4c43aa2bad831dc166694c", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-signer", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "signer-0.7.0" + }, + "platform": { + "architecture": "386", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-i386:8a39d1f803b6bd6db6a5a231b5a4e2fc1d7fc13a10e22dd28cb25e16429b16fd" + } + ] + } + }, + "oisupport/staging-ppc64le:c72366c942fc80dc984291207128f5df861e483aa0233ce6495c8a6e1c5d0918": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:f4e1aff8b7e19c3842fb78b2be1e65ec9290e7c6783f087c813987b36400587a", + "size": 1267 + }, + "ImageRef": "docker.io/oisupport/staging-ppc64le:c72366c942fc80dc984291207128f5df861e483aa0233ce6495c8a6e1c5d0918" + }, + "arches": { + "ppc64le": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:ead953dd52edeea966545664b2c9fec2e1bb4d3a79671f95190819095351f641", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-server", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "server-0.7.0" + }, + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-ppc64le:c72366c942fc80dc984291207128f5df861e483aa0233ce6495c8a6e1c5d0918" + } + ] + } + }, + "oisupport/staging-ppc64le:fe5b5efb1e03f722dcda7652bd3f61f4f727dc2e9fe52d03ca626a9cf1ac849f": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:63b53ff8f8a08587f2dfe1f9dfbfd3bb3dc1e99ccf00cdf4c7c879ff947d50b3", + "size": 1267 + }, + "ImageRef": "docker.io/oisupport/staging-ppc64le:fe5b5efb1e03f722dcda7652bd3f61f4f727dc2e9fe52d03ca626a9cf1ac849f" + }, + "arches": { + "ppc64le": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:8eb5fb1e9d7895f95d2c6485df3c1d62ae867114699be79bf388b931c587865d", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-signer", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "signer-0.7.0" + }, + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-ppc64le:fe5b5efb1e03f722dcda7652bd3f61f4f727dc2e9fe52d03ca626a9cf1ac849f" + } + ] + } + }, + "oisupport/staging-s390x:7ea69d395a11993aaef4cd29dd58bbcfe6189e984c367ed781fb4dd4bdb9ff60": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:562fb5f01ca7174dc55991dbfab646c5c742a2079f5792762700b25fef76bfbd", + "size": 1265 + }, + "ImageRef": "docker.io/oisupport/staging-s390x:7ea69d395a11993aaef4cd29dd58bbcfe6189e984c367ed781fb4dd4bdb9ff60" + }, + "arches": { + "s390x": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:23b5119f61f785a5bc3d0e08d81069495e7b7812a69f15f6894ff355737dee38", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-signer", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "signer-0.7.0" + }, + "platform": { + "architecture": "s390x", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-s390x:7ea69d395a11993aaef4cd29dd58bbcfe6189e984c367ed781fb4dd4bdb9ff60" + } + ] + } + }, + "oisupport/staging-s390x:bcf9a20ccb4e460c190294d900a06bfce1b9d9c703358b3f87ae0de4b30abf11": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:8b50da43831c731e8504c0b11dfb1573b3096428c780539b77db32ae76653199", + "size": 1265 + }, + "ImageRef": "docker.io/oisupport/staging-s390x:bcf9a20ccb4e460c190294d900a06bfce1b9d9c703358b3f87ae0de4b30abf11" + }, + "arches": { + "s390x": [ + { + "Desc": { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:66118e7982bb814a9d939808797fc0e073194732d5bcdb0afaa98c515d17ab23", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-server", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "server-0.7.0" + }, + "platform": { + "architecture": "s390x", + "os": "linux" + } + }, + "ImageRef": "docker.io/oisupport/staging-s390x:bcf9a20ccb4e460c190294d900a06bfce1b9d9c703358b3f87ae0de4b30abf11" + } + ] + } + }, + "oisupport/staging-windows-amd64:5efe68ed819e46db05a9b50e99c758feaf96df829939aca311b55e42df521acf": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:040cdf6cf38b4693b30bb38cae806e8b456d4458c2c257fb26d48e45c83b1021", + "size": 3462 + }, + "ImageRef": "docker.io/oisupport/staging-windows-amd64:5efe68ed819e46db05a9b50e99c758feaf96df829939aca311b55e42df521acf" + }, + "arches": { + "windows-amd64": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:040cdf6cf38b4693b30bb38cae806e8b456d4458c2c257fb26d48e45c83b1021", + "size": 3462, + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.17763.5122" + } + }, + "ImageRef": "docker.io/oisupport/staging-windows-amd64:5efe68ed819e46db05a9b50e99c758feaf96df829939aca311b55e42df521acf" + } + ] + } + }, + "oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e": { + "resolved": { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:69aba7120e3f4014bfa80f4eae2cfc9698dcb6b8a5d64daf06de4039a19846ce", + "size": 3462 + }, + "ImageRef": "docker.io/oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e" + }, + "arches": { + "windows-amd64": [ + { + "Desc": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:69aba7120e3f4014bfa80f4eae2cfc9698dcb6b8a5d64daf06de4039a19846ce", + "size": 3462, + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.20348.2113" + } + }, + "ImageRef": "docker.io/oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e" + } + ] + } + } + } +} diff --git a/.test/test.sh b/.test/test.sh index 6f81388..3a28ac3 100755 --- a/.test/test.sh +++ b/.test/test.sh @@ -13,4 +13,4 @@ set -- docker:cli docker:dind docker:windowsservercore notary # a little bit of # NOTE: we are explicitly *not* pinning "golang:1.19-alpine3.16" so that this also tests unpinned parent behavior (that image is deprecated so should stay unchanging) time "$dir/../sources.sh" "$@" > "$dir/sources.json" -time "$dir/../builds.sh" "$dir/sources.json" > "$dir/builds.json" +time "$dir/../builds.sh" --cache "$dir/cache-builds.json" "$dir/sources.json" > "$dir/builds.json" diff --git a/builds.go b/builds.go index be44ac2..1441844 100644 --- a/builds.go +++ b/builds.go @@ -4,22 +4,26 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "io" "os" "os/exec" "strings" "sync" + "time" "github.com/docker-library/meta-scripts/om" c8derrdefs "github.com/containerd/containerd/errdefs" + c8derrs "github.com/containerd/containerd/remotes/errors" "github.com/docker-library/bashbrew/registry" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" // this is used by containerd libraries, so we need to set the default log level for it + "golang.org/x/time/rate" ) -var concurrency = 50 +var concurrency = 1000 type MetaSource struct { SourceID string `json:"sourceId"` @@ -43,8 +47,8 @@ type RemoteResolvedFull struct { } type BuildIDParts struct { - SourceID string `json:"sourceId"` - Arch string `json:"arch"` + SourceID string `json:"sourceId"` + Arch string `json:"arch"` Parents om.OrderedMap[string] `json:"parents"` } @@ -62,31 +66,80 @@ type MetaBuild struct { var ( // keys are image/tag names, values are functions that return either cacheResolveType or error cacheResolve = sync.Map{} + cacheFile string + + registryRateLimiter = rate.NewLimiter(500/rate.Limit((1*time.Minute).Seconds()), 100) // stick to at most 500/min in registry/Hub requests (and allow an immediate burst of 100) ) type cacheResolveType struct { - r *registry.ResolvedObject - arches map[string][]registry.ResolvedObject + Resolved *registry.ResolvedObject `json:"resolved"` + Arches map[string][]registry.ResolvedObject `json:"arches"` } -func resolveRemoteArch(ctx context.Context, img string, arch string) (*RemoteResolvedFull, error) { - cacheFunc, _ := cacheResolve.LoadOrStore(img, sync.OnceValues(func() (*cacheResolveType, error) { +func resolveRemoteArch(ctx context.Context, img string, arch string, diskCacheForSure bool) (*RemoteResolvedFull, error) { + cacheFunc, wasCached := cacheResolve.LoadOrStore(img, sync.OnceValues(func() (*cacheResolveType, error) { var ( ret = cacheResolveType{} err error ) - ret.r, err = registry.Resolve(ctx, img) - if c8derrdefs.IsNotFound(err) { - return nil, nil - } else if err != nil { - return nil, err + shouldRetry := func(err error) bool { + if err == nil { + return false + } + var ret bool + var statusErr *c8derrs.ErrUnexpectedStatus + if errors.As(err, &statusErr) { + // "Too Many Requests" (new-style containerd error) + ret = statusErr.StatusCode == 429 + } else { + // another (older) flavor of 429; https://github.com/containerd/containerd/tree/v1.6.19/remotes/docker/resolver.go#L302 + ret = strings.Contains(err.Error(), "429 Too Many Requests") + } + if ret { + for i := registryRateLimiter.Tokens(); i > 0; i-- { + // just eat all available tokens and starve out the rate limiter + _ = registryRateLimiter.Allow() + } + } + return ret } - // TODO more efficient lookup of single architecture? (probably doesn't matter much, and then we have to have two independent caches) - ret.arches, err = ret.r.Architectures(ctx) - if err != nil { - return nil, err + for { + err = registryRateLimiter.Wait(ctx) + if err != nil { + return nil, err + } + + ret.Resolved, err = registry.Resolve(ctx, img) + if c8derrdefs.IsNotFound(err) { + return nil, nil + } else if shouldRetry(err) { + fmt.Fprintf(os.Stderr, "warning: lookup %q errored (%q); will retry...\n", img, err) + continue + } else if err != nil { + return nil, err + } + + break + } + + for { + err = registryRateLimiter.Wait(ctx) + if err != nil { + return nil, err + } + + // TODO more efficient lookup of single architecture? (probably doesn't matter much, and then we have to have two independent caches) + ret.Arches, err = ret.Resolved.Architectures(ctx) + if shouldRetry(err) { + fmt.Fprintf(os.Stderr, "warning: lookup arches for %q errored (%q); will retry...\n", img, err) + continue + } else if err != nil { + return nil, err + } + + break } return &ret, nil @@ -98,8 +151,25 @@ func resolveRemoteArch(ctx context.Context, img string, arch string) (*RemoteRes if cache == nil { return nil, nil } - r := cache.r - rArches := cache.arches + + r := cache.Resolved + rArches := cache.Arches + + if !wasCached { + fmt.Fprintf(os.Stderr, "NOTE: lookup %s -> %s\n", img, r.Desc.Digest) + } + + if !diskCacheForSure { + // if we don't know we should cache this lookup for sure, the answer is whether it's a by-digest lookup :) + diskCacheForSure = strings.Contains(img, "@") + } + if diskCacheForSure { + saveCacheMutex.Lock() + if saveCache != nil { + saveCache.Resolve[img] = cache + } + saveCacheMutex.Unlock() + } if _, ok := rArches[arch]; !ok { // TODO this should probably be just like a 404, right? (it's effectively a 404, even if it's not literally a 404) @@ -128,9 +198,97 @@ func resolveRemoteArch(ctx context.Context, img string, arch string) (*RemoteRes return resolved, nil } +type cacheFileContents struct { + Resolve map[string]*cacheResolveType `json:"resolve"` +} + +var ( + saveCache *cacheFileContents + saveCacheMutex sync.Mutex +) + +func loadCacheFromFile() error { + if cacheFile == "" { + return nil + } + + // now that we know we have a file we want cache to go into (and come from), let's initialize the "saveCache" (which will be written when the whole process is done / we're successful, and *only* caches staging images) + saveCacheMutex.Lock() + saveCache = &cacheFileContents{Resolve: map[string]*cacheResolveType{}} + saveCacheMutex.Unlock() + + f, err := os.Open(cacheFile) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + defer f.Close() + + var cache cacheFileContents + err = json.NewDecoder(f).Decode(&cache) // *technically*, this will silently ignore garbage (or extra documents) at the end of the file, but it's for our cache file so it's not really an issue for us (the only input to this should be our own output) + if err != nil { + return err + } + + for img, r := range cache.Resolve { + r := r // https://github.com/golang/go/issues/60078 + fun, _ := cacheResolve.LoadOrStore(img, sync.OnceValues(func() (*cacheResolveType, error) { + return r, nil + })) + r2, err := fun.(func() (*cacheResolveType, error))() + if err != nil { + // this should never happen (hence panic vs return) 🙈 + panic(err) + } + if r2 != r { + panic("r2 != r??? " + img) + } + } + + return nil +} + +func saveCacheToFile() error { + saveCacheMutex.Lock() + defer saveCacheMutex.Unlock() + + if saveCache == nil || cacheFile == "" { + return nil + } + + f, err := os.Create(cacheFile) + if err != nil { + return err + } + defer f.Close() + + enc := json.NewEncoder(f) + enc.SetIndent("", "\t") + + err = enc.Encode(saveCache) + if err != nil { + return err + } + + return nil +} + func main() { sourcesJsonFile := os.Args[1] // "sources.json" + // support "--cache foo.json" and "--cache=foo.json" + if sourcesJsonFile == "--cache" && len(os.Args) >= 4 { + cacheFile = os.Args[2] + sourcesJsonFile = os.Args[3] + } else if cf, ok := strings.CutPrefix(sourcesJsonFile, "--cache="); ok && len(os.Args) >= 3 { + cacheFile = cf + sourcesJsonFile = os.Args[2] + } + if err := loadCacheFromFile(); err != nil { + panic(err) + } + stagingTemplate := os.Getenv("BASHBREW_STAGING_TEMPLATE") // "oisupport/staging-ARCH:BUILD" if !strings.Contains(stagingTemplate, "BUILD") { panic("invalid BASHBREW_STAGING_TEMPLATE (missing BUILD)") @@ -210,7 +368,7 @@ func main() { lookup += "@" + *parent.Pin } - resolved, err = resolveRemoteArch(context.TODO(), lookup, build.Build.Arch) + resolved, err = resolveRemoteArch(context.TODO(), lookup, build.Build.Arch, false) if err != nil { panic(err) } @@ -237,7 +395,7 @@ func main() { build.Build.Img = strings.ReplaceAll(strings.ReplaceAll(stagingTemplate, "BUILD", build.BuildID), "ARCH", build.Build.Arch) // "oisupport/staging-amd64:xxxx" - build.Build.Resolved, err = resolveRemoteArch(context.TODO(), build.Build.Img, build.Build.Arch) + build.Build.Resolved, err = resolveRemoteArch(context.TODO(), build.Build.Img, build.Build.Arch, true) if err != nil { panic(err) } @@ -290,4 +448,8 @@ func main() { } fmt.Println() fmt.Println("}") + + if err := saveCacheToFile(); err != nil { + panic(err) + } } diff --git a/go.mod b/go.mod index fed484b..20c3649 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/docker-library/bashbrew v0.1.9 github.com/opencontainers/image-spec v1.1.0-rc2.0.20221013174636-8159c8264e2e github.com/sirupsen/logrus v1.9.0 + golang.org/x/time v0.5.0 ) require ( diff --git a/go.sum b/go.sum index 785c1b1..2ed05f0 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=