Merge pull request #23 from tinkerbell/release-binaries

Push binaries as release artifact

.github/workflows/ci.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
 
 jobs:
-  validation:
+  build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

.github/workflows/tags.yaml

@@ -0,0 +1,33 @@
+on:
+  push:
+    tags:
+      - "v*"
+name: Create release
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - run: sudo apt install -y libgpgme-dev libassuan-dev libdevmapper-dev
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "1.15.5"
+      - run: ./script/release-binaries.sh
+        name: get binaries
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          draft: false
+          prerelease: false
+      - name: Upload release binaries
+        uses: alexellis/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          asset_paths: '["./out/release/*"]'

.gitignore

@@ -1 +1,2 @@
 envrc
+out

README.md

@@ -16,3 +16,44 @@ that will may change a bit how Tinkerbell works.
 
 We are keeping the number of bc break as low as possible but in the current
 state they are expected.
+
+## Binary release
+
+As part of a new release for sandbox we want to push binaries to GitHub Release
+in this way the community will be able to use them if needed.
+
+We build Docker images across many architectures, each of them in its own
+repository: boots, hegel, tink and so on.
+
+Sandbox is just a collection of those services and we follow the same pattern
+for getting binaries as well.
+
+There is a go program available in `./cmd/getbinariesfromquay/main.go`. You can
+run it with `go run` or build it with `go build`:
+
+```terminal
+$ go run cmd/getbinariesfromquay/main.go -h
+  -binary-to-copy string
+        The location of the binary you want to copy from inside the image. (default "/usr/bin/hegel")
+  -image string
+        The image you want to download binaries from. It has to be a multi stage image. (default "docker://quay.io/tinkerbell/hegel")
+  -out string
+        The directory that will be used to store the release binaries (default "./out")
+  -program string
+        The name of the program you are extracing binaries for. (eg tink-worker, hegel, tink-server, tink, boots) (default "hegel")
+```
+
+By default it uses the image running on Quay for Hegel and it gets the binary
+`/usr/bin/hegel` from there. The directory `./out` is used to store images and
+binaries inside `./out/releases`.
+
+To get the binaries for example for boots you can run:
+
+```terminal
+$ go run cmd/getbinariesfromquay/main.go \
+    -binary-to-copy /usr/bin/boots \
+    -image docker://quay.io/tinkerbell/boots:sha-9625559b \
+    -program boots
+```
+
+You will find them in `./out/release`

cmd/getbinariesfromquay/docker/image.go

@@ -0,0 +1,76 @@
+package docker
+
+import (
+	"context"
+
+	"github.com/containers/image/v5/copy"
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/signature"
+	"github.com/containers/image/v5/transports/alltransports"
+	"github.com/containers/image/v5/types"
+)
+
+type Image struct {
+	src types.ImageSource
+	ref types.ImageReference
+	sys *types.SystemContext
+}
+
+func ImageFromName(ctx context.Context, sys *types.SystemContext, name string) (*Image, error) {
+	imageRef, err := alltransports.ParseImageName(name)
+	if err != nil {
+		return nil, err
+	}
+	src, err := imageRef.NewImageSource(ctx, sys)
+	if err != nil {
+		return nil, err
+	}
+	return &Image{
+		src: src,
+		ref: imageRef,
+		sys: sys,
+	}, nil
+}
+
+func (img *Image) GetManifest(ctx context.Context) ([]byte, string, error) {
+	rawManifest, _, err := img.src.GetManifest(ctx, nil)
+	if err != nil {
+		return nil, "", err
+	}
+	return rawManifest, manifest.GuessMIMEType(rawManifest), nil
+}
+
+func (img *Image) Copy(ctx context.Context, dst string) error {
+	destRef, err := alltransports.ParseImageName(dst)
+	if err != nil {
+		return err
+	}
+	pc, err := signature.NewPolicyContext(&signature.Policy{
+		Default: []signature.PolicyRequirement{
+			signature.NewPRInsecureAcceptAnything(),
+		},
+	})
+	if err != nil {
+		return err
+	}
+	_, err = copy.Image(ctx, pc, destRef, img.ref, &copy.Options{SourceCtx: img.sys})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+type SchemaV2List struct {
+	MediaType     string `json:"mediaType"`
+	SchemaVersion int    `json:"schemaVersion"`
+	Manifests     []struct {
+		MediaType string `json:"mediaType"`
+		Digest    string `json:"digest"`
+		Size      int    `json:"size"`
+		Platform  struct {
+			Architecture string `json:"architecture"`
+			Os           string `json:"os"`
+			Variant      string `json:"variant"`
+		} `json:"platform,omitempty"`
+	} `json:"manifests"`
+}

cmd/getbinariesfromquay/main.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+
+	"github.com/containers/image/manifest"
+	"github.com/containers/image/v5/types"
+	"github.com/tinkerbell/sandbox/cmd/getbinariesfromquay/docker"
+	"github.com/tinkerbell/sandbox/cmd/getbinariesfromquay/tar"
+)
+
+type Config struct {
+	ProgramName  string
+	OutDirectory string
+	Image        string
+	Binary       string
+}
+
+var config = Config{}
+
+func init() {
+	flag.StringVar(&config.ProgramName, "program", "hegel", "The name of the program you are extracing binaries for. (eg tink-worker, hegel, tink-server, tink, boots)")
+	flag.StringVar(&config.OutDirectory, "out", "./out", "The directory that will be used to store the release binaries")
+	flag.StringVar(&config.Image, "image", "docker://quay.io/tinkerbell/hegel", "The image you want to download binaries from. It has to be a multi stage image.")
+	flag.StringVar(&config.Binary, "binary-to-copy", "/usr/bin/hegel", "The location of the binary you want to copy from inside the image.")
+}
+
+func main() {
+	flag.Parse()
+
+	println("Extracing binary: " + config.Binary)
+	println("From Image: " + config.Image)
+
+	imageR := config.Image
+	binaryToCopy := config.Binary
+	baseDir, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	programName := config.ProgramName
+	outDir := path.Join(baseDir, config.OutDirectory)
+	releaseDir := path.Join(outDir, "release")
+	err = os.MkdirAll(releaseDir, 0755)
+	if err != nil {
+		log.Fatal(err)
+	}
+	println("Binaries will be located in: " + releaseDir)
+	imgsDir := path.Join(outDir, "imgs")
+	err = os.MkdirAll(imgsDir, 0755)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	ctx := context.Background()
+	img, err := docker.ImageFromName(ctx, &types.SystemContext{}, imageR)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	rawManifest, mt, err := img.GetManifest(ctx)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if mt != manifest.DockerV2ListMediaType {
+		log.Fatal("manifest not supported, it is not a multi arch image")
+	}
+
+	archList := docker.SchemaV2List{}
+	err = json.Unmarshal(rawManifest, &archList)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, arch := range archList.Manifests {
+		imgDir := fmt.Sprintf("%s-%s-%s", programName, arch.Platform.Os, arch.Platform.Architecture)
+		println("Extracting ", imgDir)
+		syss := &types.SystemContext{
+			ArchitectureChoice: arch.Platform.Architecture,
+			OSChoice:           arch.Platform.Os,
+		}
+		if arch.Platform.Variant != "" {
+			syss.VariantChoice = arch.Platform.Variant
+			imgDir = imgDir + "-" + arch.Platform.Variant
+		}
+		archImg, err := docker.ImageFromName(ctx, syss, imageR)
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = archImg.Copy(ctx, fmt.Sprintf("dir:%s", path.Join(imgsDir, imgDir)))
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = copyBinaryFromLastLayer(path.Join(imgsDir, imgDir), path.Join(releaseDir, imgDir), binaryToCopy)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func copyBinaryFromLastLayer(src, dest, binaryPath string) error {
+	b, err := ioutil.ReadFile(path.Join(src, "manifest.json"))
+	if err != nil {
+		return err
+	}
+	man, err := manifest.FromBlob(b, manifest.DockerV2Schema2MediaType)
+	if err != nil {
+		return err
+	}
+	last := man.LayerInfos()[len(man.LayerInfos())-1]
+	layerTar, err := os.Open(path.Join(src, last.Digest.String()[7:]))
+	if err != nil {
+		return err
+	}
+	err = tar.Untar(src, layerTar)
+	if err != nil {
+		return err
+	}
+
+	input, err := ioutil.ReadFile(path.Join(src, binaryPath))
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(dest, input, 0755)
+	if err != nil {
+		return err
+	}
+	return nil
+}

cmd/getbinariesfromquay/tar/untar.go

@@ -0,0 +1,48 @@
+package tar
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func Untar(dst string, r io.Reader) error {
+	gzr, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+	tr := tar.NewReader(gzr)
+
+	for {
+		header, err := tr.Next()
+		switch {
+		case err == io.EOF:
+			return nil
+		case err != nil:
+			return err
+		case header == nil:
+			continue
+		}
+		target := filepath.Join(dst, header.Name)
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+		case tar.TypeReg:
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(f, tr); err != nil {
+				return err
+			}
+			f.Close()
+		}
+	}
+}

current_versions.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# This file gets used from generate-envrc.sh but it is also used standalone by
+# automation that needs to get the version of the programs currently supported
+# in sandbox
+
+export OSIE_DOWNLOAD_LINK=https://tinkerbell-oss.s3.amazonaws.com/osie-uploads/osie-v0-n=366,c=1aec189,b=master.tar.gz
+export TINKERBELL_TINK_SERVER_IMAGE=quay.io/tinkerbell/tink:sha-0e8e5733
+export TINKERBELL_TINK_CLI_IMAGE=quay.io/tinkerbell/tink-cli:sha-0e8e5733
+export TINKERBELL_TINK_BOOTS_IMAGE=quay.io/tinkerbell/boots:sha-e81a291c
+export TINKERBELL_TINK_HEGEL_IMAGE=quay.io/tinkerbell/hegel:sha-c17b512f
+export TINKERBELL_TINK_WORKER_IMAGE=quay.io/tinkerbell/tink-worker:sha-0e8e5733

generate-envrc.sh

@@ -11,6 +11,8 @@ fi
 
 ERR="${RED:-}ERROR:${RESET:-}"
 
+source ./current_versions.sh
+
 err() (
 	if [ -z "${1:-}" ]; then
 		cat >&2
@@ -54,12 +56,12 @@ generate_envrc() (
 	cat <<EOF
 # Tinkerbell Stack version
 
-export OSIE_DOWNLOAD_LINK=https://tinkerbell-oss.s3.amazonaws.com/osie-uploads/osie-v0-n=366,c=1aec189,b=master.tar.gz
-export TINKERBELL_TINK_SERVER_IMAGE=quay.io/tinkerbell/tink:sha-0e8e5733
-export TINKERBELL_TINK_CLI_IMAGE=quay.io/tinkerbell/tink-cli:sha-0e8e5733
-export TINKERBELL_TINK_BOOTS_IMAGE=quay.io/tinkerbell/boots:sha-e81a291c
-export TINKERBELL_TINK_HEGEL_IMAGE=quay.io/tinkerbell/hegel:sha-c17b512f
-export TINKERBELL_TINK_WORKER_IMAGE=quay.io/tinkerbell/tink-worker:sha-0e8e5733
+export OSIE_DOWNLOAD_LINK=${OSIE_DOWNLOAD_LINK}
+export TINKERBELL_TINK_SERVER_IMAGE=${TINKERBELL_TINK_SERVER_IMAGE}
+export TINKERBELL_TINK_CLI_IMAGE=${TINKERBELL_TINK_CLI_IMAGE}
+export TINKERBELL_TINK_BOOTS_IMAGE=${TINKERBELL_TINK_BOOTS_IMAGE}
+export TINKERBELL_TINK_HEGEL_IMAGE=${TINKERBELL_TINK_HEGEL_IMAGE}
+export TINKERBELL_TINK_WORKER_IMAGE=${TINKERBELL_TINK_WORKER_IMAGE}
 
 # Network interface for Tinkerbell's network
 export TINKERBELL_NETWORK_INTERFACE="$tink_interface"