From f4e8404e8a09e5256e99e268283c1b6165383ec1 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 3 Aug 2023 01:20:50 +0000 Subject: [PATCH] Add support for more transports Before only raw buildkitd unix socket or docker socket was supported. This adds support for: - Docker with custom socket path - Buildx instances (defaults to the currently selected buildx builder instance) - Connect over a docker container which works similarly to buildx but does not require a buildx config. For the buildx support, currently only buildx instances over docker-containers are supported. Support other buildx drivers is possible (e.g. the kubernetes driver), but just needs some working through what's there. Signed-off-by: Brian Goff --- go.mod | 1 + go.sum | 2 + pkg/buildkit/buildkit.go | 66 +-------- pkg/buildkit/buildkit_test.go | 2 +- pkg/buildkit/drivers.go | 265 ++++++++++++++++++++++++++++++++++ 5 files changed, 271 insertions(+), 65 deletions(-) create mode 100644 pkg/buildkit/drivers.go diff --git a/go.mod b/go.mod index a37aeb65..9b9e0a36 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aquasecurity/trivy v0.44.0 github.com/containerd/console v1.0.3 github.com/containerd/containerd v1.7.3 + github.com/cpuguy83/dockercfg v0.3.1 github.com/cpuguy83/go-docker/buildkitopt v0.1.1 github.com/distribution/distribution v2.8.2+incompatible github.com/docker/cli v24.0.5+incompatible diff --git a/go.sum b/go.sum index f2d774fa..0fc63dba 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,8 @@ github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtO github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-docker v0.1.2 h1:bHyfwV+yMw+IP2iL/0T5WU/g7Bpj+yhdlwoe6LEwA+A= github.com/cpuguy83/go-docker v0.1.2/go.mod h1:tZURUlegjsgYPhkfkuPCeVrp1ocirH4P1yUexBpP25g= github.com/cpuguy83/go-docker/buildkitopt v0.1.1 h1:zzg3taUURaoBzyowOqTtH1EoH3251oyceLS3202HF/M= diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index 0f284c4a..81dc7075 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -7,8 +7,6 @@ package buildkit import ( "context" - "errors" - "fmt" "io" "net/http" "os" @@ -16,17 +14,12 @@ import ( "github.com/containerd/console" "github.com/containerd/containerd/remotes/docker" - "github.com/cpuguy83/go-docker/buildkitopt" - "github.com/cpuguy83/go-docker/transport" "github.com/docker/cli/cli/config" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" - gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" - "github.com/moby/buildkit/solver/pb" - "github.com/moby/buildkit/util/apicaps" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/imageutil" "github.com/moby/buildkit/util/progress/progressui" @@ -39,10 +32,6 @@ import ( "golang.org/x/sync/errgroup" ) -const ( - DefaultAddr = "unix:///run/buildkit/buildkitd.sock" -) - type Config struct { ImageName string Client *client.Client @@ -112,65 +101,14 @@ func resolveImageConfig(ctx context.Context, ref string, platform *ispec.Platfor return dgst, config, nil } -var errMissingCap = fmt.Errorf("missing required buildkit functionality") - -// Buildkit llb ops required for copacetic -var requiredCaps = []apicaps.CapID{pb.CapMergeOp, pb.CapDiffOp} - // NewClient returns a new buildkit client with the given addr. // If addr is empty it will first try to connect to docker's buildkit instance and then fallback to DefaultAddr. func NewClient(ctx context.Context, addr string) (*client.Client, error) { - defaultOpts := []client.ClientOpt{client.WithFailFast()} - bkOpts := defaultOpts - - if addr == "" { - // First try using docker's buildkit instance if available - tr, err := transport.DefaultTransport() - if err != nil { - log.WithError(err).Info("Failed to initialize docker transport, falling back to default buildkit address") - addr = DefaultAddr - } else { - bkOpts = append(bkOpts, buildkitopt.FromDocker(tr)...) - } - } - - c, err := client.New(ctx, addr, bkOpts...) + driver, err := getDriver(addr) if err != nil { return nil, err } - - _, err = c.Build(ctx, client.SolveOpt{}, "", func(ctx context.Context, client gateway.Client) (*gateway.Result, error) { - capset := client.BuildOpts().LLBCaps - var err error - for _, cap := range requiredCaps { - err = errors.Join(err, capset.Supports(cap)) - } - if err != nil { - return nil, errors.Join(err, errMissingCap) - } - return &gateway.Result{}, nil - }, nil) - if err == nil { - return c, nil - } - c.Close() - - if addr != "" { - return nil, err - } - - if !errors.Is(err, errMissingCap) { - log.WithError(err).Warn("Failed to check buildkit LLB merge operation support, falling back to default buildkit address") - } else { - log.WithError(err).Warn("Docker integrated buildkit is too old, falling back to default buildkit address") - } - - c, err = client.New(ctx, DefaultAddr, defaultOpts...) - if err != nil { - return nil, fmt.Errorf("failed to initialize buildkit client: %w", err) - } - - return c, nil + return driver(ctx) } func InitializeBuildkitConfig(ctx context.Context, client *client.Client, image string, manifest *types.UpdateManifest) (*Config, error) { diff --git a/pkg/buildkit/buildkit_test.go b/pkg/buildkit/buildkit_test.go index cf4fdfc4..27df3693 100644 --- a/pkg/buildkit/buildkit_test.go +++ b/pkg/buildkit/buildkit_test.go @@ -88,7 +88,7 @@ func newMockBuildkitAPI(t *testing.T, caps ...apicaps.CapID) string { caps: capList, }) - go srv.Serve(l) + go srv.Serve(l) // nolint:errcheck control := &mockControlServer{ ControlServer: &controlapi.UnimplementedControlServer{}, diff --git a/pkg/buildkit/drivers.go b/pkg/buildkit/drivers.go new file mode 100644 index 00000000..bf019250 --- /dev/null +++ b/pkg/buildkit/drivers.go @@ -0,0 +1,265 @@ +package buildkit + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "sync/atomic" + + "github.com/cpuguy83/dockercfg" + "github.com/cpuguy83/go-docker" + "github.com/cpuguy83/go-docker/buildkitopt" + "github.com/cpuguy83/go-docker/container" + "github.com/cpuguy83/go-docker/transport" + "github.com/moby/buildkit/client" + gateway "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/apicaps" + log "github.com/sirupsen/logrus" +) + +const ( + DefaultAddr = "unix:///run/buildkit/buildkitd.sock" +) + +var ( + errMissingCap = fmt.Errorf("missing required buildkit functionality") + // requiredCaps are buildkit llb ops required to function. + requiredCaps = []apicaps.CapID{pb.CapMergeOp, pb.CapDiffOp} +) + +type driverFunc func(context.Context) (*client.Client, error) + +func validateClient(ctx context.Context, c *client.Client) error { + _, err := c.Build(ctx, client.SolveOpt{}, "", func(ctx context.Context, client gateway.Client) (*gateway.Result, error) { + capset := client.BuildOpts().LLBCaps + var err error + for _, cap := range requiredCaps { + err = errors.Join(err, capset.Supports(cap)) + } + if err != nil { + return nil, errors.Join(err, errMissingCap) + } + return &gateway.Result{}, nil + }, nil) + return err +} + +func withValidateClient(f driverFunc) driverFunc { + if f == nil { + return nil + } + + return func(ctx context.Context) (*client.Client, error) { + client, err := f(ctx) + if err != nil { + return nil, err + } + if err := validateClient(ctx, client); err != nil { + client.Close() + return nil, err + } + return client, nil + } +} + +func getDriver(spec string) (ret driverFunc, _ error) { + defer func() { + ret = withValidateClient(ret) + }() + + name, addr, ok := strings.Cut(spec, "://") + if !ok { + if spec != "" { + return nil, errors.New("invalid driver spec") + } + return autoDriver, nil + } + + switch name { + case "docker": + return dockerDriver(addr), nil + case "buildx": + return buildxDriver(addr), nil + case "unix": + return buildkitdDriver(addr), nil + case "docker-container": + if addr == "" { + return nil, errors.New("docker-container driver requires a container name or id") + } + return dockerContainerDriver("", addr), nil + } + + return nil, errors.New("invalid driver") +} + +func buildxDriver(addr string) driverFunc { + return func(ctx context.Context) (*client.Client, error) { + configPath, err := dockercfg.ConfigPath() + if err != nil { + return nil, err + } + + base := filepath.Join(filepath.Dir(configPath), "buildx") + if addr == "" { + dt, err := os.ReadFile(filepath.Join(base, "current")) + if err != nil { + return nil, err + } + type ref struct { + Name string `json:"name"` + } + var r ref + if err := json.Unmarshal(dt, &r); err != nil { + return nil, fmt.Errorf("could not unmarshal buildx config: %w", err) + } + addr = r.Name + } + + dt, err := os.ReadFile(filepath.Join(base, "instances", addr)) + if err != nil { + return nil, err + } + type buildxConfig struct { + Driver string + Nodes []struct { + Name string + Endpoint string + } + } + var cfg buildxConfig + if err := json.Unmarshal(dt, &cfg); err != nil { + return nil, fmt.Errorf("could not unmarshal buildx instance config: %w", err) + } + + if cfg.Driver != "docker-container" { + return nil, fmt.Errorf("unsupported buildx driver: %s", cfg.Driver) + } + + if len(cfg.Nodes) == 0 { + return nil, errors.New("no nodes configured for buildx instance") + } + + log.WithFields(log.Fields{ + "driver": cfg.Driver, + "endpoint": cfg.Nodes[0].Endpoint, + "name": cfg.Nodes[0].Name, + }).Debug("Connect to buildx instance") + return dockerContainerDriver(cfg.Nodes[0].Endpoint, "buildx_buildkit_"+cfg.Nodes[0].Name)(ctx) + } +} + +func autoDriver(ctx context.Context) (*client.Client, error) { + var retErr error + + log.Debug("Tryin docker driver") + client, err := dockerDriver("")(ctx) + if err == nil { + err = validateClient(ctx, client) + if err == nil { + return client, nil + } + client.Close() + } + retErr = errors.Join(retErr, fmt.Errorf("could not use docker driver: %w", err)) + + log.Debug("Trying buildx driver") + client, err = buildxDriver("")(ctx) + if err == nil { + err = validateClient(ctx, client) + if err == nil { + return client, nil + } + client.Close() + } + retErr = errors.Join(retErr, fmt.Errorf("could not use buildx driver: %w", err)) + + log.Debug("Trying buildkitd driver") + client, err = buildkitdDriver("")(ctx) + if err == nil { + if err := validateClient(ctx, client); err == nil { + return client, nil + } + client.Close() + } + return nil, errors.Join(retErr, fmt.Errorf("could not use buildkitd driver: %w", err)) +} + +func buildkitdDriver(addr string) driverFunc { + return func(ctx context.Context) (*client.Client, error) { + if addr == "" { + addr = DefaultAddr + } + return client.New(ctx, "unix://"+addr, client.WithFailFast()) + } +} + +func getDockerTransport(addr string) (transport.Doer, error) { + if addr == "" { + addr = os.Getenv("DOCKER_HOST") + } + + if addr == "" { + return transport.DefaultTransport() + } + return transport.FromConnectionString(addr) +} + +func dockerDriver(addr string) driverFunc { + return func(ctx context.Context) (*client.Client, error) { + defaultOpts := []client.ClientOpt{client.WithFailFast()} + bkOpts := defaultOpts + + tr, err := getDockerTransport(addr) + if err != nil { + return nil, err + } + + bkOpts = append(bkOpts, buildkitopt.FromDocker(tr)...) + c, err := client.New(ctx, addr, bkOpts...) + if err != nil { + return nil, err + } + return c, nil + } +} + +func dockerContainerDriver(host, addr string) driverFunc { + return func(ctx context.Context) (*client.Client, error) { + tr, err := getDockerTransport(host) + if err != nil { + return nil, err + } + + cli := docker.NewClient(docker.WithTransport(tr)) + c := cli.ContainerService().NewContainer(ctx, addr) + + conn1, conn2 := net.Pipe() + ep, err := c.Exec(ctx, container.WithExecCmd("buildctl", "dial-stdio"), func(cfg *container.ExecConfig) { + cfg.Stdin = conn1 + cfg.Stdout = conn1 + cfg.Stderr = conn1 + }) + if err != nil { + return nil, err + } + if err := ep.Start(ctx); err != nil { + return nil, err + } + + var opts []client.ClientOpt + var counter int64 + opts = append(opts, client.WithContextDialer(func(context.Context, string) (net.Conn, error) { + if atomic.AddInt64(&counter, 1) > 1 { + return nil, net.ErrClosed + } + return conn2, nil + })) + return client.New(ctx, "", opts...) + } +}