diff --git a/go.mod b/go.mod index 8156277c..5ab1eff0 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/aquasecurity/trivy v0.44.1 github.com/containerd/console v1.0.3 github.com/containerd/containerd v1.7.3 - github.com/cpuguy83/go-docker/buildkitopt v0.1.1 + github.com/cpuguy83/dockercfg v0.3.1 github.com/distribution/distribution v2.8.2+incompatible github.com/docker/cli v24.0.5+incompatible github.com/google/go-containerregistry v0.16.1 diff --git a/go.sum b/go.sum index b5d08abf..5d792624 100644 --- a/go.sum +++ b/go.sum @@ -92,10 +92,10 @@ github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtO github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-docker v0.1.2 h1:bHyfwV+yMw+IP2iL/0T5WU/g7Bpj+yhdlwoe6LEwA+A= github.com/cpuguy83/go-docker v0.1.2/go.mod h1:tZURUlegjsgYPhkfkuPCeVrp1ocirH4P1yUexBpP25g= -github.com/cpuguy83/go-docker/buildkitopt v0.1.1 h1:zzg3taUURaoBzyowOqTtH1EoH3251oyceLS3202HF/M= -github.com/cpuguy83/go-docker/buildkitopt v0.1.1/go.mod h1:b4JdmqXsUZV8phNhTvShbMvFrwz80ZM3wVLAnXX41A0= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index 3574969a..d86bb8a8 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -7,8 +7,6 @@ package buildkit import ( "context" - "errors" - "fmt" "io" "net/http" "os" @@ -16,17 +14,12 @@ import ( "github.com/containerd/console" "github.com/containerd/containerd/remotes/docker" - "github.com/cpuguy83/go-docker/buildkitopt" - "github.com/cpuguy83/go-docker/transport" "github.com/docker/cli/cli/config" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" - gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" - "github.com/moby/buildkit/solver/pb" - "github.com/moby/buildkit/util/apicaps" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/imageutil" "github.com/moby/buildkit/util/progress/progressui" @@ -39,10 +32,6 @@ import ( "golang.org/x/sync/errgroup" ) -const ( - DefaultAddr = "unix:///run/buildkit/buildkitd.sock" -) - type Config struct { ImageName string Client *client.Client @@ -112,63 +101,21 @@ func resolveImageConfig(ctx context.Context, ref string, platform *ispec.Platfor return dgst, config, nil } -var errMissingCap = fmt.Errorf("missing required buildkit functionality") - -// Buildkit llb ops required for copacetic -var requiredCaps = []apicaps.CapID{pb.CapMergeOp, pb.CapDiffOp} - // NewClient returns a new buildkit client with the given addr. // If addr is empty it will first try to connect to docker's buildkit instance and then fallback to DefaultAddr. func NewClient(ctx context.Context, addr string) (*client.Client, error) { - var bkOpts []client.ClientOpt if addr == "" { - // First try using docker's buildkit instance if available - tr, err := transport.DefaultTransport() - if err != nil { - log.WithError(err).Info("Failed to initialize docker transport, falling back to default buildkit address") - addr = DefaultAddr - } else { - bkOpts = append(bkOpts, buildkitopt.FromDocker(tr)...) - } + return autoClient(ctx) } - - c, err := client.New(ctx, addr, bkOpts...) + client, err := client.New(ctx, addr) if err != nil { return nil, err } - - _, err = c.Build(ctx, client.SolveOpt{}, "", func(ctx context.Context, client gateway.Client) (*gateway.Result, error) { - capset := client.BuildOpts().LLBCaps - var err error - for _, cap := range requiredCaps { - err = errors.Join(err, capset.Supports(cap)) - } - if err != nil { - return nil, errors.Join(err, errMissingCap) - } - return &gateway.Result{}, nil - }, nil) - if err == nil { - return c, nil - } - c.Close() - - if addr != "" { + if err := validateClient(ctx, client); err != nil { + client.Close() return nil, err } - - if !errors.Is(err, errMissingCap) { - log.WithError(err).Warn("Failed to check buildkit LLB merge operation support, falling back to default buildkit address") - } else { - log.WithError(err).Warn("Docker integrated buildkit is too old, falling back to default buildkit address") - } - - c, err = client.New(ctx, DefaultAddr) - if err != nil { - return nil, fmt.Errorf("failed to initialize buildkit client: %w", err) - } - - return c, nil + return client, nil } func InitializeBuildkitConfig(ctx context.Context, client *client.Client, image string, manifest *types.UpdateManifest) (*Config, error) { diff --git a/pkg/buildkit/buildkit_test.go b/pkg/buildkit/buildkit_test.go index cf4fdfc4..27df3693 100644 --- a/pkg/buildkit/buildkit_test.go +++ b/pkg/buildkit/buildkit_test.go @@ -88,7 +88,7 @@ func newMockBuildkitAPI(t *testing.T, caps ...apicaps.CapID) string { caps: capList, }) - go srv.Serve(l) + go srv.Serve(l) // nolint:errcheck control := &mockControlServer{ ControlServer: &controlapi.UnimplementedControlServer{}, diff --git a/pkg/buildkit/connhelpers/buildx.go b/pkg/buildkit/connhelpers/buildx.go new file mode 100644 index 00000000..0f0c6a99 --- /dev/null +++ b/pkg/buildkit/connhelpers/buildx.go @@ -0,0 +1,158 @@ +package connhelpers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math/rand" + "net" + "net/url" + "os" + "os/exec" + "path/filepath" + + "github.com/cpuguy83/dockercfg" + "github.com/cpuguy83/go-docker" + "github.com/cpuguy83/go-docker/container" + "github.com/cpuguy83/go-docker/errdefs" + "github.com/moby/buildkit/client/connhelper" + log "github.com/sirupsen/logrus" +) + +func init() { + connhelper.Register("buildx", Buildx) +} + +type buildxConfig struct { + Driver string + Nodes []struct { + Name string + Endpoint string + } +} + +// Buildx returns a buildkit connection helper for connecting to a buildx instance. +// Only "docker-container" buildkit instances are currently supported. +// If there are multiple nodes configured, one will be chosen at random. +func Buildx(u *url.URL) (*connhelper.ConnectionHelper, error) { + if u.Path != "" { + return nil, fmt.Errorf("buildx driver does not support path elements: %s", u.Path) + } + return &connhelper.ConnectionHelper{ + ContextDialer: buildxContextDialer(u.Host), + }, nil +} + +func buildxContextDialer(builder string) func(context.Context, string) (net.Conn, error) { + return func(ctx context.Context, _ string) (net.Conn, error) { + configPath, err := dockercfg.ConfigPath() + if err != nil { + return nil, err + } + + if builder == "" { + // Standard env for setting a buildx builder name to use + // This is used by buildx so we should use it too. + builder = os.Getenv("BUILDX_BUILDER") + } + + base := filepath.Join(filepath.Dir(configPath), "buildx") + if builder == "" { + dt, err := os.ReadFile(filepath.Join(base, "current")) + if err != nil { + return nil, err + } + type ref struct { + Name string `json:"name"` + } + var r ref + if err := json.Unmarshal(dt, &r); err != nil { + return nil, fmt.Errorf("could not unmarshal buildx config: %w", err) + } + builder = r.Name + } + + // Note: buildx inspect does not return json here, so we can't use the output directly + cmd := exec.CommandContext(ctx, "docker", "buildx", "inspect", "--bootstrap", builder) + errBuf := bytes.NewBuffer(nil) + cmd.Stderr = errBuf + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("could not inspect buildx instance: %w: %s", err, errBuf.String()) + } + + // Read the config from the buildx instance + dt, err := os.ReadFile(filepath.Join(base, "instances", builder)) + if err != nil { + return nil, err + } + + var cfg buildxConfig + if err := json.Unmarshal(dt, &cfg); err != nil { + return nil, fmt.Errorf("could not unmarshal buildx instance config: %w", err) + } + + if cfg.Driver != "docker-container" { + return nil, fmt.Errorf("unsupported buildx driver: %s", cfg.Driver) + } + + if len(cfg.Nodes) == 0 { + return nil, errors.New("no nodes configured for buildx instance") + } + + log.WithFields(log.Fields{ + "driver": cfg.Driver, + "endpoint": cfg.Nodes[0].Endpoint, + "name": cfg.Nodes[0].Name, + }).Debug("Connect to buildx instance") + + nodes := cfg.Nodes + if len(nodes) > 1 { + rand.Shuffle(len(nodes), func(i, j int) { + nodes[i], nodes[j] = nodes[j], nodes[i] + }) + } + return containerContextDialer(ctx, nodes[0].Endpoint, "buildx_buildkit_"+nodes[0].Name) + } +} + +func containerContextDialer(ctx context.Context, host, name string) (net.Conn, error) { + tr, err := getDockerTransport(host) + if err != nil { + return nil, err + } + + cli := docker.NewClient(docker.WithTransport(tr)) + c := cli.ContainerService().NewContainer(ctx, name) + + conn1, conn2 := net.Pipe() + ep, err := c.Exec(ctx, container.WithExecCmd("buildctl", "dial-stdio"), func(cfg *container.ExecConfig) { + cfg.Stdin = conn1 + cfg.Stdout = conn1 + cfg.Stderr = conn1 + }) + if err != nil { + if errdefs.IsNotFound(err) { + return nil, fmt.Errorf("could not find container %s: %w", name, err) + } + if err2 := c.Start(ctx); err2 != nil { + return nil, err + } + + ep, err = c.Exec(ctx, container.WithExecCmd("buildctl", "dial-stdio"), func(cfg *container.ExecConfig) { + cfg.Stdin = conn1 + cfg.Stdout = conn1 + cfg.Stderr = conn1 + }) + if err != nil { + return nil, err + } + } + + if err := ep.Start(ctx); err != nil { + return nil, fmt.Errorf("could not start exec proxy: %w", err) + } + + return conn2, nil +} diff --git a/pkg/buildkit/connhelpers/docker.go b/pkg/buildkit/connhelpers/docker.go new file mode 100644 index 00000000..4824861c --- /dev/null +++ b/pkg/buildkit/connhelpers/docker.go @@ -0,0 +1,42 @@ +package connhelpers + +import ( + "context" + "net" + "net/http" + "net/url" + "os" + "path" + + "github.com/cpuguy83/go-docker/transport" + "github.com/cpuguy83/go-docker/version" + "github.com/moby/buildkit/client/connhelper" +) + +func init() { + connhelper.Register("docker", Docker) +} + +// Docker returns a buildkit connection helper for connecting to a docker daemon. +func Docker(u *url.URL) (*connhelper.ConnectionHelper, error) { + tr, err := getDockerTransport(path.Join(u.Host, u.Path)) + if err != nil { + return nil, err + } + + return &connhelper.ConnectionHelper{ + ContextDialer: func(ctx context.Context, addr string) (net.Conn, error) { + return tr.DoRaw(ctx, http.MethodPost, version.Join(ctx, "/grpc"), transport.WithUpgrade("h2c")) + }, + }, nil +} + +func getDockerTransport(addr string) (transport.Doer, error) { + if addr == "" { + addr = os.Getenv("DOCKER_HOST") + } + if addr == "" { + return transport.DefaultTransport() + } + return transport.FromConnectionString(addr) +} diff --git a/pkg/buildkit/drivers.go b/pkg/buildkit/drivers.go new file mode 100644 index 00000000..2dccf159 --- /dev/null +++ b/pkg/buildkit/drivers.go @@ -0,0 +1,93 @@ +package buildkit + +import ( + "context" + "errors" + "fmt" + "net" + "net/url" + + "github.com/moby/buildkit/client" + gateway "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/apicaps" + "github.com/project-copacetic/copacetic/pkg/buildkit/connhelpers" + log "github.com/sirupsen/logrus" +) + +const ( + DefaultAddr = "unix:///run/buildkit/buildkitd.sock" +) + +var ( + errMissingCap = fmt.Errorf("missing required buildkit functionality") + // requiredCaps are buildkit llb ops required to function. + requiredCaps = []apicaps.CapID{pb.CapMergeOp, pb.CapDiffOp} +) + +func autoClient(ctx context.Context, opts ...client.ClientOpt) (*client.Client, error) { + var retErr error + + newClient := func(ctx context.Context, dialer func(context.Context, string) (net.Conn, error)) (*client.Client, error) { + client, err := client.New(ctx, "", append(opts, client.WithContextDialer(dialer))...) + if err == nil { + err = validateClient(ctx, client) + if err == nil { + return client, nil + } + client.Close() + } + return nil, err + } + + log.Debug("Trying docker driver") + h, err := connhelpers.Docker(&url.URL{}) + if err != nil { + return nil, err + } + c, err := newClient(ctx, h.ContextDialer) + if err == nil { + return c, nil + } + log.WithError(err).Debug("Could not use docker driver") + retErr = errors.Join(retErr, fmt.Errorf("could not use docker driver: %w", err)) + + log.Debug("Trying buildx driver") + h, err = connhelpers.Buildx(&url.URL{}) + if err != nil { + return nil, err + } + + c, err = newClient(ctx, h.ContextDialer) + if err == nil { + return c, nil + } + log.WithError(err).Debug("Could not use buildx driver") + retErr = errors.Join(retErr, fmt.Errorf("could not use buildx driver: %w", err)) + + log.Debug("Trying default buildkit addr") + c, err = client.New(ctx, DefaultAddr, opts...) + if err == nil { + if err := validateClient(ctx, c); err == nil { + return c, nil + } + c.Close() + } + log.WithError(err).Debug("Could not use buildkitd driver") + return nil, errors.Join(retErr, fmt.Errorf("could not use buildkitd driver: %w", err)) +} + +func validateClient(ctx context.Context, c *client.Client) error { + _, err := c.Build(ctx, client.SolveOpt{}, "", func(ctx context.Context, client gateway.Client) (*gateway.Result, error) { + capset := client.BuildOpts().LLBCaps + var err error + for _, cap := range requiredCaps { + err = errors.Join(err, capset.Supports(cap)) + } + if err != nil { + return nil, errors.Join(err, errMissingCap) + } + return &gateway.Result{}, nil + }, nil) + return err +} diff --git a/pkg/patch/cmd.go b/pkg/patch/cmd.go index c6f4016b..26029bb1 100644 --- a/pkg/patch/cmd.go +++ b/pkg/patch/cmd.go @@ -11,6 +11,13 @@ import ( "github.com/project-copacetic/copacetic/pkg/buildkit" "github.com/spf13/cobra" + + // Register connection helpers for buildkit. + _ "github.com/moby/buildkit/client/connhelper/dockercontainer" + _ "github.com/moby/buildkit/client/connhelper/kubepod" + _ "github.com/moby/buildkit/client/connhelper/nerdctlcontainer" + _ "github.com/moby/buildkit/client/connhelper/podmancontainer" + _ "github.com/moby/buildkit/client/connhelper/ssh" ) type patchArgs struct {