Merge pull request #489 from fluxcd/verify-artifact-checksum

Verify artifacts integrity

Makefile

@@ -2,7 +2,7 @@
 IMG ?= fluxcd/kustomize-controller:latest
 # Produce CRDs that work back to Kubernetes 1.16
 CRD_OPTIONS ?= crd:crdVersions=v1
-SOURCE_VER ?= v0.17.2
+SOURCE_VER ?= v0.18.0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))

config/default/kustomization.yaml

@@ -2,8 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: kustomize-system
 resources:
-- https://github.com/fluxcd/source-controller/releases/download/v0.17.2/source-controller.crds.yaml
-- https://github.com/fluxcd/source-controller/releases/download/v0.17.2/source-controller.deployment.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v0.18.0/source-controller.crds.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v0.18.0/source-controller.deployment.yaml
 - ../crd
 - ../rbac
 - ../manager

controllers/kustomization_controller.go

@@ -19,7 +19,10 @@ package controllers
 import (
  "bytes"
  "context"
+ "crypto/sha1"
+ "crypto/sha256"
  "fmt"
+ "io"
  "net/http"
  "net/url"
  "os"
@@ -296,7 +299,7 @@ func (r *KustomizationReconciler) reconcile(
  defer os.RemoveAll(tmpDir)
 
  // download artifact and extract files
- err = r.download(source.GetArtifact().URL, tmpDir)
+ err = r.download(source.GetArtifact(), tmpDir)
  if err != nil {
  return kustomizev1.KustomizationNotReady(
  kustomization,
@@ -495,7 +498,8 @@ func (r *KustomizationReconciler) checkDependencies(kustomization kustomizev1.Ku
  return nil
 }
 
-func (r *KustomizationReconciler) download(artifactURL string, tmpDir string) error {
+func (r *KustomizationReconciler) download(artifact *sourcev1.Artifact, tmpDir string) error {
+ artifactURL := artifact.URL
  if hostname := os.Getenv("SOURCE_CONTROLLER_LOCALHOST"); hostname != "" {
  u, err := url.Parse(artifactURL)
  if err != nil {
@@ -521,14 +525,43 @@ func (r *KustomizationReconciler) download(artifactURL string, tmpDir string) er
  return fmt.Errorf("failed to download artifact from %s, status: %s", artifactURL, resp.Status)
  }
 
+ var buf bytes.Buffer
+
+ // verify checksum matches origin
+ if err := r.verifyArtifact(artifact, &buf, resp.Body); err != nil {
+ return err
+ }
+
  // extract
- if _, err = untar.Untar(resp.Body, tmpDir); err != nil {
+ if _, err = untar.Untar(&buf, tmpDir); err != nil {
  return fmt.Errorf("failed to untar artifact, error: %w", err)
  }
 
  return nil
 }
 
+func (r *KustomizationReconciler) verifyArtifact(artifact *sourcev1.Artifact, buf *bytes.Buffer, reader io.Reader) error {
+ hasher := sha256.New()
+
+ // for backwards compatibility with source-controller v0.17.2 and older
+ if len(artifact.Checksum) == 40 {
+ hasher = sha1.New()
+ }
+
+ // compute checksum
+ mw := io.MultiWriter(hasher, buf)
+ if _, err := io.Copy(mw, reader); err != nil {
+ return err
+ }
+
+ if checksum := fmt.Sprintf("%x", hasher.Sum(nil)); checksum != artifact.Checksum {
+ return fmt.Errorf("failed to verify artifact: computed checksum '%s' doesn't match advertised '%s'",
+ checksum, artifact.Checksum)
+ }
+
+ return nil
+}
+
 func (r *KustomizationReconciler) getSource(ctx context.Context, kustomization kustomizev1.Kustomization) (sourcev1.Source, error) {
  var source sourcev1.Source
  sourceNamespace := kustomization.GetNamespace()

controllers/kustomization_decryptor_test.go

@@ -43,16 +43,12 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
  err = createKubeConfigSecret(id)
  g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
 
- artifactFile := "sops-" + randStringRunes(5)
- artifactChecksum, err := createArtifact(testServer, "testdata/sops", artifactFile)
- g.Expect(err).ToNot(HaveOccurred())
- artifactURL, err := testServer.URLForFile(artifactFile)
+ artifactName := "sops-" + randStringRunes(5)
+ artifactChecksum, err := createArtifact(testServer, "testdata/sops", artifactName)
  g.Expect(err).ToNot(HaveOccurred())
 
- overlayArtifactFile := "sops-" + randStringRunes(5)
- overlayChecksum, err := createArtifact(testServer, "testdata/test-dotenv", overlayArtifactFile)
- g.Expect(err).ToNot(HaveOccurred())
- overlayArtifactUrl, err := testServer.URLForFile(overlayArtifactFile)
+ overlayArtifactName := "sops-" + randStringRunes(5)
+ overlayChecksum, err := createArtifact(testServer, "testdata/test-dotenv", overlayArtifactName)
  g.Expect(err).ToNot(HaveOccurred())
 
  repositoryName := types.NamespacedName{
@@ -65,10 +61,10 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, artifactURL, "main/"+artifactChecksum, artifactChecksum)
+ err = applyGitRepository(repositoryName, artifactName, "main/"+artifactChecksum)
  g.Expect(err).NotTo(HaveOccurred())
 
- err = applyGitRepository(overlayRepositoryName, overlayArtifactUrl, "main/"+overlayChecksum, overlayChecksum)
+ err = applyGitRepository(overlayRepositoryName, overlayArtifactName, "main/"+overlayChecksum)
  g.Expect(err).NotTo(HaveOccurred())
 
  pgpKey, err := os.ReadFile("testdata/sops/pgp.asc")
@@ -180,7 +176,7 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
  t.Run("does not emit change events for identical secrets", func(t *testing.T) {
  resultK := &kustomizev1.Kustomization{}
  revision := "v2.0.0"
- err = applyGitRepository(repositoryName, artifactURL, revision, artifactChecksum+"v2")
+ err = applyGitRepository(repositoryName, artifactName, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {

controllers/kustomization_dependson_test.go

@@ -126,8 +126,6 @@ spec:
  artifact, err := testServer.ArtifactFromFiles(manifests(id, id))
  g.Expect(err).NotTo(HaveOccurred())
 
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
-
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("dep-%s", randStringRunes(5)),
  Namespace: id,
@@ -178,7 +176,7 @@ spec:
  })
 
  t.Run("reconciles when source is found", func(t *testing.T) {
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {

controllers/kustomization_force_test.go

@@ -63,16 +63,13 @@ stringData:
  }
 
  artifact, err := testServer.ArtifactFromFiles(manifests(id, randStringRunes(5)))
- g.Expect(err).NotTo(HaveOccurred())
-
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
 
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("force-%s", randStringRunes(5)),
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  kustomizationKey := types.NamespacedName{
@@ -127,9 +124,8 @@ stringData:
  t.Run("fails to update immutable secret", func(t *testing.T) {
  artifact, err := testServer.ArtifactFromFiles(manifests(id, randStringRunes(5)))
  g.Expect(err).NotTo(HaveOccurred())
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
  revision := "v2.0.0"
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {
@@ -150,9 +146,8 @@ stringData:
  t.Run("recreates immutable secret", func(t *testing.T) {
  artifact, err := testServer.ArtifactFromFiles(manifests(id, randStringRunes(5)))
  g.Expect(err).NotTo(HaveOccurred())
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
  revision := "v3.0.0"
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() error {

controllers/kustomization_inventory_test.go

@@ -75,14 +75,12 @@ stringData:
  artifact, err := testServer.ArtifactFromFiles(manifests(id, id))
  g.Expect(err).NotTo(HaveOccurred())
 
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
-
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("inv-%s", randStringRunes(5)),
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  kustomizationKey := types.NamespacedName{
@@ -180,7 +178,7 @@ stringData:
  })
  g.Expect(k8sClient.Update(context.Background(), configMapClone)).To(Succeed())
 
- err = applyGitRepository(repositoryName, url, testRev, "")
+ err = applyGitRepository(repositoryName, artifact, testRev)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {
@@ -203,7 +201,7 @@ stringData:
  })
  g.Expect(k8sClient.Update(context.Background(), configMapClone)).To(Succeed())
 
- err = applyGitRepository(repositoryName, url, testRev, "")
+ err = applyGitRepository(repositoryName, artifact, testRev)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {
@@ -229,8 +227,8 @@ stringData:
 
  artifact, err := testServer.ArtifactFromFiles(manifests(testId, id))
  g.Expect(err).NotTo(HaveOccurred())
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
- err = applyGitRepository(repositoryName, url, testRev, "")
+
+ err = applyGitRepository(repositoryName, artifact, testRev)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {

controllers/kustomization_prune_test.go

@@ -77,14 +77,12 @@ data:
  artifact, err := testServer.ArtifactFromFiles(manifests(id, id))
  g.Expect(err).NotTo(HaveOccurred())
 
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
-
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("gc-%s", randStringRunes(5)),
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  kustomizationKey := types.NamespacedName{
@@ -134,9 +132,8 @@ data:
  newID := randStringRunes(5)
  artifact, err := testServer.ArtifactFromFiles(manifests(newID, newID))
  g.Expect(err).NotTo(HaveOccurred())
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
  revision := "v2.0.0"
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {
@@ -208,14 +205,12 @@ data:
  artifact, err := testServer.ArtifactFromFiles(manifests(id, id))
  g.Expect(err).NotTo(HaveOccurred())
 
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
-
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("gc-%s", randStringRunes(5)),
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  kustomizationKey := types.NamespacedName{
@@ -260,9 +255,9 @@ data:
  t.Run("deletes stale objects", func(t *testing.T) {
  artifact, err := testServer.ArtifactFromFiles([]testserver.File{})
  g.Expect(err).NotTo(HaveOccurred())
- url = fmt.Sprintf("%s/%s", testServer.URL(), artifact)
+
  revision = "v2.0.0"
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {
@@ -285,9 +280,8 @@ kind: Kustomization
  }
  artifact, err := testServer.ArtifactFromFiles(empty)
  g.Expect(err).NotTo(HaveOccurred())
- url = fmt.Sprintf("%s/%s", testServer.URL(), artifact)
  revision = "v3.0.0"
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {
@@ -301,9 +295,8 @@ kind: Kustomization
  t.Run("restores objects", func(t *testing.T) {
  artifact, err := testServer.ArtifactFromFiles(manifests(id, id))
  g.Expect(err).NotTo(HaveOccurred())
- url = fmt.Sprintf("%s/%s", testServer.URL(), artifact)
  revision = "v4.0.0"
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  g.Eventually(func() bool {
@@ -356,14 +349,12 @@ data:
  artifact, err := testServer.ArtifactFromFiles(manifests(id, id))
  g.Expect(err).NotTo(HaveOccurred())
 
- url := fmt.Sprintf("%s/%s", testServer.URL(), artifact)
-
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("gc-%s", randStringRunes(5)),
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, url, revision, "")
+ err = applyGitRepository(repositoryName, artifact, revision)
  g.Expect(err).NotTo(HaveOccurred())
 
  kustomizationKey := types.NamespacedName{

controllers/kustomization_transformer_test.go

@@ -53,15 +53,13 @@ func TestKustomizationReconciler_KustomizeTransformer(t *testing.T) {
  artifactFile := "patch-" + randStringRunes(5)
  artifactChecksum, err := createArtifact(testServer, "testdata/transformers", artifactFile)
  g.Expect(err).ToNot(HaveOccurred())
- artifactURL, err := testServer.URLForFile(artifactFile)
- g.Expect(err).ToNot(HaveOccurred())
 
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("%s", randStringRunes(5)),
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, artifactURL, "main/"+artifactChecksum, artifactChecksum)
+ err = applyGitRepository(repositoryName, artifactFile, "main/"+artifactChecksum)
  g.Expect(err).NotTo(HaveOccurred())
 
  kustomizationKey := types.NamespacedName{
@@ -178,15 +176,13 @@ func TestKustomizationReconciler_KustomizeTransformerFiles(t *testing.T) {
  artifactFile := "patch-" + randStringRunes(5)
  artifactChecksum, err := createArtifact(testServer, "testdata/file-transformer", artifactFile)
  g.Expect(err).ToNot(HaveOccurred())
- artifactURL, err := testServer.URLForFile(artifactFile)
- g.Expect(err).ToNot(HaveOccurred())
 
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("%s", randStringRunes(5)),
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, artifactURL, "main/"+artifactChecksum, artifactChecksum)
+ err = applyGitRepository(repositoryName, artifactFile, "main/"+artifactChecksum)
  g.Expect(err).NotTo(HaveOccurred())
 
  kustomizationKey := types.NamespacedName{
@@ -299,15 +295,13 @@ func TestKustomizationReconciler_FluxTransformers(t *testing.T) {
  artifactFile := "patch-" + randStringRunes(5)
  artifactChecksum, err := createArtifact(testServer, "testdata/patch", artifactFile)
  g.Expect(err).ToNot(HaveOccurred())
- artifactURL, err := testServer.URLForFile(artifactFile)
- g.Expect(err).ToNot(HaveOccurred())
 
  repositoryName := types.NamespacedName{
  Name: fmt.Sprintf("%s", randStringRunes(5)),
  Namespace: id,
  }
 
- err = applyGitRepository(repositoryName, artifactURL, "main/"+artifactChecksum, artifactChecksum)
+ err = applyGitRepository(repositoryName, artifactFile, "main/"+artifactChecksum)
  g.Expect(err).NotTo(HaveOccurred())
 
  kustomizationKey := types.NamespacedName{