Use minio to emulate S3 for testing

nyaruka · Jul 25, 2024 · 899a95b · 899a95b
1 parent 8725c72
commit 899a95b
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 8 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,6 +18,16 @@ jobs:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
+      minio:
+        image: bitnami/minio:latest
+        env:
+          MINIO_ROOT_USER: root
+          MINIO_ROOT_PASSWORD: nyarukanyaruka
+          MINIO_DEFAULT_BUCKETS: temba-archives
+        ports:
+          - 9000:9000     
+        options: --health-cmd 'mc ready local' --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -30,8 +40,9 @@ jobs:
       - name: Run tests
         run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./...
         env:
-          ARCHIVER_AWS_ACCESS_KEY_ID: ${{ secrets.ARCHIVER_AWS_ACCESS_KEY_ID }}
-          ARCHIVER_AWS_SECRET_ACCESS_KEY: ${{ secrets.ARCHIVER_AWS_SECRET_ACCESS_KEY }}
+          ARCHIVER_AWS_ACCESS_KEY_ID: root
+          ARCHIVER_AWS_SECRET_ACCESS_KEY: nyarukanyaruka
+          ARCHIVER_S3_FORCE_PATH_STYLE: true
 
       - name: Upload coverage
         if: success()

diff --git a/archives/config.go b/archives/config.go
@@ -12,8 +12,9 @@ type Config struct {
 	AWSSecretAccessKey string `help:"secret access key to use for AWS services"`
 	AWSRegion          string `help:"region to use for AWS services, e.g. us-east-1"`
 
-	S3Endpoint string `help:"the S3 endpoint we will write archives to"`
-	S3Bucket   string `help:"the S3 bucket we will write archives to"`
+	S3Endpoint       string `help:"the S3 endpoint we will write archives to"`
+	S3Bucket         string `help:"the S3 bucket we will write archives to"`
+	S3ForcePathStyle bool   `help:"S3 should used /bucket/path style URLs"`
 
 	TempDir       string `help:"directory where temporary archive files are written"`
 	KeepFiles     bool   `help:"whether we should keep local archive files after upload (default false)"`
@@ -43,8 +44,9 @@ func NewDefaultConfig() *Config {
 		AWSSecretAccessKey: "",
 		AWSRegion:          "us-east-1",
 
-		S3Endpoint: "https://s3.amazonaws.com",
-		S3Bucket:   "dl-archiver-test",
+		S3Endpoint:       "https://s3.amazonaws.com",
+		S3Bucket:         "dl-archiver-test",
+		S3ForcePathStyle: false,
 
 		TempDir:       "/tmp",
 		KeepFiles:     false,

diff --git a/archives/s3.go b/archives/s3.go
@@ -31,8 +31,9 @@ const chunkSizeBytes = 1e9 // 1GB
 // NewS3Client creates a new s3 client from the passed in config, testing it as necessary
 func NewS3Client(config *Config) (s3iface.S3API, error) {
 	s3config := &aws.Config{
-		Region:   aws.String(config.AWSRegion),
-		Endpoint: aws.String(config.S3Endpoint),
+		Region:           aws.String(config.AWSRegion),
+		Endpoint:         aws.String(config.S3Endpoint),
+		S3ForcePathStyle: aws.Bool(config.S3ForcePathStyle),
 	}
 	if config.AWSAccessKeyID != "" {
 		s3config.Credentials = credentials.NewStaticCredentials(config.AWSAccessKeyID, config.AWSSecretAccessKey, "")