Merge pull request #4030 from ChimneySwift/generic-s3

Allow generic S3 endpoints for alternative services
webbukkit · Dec 13, 2023 · b0e56d3 · b0e56d3
2 parents 20700c2 + 1ba6dd4
commit b0e56d3
Show file tree

Hide file tree

Showing 25 changed files with 56 additions and 18 deletions.
diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle
@@ -19,11 +19,13 @@ dependencies {
  implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
  implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
  implementation 'org.postgresql:postgresql:42.2.18'
- implementation 'io.github.linktosriram:s3-lite-core:0.2.0'
- implementation 'io.github.linktosriram:s3-lite-api:0.2.0'
- implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0'
- implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0'
- implementation 'io.github.linktosriram:s3-lite-util:0.2.0'
+ implementation 'io.github.linktosriram.s3lite:core:0.0.2-SNAPSHOT'
+ implementation 'io.github.linktosriram.s3lite:api:0.0.2-SNAPSHOT'
+ implementation 'io.github.linktosriram.s3lite:http-client-url-connection:0.0.2-SNAPSHOT'
+ implementation 'io.github.linktosriram.s3lite:http-client-spi:0.0.2-SNAPSHOT'
+ implementation 'io.github.linktosriram.s3lite:util:0.0.2-SNAPSHOT'
+ implementation 'jakarta.xml.bind:jakarta.xml.bind-api:3.0.1'
+ implementation 'com.sun.xml.bind:jaxb-impl:3.0.0'
 }
 
 processResources {
@@ -58,11 +60,13 @@ shadowJar {
  include(dependency('org.eclipse.jetty::'))
  include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
  include(dependency('org.postgresql:postgresql:'))
- include(dependency('io.github.linktosriram:s3-lite-core:'))
- include(dependency('io.github.linktosriram:s3-lite-api:'))
- include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:'))
- include(dependency('io.github.linktosriram:s3-lite-http-client-spi:'))
- include(dependency('io.github.linktosriram:s3-lite-util:'))
+ include(dependency('io.github.linktosriram.s3lite:core:'))
+ include(dependency('io.github.linktosriram.s3lite:api:'))
+ include(dependency('io.github.linktosriram.s3lite:http-client-url-connection:'))
+ include(dependency('io.github.linktosriram.s3lite:http-client-spi:'))
+ include(dependency('io.github.linktosriram.s3lite:util:'))
+ include(dependency('jakarta.xml.bind::'))
+ include(dependency('com.sun.xml.bind::'))
  include(dependency(':DynmapCoreAPI'))
  exclude("META-INF/maven/**")
  exclude("META-INF/services/**")

diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java
@@ -1,6 +1,7 @@
 package org.dynmap.storage.aws_s3;
 
 import java.io.IOException;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -139,7 +140,7 @@ public boolean write(long hash, BufferOutputStream encImage, long timestamp) {
  else {
  PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType())
  .addMetadata("x-dynmap-hash", Long.toHexString(hash)).addMetadata("x-dynmap-ts", Long.toString(timestamp)).build();
- s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+ s3.putObject(req, RequestBody.fromBytes(encImage.buf));
  }
  done = true;
  } catch (S3Exception x) {
@@ -221,7 +222,7 @@ public String toString() {
  }
 
  private String bucketname;
- private String region;
+ private Region region;
  private String access_key_id;
  private String secret_access_key;
  private String prefix;
@@ -248,10 +249,20 @@ public boolean init(DynmapCore core) {
  }
  // Get our settings
  bucketname = core.configuration.getString("storage/bucketname", "dynmap");
- region = core.configuration.getString("storage/region", "us-east-1");
  access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID"));
  secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY"));
  prefix = core.configuration.getString("storage/prefix", "");
+
+ // Either use a custom region, or one of the default AWS regions
+ String region_name = core.configuration.getString("storage/region", "us-east-1");
+ String region_endpoint = core.configuration.getString("storage/override_endpoint", "");
+
+ if (region_endpoint.length() > 0) {
+ region = Region.of(region_name, URI.create(region_endpoint));
+ } else {
+ region = Region.fromString(region_name);
+ }
+
  if ((prefix.length() > 0) && (prefix.charAt(prefix.length()-1) != '/')) {
  prefix += '/';
  }
@@ -518,7 +529,7 @@ public boolean setPlayerFaceImage(String playername, FaceType facetype,
  }
  else {
  PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
- s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+ s3.putObject(req, RequestBody.fromBytes(encImage.buf));
  }
  done = true;
  } catch (S3Exception x) {
@@ -571,7 +582,7 @@ public boolean setMarkerImage(String markerid, BufferOutputStream encImage) {
  }
  else {
  PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
- s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
+ s3.putObject(req, RequestBody.fromBytes(encImage.buf));
  }
  done = true;
  } catch (S3Exception x) {
@@ -734,7 +745,7 @@ else if (fileid.endsWith(".js")) {
  ct = "application/x-javascript";
  }
  PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build();
- s3.putObject(req, RequestBody.fromBytes(content.buf, content.len));
+ s3.putObject(req, RequestBody.fromBytes(content.buf));
  standalone_cache.put(fileid, digest);
  }
  done = true;
@@ -763,7 +774,7 @@ private S3Client getConnection() throws S3Exception, StorageShutdownException {
  if (cpoolCount < POOLSIZE) { // Still more we can have
  c = new DefaultS3ClientBuilder()
  .credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
- .region(Region.fromString(region))
+ .region(region)
  .httpClient(URLConnectionSdkHttpClient.create())
  .build();
  if (c == null) {

diff --git a/build.gradle b/build.gradle
@@ -30,6 +30,7 @@ allprojects {
  mavenLocal()
  maven { url 'https://libraries.minecraft.net/' }
  maven { url "https://oss.sonatype.org/content/repositories/releases" }
+ maven { url "https://oss.sonatype.org/content/repositories/snapshots" }
  maven { url "https://repo.mikeprimm.com" }
  maven { url "https://repo.maven.apache.org/maven2" }
  maven { url "https://hub.spigotmc.org/nexus/content/repositories/snapshots/" }

diff --git a/fabric-1.14.4/src/main/resources/configuration.txt b/fabric-1.14.4/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.15.2/src/main/resources/configuration.txt b/fabric-1.15.2/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.16.4/src/main/resources/configuration.txt b/fabric-1.16.4/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.17.1/src/main/resources/configuration.txt b/fabric-1.17.1/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.18.2/src/main/resources/configuration.txt b/fabric-1.18.2/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.19.1/src/main/resources/configuration.txt b/fabric-1.19.1/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.19.3/src/main/resources/configuration.txt b/fabric-1.19.3/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.19.4/src/main/resources/configuration.txt b/fabric-1.19.4/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.19/src/main/resources/configuration.txt b/fabric-1.19/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.20.2/src/main/resources/configuration.txt b/fabric-1.20.2/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/fabric-1.20/src/main/resources/configuration.txt b/fabric-1.20/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.14.4/src/main/resources/configuration.txt b/forge-1.14.4/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.15.2/src/main/resources/configuration.txt b/forge-1.15.2/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.16.5/src/main/resources/configuration.txt b/forge-1.16.5/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.17.1/src/main/resources/configuration.txt b/forge-1.17.1/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.18.2/src/main/resources/configuration.txt b/forge-1.18.2/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.19.2/src/main/resources/configuration.txt b/forge-1.19.2/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.19.3/src/main/resources/configuration.txt b/forge-1.19.3/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.19/src/main/resources/configuration.txt b/forge-1.19/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.20.2/src/main/resources/configuration.txt b/forge-1.20.2/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/forge-1.20/src/main/resources/configuration.txt b/forge-1.20/src/main/resources/configuration.txt
@@ -47,6 +47,7 @@ storage:
  #aws_access_key_id: "<aws-access-key-id>"
  #aws_secret_access_key: "<aws-secret-access-key>" 
  #prefix: ""
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent

diff --git a/spigot/src/main/resources/configuration.txt b/spigot/src/main/resources/configuration.txt
@@ -46,7 +46,8 @@ storage:
  #bucketname: "dynmap-bucket-name"
  #region: us-east-1
  #aws_access_key_id: "<aws-access-key-id>"
- #aws_secret_access_key: "<aws-secret-access-key>" 
+ #aws_secret_access_key: "<aws-secret-access-key>"
+ #override_endpoint: ""
 
 components:
  - class: org.dynmap.ClientConfigurationComponent