s3 credential

LICENSE

@@ -251,6 +251,9 @@
    ./iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergHiveCachedClientPool.java
    ./gradlew
 
+   Apache Polaris
+   ./credential/src/main/java/org/apache/gravitino/credential/aws/S3TokenProvider.java
+
    Apache Paimon
    ./catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java
 

build.gradle.kts

@@ -741,7 +741,7 @@ tasks {
   register("copySubprojectDependencies", Copy::class) {
     subprojects.forEach() {
       if (!it.name.startsWith("catalog") &&
-        !it.name.startsWith("authorization") &&
+        !it.name.startsWith("authorization") && !it.name.startsWith("credential") &&
         !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" &&
         it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink")
       ) {
@@ -756,6 +756,7 @@ tasks {
       if (!it.name.startsWith("catalog") &&
         !it.name.startsWith("client") &&
         !it.name.startsWith("authorization") &&
+        !it.name.startsWith("credential") &&
         !it.name.startsWith("filesystem") &&
         !it.name.startsWith("spark") &&
         !it.name.startsWith("iceberg") &&

catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java

@@ -40,6 +40,7 @@ public class IcebergConstants {
   public static final String ICEBERG_S3_ENDPOINT = "s3.endpoint";
   public static final String ICEBERG_S3_ACCESS_KEY_ID = "s3.access-key-id";
   public static final String ICEBERG_S3_SECRET_ACCESS_KEY = "s3.secret-access-key";
+  public static final String ICEBERG_S3_TOKEN = "s3.session-token";
   public static final String AWS_S3_REGION = "client.region";
 
   public static final String ICEBERG_OSS_ENDPOINT = "oss.endpoint";

catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java

@@ -23,5 +23,8 @@ public class CredentialConstants {
   public static final String CREDENTIAL_TYPE = "credential-type";
   public static final String EXPIRE_TIME = "expire-time";
 
+  public static final String S3_TOKEN_CREDENTIAL_TYPE = "s3-token";
+  public static final String S3_SECRET_KEY_CREDENTIAL_TYPE = "s3-secret-key";
+
   private CredentialConstants() {}
 }

catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java

@@ -30,6 +30,12 @@ public class S3Properties {
   public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key";
   // The region of the S3 service.
   public static final String GRAVITINO_S3_REGION = "s3-region";
+  // S3 role arn
+  public static final String GRAVITINO_S3_ROLE_ARN = "s3-role-arn";
+  // S3 token
+  public static final String GRAVITINO_S3_TOKEN = "s3-session-token";
+  // S3 external id
+  public static final String GRAVITINO_S3_EXTERNAL_ID = "s3-external-id";
 
   private S3Properties() {}
 }

common/src/main/java/org/apache/gravitino/credential/CredentialUtils.java

@@ -0,0 +1,55 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.storage.S3Properties;
+
+public class CredentialUtils {
+  private static Map<String, String> icebergCredentialPropertyMap =
+      ImmutableMap.of(
+          S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID,
+          S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY,
+              IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY,
+          S3Properties.GRAVITINO_S3_TOKEN, IcebergConstants.ICEBERG_S3_TOKEN);
+
+  public static Map<String, String> toIcebergProperties(Credential credential) {
+    if (credential instanceof S3TokenCredential || credential instanceof S3SecretKeyCredential) {
+      return transformProperties(credential.getCredentialInfo(), icebergCredentialPropertyMap);
+    }
+    throw new UnsupportedOperationException(
+        "Couldn't convert " + credential.getCredentialType() + " credential to Iceberg properties");
+  }
+
+  private static Map<String, String> transformProperties(
+      Map<String, String> originProperties, Map<String, String> transformMap) {
+    HashMap<String, String> properties = new HashMap();
+    originProperties.forEach(
+        (k, v) -> {
+          if (transformMap.containsKey(k)) {
+            properties.put(transformMap.get(k), v);
+          }
+        });
+    return properties;
+  }
+}

common/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java

@@ -0,0 +1,56 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.gravitino.storage.S3Properties;
+
+public class S3SecretKeyCredential implements Credential {
+  private String accessKeyId;
+  private String secretAccessKey;
+
+  public S3SecretKeyCredential(String accessKeyId, String secretAccessKey) {
+    Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null");
+    Preconditions.checkNotNull(secretAccessKey, "S3 secret access key should not null");
+
+    this.accessKeyId = accessKeyId;
+    this.secretAccessKey = secretAccessKey;
+  }
+
+  @Override
+  public String getCredentialType() {
+    return CredentialConstants.S3_SECRET_KEY_CREDENTIAL_TYPE;
+  }
+
+  @Override
+  public long getExpireTime() {
+    return 0;
+  }
+
+  @Override
+  public Map<String, String> getCredentialInfo() {
+    return (new ImmutableMap.Builder<String, String>())
+        .put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId)
+        .put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey)
+        .build();
+  }
+}

common/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java

@@ -0,0 +1,63 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.gravitino.storage.S3Properties;
+
+public class S3TokenCredential implements Credential {
+  private String accessKeyId;
+  private String secretAccessKey;
+  private String sessionToken;
+  private long expireMs;
+
+  public S3TokenCredential(
+      String accessKeyId, String secretAccessKey, String sessionToken, long expireMs) {
+    Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null");
+    Preconditions.checkNotNull(secretAccessKey, "S3 secret access key should not null");
+    Preconditions.checkNotNull(sessionToken, "S3 session token should not null");
+
+    this.accessKeyId = accessKeyId;
+    this.secretAccessKey = secretAccessKey;
+    this.sessionToken = sessionToken;
+    this.expireMs = expireMs;
+  }
+
+  @Override
+  public String getCredentialType() {
+    return CredentialConstants.S3_TOKEN_CREDENTIAL_TYPE;
+  }
+
+  @Override
+  public long getExpireTime() {
+    return expireMs;
+  }
+
+  @Override
+  public Map<String, String> getCredentialInfo() {
+    return (new ImmutableMap.Builder<String, String>())
+        .put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId)
+        .put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey)
+        .put(S3Properties.GRAVITINO_S3_TOKEN, sessionToken)
+        .build();
+  }
+}

common/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java

@@ -0,0 +1,54 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestCredentialUtils {
+
+  @Test
+  void testToIcebergProperties() {
+    S3TokenCredential s3TokenCredential = new S3TokenCredential("key", "secret", "token", 0);
+    Map<String, String> icebergProperties = CredentialUtils.toIcebergProperties(s3TokenCredential);
+    Map<String, String> expectedProperties =
+        ImmutableMap.of(
+            IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID,
+            "key",
+            IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY,
+            "secret",
+            IcebergConstants.ICEBERG_S3_TOKEN,
+            "token");
+    Assertions.assertEquals(expectedProperties, icebergProperties);
+
+    S3SecretKeyCredential secretKeyCredential = new S3SecretKeyCredential("key", "secret");
+    icebergProperties = CredentialUtils.toIcebergProperties(secretKeyCredential);
+    expectedProperties =
+        ImmutableMap.of(
+            IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID,
+            "key",
+            IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY,
+            "secret");
+    Assertions.assertEquals(expectedProperties, icebergProperties);
+  }
+}

core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java

@@ -0,0 +1,110 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package org.apache.gravitino.credential.config;
+
+import java.util.Map;
+import javax.validation.constraints.NotNull;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.Config;
+import org.apache.gravitino.config.ConfigBuilder;
+import org.apache.gravitino.config.ConfigConstants;
+import org.apache.gravitino.config.ConfigEntry;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.storage.S3Properties;
+
+public class S3CredentialConfig extends Config {
+
+  public static final ConfigEntry<String> S3_REGION =
+      new ConfigBuilder(S3Properties.GRAVITINO_S3_REGION)
+          .doc("The region of the S3 service")
+          .version(ConfigConstants.VERSION_0_7_0)
+          .stringConf()
+          .create();
+
+  public static final ConfigEntry<String> S3_ACCESS_KEY_ID =
+      new ConfigBuilder(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID)
+          .doc("The static access key ID used to access S3 data")
+          .version(ConfigConstants.VERSION_0_7_0)
+          .stringConf()
+          .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG)
+          .create();
+
+  public static final ConfigEntry<String> S3_SECRET_ACCESS_KEY =
+      new ConfigBuilder(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY)
+          .doc("The static secret access key used to access S3 data")
+          .version(ConfigConstants.VERSION_0_7_0)
+          .stringConf()
+          .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG)
+          .create();
+
+  public static final ConfigEntry<String> S3_ROLE_ARN =
+      new ConfigBuilder(S3Properties.GRAVITINO_S3_ROLE_ARN)
+          .doc("S3 role arn")
+          .version(ConfigConstants.VERSION_0_7_0)
+          .stringConf()
+          .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG)
+          .create();
+
+  public static final ConfigEntry<String> S3_EXTERNAL_ID =
+      new ConfigBuilder(S3Properties.GRAVITINO_S3_EXTERNAL_ID)
+          .doc("S3 external ID")
+          .version(ConfigConstants.VERSION_0_7_0)
+          .stringConf()
+          .create();
+
+  public static final ConfigEntry<Integer> S3_TOKEN_EXPIRE_SECS =
+      new ConfigBuilder(CredentialConstants.EXPIRE_TIME)
+          .doc("S3 token expire seconds")
+          .version(ConfigConstants.VERSION_0_7_0)
+          .intConf()
+          .createWithDefault(3600);
+
+  public S3CredentialConfig(Map<String, String> properties) {
+    super(false);
+    loadFromMap(properties, k -> true);
+  }
+
+  @NotNull
+  public String s3RoleArn() {
+    return this.get(S3_ROLE_ARN);
+  }
+
+  @NotNull
+  public String accessKeyID() {
+    return this.get(S3_ACCESS_KEY_ID);
+  }
+
+  @NotNull
+  public String secretAccessKey() {
+    return this.get(S3_SECRET_ACCESS_KEY);
+  }
+
+  public String region() {
+    return this.get(S3_REGION);
+  }
+
+  public String externalID() {
+    return this.get(S3_EXTERNAL_ID);
+  }
+
+  public Integer tokenExpireSecs() {
+    return this.get(S3_TOKEN_EXPIRE_SECS);
+  }
+}