From 44f0f1d3a1e3d2693edbb89f170b08c01124d2bd Mon Sep 17 00:00:00 2001 From: fanng Date: Wed, 18 Sep 2024 16:22:00 +0800 Subject: [PATCH] s3 credential --- LICENSE | 3 + build.gradle.kts | 3 +- .../lakehouse/iceberg/IcebergConstants.java | 1 + .../credential/CredentialConstants.java | 3 + .../gravitino/storage/S3Properties.java | 6 + .../gravitino/credential/CredentialUtils.java | 55 ++ .../credential/S3SecretKeyCredential.java | 56 ++ .../credential/S3TokenCredential.java | 63 +++ .../credential/TestCredentialUtils.java | 54 ++ .../credential/config/S3CredentialConfig.java | 110 ++++ credential/build.gradle.kts | 22 + credential/s3/build.gradle.kts | 85 +++ .../credential/aws/S3SecretKeyProvider.java | 54 ++ .../credential/aws/S3TokenProvider.java | 241 +++++++++ ...he.gravitino.credential.CredentialProvider | 20 + .../credential/aws/TestS3TokenProvider.java1 | 482 ++++++++++++++++++ gradle/libs.versions.toml | 2 + .../common/ops/IcebergCatalogWrapper.java | 7 +- iceberg/iceberg-rest-server/build.gradle.kts | 2 + settings.gradle.kts | 1 + 20 files changed, 1266 insertions(+), 4 deletions(-) create mode 100644 common/src/main/java/org/apache/gravitino/credential/CredentialUtils.java create mode 100644 common/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java create mode 100644 common/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java create mode 100644 common/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java create mode 100644 credential/build.gradle.kts create mode 100644 credential/s3/build.gradle.kts create mode 100644 credential/s3/src/main/java/org/apache/gravitino/credential/aws/S3SecretKeyProvider.java create mode 100644 credential/s3/src/main/java/org/apache/gravitino/credential/aws/S3TokenProvider.java create mode 100644 credential/s3/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider create mode 100644 credential/s3/src/test/java/org/apache/gravitino/credential/aws/TestS3TokenProvider.java1 diff --git a/LICENSE b/LICENSE index 8b3617a3404..d4d145ba137 100644 --- a/LICENSE +++ b/LICENSE @@ -251,6 +251,9 @@ ./iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergHiveCachedClientPool.java ./gradlew + Apache Polaris + ./credential/src/main/java/org/apache/gravitino/credential/aws/S3TokenProvider.java + Apache Paimon ./catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java diff --git a/build.gradle.kts b/build.gradle.kts index e18b5c56ca9..a4edbd2320f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -741,7 +741,7 @@ tasks { register("copySubprojectDependencies", Copy::class) { subprojects.forEach() { if (!it.name.startsWith("catalog") && - !it.name.startsWith("authorization") && + !it.name.startsWith("authorization") && !it.name.startsWith("credential") && !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") ) { @@ -756,6 +756,7 @@ tasks { if (!it.name.startsWith("catalog") && !it.name.startsWith("client") && !it.name.startsWith("authorization") && + !it.name.startsWith("credential") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java index 21462b9ca91..755195a70dc 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java @@ -40,6 +40,7 @@ public class IcebergConstants { public static final String ICEBERG_S3_ENDPOINT = "s3.endpoint"; public static final String ICEBERG_S3_ACCESS_KEY_ID = "s3.access-key-id"; public static final String ICEBERG_S3_SECRET_ACCESS_KEY = "s3.secret-access-key"; + public static final String ICEBERG_S3_TOKEN = "s3.session-token"; public static final String AWS_S3_REGION = "client.region"; public static final String ICEBERG_OSS_ENDPOINT = "oss.endpoint"; diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index 2e431b6d090..d4cc76e7f1c 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -23,5 +23,8 @@ public class CredentialConstants { public static final String CREDENTIAL_TYPE = "credential-type"; public static final String EXPIRE_TIME = "expire-time"; + public static final String S3_TOKEN_CREDENTIAL_TYPE = "s3-token"; + public static final String S3_SECRET_KEY_CREDENTIAL_TYPE = "s3-secret-key"; + private CredentialConstants() {} } diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java index 9775bebc8a5..1351219719e 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java @@ -30,6 +30,12 @@ public class S3Properties { public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; // The region of the S3 service. public static final String GRAVITINO_S3_REGION = "s3-region"; + // S3 role arn + public static final String GRAVITINO_S3_ROLE_ARN = "s3-role-arn"; + // S3 token + public static final String GRAVITINO_S3_TOKEN = "s3-session-token"; + // S3 external id + public static final String GRAVITINO_S3_EXTERNAL_ID = "s3-external-id"; private S3Properties() {} } diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialUtils.java new file mode 100644 index 00000000000..9c885c944f5 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialUtils.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.storage.S3Properties; + +public class CredentialUtils { + private static Map icebergCredentialPropertyMap = + ImmutableMap.of( + S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID, + S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, + IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY, + S3Properties.GRAVITINO_S3_TOKEN, IcebergConstants.ICEBERG_S3_TOKEN); + + public static Map toIcebergProperties(Credential credential) { + if (credential instanceof S3TokenCredential || credential instanceof S3SecretKeyCredential) { + return transformProperties(credential.getCredentialInfo(), icebergCredentialPropertyMap); + } + throw new UnsupportedOperationException( + "Couldn't convert " + credential.getCredentialType() + " credential to Iceberg properties"); + } + + private static Map transformProperties( + Map originProperties, Map transformMap) { + HashMap properties = new HashMap(); + originProperties.forEach( + (k, v) -> { + if (transformMap.containsKey(k)) { + properties.put(transformMap.get(k), v); + } + }); + return properties; + } +} diff --git a/common/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java b/common/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java new file mode 100644 index 00000000000..a7b73e92f67 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.S3Properties; + +public class S3SecretKeyCredential implements Credential { + private String accessKeyId; + private String secretAccessKey; + + public S3SecretKeyCredential(String accessKeyId, String secretAccessKey) { + Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null"); + Preconditions.checkNotNull(secretAccessKey, "S3 secret access key should not null"); + + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + } + + @Override + public String getCredentialType() { + return CredentialConstants.S3_SECRET_KEY_CREDENTIAL_TYPE; + } + + @Override + public long getExpireTime() { + return 0; + } + + @Override + public Map getCredentialInfo() { + return (new ImmutableMap.Builder()) + .put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId) + .put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey) + .build(); + } +} diff --git a/common/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java b/common/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java new file mode 100644 index 00000000000..6913fb6d0c5 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.S3Properties; + +public class S3TokenCredential implements Credential { + private String accessKeyId; + private String secretAccessKey; + private String sessionToken; + private long expireMs; + + public S3TokenCredential( + String accessKeyId, String secretAccessKey, String sessionToken, long expireMs) { + Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null"); + Preconditions.checkNotNull(secretAccessKey, "S3 secret access key should not null"); + Preconditions.checkNotNull(sessionToken, "S3 session token should not null"); + + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + this.sessionToken = sessionToken; + this.expireMs = expireMs; + } + + @Override + public String getCredentialType() { + return CredentialConstants.S3_TOKEN_CREDENTIAL_TYPE; + } + + @Override + public long getExpireTime() { + return expireMs; + } + + @Override + public Map getCredentialInfo() { + return (new ImmutableMap.Builder()) + .put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId) + .put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey) + .put(S3Properties.GRAVITINO_S3_TOKEN, sessionToken) + .build(); + } +} diff --git a/common/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java b/common/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java new file mode 100644 index 00000000000..f7b58a06cd4 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCredentialUtils { + + @Test + void testToIcebergProperties() { + S3TokenCredential s3TokenCredential = new S3TokenCredential("key", "secret", "token", 0); + Map icebergProperties = CredentialUtils.toIcebergProperties(s3TokenCredential); + Map expectedProperties = + ImmutableMap.of( + IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID, + "key", + IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY, + "secret", + IcebergConstants.ICEBERG_S3_TOKEN, + "token"); + Assertions.assertEquals(expectedProperties, icebergProperties); + + S3SecretKeyCredential secretKeyCredential = new S3SecretKeyCredential("key", "secret"); + icebergProperties = CredentialUtils.toIcebergProperties(secretKeyCredential); + expectedProperties = + ImmutableMap.of( + IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID, + "key", + IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY, + "secret"); + Assertions.assertEquals(expectedProperties, icebergProperties); + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java new file mode 100644 index 00000000000..e00f4c8222a --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential.config; + +import java.util.Map; +import javax.validation.constraints.NotNull; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Config; +import org.apache.gravitino.config.ConfigBuilder; +import org.apache.gravitino.config.ConfigConstants; +import org.apache.gravitino.config.ConfigEntry; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.storage.S3Properties; + +public class S3CredentialConfig extends Config { + + public static final ConfigEntry S3_REGION = + new ConfigBuilder(S3Properties.GRAVITINO_S3_REGION) + .doc("The region of the S3 service") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .create(); + + public static final ConfigEntry S3_ACCESS_KEY_ID = + new ConfigBuilder(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID) + .doc("The static access key ID used to access S3 data") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry S3_SECRET_ACCESS_KEY = + new ConfigBuilder(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY) + .doc("The static secret access key used to access S3 data") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry S3_ROLE_ARN = + new ConfigBuilder(S3Properties.GRAVITINO_S3_ROLE_ARN) + .doc("S3 role arn") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry S3_EXTERNAL_ID = + new ConfigBuilder(S3Properties.GRAVITINO_S3_EXTERNAL_ID) + .doc("S3 external ID") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .create(); + + public static final ConfigEntry S3_TOKEN_EXPIRE_SECS = + new ConfigBuilder(CredentialConstants.EXPIRE_TIME) + .doc("S3 token expire seconds") + .version(ConfigConstants.VERSION_0_7_0) + .intConf() + .createWithDefault(3600); + + public S3CredentialConfig(Map properties) { + super(false); + loadFromMap(properties, k -> true); + } + + @NotNull + public String s3RoleArn() { + return this.get(S3_ROLE_ARN); + } + + @NotNull + public String accessKeyID() { + return this.get(S3_ACCESS_KEY_ID); + } + + @NotNull + public String secretAccessKey() { + return this.get(S3_SECRET_ACCESS_KEY); + } + + public String region() { + return this.get(S3_REGION); + } + + public String externalID() { + return this.get(S3_EXTERNAL_ID); + } + + public Integer tokenExpireSecs() { + return this.get(S3_TOKEN_EXPIRE_SECS); + } +} diff --git a/credential/build.gradle.kts b/credential/build.gradle.kts new file mode 100644 index 00000000000..043fbfec673 --- /dev/null +++ b/credential/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +tasks.all { + enabled = false +} \ No newline at end of file diff --git a/credential/s3/build.gradle.kts b/credential/s3/build.gradle.kts new file mode 100644 index 00000000000..40ff94ecc1c --- /dev/null +++ b/credential/s3/build.gradle.kts @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "credential-s3" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +dependencies { + implementation(project(":catalogs:catalog-common")) + implementation(project(":core")) { + exclude("*") + } + implementation(project(":common")) { + exclude("*") + } + implementation(libs.bundles.log4j) + implementation(libs.commons.lang3) + implementation(libs.guava) + + annotationProcessor(libs.lombok) + + // todo: change to compile only + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:iam") + implementation("software.amazon.awssdk:iam-policy-builder") + implementation("software.amazon.awssdk:s3") + implementation("software.amazon.awssdk:sts") + implementation("software.amazon.awssdk:kms") + + // compileOnly(platform(libs.awssdk.bom)) + // compileOnly("software.amazon.awssdk:iam") + // compileOnly("software.amazon.awssdk:iam-policy-builder") + // compileOnly("software.amazon.awssdk:s3") + // compileOnly("software.amazon.awssdk:sts") + // compileOnly("software.amazon.awssdk:kms") + + compileOnly(libs.lombok) + + testImplementation(platform(libs.awssdk.bom)) + testImplementation("software.amazon.awssdk:iam") + testImplementation("software.amazon.awssdk:iam-policy-builder") + testImplementation("software.amazon.awssdk:s3") + testImplementation("software.amazon.awssdk:sts") + testImplementation("software.amazon.awssdk:kms") + + testImplementation(libs.mockito.core) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.slf4j.api) + + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks.test { + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/test/**") + } else { + dependsOn(tasks.jar) + } +} + +tasks.getByName("generateMetadataFileForMavenJavaPublication") { + dependsOn("copyDepends") +} diff --git a/credential/s3/src/main/java/org/apache/gravitino/credential/aws/S3SecretKeyProvider.java b/credential/s3/src/main/java/org/apache/gravitino/credential/aws/S3SecretKeyProvider.java new file mode 100644 index 00000000000..03e23a8193d --- /dev/null +++ b/credential/s3/src/main/java/org/apache/gravitino/credential/aws/S3SecretKeyProvider.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential.aws; + +import java.util.Map; +import org.apache.gravitino.credential.Context; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.S3SecretKeyCredential; +import org.apache.gravitino.credential.config.S3CredentialConfig; + +public class S3SecretKeyProvider implements CredentialProvider { + + private String accessKey; + private String secretKey; + + @Override + public void initialize(Map properties) { + S3CredentialConfig s3CredentialConfig = new S3CredentialConfig(properties); + this.accessKey = s3CredentialConfig.accessKeyID(); + this.secretKey = s3CredentialConfig.secretAccessKey(); + } + + @Override + public void stop() {} + + @Override + public String credentialType() { + return CredentialConstants.S3_SECRET_KEY_CREDENTIAL_TYPE; + } + + @Override + public Credential getCredential(Context context) { + return new S3SecretKeyCredential(accessKey, secretKey); + } +} diff --git a/credential/s3/src/main/java/org/apache/gravitino/credential/aws/S3TokenProvider.java b/credential/s3/src/main/java/org/apache/gravitino/credential/aws/S3TokenProvider.java new file mode 100644 index 00000000000..bffec96b37b --- /dev/null +++ b/credential/s3/src/main/java/org/apache/gravitino/credential/aws/S3TokenProvider.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential.aws; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.credential.Context; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.LocationContext; +import org.apache.gravitino.credential.S3TokenCredential; +import org.apache.gravitino.credential.config.S3CredentialConfig; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.policybuilder.iam.IamStatement; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; +import software.amazon.awssdk.services.sts.model.Credentials; + +// io/polaris/core/storage/aws/AwsCredentialsStorageIntegration +public class S3TokenProvider implements CredentialProvider { + private StsClient stsClient; + private String roleArn; + private String externalID; + private int tokenExpireSecs; + + @Override + public void initialize(Map properties) { + S3CredentialConfig s3CredentialConfig = new S3CredentialConfig(properties); + this.roleArn = s3CredentialConfig.s3RoleArn(); + this.externalID = s3CredentialConfig.externalID(); + this.tokenExpireSecs = s3CredentialConfig.tokenExpireSecs(); + this.stsClient = createStsClient(s3CredentialConfig); + } + + @Override + public void stop() { + if (stsClient != null) { + stsClient.close(); + } + } + + @Override + public String credentialType() { + return CredentialConstants.S3_TOKEN_CREDENTIAL_TYPE; + } + + @Override + public Credential getCredential(Context context) { + if (!(context instanceof LocationContext)) { + return null; + } + LocationContext locationContext = (LocationContext) context; + Credentials s3Credentials = + getS3Token( + roleArn, + locationContext.getReadLocations(), + locationContext.getWriteLocations(), + locationContext.getUserName()); + return new S3TokenCredential( + s3Credentials.accessKeyId(), + s3Credentials.secretAccessKey(), + s3Credentials.sessionToken(), + s3Credentials.expiration().toEpochMilli()); + } + + private StsClient createStsClient(S3CredentialConfig s3CredentialConfig) { + AwsCredentialsProvider credentialsProvider = + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + s3CredentialConfig.accessKeyID(), s3CredentialConfig.secretAccessKey())); + String region = s3CredentialConfig.region(); + if (StringUtils.isNotBlank(region)) { + return StsClient.builder() + .credentialsProvider(credentialsProvider) + .region(Region.of(region)) + .build(); + } else { + return StsClient.builder().credentialsProvider(credentialsProvider).build(); + } + } + + private IamPolicy getPolicy( + String roleArn, Set readLocations, Set writeLocations) { + IamPolicy.Builder policyBuilder = IamPolicy.builder(); + IamStatement.Builder allowGetObjectStatementBuilder = + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:GetObject") + .addAction("s3:GetObjectVersion"); + Map bucketListStatmentBuilder = new HashMap<>(); + Map bucketGetLocationStatmentBuilder = new HashMap<>(); + + String arnPrefix = getArnPrefixFor(roleArn); + Stream.concat(readLocations.stream(), writeLocations.stream()) + .distinct() + .forEach( + location -> { + URI uri = URI.create(location); + allowGetObjectStatementBuilder.addResource( + IamResource.create(getS3UriWithArn(arnPrefix, uri))); + String bucket = arnPrefix + getBucket(uri); + bucketListStatmentBuilder + .computeIfAbsent( + bucket, + (String key) -> + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:ListBucket") + .addResource(key)) + .addCondition( + IamConditionOperator.STRING_LIKE, + "s3:prefix", + concatFilePrefixes(trimLeadingSlash(uri.getPath()), "*", "/")); + bucketGetLocationStatmentBuilder.computeIfAbsent( + bucket, + key -> + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:GetBucketLocation") + .addResource(key)); + }); + + if (!writeLocations.isEmpty()) { + IamStatement.Builder allowPutObjectStatementBuilder = + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:PutObject") + .addAction("s3:DeleteObject"); + writeLocations.forEach( + location -> { + URI uri = URI.create(location); + allowPutObjectStatementBuilder.addResource( + IamResource.create(getS3UriWithArn(arnPrefix, uri))); + }); + policyBuilder.addStatement(allowPutObjectStatementBuilder.build()); + } + if (!bucketListStatmentBuilder.isEmpty()) { + bucketListStatmentBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + } else { + // add list privilege with 0 resources + policyBuilder.addStatement( + IamStatement.builder().effect(IamEffect.ALLOW).addAction("s3:ListBucket").build()); + } + + bucketGetLocationStatmentBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + return policyBuilder.addStatement(allowGetObjectStatementBuilder.build()).build(); + } + + private String getS3UriWithArn(String arnPrefix, URI uri) { + return arnPrefix + concatFilePrefixes(parseS3Path(uri), "*", "/"); + } + + private String getArnPrefixFor(String roleArn) { + if (roleArn.contains("aws-cn")) { + return "arn:aws-cn:s3:::"; + } else if (roleArn.contains("aws-us-gov")) { + return "arn:aws-us-gov:s3:::"; + } else { + return "arn:aws:s3:::"; + } + } + + public static String concatFilePrefixes(String leftPath, String rightPath, String fileSep) { + if (leftPath.endsWith(fileSep) && rightPath.startsWith(fileSep)) { + return leftPath + rightPath.substring(1); + } else if (!leftPath.endsWith(fileSep) && !rightPath.startsWith(fileSep)) { + return leftPath + fileSep + rightPath; + } else { + return leftPath + rightPath; + } + } + + private static String parseS3Path(URI uri) { + String bucket = uri.getHost(); + String path = trimLeadingSlash(uri.getPath()); + return String.join( + "/", Stream.of(bucket, path).filter(Objects::nonNull).toArray(String[]::new)); + } + + private static String trimLeadingSlash(String path) { + if (path.startsWith("/")) { + path = path.substring(1); + } + return path; + } + + private static String getBucket(URI uri) { + return uri.getHost(); + } + + private Credentials getS3Token( + String roleArn, Set readLocations, Set writeLocations, String userName) { + IamPolicy policy = getPolicy(roleArn, readLocations, writeLocations); + AssumeRoleRequest.Builder builder = + AssumeRoleRequest.builder() + .roleArn(roleArn) + .roleSessionName("gravitino_" + userName) + .durationSeconds(tokenExpireSecs) + .policy(policy.toJson()); + if (StringUtils.isNotBlank(externalID)) { + builder.externalId(externalID); + } + AssumeRoleResponse response = stsClient.assumeRole(builder.build()); + return response.credentials(); + } +} diff --git a/credential/s3/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/credential/s3/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider new file mode 100644 index 00000000000..78edd79840c --- /dev/null +++ b/credential/s3/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.credential.aws.S3TokenProvider +org.apache.gravitino.credential.aws.S3SecretKeyProvider \ No newline at end of file diff --git a/credential/s3/src/test/java/org/apache/gravitino/credential/aws/TestS3TokenProvider.java1 b/credential/s3/src/test/java/org/apache/gravitino/credential/aws/TestS3TokenProvider.java1 new file mode 100644 index 00000000000..7bec12b228c --- /dev/null +++ b/credential/s3/src/test/java/org/apache/gravitino/credential/aws/TestS3TokenProvider.java1 @@ -0,0 +1,482 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential.aws; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; +import software.amazon.awssdk.services.sts.model.Credentials; + +public class TestS3TokenProvider { + + public static final AssumeRoleResponse ASSUME_ROLE_RESPONSE = + AssumeRoleResponse.builder() + .credentials( + Credentials.builder() + .accessKeyId("accessKey") + .secretAccessKey("secretKey") + .sessionToken("sess") + .build()) + .build(); + + public static final String AWS_PARTITION = "aws"; + + @Test + public void testGetSubscopedCreds() { + StsClient stsClient = Mockito.mock(StsClient.class); + String roleARN = "arn:aws:iam::012345678901:role/jdoe"; + String externalId = "externalId"; + Mockito.when(stsClient.assumeRole(Mockito.isA(AssumeRoleRequest.class))) + .thenAnswer( + invocation -> { + assertThat(invocation.getArguments()[0]) + .isInstanceOf(AssumeRoleRequest.class) + .asInstanceOf(InstanceOfAssertFactories.type(AssumeRoleRequest.class)) + .returns(externalId, AssumeRoleRequest::externalId) + .returns(roleARN, AssumeRoleRequest::roleArn); + return ASSUME_ROLE_RESPONSE; + }); + String warehouseDir = "s3://bucket/path/to/warehouse"; + EnumMap credentials = + new AwsCredentialsStorageIntegration(stsClient) + .getSubscopedCreds( + Mockito.mock(PolarisDiagnostics.class), + new AwsStorageConfigurationInfo( + PolarisStorageConfigurationInfo.StorageType.S3, + List.of(warehouseDir), + roleARN, + externalId), + true, + Set.of(warehouseDir + "/namespace/table"), + Set.of(warehouseDir + "/namespace/table")); + assertThat(credentials) + .isNotEmpty() + .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") + .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") + .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey"); + } + + @ParameterizedTest + @ValueSource(strings = {AWS_PARTITION, "aws-cn", "aws-us-gov"}) + public void testGetSubscopedCredsInlinePolicy(String awsPartition) { + PolarisStorageConfigurationInfo.StorageType storageType = + PolarisStorageConfigurationInfo.StorageType.S3; + String roleARN; + switch (awsPartition) { + case AWS_PARTITION: + roleARN = "arn:aws:iam::012345678901:role/jdoe"; + break; + case "aws-cn": + roleARN = "arn:aws-cn:iam::012345678901:role/jdoe"; + break; + case "aws-us-gov": + roleARN = "arn:aws-us-gov:iam::012345678901:role/jdoe"; + break; + default: + throw new IllegalArgumentException("Unknown aws partition: " + awsPartition); + } + ; + StsClient stsClient = Mockito.mock(StsClient.class); + String externalId = "externalId"; + String bucket = "bucket"; + String warehouseKeyPrefix = "path/to/warehouse"; + String firstPath = warehouseKeyPrefix + "/namespace/table"; + String secondPath = warehouseKeyPrefix + "/oldnamespace/table"; + Mockito.when(stsClient.assumeRole(Mockito.isA(AssumeRoleRequest.class))) + .thenAnswer( + invocation -> { + assertThat(invocation.getArguments()[0]) + .isInstanceOf(AssumeRoleRequest.class) + .asInstanceOf(InstanceOfAssertFactories.type(AssumeRoleRequest.class)) + .extracting(AssumeRoleRequest::policy) + .extracting(IamPolicy::fromJson) + .satisfies( + policy -> { + assertThat(policy) + .extracting(IamPolicy::statements) + .asInstanceOf(InstanceOfAssertFactories.list(IamStatement.class)) + .hasSize(4) + .satisfiesExactly( + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .returns( + List.of( + IamResource.create( + s3Arn(awsPartition, bucket, firstPath))), + IamStatement::resources) + .returns( + List.of( + IamAction.create("s3:PutObject"), + IamAction.create("s3:DeleteObject")), + IamStatement::actions), + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .returns( + List.of( + IamResource.create( + s3Arn(awsPartition, bucket, null))), + IamStatement::resources) + .returns( + List.of(IamAction.create("s3:ListBucket")), + IamStatement::actions) + .returns( + List.of( + IamResource.create( + s3Arn(awsPartition, bucket, null))), + IamStatement::resources) + .satisfies( + st -> + assertThat(st.conditions()) + .containsExactlyInAnyOrder( + IamCondition.builder() + .operator( + IamConditionOperator.STRING_LIKE) + .key("s3:prefix") + .value(secondPath + "/*") + .build(), + IamCondition.builder() + .operator( + IamConditionOperator.STRING_LIKE) + .key("s3:prefix") + .value(firstPath + "/*") + .build())), + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .satisfies( + st -> + assertThat(st.resources()) + .contains( + IamResource.create( + s3Arn(awsPartition, bucket, null)))) + .returns( + List.of(IamAction.create("s3:GetBucketLocation")), + IamStatement::actions), + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .satisfies( + st -> + assertThat(st.resources()) + .containsExactlyInAnyOrder( + IamResource.create( + s3Arn(awsPartition, bucket, firstPath)), + IamResource.create( + s3Arn( + awsPartition, bucket, secondPath)))) + .returns( + List.of( + IamAction.create("s3:GetObject"), + IamAction.create("s3:GetObjectVersion")), + IamStatement::actions)); + }); + return ASSUME_ROLE_RESPONSE; + }); + EnumMap credentials = + new AwsCredentialsStorageIntegration(stsClient) + .getSubscopedCreds( + Mockito.mock(PolarisDiagnostics.class), + new AwsStorageConfigurationInfo( + storageType, + List.of(s3Path(bucket, warehouseKeyPrefix, storageType)), + roleARN, + externalId), + true, + Set.of( + s3Path(bucket, firstPath, storageType), + s3Path(bucket, secondPath, storageType)), + Set.of(s3Path(bucket, firstPath, storageType))); + assertThat(credentials) + .isNotEmpty() + .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") + .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") + .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey"); + } + + @Test + public void testGetSubscopedCredsInlinePolicyWithoutList() { + StsClient stsClient = Mockito.mock(StsClient.class); + String roleARN = "arn:aws:iam::012345678901:role/jdoe"; + String externalId = "externalId"; + String bucket = "bucket"; + String warehouseKeyPrefix = "path/to/warehouse"; + String firstPath = warehouseKeyPrefix + "/namespace/table"; + String secondPath = warehouseKeyPrefix + "/oldnamespace/table"; + Mockito.when(stsClient.assumeRole(Mockito.isA(AssumeRoleRequest.class))) + .thenAnswer( + invocation -> { + assertThat(invocation.getArguments()[0]) + .isInstanceOf(AssumeRoleRequest.class) + .asInstanceOf(InstanceOfAssertFactories.type(AssumeRoleRequest.class)) + .extracting(AssumeRoleRequest::policy) + .extracting(IamPolicy::fromJson) + .satisfies( + policy -> { + assertThat(policy) + .extracting(IamPolicy::statements) + .asInstanceOf(InstanceOfAssertFactories.list(IamStatement.class)) + .hasSize(3) + .satisfiesExactly( + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .returns( + List.of( + IamResource.create( + s3Arn(AWS_PARTITION, bucket, firstPath))), + IamStatement::resources) + .returns( + List.of( + IamAction.create("s3:PutObject"), + IamAction.create("s3:DeleteObject")), + IamStatement::actions), + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .satisfies( + st -> + assertThat(st.resources()) + .contains( + IamResource.create( + s3Arn(AWS_PARTITION, bucket, null)))) + .returns( + List.of(IamAction.create("s3:GetBucketLocation")), + IamStatement::actions), + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .satisfies( + st -> + assertThat(st.resources()) + .containsExactlyInAnyOrder( + IamResource.create( + s3Arn( + AWS_PARTITION, bucket, firstPath)), + IamResource.create( + s3Arn( + AWS_PARTITION, + bucket, + secondPath)))) + .returns( + List.of( + IamAction.create("s3:GetObject"), + IamAction.create("s3:GetObjectVersion")), + IamStatement::actions)); + }); + return ASSUME_ROLE_RESPONSE; + }); + PolarisStorageConfigurationInfo.StorageType storageType = + PolarisStorageConfigurationInfo.StorageType.S3; + EnumMap credentials = + new AwsCredentialsStorageIntegration(stsClient) + .getSubscopedCreds( + Mockito.mock(PolarisDiagnostics.class), + new AwsStorageConfigurationInfo( + PolarisStorageConfigurationInfo.StorageType.S3, + List.of(s3Path(bucket, warehouseKeyPrefix, storageType)), + roleARN, + externalId), + false, /* allowList = false*/ + Set.of( + s3Path(bucket, firstPath, storageType), + s3Path(bucket, secondPath, storageType)), + Set.of(s3Path(bucket, firstPath, storageType))); + assertThat(credentials) + .isNotEmpty() + .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") + .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") + .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey"); + } + + @Test + public void testGetSubscopedCredsInlinePolicyWithoutWrites() { + StsClient stsClient = Mockito.mock(StsClient.class); + String roleARN = "arn:aws:iam::012345678901:role/jdoe"; + String externalId = "externalId"; + String bucket = "bucket"; + String warehouseKeyPrefix = "path/to/warehouse"; + String firstPath = warehouseKeyPrefix + "/namespace/table"; + String secondPath = warehouseKeyPrefix + "/oldnamespace/table"; + Mockito.when(stsClient.assumeRole(Mockito.isA(AssumeRoleRequest.class))) + .thenAnswer( + invocation -> { + assertThat(invocation.getArguments()[0]) + .isInstanceOf(AssumeRoleRequest.class) + .asInstanceOf(InstanceOfAssertFactories.type(AssumeRoleRequest.class)) + .extracting(AssumeRoleRequest::policy) + .extracting(IamPolicy::fromJson) + .satisfies( + policy -> { + assertThat(policy) + .extracting(IamPolicy::statements) + .asInstanceOf(InstanceOfAssertFactories.list(IamStatement.class)) + .hasSize(3) + .satisfiesExactly( + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .returns( + List.of( + IamResource.create( + s3Arn(AWS_PARTITION, bucket, null))), + IamStatement::resources) + .returns( + List.of(IamAction.create("s3:ListBucket")), + IamStatement::actions), + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .satisfies( + st -> + assertThat(st.resources()) + .contains( + IamResource.create( + s3Arn(AWS_PARTITION, bucket, null)))) + .returns( + List.of(IamAction.create("s3:GetBucketLocation")), + IamStatement::actions), + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .satisfies( + st -> + assertThat(st.resources()) + .containsExactlyInAnyOrder( + IamResource.create( + s3Arn( + AWS_PARTITION, bucket, firstPath)), + IamResource.create( + s3Arn( + AWS_PARTITION, + bucket, + secondPath)))) + .returns( + List.of( + IamAction.create("s3:GetObject"), + IamAction.create("s3:GetObjectVersion")), + IamStatement::actions)); + }); + return ASSUME_ROLE_RESPONSE; + }); + PolarisStorageConfigurationInfo.StorageType storageType = + PolarisStorageConfigurationInfo.StorageType.S3; + EnumMap credentials = + new AwsCredentialsStorageIntegration(stsClient) + .getSubscopedCreds( + Mockito.mock(PolarisDiagnostics.class), + new AwsStorageConfigurationInfo( + storageType, + List.of(s3Path(bucket, warehouseKeyPrefix, storageType)), + roleARN, + externalId), + true, /* allowList = true */ + Set.of( + s3Path(bucket, firstPath, storageType), + s3Path(bucket, secondPath, storageType)), + Set.of()); + assertThat(credentials) + .isNotEmpty() + .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") + .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") + .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey"); + } + + @Test + public void testGetSubscopedCredsInlinePolicyWithEmptyReadAndWrite() { + StsClient stsClient = Mockito.mock(StsClient.class); + String roleARN = "arn:aws:iam::012345678901:role/jdoe"; + String externalId = "externalId"; + String bucket = "bucket"; + String warehouseKeyPrefix = "path/to/warehouse"; + Mockito.when(stsClient.assumeRole(Mockito.isA(AssumeRoleRequest.class))) + .thenAnswer( + invocation -> { + assertThat(invocation.getArguments()[0]) + .isInstanceOf(AssumeRoleRequest.class) + .asInstanceOf(InstanceOfAssertFactories.type(AssumeRoleRequest.class)) + .extracting(AssumeRoleRequest::policy) + .extracting(IamPolicy::fromJson) + .satisfies( + policy -> { + assertThat(policy) + .extracting(IamPolicy::statements) + .asInstanceOf(InstanceOfAssertFactories.list(IamStatement.class)) + .hasSize(2) + .satisfiesExactly( + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .returns(List.of(), IamStatement::resources) + .returns( + List.of(IamAction.create("s3:ListBucket")), + IamStatement::actions), + statement -> + assertThat(statement) + .returns(IamEffect.ALLOW, IamStatement::effect) + .satisfies( + st -> assertThat(st.resources()).containsExactly()) + .returns( + List.of( + IamAction.create("s3:GetObject"), + IamAction.create("s3:GetObjectVersion")), + IamStatement::actions)); + }); + return ASSUME_ROLE_RESPONSE; + }); + EnumMap credentials = + new AwsCredentialsStorageIntegration(stsClient) + .getSubscopedCreds( + Mockito.mock(PolarisDiagnostics.class), + new AwsStorageConfigurationInfo( + PolarisStorageConfigurationInfo.StorageType.S3, + List.of( + s3Path( + bucket, + warehouseKeyPrefix, + PolarisStorageConfigurationInfo.StorageType.S3)), + roleARN, + externalId), + true, /* allowList = true */ + Set.of(), + Set.of()); + assertThat(credentials) + .isNotEmpty() + .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") + .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") + .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey"); + } + + private static String s3Arn(String partition, String bucket, String keyPrefix) { + String bucketArn = "arn:" + partition + ":s3:::" + bucket; + if (keyPrefix == null) { + return bucketArn; + } + return bucketArn + "/" + keyPrefix + "/*"; + } + + private static String s3Path( + String bucket, String keyPrefix, PolarisStorageConfigurationInfo.StorageType storageType) { + return storageType.getPrefix() + bucket + "/" + keyPrefix; + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6790962cf3d..359698f6116 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,6 +17,7 @@ # under the License. # [versions] +awssdk-bom = "2.28.3" junit = "5.8.1" protoc = "3.24.4" jackson = "2.15.2" @@ -104,6 +105,7 @@ datanucleus-rdbms = "4.1.19" datanucleus-jdo = "3.2.0-m3" [libraries] +awssdk-bom= { group = "software.amazon.awssdk", name = "bom", version.ref = "awssdk-bom" } protobuf-java = { group = "com.google.protobuf", name = "protobuf-java", version.ref = "protoc" } protobuf-java-util = { group = "com.google.protobuf", name = "protobuf-java-util", version.ref = "protoc" } jackson-databind = { group = "com.fasterxml.jackson.core", name = "jackson-databind", version.ref = "jackson" } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java index d1a0f986474..a4433f4c6b5 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java @@ -36,6 +36,7 @@ import org.apache.gravitino.credential.Credential; import org.apache.gravitino.credential.CredentialManager; import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.CredentialUtils; import org.apache.gravitino.credential.LocationContext; import org.apache.gravitino.iceberg.common.IcebergCatalogBackend; import org.apache.gravitino.iceberg.common.IcebergConfig; @@ -80,8 +81,6 @@ public class IcebergCatalogWrapper implements AutoCloseable { ImmutableSet.of( IcebergConstants.IO_IMPL, IcebergConstants.AWS_S3_REGION, - IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID, - IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY, IcebergConstants.ICEBERG_S3_ENDPOINT, IcebergConstants.ICEBERG_OSS_ENDPOINT, IcebergConstants.ICEBERG_OSS_ACCESS_KEY_ID, @@ -296,7 +295,9 @@ private Map getCatalogConfigToClient(String location) { credentialProvider.ifPresent( provider -> { Credential credential = provider.getCredential(locationContext); - configs.putAll(credential.toProperties()); + if (credential != null) { + configs.putAll(CredentialUtils.toIcebergProperties(credential)); + } }); return configs; } diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index 594e6d04208..78e0a114e4a 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -40,6 +40,8 @@ dependencies { implementation(project(":common")) { exclude("*") } + // todo remove + implementation(project(":credential:s3")) implementation(project(":iceberg:iceberg-common")) implementation(project(":server-common")) { exclude("*") diff --git a/settings.gradle.kts b/settings.gradle.kts index dc49f486726..cf41d5f2beb 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -69,3 +69,4 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") +include("credential:s3")