From 55e8df3d8a8b81e0647ae1741c720eedb9bff847 Mon Sep 17 00:00:00 2001 From: fanng Date: Wed, 18 Sep 2024 16:22:00 +0800 Subject: [PATCH 01/10] s3 credential --- .../credential/S3SecretKeyCredential.java | 71 ++++++ .../credential/S3TokenCredential.java | 86 +++++++ bundles/aws-bundle/build.gradle.kts | 8 + .../s3/credential/S3SecretKeyProvider.java | 54 ++++ .../s3/credential/S3TokenProvider.java | 241 ++++++++++++++++++ ...he.gravitino.credential.CredentialProvider | 20 ++ .../lakehouse/iceberg/IcebergConstants.java | 1 + .../credential/CredentialConstants.java | 2 + .../gravitino/storage/S3Properties.java | 4 + .../credential/CredentialPropertyUtils.java | 19 +- .../TestCredentialPropertiesUtils.java | 54 ++++ core/build.gradle.kts | 1 + .../credential/config/S3CredentialConfig.java | 110 ++++++++ gradle/libs.versions.toml | 7 + .../common/ops/IcebergCatalogWrapper.java | 2 - iceberg/iceberg-rest-server/build.gradle.kts | 2 + .../service/rest/IcebergTableOperations.java | 17 +- .../integration/test/IcebergRESTS3IT.java | 127 +++++++++ 18 files changed, 820 insertions(+), 6 deletions(-) create mode 100644 api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java create mode 100644 api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java create mode 100644 bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java create mode 100644 bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java create mode 100644 bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider create mode 100644 common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java create mode 100644 iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java diff --git a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java new file mode 100644 index 00000000000..dc7474f3834 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; + +/** S3 secret key credential. */ +public class S3SecretKeyCredential implements Credential { + + /** S3 secret key credential type. */ + public static final String S3_SECRET_KEY_CREDENTIAL_TYPE = "s3-secret-key"; + /** The static access key ID used to access S3 data. */ + public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id"; + /** The static secret access key used to access S3 data. */ + public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; + + private String accessKeyId; + private String secretAccessKey; + + /** + * Constructs an instance of {@link S3SecretKeyCredential} with the specified AWS S3 access key ID + * and secret access key. + * + * @param accessKeyId the AWS S3 access key ID used for authentication. + * @param secretAccessKey the AWS S3 secret access key used for authentication. + */ + public S3SecretKeyCredential(String accessKeyId, String secretAccessKey) { + Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null"); + Preconditions.checkNotNull(secretAccessKey, "S3 secret access key should not null"); + + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + } + + @Override + public String credentialType() { + return S3_SECRET_KEY_CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return 0; + } + + @Override + public Map credentialInfo() { + return (new ImmutableMap.Builder()) + .put(GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId) + .put(GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey) + .build(); + } +} diff --git a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java new file mode 100644 index 00000000000..fe448b38d5b --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; + +/** S3 token credential. */ +public class S3TokenCredential implements Credential { + + /** S3 token credential type. */ + public static final String S3_TOKEN_CREDENTIAL_TYPE = "s3-token"; + /** The static access key ID used to access S3 data. */ + public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id"; + /** The static secret access key used to access S3 data. */ + public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; + /** S3 session token. */ + public static final String GRAVITINO_S3_TOKEN = "s3-session-token"; + + private String accessKeyId; + private String secretAccessKey; + private String sessionToken; + private long expireTimeInMS; + + /** + * Constructs an instance of {@link S3SecretKeyCredential} with the specified AWS S3 access key ID + * and secret access key. + * + * @param accessKeyId the AWS S3 access key ID used for authentication. + * @param secretAccessKey the AWS S3 secret access key used for authentication. + * @param sessionToken AWS S3 access key ID used for authentication. + * @param expireTimeInMS AWS S3 access key ID used for authentication. + */ + public S3TokenCredential( + String accessKeyId, String secretAccessKey, String sessionToken, long expireTimeInMS) { + Preconditions.checkArgument( + StringUtils.isNotBlank(accessKeyId), "S3 access key Id should not be empty"); + Preconditions.checkArgument( + StringUtils.isNotBlank(secretAccessKey), "S3 secret access key should not be empty"); + Preconditions.checkArgument( + StringUtils.isNotBlank(sessionToken), "S3 session token should not be empty"); + + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + this.sessionToken = sessionToken; + this.expireTimeInMS = expireTimeInMS; + } + + @Override + public String credentialType() { + return S3_TOKEN_CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return expireTimeInMS; + } + + @Override + public Map credentialInfo() { + return (new ImmutableMap.Builder()) + .put(GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId) + .put(GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey) + .put(GRAVITINO_S3_TOKEN, sessionToken) + .build(); + } +} diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts index 741bdc414e1..a2671b05812 100644 --- a/bundles/aws-bundle/build.gradle.kts +++ b/bundles/aws-bundle/build.gradle.kts @@ -25,8 +25,16 @@ plugins { } dependencies { + compileOnly(project(":api")) + compileOnly(project(":core")) + compileOnly(project(":catalogs:catalog-common")) compileOnly(project(":catalogs:catalog-hadoop")) compileOnly(libs.hadoop3.common) + + implementation(libs.aws.iam) + implementation(libs.aws.policy) + implementation(libs.aws.sts) + implementation(libs.aws.kms) implementation(libs.hadoop3.aws) } diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java b/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java new file mode 100644 index 00000000000..9b4da890054 --- /dev/null +++ b/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.s3.credential; + +import java.util.Map; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialContext; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.S3SecretKeyCredential; +import org.apache.gravitino.credential.config.S3CredentialConfig; + +/** Generate S3 access key and secret key to access S3 data. */ +public class S3SecretKeyProvider implements CredentialProvider { + + private String accessKey; + private String secretKey; + + @Override + public void initialize(Map properties) { + S3CredentialConfig s3CredentialConfig = new S3CredentialConfig(properties); + this.accessKey = s3CredentialConfig.accessKeyID(); + this.secretKey = s3CredentialConfig.secretAccessKey(); + } + + @Override + public void close() {} + + @Override + public String credentialType() { + return S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE; + } + + @Override + public Credential getCredential(CredentialContext context) { + return new S3SecretKeyCredential(accessKey, secretKey); + } +} diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java b/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java new file mode 100644 index 00000000000..2d20b827da9 --- /dev/null +++ b/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.s3.credential; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialContext; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.PathBasedCredentialContext; +import org.apache.gravitino.credential.S3TokenCredential; +import org.apache.gravitino.credential.config.S3CredentialConfig; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.policybuilder.iam.IamStatement; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; +import software.amazon.awssdk.services.sts.model.Credentials; + +/** Generates S3 token to access S3 data. */ +public class S3TokenProvider implements CredentialProvider { + private StsClient stsClient; + private String roleArn; + private String externalID; + private int tokenExpireSecs; + + @Override + public void initialize(Map properties) { + S3CredentialConfig s3CredentialConfig = new S3CredentialConfig(properties); + this.roleArn = s3CredentialConfig.s3RoleArn(); + this.externalID = s3CredentialConfig.externalID(); + this.tokenExpireSecs = s3CredentialConfig.tokenExpireSecs(); + this.stsClient = createStsClient(s3CredentialConfig); + } + + @Override + public void close() { + if (stsClient != null) { + stsClient.close(); + } + } + + @Override + public String credentialType() { + return S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE; + } + + @Override + public Credential getCredential(CredentialContext context) { + if (!(context instanceof PathBasedCredentialContext)) { + return null; + } + PathBasedCredentialContext pathBasedCredentialContext = (PathBasedCredentialContext) context; + Credentials s3Token = + getS3Token( + roleArn, + pathBasedCredentialContext.getReadPaths(), + pathBasedCredentialContext.getWritePaths(), + pathBasedCredentialContext.getUserName()); + return new S3TokenCredential( + s3Token.accessKeyId(), + s3Token.secretAccessKey(), + s3Token.sessionToken(), + s3Token.expiration().toEpochMilli()); + } + + private StsClient createStsClient(S3CredentialConfig s3CredentialConfig) { + AwsCredentialsProvider credentialsProvider = + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + s3CredentialConfig.accessKeyID(), s3CredentialConfig.secretAccessKey())); + String region = s3CredentialConfig.region(); + if (StringUtils.isNotBlank(region)) { + return StsClient.builder() + .credentialsProvider(credentialsProvider) + .region(Region.of(region)) + .build(); + } else { + return StsClient.builder().credentialsProvider(credentialsProvider).build(); + } + } + + private IamPolicy getPolicy( + String roleArn, Set readLocations, Set writeLocations) { + IamPolicy.Builder policyBuilder = IamPolicy.builder(); + IamStatement.Builder allowGetObjectStatementBuilder = + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:GetObject") + .addAction("s3:GetObjectVersion"); + Map bucketListStatmentBuilder = new HashMap<>(); + Map bucketGetLocationStatmentBuilder = new HashMap<>(); + + String arnPrefix = getArnPrefix(roleArn); + Stream.concat(readLocations.stream(), writeLocations.stream()) + .distinct() + .forEach( + location -> { + URI uri = URI.create(location); + allowGetObjectStatementBuilder.addResource( + IamResource.create(getS3UriWithArn(arnPrefix, uri))); + String bucketArn = arnPrefix + getBucketName(uri); + bucketListStatmentBuilder + .computeIfAbsent( + bucketArn, + (String key) -> + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:ListBucket") + .addResource(key)) + .addCondition( + IamConditionOperator.STRING_LIKE, + "s3:prefix", + concatPathWithSep(trimLeadingSlash(uri.getPath()), "*", "/")); + bucketGetLocationStatmentBuilder.computeIfAbsent( + bucketArn, + key -> + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:GetBucketLocation") + .addResource(key)); + }); + + if (!writeLocations.isEmpty()) { + IamStatement.Builder allowPutObjectStatementBuilder = + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:PutObject") + .addAction("s3:DeleteObject"); + writeLocations.forEach( + location -> { + URI uri = URI.create(location); + allowPutObjectStatementBuilder.addResource( + IamResource.create(getS3UriWithArn(arnPrefix, uri))); + }); + policyBuilder.addStatement(allowPutObjectStatementBuilder.build()); + } + if (!bucketListStatmentBuilder.isEmpty()) { + bucketListStatmentBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + } else { + // add list privilege with 0 resources + policyBuilder.addStatement( + IamStatement.builder().effect(IamEffect.ALLOW).addAction("s3:ListBucket").build()); + } + + bucketGetLocationStatmentBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + return policyBuilder.addStatement(allowGetObjectStatementBuilder.build()).build(); + } + + private String getS3UriWithArn(String arnPrefix, URI uri) { + return arnPrefix + concatPathWithSep(removeSchemaFromS3Uri(uri), "*", "/"); + } + + private String getArnPrefix(String roleArn) { + if (roleArn.contains("aws-cn")) { + return "arn:aws-cn:s3:::"; + } else if (roleArn.contains("aws-us-gov")) { + return "arn:aws-us-gov:s3:::"; + } else { + return "arn:aws:s3:::"; + } + } + + private static String concatPathWithSep(String leftPath, String rightPath, String fileSep) { + if (leftPath.endsWith(fileSep) && rightPath.startsWith(fileSep)) { + return leftPath + rightPath.substring(1); + } else if (!leftPath.endsWith(fileSep) && !rightPath.startsWith(fileSep)) { + return leftPath + fileSep + rightPath; + } else { + return leftPath + rightPath; + } + } + + // Transform 's3://bucket/path' to /bucket/path + private static String removeSchemaFromS3Uri(URI uri) { + String bucket = uri.getHost(); + String path = trimLeadingSlash(uri.getPath()); + return String.join( + "/", Stream.of(bucket, path).filter(Objects::nonNull).toArray(String[]::new)); + } + + private static String trimLeadingSlash(String path) { + if (path.startsWith("/")) { + path = path.substring(1); + } + return path; + } + + private static String getBucketName(URI uri) { + return uri.getHost(); + } + + private Credentials getS3Token( + String roleArn, Set readLocations, Set writeLocations, String userName) { + IamPolicy policy = getPolicy(roleArn, readLocations, writeLocations); + AssumeRoleRequest.Builder builder = + AssumeRoleRequest.builder() + .roleArn(roleArn) + .roleSessionName("gravitino_" + userName) + .durationSeconds(tokenExpireSecs) + .policy(policy.toJson()); + if (StringUtils.isNotBlank(externalID)) { + builder.externalId(externalID); + } + AssumeRoleResponse response = stsClient.assumeRole(builder.build()); + return response.credentials(); + } +} diff --git a/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider new file mode 100644 index 00000000000..7349688d6af --- /dev/null +++ b/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.s3.credential.S3TokenProvider +org.apache.gravitino.s3.credential.S3SecretKeyProvider \ No newline at end of file diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java index 52e665579da..4d9e99eba5f 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java @@ -40,6 +40,7 @@ public class IcebergConstants { public static final String ICEBERG_S3_ENDPOINT = "s3.endpoint"; public static final String ICEBERG_S3_ACCESS_KEY_ID = "s3.access-key-id"; public static final String ICEBERG_S3_SECRET_ACCESS_KEY = "s3.secret-access-key"; + public static final String ICEBERG_S3_TOKEN = "s3.session-token"; public static final String AWS_S3_REGION = "client.region"; public static final String ICEBERG_OSS_ENDPOINT = "oss.endpoint"; diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index a141b637eba..249fec70b4d 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -21,6 +21,8 @@ public class CredentialConstants { public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; + public static final String S3_TOKEN_CREDENTIAL_PROVIDER = "s3-token"; + public static final String TOKEN_EXPIRE_TIME = "token-expire-time"; public static final String GCS_TOKEN_CREDENTIAL_PROVIDER_TYPE = "gcs-token"; diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java index 9775bebc8a5..af37ae69073 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java @@ -30,6 +30,10 @@ public class S3Properties { public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; // The region of the S3 service. public static final String GRAVITINO_S3_REGION = "s3-region"; + // S3 role arn + public static final String GRAVITINO_S3_ROLE_ARN = "s3-role-arn"; + // S3 external id + public static final String GRAVITINO_S3_EXTERNAL_ID = "s3-external-id"; private S3Properties() {} } diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java index e380cc5d44b..c90a44db61d 100644 --- a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java @@ -19,6 +19,7 @@ package org.apache.gravitino.credential; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import java.util.HashMap; import java.util.Map; @@ -27,8 +28,21 @@ * Helper class to generate specific credential properties for different table format and engine. */ public class CredentialPropertyUtils { + + @VisibleForTesting static final String ICEBERG_S3_ACCESS_KEY_ID = "s3.access-key-id"; + @VisibleForTesting static final String ICEBERG_S3_SECRET_ACCESS_KEY = "s3.secret-access-key"; + @VisibleForTesting static final String ICEBERG_S3_TOKEN = "s3.session-token"; + private static Map icebergCredentialPropertyMap = - ImmutableMap.of(GCSTokenCredential.GCS_TOKEN_NAME, "gcs.oauth2.token"); + ImmutableMap.of( + GCSTokenCredential.GCS_TOKEN_NAME, + "gcs.oauth2.token", + S3TokenCredential.GRAVITINO_S3_ACCESS_KEY_ID, + ICEBERG_S3_ACCESS_KEY_ID, + S3TokenCredential.GRAVITINO_S3_SECRET_ACCESS_KEY, + ICEBERG_S3_SECRET_ACCESS_KEY, + S3TokenCredential.GRAVITINO_S3_TOKEN, + ICEBERG_S3_TOKEN); /** * Transforms a specific credential into a map of Iceberg properties. @@ -44,6 +58,9 @@ public static Map toIcebergProperties(Credential credential) { "gcs.oauth2.token-expires-at", String.valueOf(credential.expireTimeInMs())); return icebergGCSCredentialProperties; } + if (credential instanceof S3TokenCredential || credential instanceof S3SecretKeyCredential) { + return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); + } return credential.toProperties(); } diff --git a/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java new file mode 100644 index 00000000000..4d3ad698b51 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/credential/TestCredentialPropertiesUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCredentialPropertiesUtils { + + @Test + void testToIcebergProperties() { + S3TokenCredential s3TokenCredential = new S3TokenCredential("key", "secret", "token", 0); + Map icebergProperties = + CredentialPropertyUtils.toIcebergProperties(s3TokenCredential); + Map expectedProperties = + ImmutableMap.of( + CredentialPropertyUtils.ICEBERG_S3_ACCESS_KEY_ID, + "key", + CredentialPropertyUtils.ICEBERG_S3_SECRET_ACCESS_KEY, + "secret", + CredentialPropertyUtils.ICEBERG_S3_TOKEN, + "token"); + Assertions.assertEquals(expectedProperties, icebergProperties); + + S3SecretKeyCredential secretKeyCredential = new S3SecretKeyCredential("key", "secret"); + icebergProperties = CredentialPropertyUtils.toIcebergProperties(secretKeyCredential); + expectedProperties = + ImmutableMap.of( + CredentialPropertyUtils.ICEBERG_S3_ACCESS_KEY_ID, + "key", + CredentialPropertyUtils.ICEBERG_S3_SECRET_ACCESS_KEY, + "secret"); + Assertions.assertEquals(expectedProperties, icebergProperties); + } +} diff --git a/core/build.gradle.kts b/core/build.gradle.kts index cb07b49d971..b09c0e35889 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -25,6 +25,7 @@ plugins { dependencies { implementation(project(":api")) implementation(project(":common")) + implementation(project(":catalogs:catalog-common")) implementation(project(":meta")) implementation(libs.bundles.log4j) implementation(libs.bundles.metrics) diff --git a/core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java new file mode 100644 index 00000000000..d16e43c0841 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/config/S3CredentialConfig.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential.config; + +import java.util.Map; +import javax.validation.constraints.NotNull; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Config; +import org.apache.gravitino.config.ConfigBuilder; +import org.apache.gravitino.config.ConfigConstants; +import org.apache.gravitino.config.ConfigEntry; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.storage.S3Properties; + +public class S3CredentialConfig extends Config { + + public static final ConfigEntry S3_REGION = + new ConfigBuilder(S3Properties.GRAVITINO_S3_REGION) + .doc("The region of the S3 service") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .create(); + + public static final ConfigEntry S3_ACCESS_KEY_ID = + new ConfigBuilder(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID) + .doc("The static access key ID used to access S3 data") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry S3_SECRET_ACCESS_KEY = + new ConfigBuilder(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY) + .doc("The static secret access key used to access S3 data") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry S3_ROLE_ARN = + new ConfigBuilder(S3Properties.GRAVITINO_S3_ROLE_ARN) + .doc("S3 role arn") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + public static final ConfigEntry S3_EXTERNAL_ID = + new ConfigBuilder(S3Properties.GRAVITINO_S3_EXTERNAL_ID) + .doc("S3 external ID") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .create(); + + public static final ConfigEntry S3_TOKEN_EXPIRE_SECS = + new ConfigBuilder(CredentialConstants.TOKEN_EXPIRE_TIME) + .doc("S3 token expire seconds") + .version(ConfigConstants.VERSION_0_7_0) + .intConf() + .createWithDefault(3600); + + public S3CredentialConfig(Map properties) { + super(false); + loadFromMap(properties, k -> true); + } + + @NotNull + public String s3RoleArn() { + return this.get(S3_ROLE_ARN); + } + + @NotNull + public String accessKeyID() { + return this.get(S3_ACCESS_KEY_ID); + } + + @NotNull + public String secretAccessKey() { + return this.get(S3_SECRET_ACCESS_KEY); + } + + public String region() { + return this.get(S3_REGION); + } + + public String externalID() { + return this.get(S3_EXTERNAL_ID); + } + + public Integer tokenExpireSecs() { + return this.get(S3_TOKEN_EXPIRE_SECS); + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 830fe5e747c..27d67a434b2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,6 +17,7 @@ # under the License. # [versions] +awssdk = "2.28.3" junit = "5.8.1" protoc = "3.24.4" jackson = "2.15.2" @@ -110,6 +111,11 @@ hudi = "0.15.0" google-auth = "1.28.0" [libraries] +aws-iam = { group = "software.amazon.awssdk", name = "iam", version.ref = "awssdk" } +aws-policy = { group = "software.amazon.awssdk", name = "iam-policy-builder", version.ref = "awssdk" } +aws-s3 = { group = "software.amazon.awssdk", name = "s3", version.ref = "awssdk" } +aws-sts = { group = "software.amazon.awssdk", name = "sts", version.ref = "awssdk" } +aws-kms = { group = "software.amazon.awssdk", name = "kms", version.ref = "awssdk" } protobuf-java = { group = "com.google.protobuf", name = "protobuf-java", version.ref = "protoc" } protobuf-java-util = { group = "com.google.protobuf", name = "protobuf-java-util", version.ref = "protoc" } jackson-databind = { group = "com.fasterxml.jackson.core", name = "jackson-databind", version.ref = "jackson" } @@ -177,6 +183,7 @@ commons-collections3 = { group = "commons-collections", name = "commons-collecti commons-configuration1 = { group = "commons-configuration", name = "commons-configuration", version.ref = "commons-configuration1" } iceberg-aliyun = { group = "org.apache.iceberg", name = "iceberg-aliyun", version.ref = "iceberg" } iceberg-aws = { group = "org.apache.iceberg", name = "iceberg-aws", version.ref = "iceberg" } +iceberg-aws-bundle = { group = "org.apache.iceberg", name = "iceberg-aws-bundle", version.ref = "iceberg" } iceberg-core = { group = "org.apache.iceberg", name = "iceberg-core", version.ref = "iceberg" } iceberg-api = { group = "org.apache.iceberg", name = "iceberg-api", version.ref = "iceberg" } iceberg-hive-metastore = { group = "org.apache.iceberg", name = "iceberg-hive-metastore", version.ref = "iceberg" } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java index 95e82aa2275..f1d3b178e81 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java @@ -75,8 +75,6 @@ public class IcebergCatalogWrapper implements AutoCloseable { ImmutableSet.of( IcebergConstants.IO_IMPL, IcebergConstants.AWS_S3_REGION, - IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID, - IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY, IcebergConstants.ICEBERG_S3_ENDPOINT, IcebergConstants.ICEBERG_OSS_ENDPOINT, IcebergConstants.ICEBERG_OSS_ACCESS_KEY_ID, diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index f088ce2926d..c635ecd79e9 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -63,6 +63,7 @@ dependencies { compileOnly(libs.lombok) + testImplementation(project(":bundles:aws-bundle")) testImplementation(project(":bundles:gcp-bundle", configuration = "shadow")) testImplementation(project(":integration-test-common", "testArtifacts")) @@ -76,6 +77,7 @@ dependencies { exclude("org.rocksdb") } + testImplementation(libs.iceberg.aws.bundle) testImplementation(libs.iceberg.gcp.bundle) testImplementation(libs.jersey.test.framework.core) { exclude(group = "org.junit.jupiter") diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java index 46546bbdcaf..ce245e94511 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java @@ -133,7 +133,11 @@ public Response createTable( LoadTableResponse loadTableResponse = tableOperationDispatcher.createTable(catalogName, icebergNS, createTableRequest); if (isCredentialVending) { - return IcebergRestUtils.ok(injectCredentialConfig(catalogName, loadTableResponse)); + return IcebergRestUtils.ok( + injectCredentialConfig( + catalogName, + TableIdentifier.of(icebergNS, createTableRequest.name()), + loadTableResponse)); } else { return IcebergRestUtils.ok(loadTableResponse); } @@ -215,7 +219,8 @@ public Response loadTable( LoadTableResponse loadTableResponse = tableOperationDispatcher.loadTable(catalogName, tableIdentifier); if (isCredentialVending) { - return IcebergRestUtils.ok(injectCredentialConfig(catalogName, loadTableResponse)); + return IcebergRestUtils.ok( + injectCredentialConfig(catalogName, tableIdentifier, loadTableResponse)); } else { return IcebergRestUtils.ok(loadTableResponse); } @@ -270,7 +275,7 @@ private String SerializeUpdateTableRequest(UpdateTableRequest updateTableRequest } private LoadTableResponse injectCredentialConfig( - String catalogName, LoadTableResponse loadTableResponse) { + String catalogName, TableIdentifier tableIdentifier, LoadTableResponse loadTableResponse) { CredentialProvider credentialProvider = icebergCatalogWrapperManager.getCredentialProvider(catalogName); if (credentialProvider == null) { @@ -286,6 +291,12 @@ private LoadTableResponse injectCredentialConfig( throw new ServiceUnavailableException( "Couldn't generate credential for %s", credentialProvider.credentialType()); } + + LOG.info( + "Generate credential: {} for Iceberg table: {}", + credential.credentialType(), + tableIdentifier); + Map credentialConfig = CredentialPropertyUtils.toIcebergProperties(credential); return LoadTableResponse.builder() .withTableMetadata(loadTableResponse.tableMetadata()) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java new file mode 100644 index 00000000000..7941177a79a --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.integration.test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.DownloaderUtils; +import org.apache.gravitino.integration.test.util.ITUtils; +import org.apache.gravitino.storage.S3Properties; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.platform.commons.util.StringUtils; + +@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") +public class IcebergRESTS3IT extends IcebergRESTJdbcCatalogIT { + + private String s3Warehouse; + private String accessKey; + private String secretKey; + private String region; + private String roleArn; + private String externalId; + + @Override + void initEnv() { + this.s3Warehouse = + String.format("s3://%s/test1", getFromEnvOrDefault("GRAVITINO_S3_BUCKET", "{BUCKET_NAME}")); + this.accessKey = getFromEnvOrDefault("GRAVITINO_S3_ACCESS_KEY", "{ACCESS_KEY}"); + this.secretKey = getFromEnvOrDefault("GRAVITINO_S3_SECRET_KEY", "{SECRET_KEY}"); + this.region = getFromEnvOrDefault("GRAVITINO_S3_REGION", "ap-southeast-2"); + this.roleArn = getFromEnvOrDefault("GRAVITINO_S3_ROLE_ARN", "{ROLE_ARN}"); + this.externalId = getFromEnvOrDefault("GRAVITINO_S3_EXTERNAL_ID", ""); + if (ITUtils.isEmbedded()) { + return; + } + try { + downloadIcebergAwsBundleJar(); + } catch (IOException e) { + LOG.warn("Download Iceberg AWS bundle jar failed,", e); + throw new RuntimeException(e); + } + copyS3BundleJar(); + } + + @Override + public Map getCatalogConfig() { + HashMap m = new HashMap(); + m.putAll(getCatalogJdbcConfig()); + m.putAll(getS3Config()); + return m; + } + + public boolean supportsCredentialVending() { + return true; + } + + private Map getS3Config() { + Map configMap = new HashMap(); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + CredentialConstants.S3_TOKEN_CREDENTIAL_PROVIDER); + configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + S3Properties.GRAVITINO_S3_REGION, region); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKey); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, + secretKey); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + S3Properties.GRAVITINO_S3_ROLE_ARN, roleArn); + if (StringUtils.isNotBlank(externalId)) { + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + S3Properties.GRAVITINO_S3_EXTERNAL_ID, externalId); + } + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.IO_IMPL, + "org.apache.iceberg.aws.s3.S3FileIO"); + configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.WAREHOUSE, s3Warehouse); + + return configMap; + } + + private void downloadIcebergAwsBundleJar() throws IOException { + String icebergBundleJarName = "iceberg-aws-bundle-1.5.2.jar"; + String icebergBundleJarUri = + "https://repo1.maven.org/maven2/org/apache/iceberg/" + + "iceberg-aws-bundle/1.5.2/" + + icebergBundleJarName; + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir); + } + + private void copyS3BundleJar() { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + BaseIT.copyBundleJarsToDirectory("aws-bundle", targetDir); + } + + private String getFromEnvOrDefault(String envVar, String defaultValue) { + String envValue = System.getenv(envVar); + return Optional.ofNullable(envValue).orElse(defaultValue); + } +} From a69edb17da7e38fced8e43868e667deab7708702 Mon Sep 17 00:00:00 2001 From: fanng Date: Fri, 25 Oct 2024 16:04:18 +0800 Subject: [PATCH 02/10] fix --- bundles/aws-bundle/build.gradle.kts | 1 - bundles/gcp-bundle/build.gradle.kts | 5 +++-- gradle/libs.versions.toml | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts index a2671b05812..e1723d7af9d 100644 --- a/bundles/aws-bundle/build.gradle.kts +++ b/bundles/aws-bundle/build.gradle.kts @@ -34,7 +34,6 @@ dependencies { implementation(libs.aws.iam) implementation(libs.aws.policy) implementation(libs.aws.sts) - implementation(libs.aws.kms) implementation(libs.hadoop3.aws) } diff --git a/bundles/gcp-bundle/build.gradle.kts b/bundles/gcp-bundle/build.gradle.kts index e69ff345ea8..ae131b1c318 100644 --- a/bundles/gcp-bundle/build.gradle.kts +++ b/bundles/gcp-bundle/build.gradle.kts @@ -33,6 +33,7 @@ dependencies { compileOnly(libs.hadoop3.common) implementation(libs.commons.lang3) + implementation(libs.commons.logging) implementation(libs.hadoop3.gcs) implementation(libs.google.auth.http) implementation(libs.google.auth.credentials) @@ -44,8 +45,8 @@ tasks.withType(ShadowJar::class.java) { archiveClassifier.set("") // Relocate dependencies to avoid conflicts - relocate("org.apache.httpcomponents", "org.apache.gravitino.shaded.org.apache.httpcomponents") - relocate("org.apache.commons", "org.apache.gravitino.shaded.org.apache.commons") + relocate("org.apache.httpcomponents", "org.apache.gravitino.gcp.shaded.org.apache.httpcomponents") + relocate("org.apache.commons", "org.apache.gravitino.gcp.shaded.org.apache.commons") relocate("com.google", "org.apache.gravitino.shaded.com.google") } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 27d67a434b2..20a0a218e07 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -41,6 +41,7 @@ httpclient5 = "5.2.1" mockserver = "5.15.0" commons-lang3 = "3.14.0" commons-lang = "2.6" +commons-logging = "1.2" commons-io = "2.15.0" commons-collections4 = "4.4" commons-collections3 = "3.2.2" @@ -175,6 +176,7 @@ mockserver-netty = { group = "org.mock-server", name = "mockserver-netty", versi mockserver-client-java = { group = "org.mock-server", name = "mockserver-client-java", version.ref = "mockserver" } commons-lang = { group = "commons-lang", name = "commons-lang", version.ref = "commons-lang" } commons-lang3 = { group = "org.apache.commons", name = "commons-lang3", version.ref = "commons-lang3" } +commons-logging = { group = "commons-logging", name = "commons-logging", version.ref = "commons-logging" } commons-io = { group = "commons-io", name = "commons-io", version.ref = "commons-io" } caffeine = { group = "com.github.ben-manes.caffeine", name = "caffeine", version.ref = "caffeine" } rocksdbjni = { group = "org.rocksdb", name = "rocksdbjni", version.ref = "rocksdbjni" } From ebe23c73552641a29eb4c0a430a4aab65ee7ed66 Mon Sep 17 00:00:00 2001 From: fanng Date: Fri, 25 Oct 2024 16:07:04 +0800 Subject: [PATCH 03/10] fix --- bundles/gcp-bundle/build.gradle.kts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundles/gcp-bundle/build.gradle.kts b/bundles/gcp-bundle/build.gradle.kts index ae131b1c318..b887ef2c5a3 100644 --- a/bundles/gcp-bundle/build.gradle.kts +++ b/bundles/gcp-bundle/build.gradle.kts @@ -33,6 +33,7 @@ dependencies { compileOnly(libs.hadoop3.common) implementation(libs.commons.lang3) + // runtime used implementation(libs.commons.logging) implementation(libs.hadoop3.gcs) implementation(libs.google.auth.http) @@ -45,8 +46,8 @@ tasks.withType(ShadowJar::class.java) { archiveClassifier.set("") // Relocate dependencies to avoid conflicts - relocate("org.apache.httpcomponents", "org.apache.gravitino.gcp.shaded.org.apache.httpcomponents") - relocate("org.apache.commons", "org.apache.gravitino.gcp.shaded.org.apache.commons") + relocate("org.apache.httpcomponents", "org.apache.gravitino.shaded.org.apache.httpcomponents") + relocate("org.apache.commons", "org.apache.gravitino.shaded.org.apache.commons") relocate("com.google", "org.apache.gravitino.shaded.com.google") } From cc4ec18144f5ca41514dac9e6c6b0a97b052a047 Mon Sep 17 00:00:00 2001 From: fanng Date: Fri, 25 Oct 2024 17:37:50 +0800 Subject: [PATCH 04/10] fix --- .../s3/credential/S3TokenProvider.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java b/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java index 2d20b827da9..eda134a153f 100644 --- a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java +++ b/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java @@ -42,6 +42,7 @@ import software.amazon.awssdk.policybuilder.iam.IamStatement; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; import software.amazon.awssdk.services.sts.model.Credentials; @@ -81,7 +82,7 @@ public Credential getCredential(CredentialContext context) { } PathBasedCredentialContext pathBasedCredentialContext = (PathBasedCredentialContext) context; Credentials s3Token = - getS3Token( + createS3Token( roleArn, pathBasedCredentialContext.getReadPaths(), pathBasedCredentialContext.getWritePaths(), @@ -98,18 +99,15 @@ private StsClient createStsClient(S3CredentialConfig s3CredentialConfig) { StaticCredentialsProvider.create( AwsBasicCredentials.create( s3CredentialConfig.accessKeyID(), s3CredentialConfig.secretAccessKey())); + StsClientBuilder builder = StsClient.builder().credentialsProvider(credentialsProvider); String region = s3CredentialConfig.region(); if (StringUtils.isNotBlank(region)) { - return StsClient.builder() - .credentialsProvider(credentialsProvider) - .region(Region.of(region)) - .build(); - } else { - return StsClient.builder().credentialsProvider(credentialsProvider).build(); + builder.region(Region.of(region)); } + return builder.build(); } - private IamPolicy getPolicy( + private IamPolicy createPolicy( String roleArn, Set readLocations, Set writeLocations) { IamPolicy.Builder policyBuilder = IamPolicy.builder(); IamStatement.Builder allowGetObjectStatementBuilder = @@ -223,9 +221,9 @@ private static String getBucketName(URI uri) { return uri.getHost(); } - private Credentials getS3Token( + private Credentials createS3Token( String roleArn, Set readLocations, Set writeLocations, String userName) { - IamPolicy policy = getPolicy(roleArn, readLocations, writeLocations); + IamPolicy policy = createPolicy(roleArn, readLocations, writeLocations); AssumeRoleRequest.Builder builder = AssumeRoleRequest.builder() .roleArn(roleArn) From 3042b4db7a8d63d85dbed0d1bf363f9eb9e5c142 Mon Sep 17 00:00:00 2001 From: fanng Date: Mon, 28 Oct 2024 15:28:14 +0800 Subject: [PATCH 05/10] make it final --- .../apache/gravitino/credential/GCSTokenCredential.java | 4 ++-- .../gravitino/credential/S3SecretKeyCredential.java | 4 ++-- .../apache/gravitino/credential/S3TokenCredential.java | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/credential/GCSTokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/GCSTokenCredential.java index 98186e2dea7..59dc3b24637 100644 --- a/api/src/main/java/org/apache/gravitino/credential/GCSTokenCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/GCSTokenCredential.java @@ -33,8 +33,8 @@ public class GCSTokenCredential implements Credential { /** GCS credential property, token name. */ public static final String GCS_TOKEN_NAME = "token"; - private String token; - private long expireMs; + private final String token; + private final long expireMs; /** * @param token The GCS token. diff --git a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java index dc7474f3834..709759fec70 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java @@ -33,8 +33,8 @@ public class S3SecretKeyCredential implements Credential { /** The static secret access key used to access S3 data. */ public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; - private String accessKeyId; - private String secretAccessKey; + private final String accessKeyId; + private final String secretAccessKey; /** * Constructs an instance of {@link S3SecretKeyCredential} with the specified AWS S3 access key ID diff --git a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java index fe448b38d5b..8a3deb258fe 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java @@ -36,10 +36,10 @@ public class S3TokenCredential implements Credential { /** S3 session token. */ public static final String GRAVITINO_S3_TOKEN = "s3-session-token"; - private String accessKeyId; - private String secretAccessKey; - private String sessionToken; - private long expireTimeInMS; + private final String accessKeyId; + private final String secretAccessKey; + private final String sessionToken; + private final long expireTimeInMS; /** * Constructs an instance of {@link S3SecretKeyCredential} with the specified AWS S3 access key ID From 36f5c9f1472eaf5514ee1d869ca41d5da99dc4b1 Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 29 Oct 2024 14:55:58 +0800 Subject: [PATCH 06/10] fix comment --- .../apache/gravitino/credential/S3TokenCredential.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java index 8a3deb258fe..3a27bc57d11 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java @@ -29,10 +29,10 @@ public class S3TokenCredential implements Credential { /** S3 token credential type. */ public static final String S3_TOKEN_CREDENTIAL_TYPE = "s3-token"; - /** The static access key ID used to access S3 data. */ - public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id"; - /** The static secret access key used to access S3 data. */ - public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; + /** The session access key ID used to access S3 data. */ + public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-session-access-key-id"; + /** The session secret access key used to access S3 data. */ + public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-session-secret-access-key"; /** S3 session token. */ public static final String GRAVITINO_S3_TOKEN = "s3-session-token"; From 0eb0fa68d7707315005cca5b08defe9adc114feb Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 29 Oct 2024 15:04:41 +0800 Subject: [PATCH 07/10] fix comment --- .../credential/S3SecretKeyCredential.java | 22 +++++++++- .../credential/S3TokenCredential.java | 42 +++++++++++++++---- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java index 709759fec70..323d81d5615 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java @@ -40,8 +40,8 @@ public class S3SecretKeyCredential implements Credential { * Constructs an instance of {@link S3SecretKeyCredential} with the specified AWS S3 access key ID * and secret access key. * - * @param accessKeyId the AWS S3 access key ID used for authentication. - * @param secretAccessKey the AWS S3 secret access key used for authentication. + * @param accessKeyId the AWS S3 static access key ID. + * @param secretAccessKey the AWS S3 static secret access key. */ public S3SecretKeyCredential(String accessKeyId, String secretAccessKey) { Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null"); @@ -68,4 +68,22 @@ public Map credentialInfo() { .put(GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey) .build(); } + + /** + * Get S3 static access key ID. + * + * @return The S3 access key ID. + */ + public String accessKeyId() { + return accessKeyId; + } + + /** + * Get S3 static secret access key. + * + * @return The S3 secret access key. + */ + public String secretAccessKey() { + return secretAccessKey; + } } diff --git a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java index 3a27bc57d11..8347b77c8bc 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java @@ -30,9 +30,9 @@ public class S3TokenCredential implements Credential { /** S3 token credential type. */ public static final String S3_TOKEN_CREDENTIAL_TYPE = "s3-token"; /** The session access key ID used to access S3 data. */ - public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-session-access-key-id"; + public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id"; /** The session secret access key used to access S3 data. */ - public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-session-secret-access-key"; + public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; /** S3 session token. */ public static final String GRAVITINO_S3_TOKEN = "s3-session-token"; @@ -42,13 +42,12 @@ public class S3TokenCredential implements Credential { private final long expireTimeInMS; /** - * Constructs an instance of {@link S3SecretKeyCredential} with the specified AWS S3 access key ID - * and secret access key. + * Constructs an instance of {@link S3SecretKeyCredential} with session secret key and token. * - * @param accessKeyId the AWS S3 access key ID used for authentication. - * @param secretAccessKey the AWS S3 secret access key used for authentication. - * @param sessionToken AWS S3 access key ID used for authentication. - * @param expireTimeInMS AWS S3 access key ID used for authentication. + * @param accessKeyId the AWS S3 session access key ID. + * @param secretAccessKey the AWS S3 session secret access key. + * @param sessionToken AWS S3 session token. + * @param expireTimeInMS AWS S3 session token expire time in ms. */ public S3TokenCredential( String accessKeyId, String secretAccessKey, String sessionToken, long expireTimeInMS) { @@ -83,4 +82,31 @@ public Map credentialInfo() { .put(GRAVITINO_S3_TOKEN, sessionToken) .build(); } + + /** + * Get S3 session access key ID. + * + * @return The S3 access key ID. + */ + public String accessKeyId() { + return accessKeyId; + } + + /** + * Get S3 session secret access key. + * + * @return The S3 secret access key. + */ + public String secretAccessKey() { + return secretAccessKey; + } + + /** + * Get S3 session token. + * + * @return The S3 token. + */ + public String sessionToken() { + return sessionToken; + } } From d5536c3495227e53394cefd4f55bcab3f1e255dc Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 29 Oct 2024 15:09:09 +0800 Subject: [PATCH 08/10] fix comment --- .../credential/S3SecretKeyCredential.java | 4 ++-- .../gravitino/credential/S3TokenCredential.java | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java index 323d81d5615..924c682025c 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java @@ -40,8 +40,8 @@ public class S3SecretKeyCredential implements Credential { * Constructs an instance of {@link S3SecretKeyCredential} with the specified AWS S3 access key ID * and secret access key. * - * @param accessKeyId the AWS S3 static access key ID. - * @param secretAccessKey the AWS S3 static secret access key. + * @param accessKeyId The S3 static access key ID. + * @param secretAccessKey The S3 static secret access key. */ public S3SecretKeyCredential(String accessKeyId, String secretAccessKey) { Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null"); diff --git a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java index 8347b77c8bc..2b835e46652 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java @@ -29,9 +29,9 @@ public class S3TokenCredential implements Credential { /** S3 token credential type. */ public static final String S3_TOKEN_CREDENTIAL_TYPE = "s3-token"; - /** The session access key ID used to access S3 data. */ + /** S3 session access key ID used to access S3 data. */ public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id"; - /** The session secret access key used to access S3 data. */ + /** S3 session secret access key used to access S3 data. */ public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; /** S3 session token. */ public static final String GRAVITINO_S3_TOKEN = "s3-session-token"; @@ -44,10 +44,10 @@ public class S3TokenCredential implements Credential { /** * Constructs an instance of {@link S3SecretKeyCredential} with session secret key and token. * - * @param accessKeyId the AWS S3 session access key ID. - * @param secretAccessKey the AWS S3 session secret access key. - * @param sessionToken AWS S3 session token. - * @param expireTimeInMS AWS S3 session token expire time in ms. + * @param accessKeyId The S3 session access key ID. + * @param secretAccessKey The S3 session secret access key. + * @param sessionToken The S3 session token. + * @param expireTimeInMS The S3 session token expire time in ms. */ public S3TokenCredential( String accessKeyId, String secretAccessKey, String sessionToken, long expireTimeInMS) { @@ -104,7 +104,7 @@ public String secretAccessKey() { /** * Get S3 session token. * - * @return The S3 token. + * @return The S3 session token. */ public String sessionToken() { return sessionToken; From 4088c32a9beba7437013b605b114ddbb3ab4e4a6 Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 29 Oct 2024 17:02:45 +0800 Subject: [PATCH 09/10] use static and session name --- .../credential/S3SecretKeyCredential.java | 38 +++++++++---------- .../credential/S3TokenCredential.java | 38 ++++++++++--------- .../credential/CredentialPropertyUtils.java | 11 ++++-- 3 files changed, 48 insertions(+), 39 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java index 924c682025c..b6edd100e05 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java @@ -29,26 +29,26 @@ public class S3SecretKeyCredential implements Credential { /** S3 secret key credential type. */ public static final String S3_SECRET_KEY_CREDENTIAL_TYPE = "s3-secret-key"; /** The static access key ID used to access S3 data. */ - public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id"; + public static final String GRAVITINO_S3_STATIC_ACCESS_KEY_ID = "s3-static-access-key-id"; /** The static secret access key used to access S3 data. */ - public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; + public static final String GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY = "s3-static-secret-access-key"; - private final String accessKeyId; - private final String secretAccessKey; + private final String staticAccessKeyId; + private final String staticSecretAccessKey; /** - * Constructs an instance of {@link S3SecretKeyCredential} with the specified AWS S3 access key ID - * and secret access key. + * Constructs an instance of {@link S3SecretKeyCredential} with the static S3 access key ID and + * secret access key. * - * @param accessKeyId The S3 static access key ID. - * @param secretAccessKey The S3 static secret access key. + * @param staticAccessKeyId The S3 static access key ID. + * @param staticSecretAccessKey The S3 static secret access key. */ - public S3SecretKeyCredential(String accessKeyId, String secretAccessKey) { - Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null"); - Preconditions.checkNotNull(secretAccessKey, "S3 secret access key should not null"); + public S3SecretKeyCredential(String staticAccessKeyId, String staticSecretAccessKey) { + Preconditions.checkNotNull(staticAccessKeyId, "S3 access key Id should not null"); + Preconditions.checkNotNull(staticSecretAccessKey, "S3 secret access key should not null"); - this.accessKeyId = accessKeyId; - this.secretAccessKey = secretAccessKey; + this.staticAccessKeyId = staticAccessKeyId; + this.staticSecretAccessKey = staticSecretAccessKey; } @Override @@ -64,8 +64,8 @@ public long expireTimeInMs() { @Override public Map credentialInfo() { return (new ImmutableMap.Builder()) - .put(GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId) - .put(GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey) + .put(GRAVITINO_S3_STATIC_ACCESS_KEY_ID, staticAccessKeyId) + .put(GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY, staticSecretAccessKey) .build(); } @@ -74,8 +74,8 @@ public Map credentialInfo() { * * @return The S3 access key ID. */ - public String accessKeyId() { - return accessKeyId; + public String staticAccessKeyId() { + return staticAccessKeyId; } /** @@ -83,7 +83,7 @@ public String accessKeyId() { * * @return The S3 secret access key. */ - public String secretAccessKey() { - return secretAccessKey; + public String staticSecretAccessKey() { + return staticSecretAccessKey; } } diff --git a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java index 2b835e46652..7a8b834f2a6 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java @@ -30,36 +30,40 @@ public class S3TokenCredential implements Credential { /** S3 token credential type. */ public static final String S3_TOKEN_CREDENTIAL_TYPE = "s3-token"; /** S3 session access key ID used to access S3 data. */ - public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id"; + public static final String GRAVITINO_S3_SESSION_ACCESS_KEY_ID = "s3-session-access-key-id"; /** S3 session secret access key used to access S3 data. */ - public static final String GRAVITINO_S3_SECRET_ACCESS_KEY = "s3-secret-access-key"; + public static final String GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY = + "s3-session-secret-access-key"; /** S3 session token. */ public static final String GRAVITINO_S3_TOKEN = "s3-session-token"; - private final String accessKeyId; - private final String secretAccessKey; + private final String sessionAccessKeyId; + private final String sessionSecretAccessKey; private final String sessionToken; private final long expireTimeInMS; /** * Constructs an instance of {@link S3SecretKeyCredential} with session secret key and token. * - * @param accessKeyId The S3 session access key ID. - * @param secretAccessKey The S3 session secret access key. + * @param sessionAccessKeyId The S3 session access key ID. + * @param sessionSecretAccessKey The S3 session secret access key. * @param sessionToken The S3 session token. * @param expireTimeInMS The S3 session token expire time in ms. */ public S3TokenCredential( - String accessKeyId, String secretAccessKey, String sessionToken, long expireTimeInMS) { + String sessionAccessKeyId, + String sessionSecretAccessKey, + String sessionToken, + long expireTimeInMS) { Preconditions.checkArgument( - StringUtils.isNotBlank(accessKeyId), "S3 access key Id should not be empty"); + StringUtils.isNotBlank(sessionAccessKeyId), "S3 access key Id should not be empty"); Preconditions.checkArgument( - StringUtils.isNotBlank(secretAccessKey), "S3 secret access key should not be empty"); + StringUtils.isNotBlank(sessionSecretAccessKey), "S3 secret access key should not be empty"); Preconditions.checkArgument( StringUtils.isNotBlank(sessionToken), "S3 session token should not be empty"); - this.accessKeyId = accessKeyId; - this.secretAccessKey = secretAccessKey; + this.sessionAccessKeyId = sessionAccessKeyId; + this.sessionSecretAccessKey = sessionSecretAccessKey; this.sessionToken = sessionToken; this.expireTimeInMS = expireTimeInMS; } @@ -77,8 +81,8 @@ public long expireTimeInMs() { @Override public Map credentialInfo() { return (new ImmutableMap.Builder()) - .put(GRAVITINO_S3_ACCESS_KEY_ID, accessKeyId) - .put(GRAVITINO_S3_SECRET_ACCESS_KEY, secretAccessKey) + .put(GRAVITINO_S3_SESSION_ACCESS_KEY_ID, sessionAccessKeyId) + .put(GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY, sessionSecretAccessKey) .put(GRAVITINO_S3_TOKEN, sessionToken) .build(); } @@ -88,8 +92,8 @@ public Map credentialInfo() { * * @return The S3 access key ID. */ - public String accessKeyId() { - return accessKeyId; + public String sessionAccessKeyId() { + return sessionAccessKeyId; } /** @@ -97,8 +101,8 @@ public String accessKeyId() { * * @return The S3 secret access key. */ - public String secretAccessKey() { - return secretAccessKey; + public String sessionSecretAccessKey() { + return sessionSecretAccessKey; } /** diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java index c90a44db61d..cddd2220684 100644 --- a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java @@ -32,14 +32,19 @@ public class CredentialPropertyUtils { @VisibleForTesting static final String ICEBERG_S3_ACCESS_KEY_ID = "s3.access-key-id"; @VisibleForTesting static final String ICEBERG_S3_SECRET_ACCESS_KEY = "s3.secret-access-key"; @VisibleForTesting static final String ICEBERG_S3_TOKEN = "s3.session-token"; + @VisibleForTesting static final String ICEBERG_GCS_TOKEN = "gcs.oauth2.token"; private static Map icebergCredentialPropertyMap = ImmutableMap.of( GCSTokenCredential.GCS_TOKEN_NAME, - "gcs.oauth2.token", - S3TokenCredential.GRAVITINO_S3_ACCESS_KEY_ID, + ICEBERG_GCS_TOKEN, + S3SecretKeyCredential.GRAVITINO_S3_STATIC_ACCESS_KEY_ID, ICEBERG_S3_ACCESS_KEY_ID, - S3TokenCredential.GRAVITINO_S3_SECRET_ACCESS_KEY, + S3SecretKeyCredential.GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY, + ICEBERG_S3_SECRET_ACCESS_KEY, + S3TokenCredential.GRAVITINO_S3_SESSION_ACCESS_KEY_ID, + ICEBERG_S3_ACCESS_KEY_ID, + S3TokenCredential.GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY, ICEBERG_S3_SECRET_ACCESS_KEY, S3TokenCredential.GRAVITINO_S3_TOKEN, ICEBERG_S3_TOKEN); From 809b7f633e344de0571fd6accf93de517fe07c6a Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 29 Oct 2024 17:19:18 +0800 Subject: [PATCH 10/10] use static and session name --- .../credential/S3SecretKeyCredential.java | 34 ++++++++--------- .../credential/S3TokenCredential.java | 38 +++++++++---------- .../credential/CredentialPropertyUtils.java | 4 -- 3 files changed, 34 insertions(+), 42 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java index b6edd100e05..3063d5615f6 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3SecretKeyCredential.java @@ -29,26 +29,26 @@ public class S3SecretKeyCredential implements Credential { /** S3 secret key credential type. */ public static final String S3_SECRET_KEY_CREDENTIAL_TYPE = "s3-secret-key"; /** The static access key ID used to access S3 data. */ - public static final String GRAVITINO_S3_STATIC_ACCESS_KEY_ID = "s3-static-access-key-id"; + public static final String GRAVITINO_S3_STATIC_ACCESS_KEY_ID = "s3-access-key-id"; /** The static secret access key used to access S3 data. */ - public static final String GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY = "s3-static-secret-access-key"; + public static final String GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY = "s3-secret-access-key"; - private final String staticAccessKeyId; - private final String staticSecretAccessKey; + private final String accessKeyId; + private final String secretAccessKey; /** * Constructs an instance of {@link S3SecretKeyCredential} with the static S3 access key ID and * secret access key. * - * @param staticAccessKeyId The S3 static access key ID. - * @param staticSecretAccessKey The S3 static secret access key. + * @param accessKeyId The S3 static access key ID. + * @param secretAccessKey The S3 static secret access key. */ - public S3SecretKeyCredential(String staticAccessKeyId, String staticSecretAccessKey) { - Preconditions.checkNotNull(staticAccessKeyId, "S3 access key Id should not null"); - Preconditions.checkNotNull(staticSecretAccessKey, "S3 secret access key should not null"); + public S3SecretKeyCredential(String accessKeyId, String secretAccessKey) { + Preconditions.checkNotNull(accessKeyId, "S3 access key Id should not null"); + Preconditions.checkNotNull(secretAccessKey, "S3 secret access key should not null"); - this.staticAccessKeyId = staticAccessKeyId; - this.staticSecretAccessKey = staticSecretAccessKey; + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; } @Override @@ -64,8 +64,8 @@ public long expireTimeInMs() { @Override public Map credentialInfo() { return (new ImmutableMap.Builder()) - .put(GRAVITINO_S3_STATIC_ACCESS_KEY_ID, staticAccessKeyId) - .put(GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY, staticSecretAccessKey) + .put(GRAVITINO_S3_STATIC_ACCESS_KEY_ID, accessKeyId) + .put(GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY, secretAccessKey) .build(); } @@ -74,8 +74,8 @@ public Map credentialInfo() { * * @return The S3 access key ID. */ - public String staticAccessKeyId() { - return staticAccessKeyId; + public String accessKeyId() { + return accessKeyId; } /** @@ -83,7 +83,7 @@ public String staticAccessKeyId() { * * @return The S3 secret access key. */ - public String staticSecretAccessKey() { - return staticSecretAccessKey; + public String secretAccessKey() { + return secretAccessKey; } } diff --git a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java index 7a8b834f2a6..d30a7e14079 100644 --- a/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java +++ b/api/src/main/java/org/apache/gravitino/credential/S3TokenCredential.java @@ -30,40 +30,36 @@ public class S3TokenCredential implements Credential { /** S3 token credential type. */ public static final String S3_TOKEN_CREDENTIAL_TYPE = "s3-token"; /** S3 session access key ID used to access S3 data. */ - public static final String GRAVITINO_S3_SESSION_ACCESS_KEY_ID = "s3-session-access-key-id"; + public static final String GRAVITINO_S3_SESSION_ACCESS_KEY_ID = "s3-access-key-id"; /** S3 session secret access key used to access S3 data. */ - public static final String GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY = - "s3-session-secret-access-key"; + public static final String GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY = "s3-secret-access-key"; /** S3 session token. */ public static final String GRAVITINO_S3_TOKEN = "s3-session-token"; - private final String sessionAccessKeyId; - private final String sessionSecretAccessKey; + private final String accessKeyId; + private final String secretAccessKey; private final String sessionToken; private final long expireTimeInMS; /** * Constructs an instance of {@link S3SecretKeyCredential} with session secret key and token. * - * @param sessionAccessKeyId The S3 session access key ID. - * @param sessionSecretAccessKey The S3 session secret access key. + * @param accessKeyId The S3 session access key ID. + * @param secretAccessKey The S3 session secret access key. * @param sessionToken The S3 session token. * @param expireTimeInMS The S3 session token expire time in ms. */ public S3TokenCredential( - String sessionAccessKeyId, - String sessionSecretAccessKey, - String sessionToken, - long expireTimeInMS) { + String accessKeyId, String secretAccessKey, String sessionToken, long expireTimeInMS) { Preconditions.checkArgument( - StringUtils.isNotBlank(sessionAccessKeyId), "S3 access key Id should not be empty"); + StringUtils.isNotBlank(accessKeyId), "S3 access key Id should not be empty"); Preconditions.checkArgument( - StringUtils.isNotBlank(sessionSecretAccessKey), "S3 secret access key should not be empty"); + StringUtils.isNotBlank(secretAccessKey), "S3 secret access key should not be empty"); Preconditions.checkArgument( StringUtils.isNotBlank(sessionToken), "S3 session token should not be empty"); - this.sessionAccessKeyId = sessionAccessKeyId; - this.sessionSecretAccessKey = sessionSecretAccessKey; + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; this.sessionToken = sessionToken; this.expireTimeInMS = expireTimeInMS; } @@ -81,8 +77,8 @@ public long expireTimeInMs() { @Override public Map credentialInfo() { return (new ImmutableMap.Builder()) - .put(GRAVITINO_S3_SESSION_ACCESS_KEY_ID, sessionAccessKeyId) - .put(GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY, sessionSecretAccessKey) + .put(GRAVITINO_S3_SESSION_ACCESS_KEY_ID, accessKeyId) + .put(GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY, secretAccessKey) .put(GRAVITINO_S3_TOKEN, sessionToken) .build(); } @@ -92,8 +88,8 @@ public Map credentialInfo() { * * @return The S3 access key ID. */ - public String sessionAccessKeyId() { - return sessionAccessKeyId; + public String accessKeyId() { + return accessKeyId; } /** @@ -101,8 +97,8 @@ public String sessionAccessKeyId() { * * @return The S3 secret access key. */ - public String sessionSecretAccessKey() { - return sessionSecretAccessKey; + public String secretAccessKey() { + return secretAccessKey; } /** diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java index cddd2220684..811f04c53a7 100644 --- a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java @@ -42,10 +42,6 @@ public class CredentialPropertyUtils { ICEBERG_S3_ACCESS_KEY_ID, S3SecretKeyCredential.GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY, ICEBERG_S3_SECRET_ACCESS_KEY, - S3TokenCredential.GRAVITINO_S3_SESSION_ACCESS_KEY_ID, - ICEBERG_S3_ACCESS_KEY_ID, - S3TokenCredential.GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY, - ICEBERG_S3_SECRET_ACCESS_KEY, S3TokenCredential.GRAVITINO_S3_TOKEN, ICEBERG_S3_TOKEN);