From 2c0296952eaff2c1769021e988d72df7fcc72f03 Mon Sep 17 00:00:00 2001 From: xunliu Date: Thu, 11 Jul 2024 17:55:57 +0800 Subject: [PATCH] [#3963] feat(core): Apache Ranger Hive authorization pushdown Please enter the commit message for your changes. Lines starting --- LICENSE | 15 + .../gravitino/authorization/RoleChange.java | 162 +- .../authorization/SecurableObjects.java | 3 +- .../AuthorizationPluginException.java | 46 + .../authorization-ranger/build.gradle.kts | 113 ++ .../ranger/RangerAuthorization.java | 42 + .../ranger/RangerAuthorizationPlugin.java | 1027 +++++++++++++ .../authorization/ranger/RangerClientExt.java | 184 +++ .../authorization/ranger/RangerDefines.java | 89 ++ .../ranger/RangerHiveAuthorizationPlugin.java | 87 ++ .../ranger/defines/JsonDateSerializer.java | 42 + .../ranger/defines/RangerCommonEnums.java | 29 + .../authorization/ranger/defines/VList.java | 72 + .../ranger/defines/VXDataObject.java | 87 ++ .../authorization/ranger/defines/VXGroup.java | 98 ++ .../ranger/defines/VXGroupList.java | 49 + .../authorization/ranger/defines/VXUser.java | 122 ++ .../ranger/defines/VXUserList.java | 50 + ...o.authorization.ranger.RangerAuthorization | 19 + .../ranger/integration/test/RangerHiveIT.java | 1337 +++++++++++++++++ .../ranger/integration/test/RangerITEnv.java | 33 +- authorizations/build.gradle.kts | 22 + build.gradle.kts | 8 +- .../hive/HiveCatalogPropertiesMeta.java | 2 + .../hive/TestHiveCatalogOperations.java | 16 +- .../AuthorizationPropertiesMeta.java | 68 + .../gravitino/connector/BaseCatalog.java | 2 +- .../authorization/BaseAuthorization.java | 8 +- .../RoleAuthorizationPlugin.java | 47 + .../UserGroupAuthorizationPlugin.java | 53 +- .../mysql/TestMySQLAuthorization.java | 3 +- .../mysql/TestMySQLAuthorizationPlugin.java | 8 + .../ranger/TestRangerAuthorization.java | 3 +- .../ranger/TestRangerAuthorizationPlugin.java | 8 + dev/docker/tools/docker-connector.conf | 1 + docs/build.gradle.kts | 2 +- gradle/libs.versions.toml | 7 +- .../test/container/RangerContainer.java | 19 +- integration-test/build.gradle.kts | 10 - .../authorization/ranger/RangerDefines.java | 51 - .../authorization/ranger/RangerHiveIT.java | 194 --- settings.gradle.kts | 1 + 42 files changed, 3896 insertions(+), 343 deletions(-) create mode 100644 api/src/main/java/org/apache/gravitino/exceptions/AuthorizationPluginException.java create mode 100644 authorizations/authorization-ranger/build.gradle.kts create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExt.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerDefines.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHiveAuthorizationPlugin.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/JsonDateSerializer.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/RangerCommonEnums.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VList.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXDataObject.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXGroup.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXGroupList.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXUser.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXUserList.java create mode 100644 authorizations/authorization-ranger/src/main/resources/META-INF/services/org.apache.gravitino.authorization.ranger.RangerAuthorization create mode 100644 authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java rename integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/AbstractRangerIT.java => authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java (92%) create mode 100644 authorizations/build.gradle.kts create mode 100644 core/src/main/java/org/apache/gravitino/connector/AuthorizationPropertiesMeta.java delete mode 100644 integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerDefines.java delete mode 100644 integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java diff --git a/LICENSE b/LICENSE index 2d4b82eb2be..1f19688b9f5 100644 --- a/LICENSE +++ b/LICENSE @@ -293,3 +293,18 @@ Kyligence/kylinpy ./clients/client-python/gravitino/utils/exceptions.py ./clients/client-python/gravitino/utils/http_client.py + + Apache Ranger + ./security-admin/src/main/java/org/apache/ranger/defines/JsonDateSerializer.java + ./security-admin/src/main/java/org/apache/ranger/defines/RangerCommonEnums.java + ./security-admin/src/main/java/org/apache/ranger/view/VList.java + ./security-admin/src/main/java/org/apache/ranger/view/VXGroupList.java + ./security-admin/src/main/java/org/apache/ranger/view/VXUserPermission.java + ./security-admin/src/main/java/org/apache/ranger/view/VXDataObject.java + ./security-admin/src/main/java/org/apache/ranger/view/VXGroup.java + ./security-admin/src/main/java/org/apache/ranger/view/VXGroupList.java + ./security-admin/src/main/java/org/apache/ranger/view/VXGroupPermission.java + ./security-admin/src/main/java/org/apache/ranger/view/VXPortalUser.java + ./security-admin/src/main/java/org/apache/ranger/view/VXUser.java + ./security-admin/src/main/java/org/apache/ranger/view/VXUserList.java + ./security-admin/src/main/java/org/apache/ranger/view/VXUserPermission.java diff --git a/api/src/main/java/org/apache/gravitino/authorization/RoleChange.java b/api/src/main/java/org/apache/gravitino/authorization/RoleChange.java index 9004193f2d3..daf2d3c12ab 100644 --- a/api/src/main/java/org/apache/gravitino/authorization/RoleChange.java +++ b/api/src/main/java/org/apache/gravitino/authorization/RoleChange.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.authorization; +import java.util.Objects; import org.apache.gravitino.annotation.Evolving; /** The RoleChange interface defines the public API for managing roles in an authorization. */ @@ -26,31 +27,57 @@ public interface RoleChange { /** * Create a RoleChange to add a securable object into a role. * + * @param roleName The role name. * @param securableObject The securable object. * @return return a RoleChange for the add securable object. */ - static RoleChange addSecurableObject(SecurableObject securableObject) { - return new AddSecurableObject(securableObject); + static RoleChange addSecurableObject(String roleName, SecurableObject securableObject) { + return new AddSecurableObject(roleName, securableObject); } /** * Create a RoleChange to remove a securable object from a role. * + * @param roleName The role name. * @param securableObject The securable object. * @return return a RoleChange for the add securable object. */ - static RoleChange removeSecurableObject(SecurableObject securableObject) { - return new RemoveSecurableObject(securableObject); + static RoleChange removeSecurableObject(String roleName, SecurableObject securableObject) { + return new RemoveSecurableObject(roleName, securableObject); + } + + /** + * Update a securable object RoleChange. + * + * @param roleName The role name. + * @param securableObject The securable object. + * @param newSecurableObject The new securable object. + * @return return a RoleChange for the update securable object. + */ + static RoleChange updateSecurableObject( + String roleName, SecurableObject securableObject, SecurableObject newSecurableObject) { + return new UpdateSecurableObject(roleName, securableObject, newSecurableObject); } /** A AddSecurableObject to add securable object to role. */ final class AddSecurableObject implements RoleChange { + private final String roleName; private final SecurableObject securableObject; - private AddSecurableObject(SecurableObject securableObject) { + private AddSecurableObject(String roleName, SecurableObject securableObject) { + this.roleName = roleName; this.securableObject = securableObject; } + /** + * Returns the role name to be added. + * + * @return return a role name. + */ + public String getRoleName() { + return this.roleName; + } + /** * Returns the securable object to be added. * @@ -65,14 +92,15 @@ public SecurableObject getSecurableObject() { * based on the add securable object to role. * * @param o The object to compare with this instance. - * @return true if the given object represents the same add securable object; false otherwise. + * @return true if the given object represents the same as the add securable object; false + * otherwise. */ @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; AddSecurableObject that = (AddSecurableObject) o; - return securableObject.equals(that.securableObject); + return roleName.equals(that.roleName) && securableObject.equals(that.securableObject); } /** @@ -83,7 +111,7 @@ public boolean equals(Object o) { */ @Override public int hashCode() { - return securableObject.hashCode(); + return Objects.hash(roleName, securableObject); } /** @@ -94,18 +122,29 @@ public int hashCode() { */ @Override public String toString() { - return "ADDSECURABLEOBJECT " + securableObject; + return "ADDSECURABLEOBJECT " + roleName + " " + securableObject; } } /** A RemoveSecurableObject to remove securable object from role. */ final class RemoveSecurableObject implements RoleChange { + private final String roleName; private final SecurableObject securableObject; - private RemoveSecurableObject(SecurableObject securableObject) { + private RemoveSecurableObject(String roleName, SecurableObject securableObject) { + this.roleName = roleName; this.securableObject = securableObject; } + /** + * Returns the role name. + * + * @return return a role name. + */ + public String getRoleName() { + return roleName; + } + /** * Returns the securable object to be added. * @@ -120,14 +159,15 @@ public SecurableObject getSecurableObject() { * is based on the add securable object to role. * * @param o The object to compare with this instance. - * @return true if the given object represents the same add securable object; false otherwise. + * @return true if the given object represents the same as remove securable object; false + * otherwise. */ @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RemoveSecurableObject that = (RemoveSecurableObject) o; - return securableObject.equals(that.securableObject); + return roleName.equals(that.roleName) && securableObject.equals(that.securableObject); } /** @@ -138,7 +178,7 @@ public boolean equals(Object o) { */ @Override public int hashCode() { - return securableObject.hashCode(); + return Objects.hash(roleName, securableObject); } /** @@ -149,7 +189,101 @@ public int hashCode() { */ @Override public String toString() { - return "REMOVESECURABLEOBJECT " + securableObject; + return "REMOVESECURABLEOBJECT " + roleName + " " + securableObject; + } + } + + /** + * A UpdateSecurableObject is to update securable object's privilege from role.
+ * The securable object's metadata entity must be the same as new securable object's metadata + * entity.
+ * The securable object's privilege must be different as new securable object's privilege.
+ */ + final class UpdateSecurableObject implements RoleChange { + private final String roleName; + private final SecurableObject securableObject; + private final SecurableObject newSecurableObject; + + private UpdateSecurableObject( + String roleName, SecurableObject securableObject, SecurableObject newSecurableObject) { + if (!securableObject.fullName().equals(newSecurableObject.fullName())) { + throw new IllegalArgumentException( + "The securable object's metadata entity must be same as new securable object's metadata entity."); + } + if (securableObject.privileges().containsAll(newSecurableObject.privileges())) { + throw new IllegalArgumentException( + "The updated securable object's privilege must be different as new securable object's privilege."); + } + this.roleName = roleName; + this.securableObject = securableObject; + this.newSecurableObject = newSecurableObject; + } + + /** + * Returns the role name. + * + * @return return a role name. + */ + public String getRoleName() { + return roleName; + } + + /** + * Returns the securable object to be updated. + * + * @return return a securable object. + */ + public SecurableObject getSecurableObject() { + return this.securableObject; + } + + /** + * Returns the new securable object. + * + * @return return a securable object. + */ + public SecurableObject getNewSecurableObject() { + return this.newSecurableObject; + } + + /** + * Compares this UpdateSecurableObject instance with another object for equality. The comparison + * is based on the old securable object and new securable object. + * + * @param o The object to compare with this instance. + * @return true if the given object represents the same update securable object; false + * otherwise. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UpdateSecurableObject that = (UpdateSecurableObject) o; + return roleName.equals(that.roleName) + && securableObject.equals(that.securableObject) + && newSecurableObject.equals(that.newSecurableObject); + } + + /** + * Generates a hash code for this UpdateSecurableObject instance. The hash code is based on the + * old securable object and new securable object. + * + * @return A hash code value for this update securable object operation. + */ + @Override + public int hashCode() { + return Objects.hash(roleName, securableObject, newSecurableObject); + } + + /** + * Returns a string representation of the UpdateSecurableObject instance. This string format + * includes the class name followed by the add securable object operation. + * + * @return A string representation of the RemoveSecurableObject instance. + */ + @Override + public String toString() { + return "UPDATESECURABLEOBJECT " + roleName + " " + securableObject + " " + newSecurableObject; } } } diff --git a/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java b/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java index c5ab814eb32..2ec594bcd05 100644 --- a/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java +++ b/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java @@ -31,7 +31,8 @@ /** The helper class for {@link SecurableObject}. */ public class SecurableObjects { - private static final Splitter DOT_SPLITTER = Splitter.on('.'); + /** The splitter for splitting the names. */ + public static final Splitter DOT_SPLITTER = Splitter.on('.'); /** * Create the metalake {@link SecurableObject} with the given metalake name and privileges. diff --git a/api/src/main/java/org/apache/gravitino/exceptions/AuthorizationPluginException.java b/api/src/main/java/org/apache/gravitino/exceptions/AuthorizationPluginException.java new file mode 100644 index 00000000000..a57944c94a6 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/AuthorizationPluginException.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** An exception thrown when an authorization plugin operation failed. */ +public class AuthorizationPluginException extends IllegalArgumentException { + + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public AuthorizationPluginException(@FormatString String message, Object... args) { + super(String.format(message, args)); + } + + /** + * Constructs a new exception with the specified cause. + * + * @param cause the cause. + */ + public AuthorizationPluginException(Throwable cause) { + super(cause); + } +} diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts new file mode 100644 index 00000000000..bb31c0570c7 --- /dev/null +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "authorization-ranger" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +dependencies { + implementation(project(":api")) { + exclude(group = "*") + } + implementation(project(":core")) + implementation(libs.bundles.log4j) + implementation(libs.commons.collections4) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.slf4j.api) + + compileOnly(libs.lombok) + implementation(libs.jackson.annotations) + implementation(libs.ranger.intg) { + exclude("org.apache.hadoop", "hadoop-common") + exclude("org.apache.hive", "hive-storage-api") + exclude("org.apache.lucene") + exclude("org.apache.solr") + exclude("org.apache.kafka") + exclude("org.elasticsearch") + exclude("org.elasticsearch.client") + exclude("org.elasticsearch.plugin") + exclude("org.apache.ranger", "ranger-plugins-audit") + exclude("org.apache.ranger", "ranger-plugins-cred") + exclude("org.apache.ranger", "ranger-plugins-classloader") + } + implementation(libs.javax.jaxb.api) { + exclude("*") + } + + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.mockito.core) + testImplementation(libs.testcontainers) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.ranger.intg) { + exclude("org.apache.hive", "hive-storage-api") + exclude("org.apache.lucene") + exclude("org.apache.solr") + exclude("org.apache.kafka") + exclude("org.elasticsearch") + exclude("org.elasticsearch.client") + exclude("org.elasticsearch.plugin") + } + testImplementation(libs.hive2.jdbc) { + exclude("org.slf4j") + } +} + +tasks { + val runtimeJars by registering(Copy::class) { + from(configurations.runtimeClasspath) + into("build/libs") + } +} + +tasks.build { + dependsOn("runtimeJars", "javadoc") +} + +tasks.test { + val skipUTs = project.hasProperty("skipTests") + if (skipUTs) { + // Only run integration tests + include("**/integration/**") + } + + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/**") + } else { + dependsOn(tasks.jar) + + doFirst { + environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.13") + environment("GRAVITINO_CI_RANGER_DOCKER_IMAGE", "datastrato/gravitino-ci-ranger:0.1.1") + } + + val init = project.extra.get("initIntegrationTest") as (Test) -> Unit + init(this) + } +} + +tasks.getByName("generateMetadataFileForMavenJavaPublication") { + dependsOn("runtimeJars") +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java new file mode 100644 index 00000000000..fe9542beb4c --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import java.util.Map; +import org.apache.gravitino.connector.authorization.AuthorizationPlugin; +import org.apache.gravitino.connector.authorization.BaseAuthorization; + +/** Implementation of a Ranger authorization in Gravitino. */ +public class RangerAuthorization extends BaseAuthorization { + @Override + public String shortName() { + return "ranger"; + } + + @Override + protected AuthorizationPlugin newPlugin(String catalogProvider, Map config) { + switch (catalogProvider) { + case "hive": + return new RangerHiveAuthorizationPlugin(catalogProvider, config); + default: + throw new IllegalArgumentException( + "Authorization plugin unsupported catalog provider: " + catalogProvider); + } + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java new file mode 100644 index 00000000000..32df643b3d1 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -0,0 +1,1027 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.Owner; +import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.RoleChange; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.ranger.defines.VXGroup; +import org.apache.gravitino.authorization.ranger.defines.VXGroupList; +import org.apache.gravitino.authorization.ranger.defines.VXUser; +import org.apache.gravitino.authorization.ranger.defines.VXUserList; +import org.apache.gravitino.connector.AuthorizationPropertiesMeta; +import org.apache.gravitino.connector.authorization.AuthorizationPlugin; +import org.apache.gravitino.exceptions.AuthorizationPluginException; +import org.apache.ranger.RangerServiceException; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerRole; +import org.apache.ranger.plugin.util.GrantRevokeRoleRequest; +import org.apache.ranger.plugin.util.SearchFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Ranger authorization operations plugin abstract class. */ +public abstract class RangerAuthorizationPlugin implements AuthorizationPlugin { + private static final Logger LOG = LoggerFactory.getLogger(RangerAuthorizationPlugin.class); + + protected String catalogProvider; + protected RangerClientExt rangerClient; + protected String rangerServiceName; + /** Mapping Gravitino privilege name to the underlying authorization system privileges. */ + protected Map> mapPrivileges = null; + // The owner privileges, the owner can do anything on the metadata object + protected Set ownerPrivileges = null; + + /** + * Because Ranger doesn't support the precise filter, Ranger will return the policy meets the + * wildcard(*,?) conditions, just like `*.*.*` policy will match `db1.table1.column1` So we need + * to manual precise filter the policies. + */ + // Search Ranger policy filter keys + protected List policyFilterKeys = null; + // Search Ranger policy precise filter keys + protected List policyPreciseFilterKeys = null; + + public static final String MANAGED_BY_GRAVITINO = "MANAGED_BY_GRAVITINO"; + + // TODO: Maybe need to move to the configuration in the future + public static final String RANGER_ADMIN_NAME = "admin"; + + public RangerAuthorizationPlugin(String catalogProvider, Map config) { + super(); + this.catalogProvider = catalogProvider; + String rangerUrl = config.get(AuthorizationPropertiesMeta.RANGER_ADMIN_URL); + String authType = config.get(AuthorizationPropertiesMeta.RANGER_AUTH_TYPE); + String username = config.get(AuthorizationPropertiesMeta.RANGER_USERNAME); + // Apache Ranger Password should be minimum 8 characters with min one alphabet and one numeric. + String password = config.get(AuthorizationPropertiesMeta.RANGER_PASSWORD); + rangerServiceName = config.get(AuthorizationPropertiesMeta.RANGER_SERVICE_NAME); + check(rangerUrl != null, "Ranger admin URL is required"); + check(authType != null, "Ranger auth type is required"); + check(username != null, "Ranger username is required"); + check(password != null, "Ranger password is required"); + check(rangerServiceName != null, "Ranger service name is required"); + + rangerClient = new RangerClientExt(rangerUrl, authType, username, password); + + initMapPrivileges(); + initOwnerPrivileges(); + initPolicyFilterKeys(); + initPreciseFilterKeys(); + } + + /** + * Different underlying permission system may have different privilege names, this function is + * used to initialize the privilege mapping. + */ + protected abstract void initMapPrivileges(); + + /** + * Different underlying permission system may have different owner privilege names, this function + * is used to initialize the owner privilege mapping. + */ + protected abstract void initOwnerPrivileges(); + + // Initial Ranger policy filter keys + protected abstract void initPolicyFilterKeys(); + // Initial Ranger policy precise filter keys + protected abstract void initPreciseFilterKeys(); + + /** + * Translate the privilege name to the corresponding privilege name in the underlying permission + * + * @param name The privilege name to translate + * @return The corresponding privilege name in the underlying permission system + */ + public Set translatePrivilege(Privilege.Name name) { + return mapPrivileges.get(name); + } + + /** + * Whether this privilege is underlying permission system supported + * + * @param name The privilege name to check + * @return true if the privilege is supported, otherwise false + */ + protected boolean checkPrivilege(Privilege.Name name) { + return mapPrivileges.containsKey(name); + } + + @FormatMethod + protected void check(boolean condition, @FormatString String message, Object... args) { + if (!condition) { + throw new AuthorizationPluginException(message, args); + } + } + + @VisibleForTesting + public List getOwnerPrivileges() { + return Lists.newArrayList(ownerPrivileges); + } + + /** + * Because Ranger does not have Role concept, Each metadata object will have a unique Ranger + * policy. we can use one or more Ranger policy to simulate the role.
+ * 1. Create a policy for each metadata object.
+ * 2. Save role name in the Policy properties.
+ * 3. Set `MANAGED_BY_GRAVITINO` label in the policy.
+ * 4. For easy manage, each privilege will create a RangerPolicyItemAccess in the policy.
+ * 5. The policy will only have one user, the user is the {OWNER} of the policy.
+ * 6. The policy will not have group.
+ */ + @Override + public Boolean onRoleCreated(Role role) throws RuntimeException { + createRangerRole(role.name()); + return onRoleUpdated( + role, + role.securableObjects().stream() + .map(securableObject -> RoleChange.addSecurableObject(role.name(), securableObject)) + .toArray(RoleChange[]::new)); + } + + @Override + public Boolean onRoleAcquired(Role role) throws RuntimeException { + Boolean findAll; + try { + findAll = + role.securableObjects().stream() + .filter( + securableObject -> { + RangerPolicy policy = findManagedPolicy(securableObject); + return policy != null; + }) + .count() + == role.securableObjects().size(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return findAll; + } + + /** + * Because one Ranger policy maybe contain multiple securable objects, so we didn't directly + * remove the policy.
+ */ + @Override + public Boolean onRoleDeleted(Role role) throws RuntimeException { + // First remove the role in the Ranger policy + onRoleUpdated( + role, + role.securableObjects().stream() + .map(securableObject -> RoleChange.removeSecurableObject(role.name(), securableObject)) + .toArray(RoleChange[]::new)); + // Lastly, remove the role in the Ranger + try { + rangerClient.deleteRole(role.name(), RANGER_ADMIN_NAME, rangerServiceName); + } catch (RangerServiceException e) { + // Ignore exception to support idempotent operation + } + return Boolean.TRUE; + } + + @Override + public Boolean onRoleUpdated(Role role, RoleChange... changes) throws RuntimeException { + for (RoleChange change : changes) { + boolean execResult; + if (change instanceof RoleChange.AddSecurableObject) { + execResult = doAddSecurableObject((RoleChange.AddSecurableObject) change); + } else if (change instanceof RoleChange.RemoveSecurableObject) { + execResult = + doRemoveSecurableObject(role.name(), (RoleChange.RemoveSecurableObject) change); + } else if (change instanceof RoleChange.UpdateSecurableObject) { + execResult = + doUpdateSecurableObject(role.name(), (RoleChange.UpdateSecurableObject) change); + } else { + throw new IllegalArgumentException( + "Unsupported role change type: " + + (change == null ? "null" : change.getClass().getSimpleName())); + } + if (!execResult) { + return Boolean.FALSE; + } + } + + return Boolean.TRUE; + } + + /** + * Set or transfer the ownership of the metadata object.
+ * + * @param metadataObject The metadata object to set the owner. + * @param preOwner The previous owner of the metadata object. If the metadata object doesn't have + * owner, then the preOwner will be null and newOwner will be not null. + * @param newOwner The new owner of the metadata object. If the metadata object already have + * owner, then the preOwner and newOwner will not be null. + */ + @Override + public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner) + throws RuntimeException { + // 1. Set the owner of the metadata object + // 2. Transfer the ownership from preOwner to newOwner of the metadata object + check(newOwner != null, "The newOwner must be not null"); + + RangerPolicy policy = findManagedPolicy(metadataObject); + if (policy != null) { + policy.getPolicyItems().stream() + .filter( + policyItem -> { + return policyItem.getAccesses().stream() + .allMatch( + policyItemAccess -> { + return ownerPrivileges.contains(policyItemAccess.getType()); + }); + }) + .forEach( + policyItem -> { + if (preOwner != null) { + if (preOwner.type() == Owner.Type.USER) { + policyItem.getUsers().removeIf(preOwner.name()::equals); + } else { + policyItem.getGroups().removeIf(preOwner.name()::equals); + } + } + if (newOwner != null) { + if (newOwner.type() == Owner.Type.USER) { + policyItem.getUsers().add(newOwner.name()); + } else { + policyItem.getGroups().add(newOwner.name()); + } + } + }); + } else { + policy = new RangerPolicy(); + policy.setService(rangerServiceName); + policy.setName(metadataObject.fullName()); + policy.setPolicyLabels(Lists.newArrayList(MANAGED_BY_GRAVITINO)); + + List nsMetadataObject = + Lists.newArrayList(SecurableObjects.DOT_SPLITTER.splitToList(metadataObject.fullName())); + if (nsMetadataObject.size() > 4) { + // The max level of the securable object is `catalog.db.table.column` + throw new RuntimeException("The securable object than 4"); + } + nsMetadataObject.remove(0); // remove `catalog` + + for (int i = 0; i < nsMetadataObject.size(); i++) { + RangerPolicy.RangerPolicyResource policyResource = + new RangerPolicy.RangerPolicyResource(nsMetadataObject.get(i)); + policy + .getResources() + .put( + i == 0 + ? RangerDefines.RESOURCE_DATABASE + : i == 1 ? RangerDefines.RESOURCE_TABLE : RangerDefines.RESOURCE_COLUMN, + policyResource); + } + + RangerPolicy finalPolicy = policy; + ownerPrivileges.stream() + .forEach( + ownerPrivilege -> { + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem + .getAccesses() + .add(new RangerPolicy.RangerPolicyItemAccess(ownerPrivilege)); + if (newOwner != null) { + if (newOwner.type() == Owner.Type.USER) { + policyItem.getUsers().add(newOwner.name()); + } else { + policyItem.getGroups().add(newOwner.name()); + } + } + finalPolicy.getPolicyItems().add(policyItem); + }); + } + try { + if (policy.getId() == null) { + rangerClient.createPolicy(policy); + } else { + rangerClient.updatePolicy(policy.getId(), policy); + } + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + + return Boolean.TRUE; + } + + /** + * Because one Ranger policy maybe contain multiple Gravitino securable objects,
+ * So we need to find the corresponding policy item mapping to set the user. + */ + @Override + public Boolean onGrantedRolesToUser(List roles, User user) throws RuntimeException { + // If the user is not exist, then create it. + onUserAdded(user); + + AtomicReference execResult = new AtomicReference<>(Boolean.TRUE); + roles.stream() + .forEach( + role -> { + createRangerRole(role.name()); + GrantRevokeRoleRequest grantRevokeRoleRequest = + createGrantRevokeRoleRequest(role.name(), user.name(), null); + try { + rangerClient.grantRole(rangerServiceName, grantRevokeRoleRequest); + } catch (RangerServiceException e) { + // ignore exception, support idempotent operation + } + + role.securableObjects().stream() + .forEach( + securableObject -> { + RangerPolicy policy = findManagedPolicy(securableObject); + if (policy == null) { + LOG.warn( + "The policy does not exist for the securable object({})", + securableObject); + execResult.set(Boolean.FALSE); + return; + } + + securableObject.privileges().stream() + .forEach( + privilege -> { + mapPrivileges + .getOrDefault(privilege.name(), Collections.emptySet()) + .stream() + .forEach( + rangerPrivilegeName -> { + policy.getPolicyItems().stream() + .forEach( + policyItem -> { + if (policyItem.getAccesses().stream() + .anyMatch( + policyItemAccess -> + policyItemAccess + .getType() + .equals( + rangerPrivilegeName))) { + if (!policyItem + .getRoles() + .contains(role.name())) { + policyItem.getRoles().add(role.name()); + } + } + }); + try { + rangerClient.updatePolicy(policy.getId(), policy); + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + }); + }); + }); + }); + + return Boolean.TRUE; + } + + /** + * Because one Ranger policy maybe contain multiple Gravitino securable objects,
+ * So we need to find the corresponding policy item mapping to remove the user. + */ + @Override + public Boolean onRevokedRolesFromUser(List roles, User user) throws RuntimeException { + roles.stream() + .forEach( + role -> { + checkRangerRole(role.name()); + GrantRevokeRoleRequest grantRevokeRoleRequest = + createGrantRevokeRoleRequest(role.name(), user.name(), null); + try { + rangerClient.revokeRole(rangerServiceName, grantRevokeRoleRequest); + } catch (RangerServiceException e) { + // Ignore exception to support idempotent operation + } + }); + + return Boolean.TRUE; + } + + /** + * Grant the roles to the group.
+ * 1. Create a group in the Ranger if the group is not exist.
+ * 2. Create a role in the Ranger if the role is not exist.
+ * 3. Add this group to the role.
+ * 4. Add the role to the policy item base the metadata object.
+ * + * @param roles The roles to grant to the group. + * @param group The group to grant the roles. + */ + @Override + public Boolean onGrantedRolesToGroup(List roles, Group group) throws RuntimeException { + // If the group is not exist, then create it. + onGroupAdded(group); + + AtomicReference execResult = new AtomicReference<>(Boolean.TRUE); + roles.stream() + .forEach( + role -> { + createRangerRole(role.name()); + GrantRevokeRoleRequest grantRevokeRoleRequest = + createGrantRevokeRoleRequest(role.name(), null, group.name()); + try { + rangerClient.grantRole(rangerServiceName, grantRevokeRoleRequest); + } catch (RangerServiceException e) { + // Ignore exception to support idempotent operation + } + + role.securableObjects().stream() + .forEach( + securableObject -> { + RangerPolicy policy = findManagedPolicy(securableObject); + if (policy == null) { + LOG.warn( + "The policy is not exist for the securable object({})", + securableObject); + execResult.set(Boolean.FALSE); + return; + } + + securableObject.privileges().stream() + .forEach( + privilege -> { + mapPrivileges + .getOrDefault(privilege.name(), Collections.emptySet()) + .stream() + .forEach( + rangerPrivilege -> { + policy.getPolicyItems().stream() + .forEach( + policyItem -> { + if (policyItem.getAccesses().stream() + .anyMatch( + policyItemAccess -> + policyItemAccess + .getType() + .equals(rangerPrivilege))) { + if (!policyItem + .getRoles() + .contains(role.name())) { + policyItem.getRoles().add(role.name()); + } + } + }); + try { + rangerClient.updatePolicy(policy.getId(), policy); + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + }); + }); + }); + }); + if (!execResult.get()) { + return Boolean.FALSE; + } + + return Boolean.TRUE; + } + + @Override + public Boolean onRevokedRolesFromGroup(List roles, Group group) throws RuntimeException { + roles.stream() + .forEach( + role -> { + checkRangerRole(role.name()); + GrantRevokeRoleRequest grantRevokeRoleRequest = + createGrantRevokeRoleRequest(role.name(), null, group.name()); + try { + rangerClient.revokeRole(rangerServiceName, grantRevokeRoleRequest); + } catch (RangerServiceException e) { + // Ignore exception to support idempotent operation + } + }); + + return Boolean.TRUE; + } + + @Override + public Boolean onUserAdded(User user) throws RuntimeException { + VXUserList list = rangerClient.searchUser(ImmutableMap.of("name", user.name())); + if (list.getListSize() > 0) { + LOG.warn("The user({}) is already exist in the Ranger!", user.name()); + return Boolean.FALSE; + } + + VXUser rangerUser = VXUser.builder().withName(user.name()).withDescription(user.name()).build(); + return rangerClient.createUser(rangerUser); + } + + @Override + public Boolean onUserRemoved(User user) throws RuntimeException { + VXUserList list = rangerClient.searchUser(ImmutableMap.of("name", user.name())); + if (list.getListSize() == 0) { + LOG.warn("The user({}) is not exist in the Ranger!", user); + return Boolean.FALSE; + } + rangerClient.deleteUser(list.getList().get(0).getId()); + return Boolean.TRUE; + } + + @Override + public Boolean onUserAcquired(User user) throws RuntimeException { + VXUserList list = rangerClient.searchUser(ImmutableMap.of("name", user.name())); + if (list.getListSize() == 0) { + LOG.warn("The user({}) is not exist in the Ranger!", user); + return Boolean.FALSE; + } + return Boolean.TRUE; + } + + @Override + public Boolean onGroupAdded(Group group) throws RuntimeException { + return rangerClient.createGroup(VXGroup.builder().withName(group.name()).build()); + } + + @Override + public Boolean onGroupRemoved(Group group) throws RuntimeException { + VXGroupList list = rangerClient.searchGroup(ImmutableMap.of("name", group.name())); + if (list.getListSize() == 0) { + LOG.warn("The group({}) is not exist in the Ranger!", group); + return Boolean.FALSE; + } + return rangerClient.deleteGroup(list.getList().get(0).getId()); + } + + @Override + public Boolean onGroupAcquired(Group group) { + VXGroupList vxGroupList = rangerClient.searchGroup(ImmutableMap.of("name", group.name())); + if (vxGroupList.getListSize() == 0) { + LOG.warn("The group({}) is not exist in the Ranger!", group); + return Boolean.FALSE; + } + return Boolean.TRUE; + } + + private boolean checkRangerRole(String roleName) throws AuthorizationPluginException { + try { + rangerClient.getRole(roleName, RANGER_ADMIN_NAME, rangerServiceName); + } catch (RangerServiceException e) { + throw new AuthorizationPluginException(e); + } + return true; + } + + private GrantRevokeRoleRequest createGrantRevokeRoleRequest( + String roleName, String userName, String groupName) { + Set users; + if (userName == null || userName.isEmpty()) { + users = new HashSet<>(); + } else { + users = new HashSet<>(Arrays.asList(userName)); + } + Set groups; + if (groupName == null || groupName.isEmpty()) { + groups = new HashSet<>(); + } else { + groups = new HashSet<>(Arrays.asList(groupName)); + } + + GrantRevokeRoleRequest roleRequest = new GrantRevokeRoleRequest(); + roleRequest.setUsers(users); + roleRequest.setGroups(groups); + roleRequest.setGrantor(RANGER_ADMIN_NAME); + roleRequest.setTargetRoles(new HashSet<>(Arrays.asList(roleName))); + return roleRequest; + } + + /** Create a Ranger role if the role is not exist. */ + private RangerRole createRangerRole(String roleName) { + RangerRole rangerRole = null; + try { + rangerRole = rangerClient.getRole(roleName, RANGER_ADMIN_NAME, rangerServiceName); + } catch (RangerServiceException e) { + // ignore exception, If the role is not exist, then create it. + } + try { + if (rangerRole == null) { + rangerRole = new RangerRole(roleName, MANAGED_BY_GRAVITINO, null, null, null); + rangerClient.createRole(rangerServiceName, rangerRole); + } + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + return rangerRole; + } + + /** + * Find the managed policy for the metadata object. + * + * @param metadataObject The metadata object to find the managed policy. + * @return The managed policy for the metadata object. + */ + @VisibleForTesting + public RangerPolicy findManagedPolicy(MetadataObject metadataObject) + throws AuthorizationPluginException { + List nsMetadataObj = + Lists.newArrayList(SecurableObjects.DOT_SPLITTER.splitToList(metadataObject.fullName())); + nsMetadataObj.remove(0); // skip `catalog` + Map policyFilter = new HashMap<>(); + Map preciseFilterKeysFilter = new HashMap<>(); + policyFilter.put(RangerDefines.SEARCH_FILTER_SERVICE_NAME, this.rangerServiceName); + policyFilter.put(SearchFilter.POLICY_LABELS_PARTIAL, MANAGED_BY_GRAVITINO); + for (int i = 0; i < nsMetadataObj.size(); i++) { + policyFilter.put(policyFilterKeys.get(i), nsMetadataObj.get(i)); + preciseFilterKeysFilter.put(policyPreciseFilterKeys.get(i), nsMetadataObj.get(i)); + } + + try { + List policies = rangerClient.findPolicies(policyFilter); + + if (!policies.isEmpty()) { + // Because Ranger doesn't support the precise filter, Ranger will return the policy meets + // the wildcard(*,?) conditions, just like `*.*.*` policy will match `db1.table1.column1` + // So we need to manual precise filter the policies. + policies = + policies.stream() + .filter( + policy -> + policy.getResources().entrySet().stream() + .allMatch( + entry -> + preciseFilterKeysFilter.containsKey(entry.getKey()) + && entry.getValue().getValues().size() == 1 + && entry + .getValue() + .getValues() + .contains(preciseFilterKeysFilter.get(entry.getKey())))) + .collect(Collectors.toList()); + } + + // Only return the policies that are delegate Gravitino management + if (policies.size() > 1) { + throw new AuthorizationPluginException( + "Each metadata object only have one Gravitino management enable policies."); + } + + RangerPolicy policy = policies.size() == 1 ? policies.get(0) : null; + // Didn't contain duplicate privilege in the delegate Gravitino management policy + if (policy != null) { + policy.getPolicyItems().forEach(this::checkPolicyItemAccess); + policy.getDenyPolicyItems().forEach(this::checkPolicyItemAccess); + policy.getRowFilterPolicyItems().forEach(this::checkPolicyItemAccess); + policy.getDataMaskPolicyItems().forEach(this::checkPolicyItemAccess); + } + + return policy; + } catch (RangerServiceException e) { + throw new AuthorizationPluginException(e); + } + } + + /** + * For easy manage, each privilege will create a RangerPolicyItemAccess in the policy. + * + * @param policyItem The policy item to check + * @throws AuthorizationPluginException If the policy item contains more than one access type + */ + void checkPolicyItemAccess(RangerPolicy.RangerPolicyItem policyItem) + throws AuthorizationPluginException { + if (policyItem.getAccesses().size() != 1) { + throw new AuthorizationPluginException( + "The access type only have one in the delegate Gravitino management policy"); + } + Map mapAccesses = new HashMap<>(); + policyItem + .getAccesses() + .forEach( + access -> { + if (mapAccesses.containsKey(access.getType()) && mapAccesses.get(access.getType())) { + throw new AuthorizationPluginException( + "Contain duplicate privilege in the delegate Gravitino management policy "); + } + mapAccesses.put(access.getType(), true); + }); + } + + /** + * Add the securable object's privilege to the policy.
+ * 1. Find the policy base the metadata object.
+ * 2. If the policy is exist and have same privilege, because support idempotent operation, so + * return true.
+ * 3. If the policy is exist but have different privilege, also return true. Because one Ranger + * policy maybe contain multiple Gravitino securable object
+ * 4. If the policy is not exist, then create a new policy.
+ */ + private boolean doAddSecurableObject(RoleChange.AddSecurableObject change) { + RangerPolicy policy = findManagedPolicy(change.getSecurableObject()); + + if (policy != null) { + // Check the policy item's accesses and roles equal the Gravition securable object's privilege + // and roleName + Set policyPrivileges = + policy.getPolicyItems().stream() + .filter(policyItem -> policyItem.getRoles().contains(change.getRoleName())) + .flatMap(policyItem -> policyItem.getAccesses().stream()) + .map(RangerPolicy.RangerPolicyItemAccess::getType) + .collect(Collectors.toSet()); + Set newPrivileges = + change.getSecurableObject().privileges().stream() + .filter(Objects::nonNull) + .flatMap(privilege -> translatePrivilege(privilege.name()).stream()) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + if (policyPrivileges.containsAll(newPrivileges)) { + LOG.info( + "The privilege({}) already added to Ranger policy({})!", + policy.getName(), + change.getSecurableObject().fullName()); + // If exist policy and have same privilege then directly return true, because support + // idempotent operation. + return true; + } + } else { + policy = new RangerPolicy(); + policy.setService(rangerServiceName); + policy.setName(change.getSecurableObject().fullName()); + policy.setPolicyLabels(Lists.newArrayList(MANAGED_BY_GRAVITINO)); + + List nsMetadataObject = + Lists.newArrayList( + SecurableObjects.DOT_SPLITTER.splitToList(change.getSecurableObject().fullName())); + if (nsMetadataObject.size() > 4) { + // The max level of the securable object is `catalog.db.table.column` + throw new RuntimeException("The securable object than 4"); + } + nsMetadataObject.remove(0); // remove `catalog` + + for (int i = 0; i < nsMetadataObject.size(); i++) { + RangerPolicy.RangerPolicyResource policyResource = + new RangerPolicy.RangerPolicyResource(nsMetadataObject.get(i)); + policy + .getResources() + .put( + i == 0 + ? RangerDefines.RESOURCE_DATABASE + : i == 1 ? RangerDefines.RESOURCE_TABLE : RangerDefines.RESOURCE_COLUMN, + policyResource); + } + } + + addPolicyItem(policy, change.getRoleName(), change.getSecurableObject()); + try { + if (policy.getId() == null) { + rangerClient.createPolicy(policy); + } else { + rangerClient.updatePolicy(policy.getId(), policy); + } + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + + return true; + } + + /** + * Remove the securable object's privilege from the policy.
+ * Because Ranger use unique metadata to location Ranger policy and manage the all privileges, So + * one Ranger policy maybe contain multiple Gravitino privilege objects.
+ * Remove Ranger policy item condition is:
+ * 1. This Ranger policy item's accesses equal the Gravition securable object's privilege.
+ * 2. This Ranger policy item's users and groups is empty.
+ * If policy didn't have any policy item, then delete this policy:
+ */ + private boolean doRemoveSecurableObject( + String roleName, RoleChange.RemoveSecurableObject change) { + RangerPolicy policy = findManagedPolicy(change.getSecurableObject()); + + if (policy != null) { + policy + .getPolicyItems() + .forEach( + policyItem -> { + boolean match = + policyItem.getAccesses().stream() + .allMatch( + // Find the policy item that match access and role + access -> { + // Use Gravitino privilege to search the Ranger policy item's access + boolean matchPrivilege = + change.getSecurableObject().privileges().stream() + .filter(Objects::nonNull) + .flatMap( + privilege -> + translatePrivilege(privilege.name()).stream()) + .filter(Objects::nonNull) + .anyMatch(privilege -> privilege.equals(access.getType())); + return matchPrivilege; + }); + if (match) { + policyItem.getRoles().removeIf(roleName::equals); + } + }); + + // If the policy does have any role and user and group, then remove it. + policy + .getPolicyItems() + .removeIf( + policyItem -> + policyItem.getRoles().isEmpty() + && policyItem.getUsers().isEmpty() + && policyItem.getGroups().isEmpty()); + + try { + if (policy.getPolicyItems().size() == 0) { + rangerClient.deletePolicy(policy.getId()); + } else { + rangerClient.updatePolicy(policy.getId(), policy); + } + } catch (RangerServiceException e) { + LOG.error("Failed to remove the policy item from the Ranger policy {}!", policy); + throw new RuntimeException(e); + } + } else { + LOG.warn( + "Cannot find the Ranger policy({}) for the Gravitino securable object({})!", + roleName, + change.getSecurableObject().fullName()); + // Don't throw exception or return false, because need support immutable operation. + } + return true; + } + + /** + * 1. Find the policy base securable object.
+ * 2. If the policy is exist, then user new securable object's privilege to update.
+ * 3. If the policy is not exist return false.
+ */ + private boolean doUpdateSecurableObject( + String roleName, RoleChange.UpdateSecurableObject change) { + RangerPolicy policy = findManagedPolicy(change.getSecurableObject()); + + if (policy != null) { + removePolicyItem(policy, roleName, change.getSecurableObject()); + addPolicyItem(policy, roleName, change.getNewSecurableObject()); + try { + if (policy.getId() == null) { + rangerClient.createPolicy(policy); + } else { + rangerClient.updatePolicy(policy.getId(), policy); + } + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + } else { + LOG.warn( + "Cannot find the policy({}) for the securable object({})!", + roleName, + change.getSecurableObject().fullName()); + return false; + } + return true; + } + + /** + * Add policy item access items base the securable object's privileges.
+ * We didn't clean the policy items, because one Ranger policy maybe contain multiple Gravitino + * securable objects.
+ */ + private void addPolicyItem( + RangerPolicy policy, String roleName, SecurableObject securableObject) { + // First check the privilege if support in the Ranger Hive + checkSecurableObject(securableObject); + + // Add the policy items by the securable object's privileges + securableObject + .privileges() + .forEach( + gravitinoPrivilege -> { + // Translate the Gravitino privilege to mapped Ranger privilege + translatePrivilege(gravitinoPrivilege.name()) + .forEach( + mappedPrivilege -> { + // Find the policy item that match access and role + boolean exist = + policy.getPolicyItems().stream() + .anyMatch( + policyItem -> { + return policyItem.getRoles().contains(roleName) + && policyItem.getAccesses().stream() + .anyMatch( + access -> + access.getType().equals(mappedPrivilege)); + }); + if (exist) { + return; + } + + RangerPolicy.RangerPolicyItem policyItem = + new RangerPolicy.RangerPolicyItem(); + RangerPolicy.RangerPolicyItemAccess access = + new RangerPolicy.RangerPolicyItemAccess(); + access.setType(mappedPrivilege); + policyItem.getAccesses().add(access); + policyItem.getRoles().add(roleName); + if (Privilege.Condition.ALLOW == gravitinoPrivilege.condition()) { + policy.getPolicyItems().add(policyItem); + } else { + policy.getDenyPolicyItems().add(policyItem); + } + }); + }); + } + + /** + * Remove policy item base the securable object's privileges and role name.
+ * We didn't directly clean the policy items, because one Ranger policy maybe contain multiple + * Gravitino privilege objects.
+ */ + private void removePolicyItem( + RangerPolicy policy, String roleName, SecurableObject securableObject) { + // First check the privilege if support in the Ranger Hive + checkSecurableObject(securableObject); + + // Delete the policy role base the securable object's privileges + policy.getPolicyItems().stream() + .forEach( + policyItem -> { + policyItem + .getAccesses() + .forEach( + access -> { + boolean matchPrivilege = + securableObject.privileges().stream() + .filter(Objects::nonNull) + .flatMap(privilege -> translatePrivilege(privilege.name()).stream()) + .filter(Objects::nonNull) + .anyMatch( + privilege -> { + return access.getType().equals(privilege); + }); + if (matchPrivilege + && !policyItem.getUsers().isEmpty() + && !policyItem.getGroups().isEmpty()) { + // Not ownership policy item, then remove the role + policyItem.getRoles().removeIf(roleName::equals); + } + }); + }); + + // Delete the policy items if the roles is empty and not ownership policy item + policy + .getPolicyItems() + .removeIf( + policyItem -> { + return policyItem.getRoles().isEmpty() + && policyItem.getUsers().isEmpty() + && policyItem.getGroups().isEmpty(); + }); + } + + private boolean checkSecurableObject(SecurableObject securableObject) { + securableObject + .privileges() + .forEach( + privilege -> { + check( + checkPrivilege(privilege.name()), + "This privilege %s is not support in the Ranger hive authorization", + privilege.name()); + }); + return true; + } + + @Override + public void close() throws IOException {} +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExt.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExt.java new file mode 100644 index 00000000000..3cd6fe3613f --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExt.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import com.google.common.collect.ImmutableMap; +import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.UniformInterfaceException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Map; +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.Response; +import org.apache.gravitino.authorization.ranger.defines.VXGroup; +import org.apache.gravitino.authorization.ranger.defines.VXGroupList; +import org.apache.gravitino.authorization.ranger.defines.VXUser; +import org.apache.gravitino.authorization.ranger.defines.VXUserList; +import org.apache.ranger.RangerClient; +import org.apache.ranger.RangerServiceException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Apache Ranger client extension
+ * The class extends the RangerClient class and provides additional methods to create, search and + * delete users and groups + */ +public class RangerClientExt extends RangerClient { + private static final Logger LOG = LoggerFactory.getLogger(RangerClientExt.class); + private static final String URI_USER_BASE = "/service/xusers/users"; + private static final String URI_USER_BY_ID = URI_USER_BASE + "/%d"; + private static final String URI_GROUP_BASE = "/service/xusers/groups"; + private static final String URI_GROUP_BY_ID = URI_GROUP_BASE + "/%d"; + private static final String URI_CREATE_EXTERNAL_USER = URI_USER_BASE + "/external"; + + // Ranger user APIs + private static final API SEARCH_USER = new API(URI_USER_BASE, HttpMethod.GET, Response.Status.OK); + private static final API CREATE_EXTERNAL_USER = + new API(URI_CREATE_EXTERNAL_USER, HttpMethod.POST, Response.Status.OK); + private static final API DELETE_USER = + new API(URI_USER_BY_ID, HttpMethod.DELETE, Response.Status.NO_CONTENT); + + // Ranger group APIs + private static final API CREATE_GROUP = + new API(URI_GROUP_BASE, HttpMethod.POST, Response.Status.OK); + private static final API SEARCH_GROUP = + new API(URI_GROUP_BASE, HttpMethod.GET, Response.Status.OK); + // private static final API GET_GROUP = new API(URI_GROUP_BY_ID, HttpMethod.GET, + // Response.Status.OK); + private static final API DELETE_GROUP = + new API(URI_GROUP_BY_ID, HttpMethod.DELETE, Response.Status.NO_CONTENT); + + // apache/ranger/intg/src/main/java/org/apache/ranger/RangerClient.java + // The private method callAPI of Ranger is called by reflection + // private T callAPI(API api, Map params, Object request, GenericType + // responseType) throws RangerServiceException + private Method callAPIMethodGenericResponseType; + + // private T callAPI(API api, Map params, Object request, Class + // responseType) throws RangerServiceException + private Method callAPIMethodClassResponseType; + // private void callAPI(API api, Map params) throws RangerServiceException + private Method callAPIMethodNonResponse; + + public RangerClientExt(String hostName, String authType, String username, String password) { + super(hostName, authType, username, password, null); + + // initialize callAPI method + try { + callAPIMethodGenericResponseType = + RangerClient.class.getDeclaredMethod( + "callAPI", API.class, Map.class, Object.class, GenericType.class); + callAPIMethodGenericResponseType.setAccessible(true); + + callAPIMethodNonResponse = + RangerClient.class.getDeclaredMethod("callAPI", API.class, Map.class); + callAPIMethodNonResponse.setAccessible(true); + + callAPIMethodClassResponseType = + RangerClient.class.getDeclaredMethod( + "callAPI", API.class, Map.class, Object.class, Class.class); + callAPIMethodClassResponseType.setAccessible(true); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + public Boolean createUser(VXUser user) throws RuntimeException { + try { + callAPIMethodClassResponseType.invoke(this, CREATE_EXTERNAL_USER, null, user, VXUser.class); + } catch (UniformInterfaceException e) { + LOG.error("Failed to create user: " + e.getResponse().getEntity(String.class)); + return Boolean.FALSE; + } catch (InvocationTargetException | IllegalAccessException e) { + Throwable cause = e.getCause(); + if (cause instanceof com.sun.jersey.api.client.UniformInterfaceException) { + com.sun.jersey.api.client.UniformInterfaceException uniformException = + (com.sun.jersey.api.client.UniformInterfaceException) cause; + int statusCode = uniformException.getResponse().getStatus(); + if (statusCode == 204) { + // Because Ranger use asynchronous create user, so the response status is 204, research + // the user to check if it is created + VXUserList userList = searchUser(ImmutableMap.of("name", user.getName())); + if (userList.getPageSize() == 0) { + throw new RuntimeException("Failed to create user: " + user.getName(), cause); + } else { + return Boolean.TRUE; + } + } else { + throw new RuntimeException(e); + } + } else { + throw new RuntimeException(e); + } + } + return Boolean.TRUE; + } + + public VXUserList searchUser(Map filter) throws RuntimeException { + try { + return (VXUserList) + callAPIMethodClassResponseType.invoke(this, SEARCH_USER, filter, null, VXUserList.class); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + public boolean deleteUser(Long userId) throws RuntimeException { + try { + Map params = ImmutableMap.of("forceDelete", "true"); + callAPIMethodNonResponse.invoke(this, DELETE_USER.applyUrlFormat(userId), params); + } catch (InvocationTargetException | IllegalAccessException | RangerServiceException e) { + throw new RuntimeException(e); + } + return true; + } + + public Boolean createGroup(VXGroup group) throws RuntimeException { + try { + callAPIMethodClassResponseType.invoke(this, CREATE_GROUP, null, group, VXGroup.class); + } catch (UniformInterfaceException e) { + LOG.error("Failed to create user: " + e.getResponse().getEntity(String.class)); + return Boolean.FALSE; + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + return Boolean.TRUE; + } + + public VXGroupList searchGroup(Map filter) throws RuntimeException { + try { + return (VXGroupList) + callAPIMethodClassResponseType.invoke( + this, SEARCH_GROUP, filter, null, VXGroupList.class); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + public boolean deleteGroup(Long groupId) throws RuntimeException { + try { + Map params = ImmutableMap.of("forceDelete", "true"); + callAPIMethodNonResponse.invoke(this, DELETE_GROUP.applyUrlFormat(groupId), params); + } catch (InvocationTargetException | IllegalAccessException | RangerServiceException e) { + throw new RuntimeException(e); + } + return true; + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerDefines.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerDefines.java new file mode 100644 index 00000000000..8bf6b17d367 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerDefines.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import org.apache.ranger.plugin.util.SearchFilter; + +public class RangerDefines { + // In the Ranger 2.4.0 + // apache/ranger/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java:L43 + public static final String IMPLICIT_CONDITION_EXPRESSION_NAME = "_expression"; + + // In the Ranger 2.4.0 + // apache/ranger/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java:L159 + // Search filter constants + public static final String SEARCH_FILTER_SERVICE_NAME = SearchFilter.SERVICE_NAME; + // Hive resource database name + public static final String RESOURCE_DATABASE = "database"; + // Hive resource table name + public static final String RESOURCE_TABLE = "table"; + // Hive resource column name + public static final String RESOURCE_COLUMN = "column"; + // HDFS resource path name + public static final String RESOURCE_PATH = "path"; + // Search filter prefix database constants + public static final String SEARCH_FILTER_DATABASE = + SearchFilter.RESOURCE_PREFIX + RESOURCE_DATABASE; + // Search filter prefix table constants + public static final String SEARCH_FILTER_TABLE = SearchFilter.RESOURCE_PREFIX + RESOURCE_TABLE; + // Search filter prefix column constants + public static final String SEARCH_FILTER_COLUMN = SearchFilter.RESOURCE_PREFIX + RESOURCE_COLUMN; + // Search filter prefix file path constants + public static final String SEARCH_FILTER_PATH = SearchFilter.RESOURCE_PREFIX + RESOURCE_PATH; + // Ranger service type HDFS + public static final String SERVICE_TYPE_HDFS = "hdfs"; // HDFS service type + // Ranger service type Hive + public static final String SERVICE_TYPE_HIVE = "hive"; // Hive service type + // {OWNER}: resource owner user variable + public static final String OWNER_USER = "{OWNER}"; + // {USER}: current user variable + public static final String CURRENT_USER = "{USER}"; + // public group + public static final String PUBLIC_GROUP = "public"; + // Read access type in the HDFS + public static final String ACCESS_TYPE_HDFS_READ = "read"; + // Write access type in the HDFS + public static final String ACCESS_TYPE_HDFS_WRITE = "write"; + // execute access type in the HDFS + public static final String ACCESS_TYPE_HDFS_EXECUTE = "execute"; + // All access type in the Hive + public static final String ACCESS_TYPE_HIVE_ALL = "all"; + // Select access type in the Hive + public static final String ACCESS_TYPE_HIVE_SELECT = "select"; + // update access type in the Hive + public static final String ACCESS_TYPE_HIVE_UPDATE = "update"; + // create access type in the Hive + public static final String ACCESS_TYPE_HIVE_CREATE = "create"; + // drop access type in the Hive + public static final String ACCESS_TYPE_HIVE_DROP = "drop"; + // alter access type in the Hive + public static final String ACCESS_TYPE_HIVE_ALTER = "alter"; + // index access type in the Hive + public static final String ACCESS_TYPE_HIVE_INDEX = "index"; + // lock access type in the Hive + public static final String ACCESS_TYPE_HIVE_LOCK = "lock"; + // read access type in the Hive + public static final String ACCESS_TYPE_HIVE_READ = "read"; + // write access type in the Hive + public static final String ACCESS_TYPE_HIVE_WRITE = "write"; + // repladmin access type in the Hive + public static final String ACCESS_TYPE_HIVE_REPLADMIN = "repladmin"; + // serviceadmin access type in the Hive + public static final String ACCESS_TYPE_HIVE_SERVICEADMIN = "serviceadmin"; +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHiveAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHiveAuthorizationPlugin.java new file mode 100644 index 00000000000..8bf6f91377b --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHiveAuthorizationPlugin.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import org.apache.gravitino.authorization.Privilege; + +/** + * RangerHiveAuthorizationPlugin is a plugin for Apache Ranger to manage the Hive authorization of + * the Apache Gravitino. + */ +public class RangerHiveAuthorizationPlugin extends RangerAuthorizationPlugin { + public RangerHiveAuthorizationPlugin(String catalogProvider, Map config) { + super(catalogProvider, config); + } + + /** + * Ranger hive's privilege have `select`, `update`, `create`, `drop`, `alter`, `index`, `lock`, + * `read`, `write`, `repladmin`, `serviceadmin`, `refresh` and `all`. Reference: + * ranger/agents-common/src/main/resources/service-defs/ranger-servicedef-hive.json + */ + @Override + protected void initMapPrivileges() { + mapPrivileges = + ImmutableMap.>builder() + .put( + Privilege.Name.CREATE_SCHEMA, + ImmutableSet.of(RangerDefines.ACCESS_TYPE_HIVE_SELECT)) + .put( + Privilege.Name.CREATE_TABLE, ImmutableSet.of(RangerDefines.ACCESS_TYPE_HIVE_CREATE)) + .put( + Privilege.Name.MODIFY_TABLE, + ImmutableSet.of( + RangerDefines.ACCESS_TYPE_HIVE_UPDATE, + RangerDefines.ACCESS_TYPE_HIVE_DROP, + RangerDefines.ACCESS_TYPE_HIVE_ALTER, + RangerDefines.ACCESS_TYPE_HIVE_WRITE)) + .put( + Privilege.Name.SELECT_TABLE, + ImmutableSet.of( + RangerDefines.ACCESS_TYPE_HIVE_READ, RangerDefines.ACCESS_TYPE_HIVE_SELECT)) + .build(); + } + + @Override + protected void initOwnerPrivileges() { + ownerPrivileges = ImmutableSet.of(RangerDefines.ACCESS_TYPE_HIVE_ALL); + } + + @Override + protected void initPolicyFilterKeys() { + policyFilterKeys = + Arrays.asList( + RangerDefines.SEARCH_FILTER_DATABASE, + RangerDefines.SEARCH_FILTER_TABLE, + RangerDefines.SEARCH_FILTER_COLUMN); + } + + @Override + protected void initPreciseFilterKeys() { + policyPreciseFilterKeys = + Arrays.asList( + RangerDefines.RESOURCE_DATABASE, + RangerDefines.RESOURCE_TABLE, + RangerDefines.RESOURCE_COLUMN); + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/JsonDateSerializer.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/JsonDateSerializer.java new file mode 100644 index 00000000000..99551b0025f --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/JsonDateSerializer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.defines; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; +import org.codehaus.jackson.JsonGenerator; +import org.codehaus.jackson.map.JsonSerializer; +import org.codehaus.jackson.map.SerializerProvider; + +/** + * Used to serialize Java.util.Date, which is not a common JSON type, so we have to create a custom + * serialize method + */ +// apache/ranger/security-admin/src/main/java/org/apache/ranger/defines/JsonDateSerializer.java +public class JsonDateSerializer extends JsonSerializer { + private static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + + @Override + public void serialize(Date date, JsonGenerator gen, SerializerProvider provider) + throws IOException { + String formattedDate = new SimpleDateFormat(DATE_FORMAT).format(date); + gen.writeString(formattedDate); + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/RangerCommonEnums.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/RangerCommonEnums.java new file mode 100644 index 00000000000..f1c70c49de7 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/RangerCommonEnums.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.defines; + +// apache/ranger/security-admin/src/main/java/org/apache/ranger/defines/RangerCommonEnums.java +public class RangerCommonEnums { + /** IS_VISIBLE is an element of enum VisibilityStatus. Its value is "IS_VISIBLE". */ + public static final int IS_VISIBLE = 1; + /** STATUS_ENABLED is an element of enum ActiveStatus. Its value is "STATUS_ENABLED". */ + public static final int STATUS_ENABLED = 1; + /** internal group */ + public static final int GROUP_INTERNAL = 0; +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VList.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VList.java new file mode 100644 index 00000000000..7f76f54e91c --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VList.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.defines; + +import java.util.List; +import lombok.Setter; + +// apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VList.java +public abstract class VList implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + /** Start index for the result. */ + @Setter protected int startIndex; + /** Page size used for the result. */ + @Setter protected int pageSize; + /** Total records in the database for the given search conditions. */ + @Setter protected long totalCount; + /** Number of rows returned for the search condition. */ + @Setter protected int resultSize; + /** Sort type. Either desc or asc. */ + @Setter protected String sortType; + /** Comma seperated list of the fields for sorting. */ + @Setter protected String sortBy; + + protected long queryTimeMS = System.currentTimeMillis(); + + /** Default constructor. This will set all the attributes to default value. */ + public VList() {} + + public abstract int getListSize(); + + public abstract List getList(); + + public int getPageSize() { + return pageSize; + } + + @Override + public String toString() { + return "VList [startIndex=" + + startIndex + + ", pageSize=" + + pageSize + + ", totalCount=" + + totalCount + + ", resultSize=" + + resultSize + + ", sortType=" + + sortType + + ", sortBy=" + + sortBy + + ", queryTimeMS=" + + queryTimeMS + + "]"; + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXDataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXDataObject.java new file mode 100644 index 00000000000..148abd6bc33 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXDataObject.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.defines; + +import java.util.Date; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +// apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXDataObject.java +@JsonAutoDetect( + getterVisibility = Visibility.NONE, + setterVisibility = Visibility.NONE, + fieldVisibility = Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class VXDataObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + /** Id of the data */ + protected Long id; + /** Date when this data was created */ + @JsonSerialize(using = JsonDateSerializer.class) + protected Date createDate; + /** Date when this data was updated */ + @JsonSerialize(using = JsonDateSerializer.class) + protected Date updateDate; + /** Owner */ + protected String owner; + /** Updated By */ + protected String updatedBy; + /** Default constructor. This will set all the attributes to default value. */ + public VXDataObject() {} + + /** + * This method sets the value to the member attribute id. You cannot set null to the + * attribute. + * + * @param id Value to set member attribute id + */ + public void setId(Long id) { + this.id = id; + } + + /** + * Returns the value for the member attribute id + * + * @return Long - value of member attribute id. + */ + public Long getId() { + return this.id; + } + + /** + * This return the bean content in string format + * + * @return formatedStr + */ + public String toString() { + String str = "VXDataObject={"; + str += super.toString(); + str += "id={" + id + "} "; + str += "createDate={" + createDate + "} "; + str += "updateDate={" + updateDate + "} "; + str += "owner={" + owner + "} "; + str += "updatedBy={" + updatedBy + "} "; + str += "}"; + return str; + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXGroup.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXGroup.java new file mode 100644 index 00000000000..1687bfdc6b3 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXGroup.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.defines; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +// apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXGroup.java +@JsonAutoDetect( + getterVisibility = JsonAutoDetect.Visibility.NONE, + setterVisibility = JsonAutoDetect.Visibility.NONE, + fieldVisibility = JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class VXGroup extends VXDataObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + /** Name */ + protected String name; + /** Description */ + protected String description; + /** Type of group This attribute is of type enum CommonEnums::XAGroupType */ + protected int groupType = 0; + + protected int groupSource = RangerCommonEnums.GROUP_INTERNAL; + /** Id of the credential store */ + protected Long credStoreId; + /** Group visibility */ + protected Integer isVisible; + /** Additional store attributes. */ + protected String otherAttributes; + /** Sync Source Attribute */ + protected String syncSource; + + /** Default constructor. This will set all the attributes to default value. */ + public VXGroup() { + groupType = 0; + isVisible = RangerCommonEnums.IS_VISIBLE; + } + + /** + * This return the bean content in string format + * + * @return formatedStr + */ + public String toString() { + String str = "VXGroup={"; + str += super.toString(); + str += "name={" + name + "} "; + str += "description={" + description + "} "; + str += "groupType={" + groupType + "} "; + str += "credStoreId={" + credStoreId + "} "; + str += "isVisible={" + isVisible + "} "; + str += "groupSrc={" + groupSource + "} "; + str += "otherAttributes={" + otherAttributes + "} "; + str += "syncSource={" + syncSource + "} "; + str += "}"; + return str; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final VXGroup vxGroup; + + public Builder() { + this.vxGroup = new VXGroup(); + } + + public Builder withName(String name) { + vxGroup.name = name; + return this; + } + + public VXGroup build() { + return vxGroup; + } + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXGroupList.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXGroupList.java new file mode 100644 index 00000000000..a13787130e0 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXGroupList.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.defines; + +import java.util.ArrayList; +import java.util.List; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +// apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXGroupList.java +@JsonAutoDetect( + getterVisibility = JsonAutoDetect.Visibility.NONE, + setterVisibility = JsonAutoDetect.Visibility.NONE, + fieldVisibility = JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class VXGroupList extends VList { + private static final long serialVersionUID = 1L; + + List vXGroups = new ArrayList<>(); + + @Override + public int getListSize() { + if (vXGroups != null) { + return vXGroups.size(); + } + return 0; + } + + @Override + public List getList() { + return vXGroups; + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXUser.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXUser.java new file mode 100644 index 00000000000..0c7f21f2c11 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXUser.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.defines; + +import java.util.Collection; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +// apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXUser.java +@JsonAutoDetect( + getterVisibility = JsonAutoDetect.Visibility.NONE, + setterVisibility = JsonAutoDetect.Visibility.NONE, + fieldVisibility = JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class VXUser extends VXDataObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + /** Name. */ + protected String name; + /** First Name. */ + protected String firstName; + /** Last Name. */ + protected String lastName; + /** Email address. */ + protected String emailAddress; + /** Password. */ + protected String password = "rangerR0cks!"; + /** Description */ + protected String description; + /** Id of the credential store */ + protected Long credStoreId; + /** List of group ids for this user. */ + protected Collection groupIdList; + + protected Collection groupNameList; + protected int status = RangerCommonEnums.STATUS_ENABLED; + protected Integer isVisible = RangerCommonEnums.IS_VISIBLE; + protected int userSource; + /** List of roles for this user */ + protected Collection userRoleList; + /** Additional store attributes. */ + protected String otherAttributes; + /** Sync Source */ + protected String syncSource; + + /** Default constructor. This will set all the attributes to default value. */ + public VXUser() { + isVisible = RangerCommonEnums.IS_VISIBLE; + } + + public String getName() { + return name; + } + + /** + * This return the bean content in string format + * + * @return formatedStr + */ + public String toString() { + String str = "VXUser={"; + str += super.toString(); + str += "name={" + name + "} "; + str += "firstName={" + firstName + "} "; + str += "lastName={" + lastName + "} "; + str += "emailAddress={" + emailAddress + "} "; + str += "description={" + description + "} "; + str += "credStoreId={" + credStoreId + "} "; + str += "isVisible={" + isVisible + "} "; + str += "groupIdList={" + groupIdList + "} "; + str += "groupNameList={" + groupNameList + "} "; + str += "roleList={" + userRoleList + "} "; + str += "otherAttributes={" + otherAttributes + "} "; + str += "syncSource={" + syncSource + "} "; + str += "}"; + return str; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final VXUser vxUser; + + public Builder() { + this.vxUser = new VXUser(); + } + + public Builder withName(String name) { + vxUser.name = name; + return this; + } + + public Builder withDescription(String description) { + vxUser.description = description; + return this; + } + + public VXUser build() { + return vxUser; + } + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXUserList.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXUserList.java new file mode 100644 index 00000000000..26e6c83598c --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/defines/VXUserList.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.defines; + +import java.util.ArrayList; +import java.util.List; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +// apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXUserList.java +@JsonAutoDetect( + getterVisibility = JsonAutoDetect.Visibility.NONE, + setterVisibility = JsonAutoDetect.Visibility.NONE, + fieldVisibility = JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class VXUserList extends VList { + private static final long serialVersionUID = 1L; + + /** List of users */ + List vXUsers = new ArrayList<>(); + + @Override + public int getListSize() { + if (vXUsers != null) { + return vXUsers.size(); + } + return 0; + } + + @Override + public List getList() { + return vXUsers; + } +} diff --git a/authorizations/authorization-ranger/src/main/resources/META-INF/services/org.apache.gravitino.authorization.ranger.RangerAuthorization b/authorizations/authorization-ranger/src/main/resources/META-INF/services/org.apache.gravitino.authorization.ranger.RangerAuthorization new file mode 100644 index 00000000000..80243251377 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/resources/META-INF/services/org.apache.gravitino.authorization.ranger.RangerAuthorization @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.authorization.ranger.RangerAuthorization \ No newline at end of file diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java new file mode 100644 index 00000000000..d82616ebb30 --- /dev/null +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -0,0 +1,1337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger.integration.test; + +import static org.apache.gravitino.authorization.SecurableObjects.DOT_SPLITTER; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.authorization.Owner; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.RoleChange; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; +import org.apache.gravitino.authorization.ranger.RangerDefines; +import org.apache.gravitino.authorization.ranger.RangerHiveAuthorizationPlugin; +import org.apache.gravitino.connector.AuthorizationPropertiesMeta; +import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.GroupEntity; +import org.apache.gravitino.meta.RoleEntity; +import org.apache.gravitino.meta.UserEntity; +import org.apache.ranger.RangerServiceException; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerRole; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag("gravitino-docker-test") +public class RangerHiveIT extends RangerITEnv { + private static final Logger LOG = LoggerFactory.getLogger(RangerHiveIT.class); + + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + private static Connection adminConnection; + private static Connection anonymousConnection; + private static final String adminUser = "gravitino"; + private static final String anonymousUser = "anonymous"; + static RangerHiveAuthorizationPlugin rangerHiveAuthPlugin; + private final AuditInfo auditInfo = + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); + + @BeforeAll + public static void setup() { + RangerITEnv.setup(); + + containerSuite.startHiveRangerContainer( + new HashMap<>( + ImmutableMap.of( + HiveContainer.HIVE_RUNTIME_VERSION, + HiveContainer.HIVE3, + RangerContainer.DOCKER_ENV_RANGER_SERVER_URL, + String.format( + "http://%s:%d", + containerSuite.getRangerContainer().getContainerIpAddress(), + RangerContainer.RANGER_SERVER_PORT), + RangerContainer.DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME, + RangerITEnv.RANGER_HIVE_REPO_NAME, + RangerContainer.DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME, + RangerITEnv.RANGER_HDFS_REPO_NAME, + HiveContainer.HADOOP_USER_NAME, + adminUser))); + + rangerHiveAuthPlugin = + new RangerHiveAuthorizationPlugin( + "hive", + ImmutableMap.of( + AuthorizationPropertiesMeta.RANGER_ADMIN_URL, + String.format( + "http://%s:%d", + containerSuite.getRangerContainer().getContainerIpAddress(), + RangerContainer.RANGER_SERVER_PORT), + AuthorizationPropertiesMeta.RANGER_AUTH_TYPE, + RangerContainer.authType, + AuthorizationPropertiesMeta.RANGER_USERNAME, + RangerContainer.rangerUserName, + AuthorizationPropertiesMeta.RANGER_PASSWORD, + RangerContainer.rangerPassword, + AuthorizationPropertiesMeta.RANGER_SERVICE_NAME, + RangerITEnv.RANGER_HIVE_REPO_NAME)); + + createRangerHdfsRepository( + containerSuite.getHiveRangerContainer().getContainerIpAddress(), true); + createRangerHiveRepository( + containerSuite.getHiveRangerContainer().getContainerIpAddress(), true); + allowAnyoneAccessHDFS(); + allowAnyoneAccessInformationSchema(); + + // Create hive connection + String url = + String.format( + "jdbc:hive2://%s:%d/default", + containerSuite.getHiveRangerContainer().getContainerIpAddress(), + HiveContainer.HIVE_SERVICE_PORT); + try { + Class.forName("org.apache.hive.jdbc.HiveDriver"); + adminConnection = DriverManager.getConnection(url, adminUser, ""); + anonymousConnection = DriverManager.getConnection(url, anonymousUser, ""); + } catch (ClassNotFoundException | SQLException e) { + throw new RuntimeException(e); + } + } + + /** + * Create a mock role with 3 securable objects
+ * 1. catalog.db1.tab1 with CREATE_TABLE privilege.
+ * 2. catalog.db1.tab2 with SELECT_TABLE privilege.
+ * 3. catalog.db1.tab3 with MODIFY_TABLE privilege.
+ * + * @param roleName The name of the role, must be unique in this test class + */ + public RoleEntity mock3TableRole(String roleName) { + SecurableObject securableObject1 = + SecurableObjects.parse( + String.format("catalog.%s.tab1", roleName), // use unique db name to avoid conflict + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateTable.allow())); + + SecurableObject securableObject2 = + SecurableObjects.parse( + String.format("catalog.%s.tab2", roleName), + SecurableObject.Type.TABLE, + Lists.newArrayList(Privileges.SelectTable.allow())); + + SecurableObject securableObject3 = + SecurableObjects.parse( + String.format("catalog.%s.tab3", roleName), + SecurableObject.Type.TABLE, + Lists.newArrayList(Privileges.ModifyTable.allow())); + + return RoleEntity.builder() + .withId(1L) + .withName(roleName) + .withAuditInfo(auditInfo) + .withSecurableObjects( + Lists.newArrayList(securableObject1, securableObject2, securableObject3)) + .build(); + } + + // Use the different db.table different privilege to test OnRoleCreated() + @Test + public void testOnRoleCreated() { + RoleEntity role = mock3TableRole(currentFunName()); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + verifyRoleInRanger(role); + } + + @Test + public void testOnRoleDeleted() { + // prepare create a role + RoleEntity role = mock3TableRole(currentFunName()); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + + // delete this role + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleDeleted(role)); + // Check if the policy is deleted + role.securableObjects().stream() + .forEach( + securableObject -> + Assertions.assertNull(rangerHiveAuthPlugin.findManagedPolicy(securableObject))); + } + + @Test + public void testOnRoleDeleted2() { + // prepare create a role + RoleEntity role = mock3TableRole(currentFunName()); + // Set metadata object owner + role.securableObjects().stream() + .forEach( + securableObject -> { + Assertions.assertTrue( + rangerHiveAuthPlugin.onOwnerSet( + securableObject, null, new MockOwner("user1", Owner.Type.USER))); + }); + + // delete this role + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleDeleted(role)); + // Because this metaobject has owner, so the policy should not be deleted + role.securableObjects().stream() + .forEach( + securableObject -> + Assertions.assertNotNull(rangerHiveAuthPlugin.findManagedPolicy(securableObject))); + } + + @Test + public void testOnRoleAcquired() { + RoleEntity role = mock3TableRole(GravitinoITUtils.genRandomName(currentFunName())); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleAcquired(role)); + } + + /** The metalake role does not to create Ranger policy. Only use it to help test */ + public RoleEntity mockCatalogRole(String roleName) { + SecurableObject securableObject1 = + SecurableObjects.parse( + "catalog", + SecurableObject.Type.CATALOG, + Lists.newArrayList(Privileges.UseCatalog.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(roleName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(securableObject1)) + .build(); + return role; + } + + @Test + public void testFindManagedPolicy() { + // Because Ranger support wildcard to match the policy, so we need to test the policy with + // wildcard + String dbName = currentFunName(); + createHivePolicy( + Lists.newArrayList(String.format("%s*", dbName), "*"), + GravitinoITUtils.genRandomName(currentFunName())); + createHivePolicy( + Lists.newArrayList(String.format("%s*", dbName), "tab*"), + GravitinoITUtils.genRandomName(currentFunName())); + createHivePolicy( + Lists.newArrayList(String.format("%s3", dbName), "*"), + GravitinoITUtils.genRandomName(currentFunName())); + createHivePolicy( + Lists.newArrayList(String.format("%s3", dbName), "tab*"), + GravitinoITUtils.genRandomName(currentFunName())); + // findManagedPolicy function use precise search, so return null + SecurableObject securableObject1 = + SecurableObjects.parse( + String.format("catalog.%s3.tab1", dbName), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateTable.allow())); + Assertions.assertNull(rangerHiveAuthPlugin.findManagedPolicy(securableObject1)); + + // Add a policy for `db3.tab1` + createHivePolicy( + Lists.newArrayList(String.format("%s3", dbName), "tab1"), + GravitinoITUtils.genRandomName(currentFunName())); + // findManagedPolicy function use precise search, so return not null + Assertions.assertNotNull(rangerHiveAuthPlugin.findManagedPolicy(securableObject1)); + } + + static void createHivePolicy(List metaObjects, String roleName) { + Assertions.assertTrue(metaObjects.size() < 4); + Map policyResourceMap = new HashMap<>(); + for (int i = 0; i < metaObjects.size(); i++) { + RangerPolicy.RangerPolicyResource policyResource = + new RangerPolicy.RangerPolicyResource(metaObjects.get(i)); + policyResourceMap.put( + i == 0 + ? RangerDefines.RESOURCE_DATABASE + : i == 1 ? RangerDefines.RESOURCE_TABLE : RangerDefines.RESOURCE_COLUMN, + policyResource); + } + + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem.setGroups(Arrays.asList(RangerDefines.PUBLIC_GROUP)); + policyItem.setAccesses( + Arrays.asList( + new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HIVE_SELECT))); + updateOrCreateRangerPolicy( + RangerDefines.SERVICE_TYPE_HIVE, + RANGER_HIVE_REPO_NAME, + roleName, + policyResourceMap, + Collections.singletonList(policyItem)); + } + + @Test + public void testRoleChangeAddSecurableObject() { + SecurableObject securableObject1 = + SecurableObjects.parse( + String.format("catalog.%s.tab1", currentFunName()), + SecurableObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateTable.allow())); + + Role mockCatalogRole = mockCatalogRole(currentFunName()); + // 1. Add a securable object to the role + Assertions.assertTrue( + rangerHiveAuthPlugin.onRoleUpdated( + mockCatalogRole, + RoleChange.addSecurableObject(mockCatalogRole.name(), securableObject1))); + + // construct a verify role to check if the role and Ranger policy is created correctly + RoleEntity verifyRole1 = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName()) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(securableObject1)) + .build(); + verifyRoleInRanger(verifyRole1); + + // 2. Multi-call Add a same entity and privilege to the role, because support idempotent + // operation, so return true + Assertions.assertTrue( + rangerHiveAuthPlugin.onRoleUpdated( + mockCatalogRole, + RoleChange.addSecurableObject(mockCatalogRole.name(), securableObject1))); + verifyRoleInRanger(verifyRole1); + + // 3. Add a same metadata but have different privilege to the role + SecurableObject securableObject3 = + SecurableObjects.parse( + securableObject1.fullName(), + SecurableObject.Type.TABLE, + Lists.newArrayList(Privileges.SelectTable.allow(), Privileges.ModifyTable.allow())); + Assertions.assertTrue( + rangerHiveAuthPlugin.onRoleUpdated( + mockCatalogRole, + RoleChange.addSecurableObject(mockCatalogRole.name(), securableObject3))); + } + + @Test + public void testRoleChangeRemoveSecurableObject() { + // Prepare a role contain 3 securable objects + String currentFunName = currentFunName(); + RoleEntity role = mock3TableRole(currentFunName()); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + + Role mockCatalogRole = mockCatalogRole(currentFunName); + // remove a securable object from role + List securableObjects = new ArrayList<>(role.securableObjects()); + while (!securableObjects.isEmpty()) { + SecurableObject removeSecurableObject = securableObjects.remove(0); + Assertions.assertTrue( + rangerHiveAuthPlugin.onRoleUpdated( + mockCatalogRole, + RoleChange.removeSecurableObject(mockCatalogRole.name(), removeSecurableObject))); + + if (!securableObjects.isEmpty()) { + // Doesn't vrify the last securable object, because the role is empty + RoleEntity verifyRole = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName()) + .withAuditInfo(auditInfo) + .withSecurableObjects(securableObjects) + .build(); + verifyRoleInRanger(verifyRole); + } + } + } + + @Test + public void testRoleChangeUpdateSecurableObject() { + SecurableObject oldSecurableObject = + SecurableObjects.parse( + "catalog.db4.tab1", + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName()) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(oldSecurableObject)) + .build(); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + + Role mockCatalogRole = mockCatalogRole(currentFunName()); + // Keep same matedata namespace and type, but change privileges + SecurableObject newSecurableObject = + SecurableObjects.parse( + oldSecurableObject.fullName(), + oldSecurableObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + Assertions.assertTrue( + rangerHiveAuthPlugin.onRoleUpdated( + mockCatalogRole, + RoleChange.updateSecurableObject( + mockCatalogRole.name(), oldSecurableObject, newSecurableObject))); + + // construct a verify role to check if the role and Ranger policy is created correctly + RoleEntity verifyRole = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName()) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(newSecurableObject)) + .build(); + verifyRoleInRanger(verifyRole); + } + + @Test + public void testOnGrantedRolesToUser() { + // prepare create a role + RoleEntity role = mock3TableRole(currentFunName()); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + + // granted role to the user1 + String userName1 = "user1"; + UserEntity userEntity1 = + UserEntity.builder() + .withId(1L) + .withName(userName1) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(role, Lists.newArrayList(userName1)); + + // multi-call to granted role to the user1 + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(role, Lists.newArrayList(userName1)); + + // granted role to the user2 + String userName2 = "user2"; + UserEntity userEntity2 = + UserEntity.builder() + .withId(1L) + .withName(userName2) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity2)); + + // Same to verify user1 and user2 + verifyRoleInRanger(role, Lists.newArrayList(userName1, userName2)); + } + + @Test + public void testOnRevokedRolesFromUser() { + // prepare create a role + RoleEntity role = mock3TableRole(currentFunName()); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + + // granted role to the user1 + String userName1 = "user1"; + UserEntity userEntity1 = + UserEntity.builder() + .withId(1L) + .withName(userName1) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(role, Lists.newArrayList(userName1)); + + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(role, null, Lists.newArrayList(userName1)); + + // multi-call to revoked role from user1 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(role, null, Lists.newArrayList(userName1)); + } + + @Test + public void testOnGrantedRolesToGroup() { + // prepare create a role + RoleEntity role = mock3TableRole(currentFunName()); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + + // granted role to the group1 + String groupName1 = "group1"; + GroupEntity groupEntity1 = + GroupEntity.builder() + .withId(1L) + .withName(groupName1) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger(role, null, null, Lists.newArrayList(groupName1)); + + // multi-call to granted role to the group1 test idempotent operation + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger(role, null, null, Lists.newArrayList(groupName1)); + + // granted role to the group2 + String groupName2 = "group2"; + GroupEntity groupEntity2 = + GroupEntity.builder() + .withId(1L) + .withName(groupName2) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity2)); + + // Same to verify group1 and group2 + verifyRoleInRanger(role, null, null, Lists.newArrayList(groupName1, groupName2)); + } + + @Test + public void testOnRevokedRolesFromGroup() { + // prepare create a role + RoleEntity role = mock3TableRole(currentFunName()); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role)); + + // granted role to the group1 + String groupName1 = "group1"; + GroupEntity groupEntity1 = + GroupEntity.builder() + .withId(1L) + .withName(groupName1) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger(role, null, null, Lists.newArrayList(groupName1)); + + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger(role, null, null, null, Lists.newArrayList(groupName1)); + + // multi-call to revoked to the group1 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger(role, null, null, null, Lists.newArrayList(groupName1)); + } + + private static class MockOwner implements Owner { + private final String name; + private final Type type; + + public MockOwner(String name, Type type) { + this.name = name; + this.type = type; + } + + @Override + public String name() { + return name; + } + + @Override + public Type type() { + return type; + } + } + + @Test + public void testOnOwnerSet() { + MetadataObject metadataObject = + MetadataObjects.parse( + String.format("catalog.%s.tab1", currentFunName()), MetadataObject.Type.TABLE); + String userName1 = "user1"; + Owner owner1 = new MockOwner(userName1, Owner.Type.USER); + Assertions.assertTrue(rangerHiveAuthPlugin.onOwnerSet(metadataObject, null, owner1)); + verifyOwnerInRanger(metadataObject, Lists.newArrayList(userName1), null, null, null); + + String userName2 = "user2"; + Owner owner2 = new MockOwner(userName2, Owner.Type.USER); + Assertions.assertTrue(rangerHiveAuthPlugin.onOwnerSet(metadataObject, owner1, owner2)); + verifyOwnerInRanger( + metadataObject, Lists.newArrayList(userName2), Lists.newArrayList(userName1), null, null); + + String groupName1 = "group1"; + Owner owner3 = new MockOwner(groupName1, Owner.Type.GROUP); + Assertions.assertTrue(rangerHiveAuthPlugin.onOwnerSet(metadataObject, owner2, owner3)); + verifyOwnerInRanger( + metadataObject, + null, + Lists.newArrayList(userName1, userName2), + Lists.newArrayList(groupName1), + null); + + String groupName2 = "group2"; + Owner owner4 = new MockOwner(groupName2, Owner.Type.GROUP); + Assertions.assertTrue(rangerHiveAuthPlugin.onOwnerSet(metadataObject, owner3, owner4)); + verifyOwnerInRanger( + metadataObject, + null, + Lists.newArrayList(userName1, userName2), + Lists.newArrayList(groupName2), + Lists.newArrayList(groupName1)); + } + + @Test + public void testCreateUser() { + UserEntity user = + UserEntity.builder() + .withId(0L) + .withName(currentFunName()) + .withAuditInfo(auditInfo) + .withRoleIds(null) + .withRoleNames(null) + .build(); + Assertions.assertTrue(rangerHiveAuthPlugin.onUserAdded(user)); + Assertions.assertTrue(rangerHiveAuthPlugin.onUserAcquired(user)); + Assertions.assertTrue(rangerHiveAuthPlugin.onUserRemoved(user)); + Assertions.assertFalse(rangerHiveAuthPlugin.onUserAcquired(user)); + } + + @Test + public void testCreateGroup() { + GroupEntity group = + GroupEntity.builder() + .withId(0L) + .withName(currentFunName()) + .withAuditInfo(auditInfo) + .withRoleIds(null) + .withRoleNames(null) + .build(); + + Assertions.assertTrue(rangerHiveAuthPlugin.onGroupAdded(group)); + Assertions.assertTrue(rangerHiveAuthPlugin.onGroupAcquired(group)); + Assertions.assertTrue(rangerHiveAuthPlugin.onGroupRemoved(group)); + Assertions.assertFalse(rangerHiveAuthPlugin.onGroupAcquired(group)); + } + + @Test + public void testCombinationOperation() { + // Create a `CreateTable` privilege role + SecurableObject securableObject1 = + SecurableObjects.parse( + String.format( + "catalog.%s.tab1", currentFunName()), // use unique db name to avoid conflict + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateTable.allow())); + + String ownerName = "owner1"; + rangerHiveAuthPlugin.onOwnerSet( + securableObject1, null, new MockOwner(ownerName, Owner.Type.USER)); + verifyOwnerInRanger(securableObject1, Lists.newArrayList(ownerName), null, null, null); + + RoleEntity role1 = + RoleEntity.builder() + .withId(1L) + .withName(GravitinoITUtils.genRandomName(currentFunName())) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(securableObject1)) + .build(); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role1)); + verifyRoleInRanger(role1); + + // Create a `SelectTable` privilege role + SecurableObject securableObject2 = + SecurableObjects.parse( + securableObject1.fullName(), // Use the same db.table to test the combination operation + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity role2 = + RoleEntity.builder() + .withId(1L) + .withName(GravitinoITUtils.genRandomName(currentFunName())) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(securableObject2)) + .build(); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role2)); + verifyRoleInRanger(role2); + + // Create a `ModifyTable` privilege role + SecurableObject securableObject3 = + SecurableObjects.parse( + securableObject1.fullName(), // Use the same db.table to test the combination operation + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.ModifyTable.allow())); + RoleEntity role3 = + RoleEntity.builder() + .withId(1L) + .withName(GravitinoITUtils.genRandomName(currentFunName())) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(securableObject3)) + .build(); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleCreated(role3)); + verifyRoleInRanger(role3); + + /** Test grant to user */ + // granted role1 to the user1 + String userName1 = "user1"; + UserEntity userEntity1 = + UserEntity.builder() + .withId(1L) + .withName(userName1) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity1)); + // multiple call to granted role1 to the user1 to test idempotent operation + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity1)); + verifyRoleInRanger(role1, Lists.newArrayList(userName1)); + + // granted role1 to the user2 + String userName2 = "user2"; + UserEntity userEntity2 = + UserEntity.builder() + .withId(1L) + .withName(userName2) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity2)); + verifyRoleInRanger(role1, Lists.newArrayList(userName1, userName2)); + + // granted role1 to the user3 + String userName3 = "user3"; + UserEntity userEntity3 = + UserEntity.builder() + .withId(1L) + .withName(userName3) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity3)); + verifyRoleInRanger(role1, Lists.newArrayList(userName1, userName2, userName3)); + + // Same granted role2 and role3 to the user1 and user2 and user3 + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity1)); + verifyRoleInRanger(role2, Lists.newArrayList(userName1)); + verifyRoleInRanger(role3, Lists.newArrayList(userName1)); + + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity2)); + verifyRoleInRanger(role2, Lists.newArrayList(userName1, userName2)); + verifyRoleInRanger(role3, Lists.newArrayList(userName1, userName2)); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity3)); + verifyRoleInRanger(role2, Lists.newArrayList(userName1, userName2, userName3)); + verifyRoleInRanger(role3, Lists.newArrayList(userName1, userName2, userName3)); + + /** Test grant to group */ + // granted role1 to the group1 + String groupName1 = "group1"; + GroupEntity groupEntity1 = + GroupEntity.builder() + .withId(1L) + .withName(groupName1) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity1)); + verifyRoleInRanger( + role1, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1)); + + // granted role1 to the group2 + String groupName2 = "group2"; + GroupEntity groupEntity2 = + GroupEntity.builder() + .withId(1L) + .withName(groupName2) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity2)); + verifyRoleInRanger( + role1, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2)); + + // granted role1 to the group3 + String groupName3 = "group3"; + GroupEntity groupEntity3 = + GroupEntity.builder() + .withId(1L) + .withName(groupName3) + .withRoleNames(Collections.emptyList()) + .withRoleIds(Collections.emptyList()) + .withAuditInfo(auditInfo) + .build(); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity3)); + verifyRoleInRanger( + role1, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2, groupName3)); + + // Same granted role2 and role3 to the group1 and group2 and group3 + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity1)); + verifyRoleInRanger( + role2, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1)); + verifyRoleInRanger( + role3, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1)); + + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity2)); + verifyRoleInRanger( + role2, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2)); + verifyRoleInRanger( + role3, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2)); + Assertions.assertTrue( + rangerHiveAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity3)); + verifyRoleInRanger( + role2, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2, groupName3)); + verifyRoleInRanger( + role3, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2, groupName3)); + + /** Test revoke from user */ + // revoke role1 from the user1 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity1)); + verifyRoleInRanger( + role1, + Lists.newArrayList(userName2, userName3), + Lists.newArrayList(userName1), + Lists.newArrayList(groupName1, groupName2, groupName3)); + + // revoke role1 from the user2 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity2)); + verifyRoleInRanger( + role1, + Lists.newArrayList(userName3), + Lists.newArrayList(userName1, userName2), + Lists.newArrayList(groupName1, groupName2, groupName3)); + + // revoke role1 from the user3 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity3)); + verifyRoleInRanger( + role1, + null, + Lists.newArrayList(userName1, userName2, userName3), + Lists.newArrayList(groupName1, groupName2, groupName3)); + + // Same revoke role2 and role3 from the user1 and user2 and user3 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity1)); + verifyRoleInRanger( + role2, Lists.newArrayList(userName2, userName3), Lists.newArrayList(userName1)); + verifyRoleInRanger( + role3, Lists.newArrayList(userName2, userName3), Lists.newArrayList(userName1)); + + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity2)); + verifyRoleInRanger( + role2, Lists.newArrayList(userName3), Lists.newArrayList(userName1, userName2)); + verifyRoleInRanger( + role3, Lists.newArrayList(userName3), Lists.newArrayList(userName1, userName2)); + + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity3)); + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity3)); + verifyRoleInRanger(role2, null, Lists.newArrayList(userName1, userName2, userName3)); + verifyRoleInRanger(role3, null, Lists.newArrayList(userName1, userName2, userName3)); + + /** Test revoke from group */ + // revoke role1 from the group1 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity1)); + verifyRoleInRanger( + role1, + null, + Lists.newArrayList(userName1, userName2, userName3), + Lists.newArrayList(groupName2, groupName3), + Lists.newArrayList(groupName1)); + + // revoke role1 from the group2 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity2)); + verifyRoleInRanger( + role1, + null, + Lists.newArrayList(userName1, userName2, userName3), + Lists.newArrayList(groupName3), + Lists.newArrayList(groupName1, groupName2)); + + // revoke role1 from the group3 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity3)); + verifyRoleInRanger( + role1, + null, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2, groupName3)); + + // Same revoke role2 and role3 from the group1 and group2 and group3 + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromGroup( + Lists.newArrayList(role2, role3), groupEntity1)); + verifyRoleInRanger( + role2, + null, + Lists.newArrayList(userName1, userName2, userName3), + Lists.newArrayList(groupName2, groupName3), + Lists.newArrayList(groupName1)); + verifyRoleInRanger( + role3, + null, + Lists.newArrayList(userName1, userName2, userName3), + Lists.newArrayList(groupName2, groupName3), + Lists.newArrayList(groupName1)); + + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromGroup( + Lists.newArrayList(role2, role3), groupEntity2)); + verifyRoleInRanger( + role2, + null, + Lists.newArrayList(userName1, userName2, userName3), + Lists.newArrayList(groupName3), + Lists.newArrayList(groupName1, groupName2)); + verifyRoleInRanger( + role3, + null, + Lists.newArrayList(userName1, userName2, userName3), + Lists.newArrayList(groupName3), + Lists.newArrayList(groupName1, groupName2)); + + Assertions.assertTrue( + rangerHiveAuthPlugin.onRevokedRolesFromGroup( + Lists.newArrayList(role2, role3), groupEntity3)); + verifyRoleInRanger( + role2, + null, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2, groupName3)); + verifyRoleInRanger( + role3, + null, + Lists.newArrayList(userName1, userName2, userName3), + null, + Lists.newArrayList(groupName1, groupName2, groupName3)); + + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleDeleted(role1)); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleDeleted(role2)); + Assertions.assertTrue(rangerHiveAuthPlugin.onRoleDeleted(role3)); + // Because these metaobject have owner, so the policy will not be deleted. + role1.securableObjects().stream() + .forEach( + securableObject -> + Assertions.assertNotNull(rangerHiveAuthPlugin.findManagedPolicy(securableObject))); + role2.securableObjects().stream() + .forEach( + securableObject -> + Assertions.assertNotNull(rangerHiveAuthPlugin.findManagedPolicy(securableObject))); + role3.securableObjects().stream() + .forEach( + securableObject -> + Assertions.assertNotNull(rangerHiveAuthPlugin.findManagedPolicy(securableObject))); + } + + private void verifyRoleInRanger( + Role role, + List includeUsers, + List excludeUsers, + List includeGroups, + List excludeGroups) { + // Verify role in RangerRole + RangerRole rangerRole = null; + try { + rangerRole = + rangerClient.getRole( + role.name(), RangerAuthorizationPlugin.RANGER_ADMIN_NAME, RANGER_HIVE_REPO_NAME); + LOG.info("rangerRole: " + rangerRole.toString()); + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + rangerRole + .getUsers() + .forEach( + user -> { + if (includeUsers != null && !includeUsers.isEmpty()) { + Assertions.assertTrue( + includeUsers.contains(user.getName()), + "includeUsersInRole: " + includeUsers + ", user: " + user.getName()); + } + if (excludeUsers != null && !excludeUsers.isEmpty()) { + Assertions.assertFalse( + excludeUsers.contains(user.getName()), + "excludeUsersInRole: " + excludeUsers.toString() + ", user: " + user.getName()); + } + }); + rangerRole + .getGroups() + .forEach( + group -> { + if (includeGroups != null && !includeGroups.isEmpty()) { + Assertions.assertTrue( + includeGroups.contains(group.getName()), + "includeGroupsInRole: " + + includeGroups.toString() + + ", group: " + + group.getName()); + } + if (excludeGroups != null && !excludeGroups.isEmpty()) { + Assertions.assertFalse( + excludeGroups.contains(group.getName()), + "excludeGroupsInRole: " + + excludeGroups.toString() + + ", group: " + + group.getName()); + } + }); + + // Verify role in RangerPolicy + role.securableObjects() + .forEach( + securableObject -> { + RangerPolicy policy; + try { + policy = + rangerClient.getPolicy( + RangerITEnv.RANGER_HIVE_REPO_NAME, securableObject.fullName()); + LOG.info("policy: " + policy.toString()); + } catch (RangerServiceException e) { + LOG.error("Failed to get policy: " + securableObject.fullName()); + throw new RuntimeException(e); + } + + securableObject + .privileges() + .forEach( + gravitinoPrivilege -> { + Set mappedPrivileges = + rangerHiveAuthPlugin.translatePrivilege(gravitinoPrivilege.name()); + + boolean match = + policy.getPolicyItems().stream() + .filter( + policyItem -> { + // Filter Ranger policy item by Gravitino privilege + return policyItem.getAccesses().stream() + .anyMatch( + access -> { + return mappedPrivileges.contains(access.getType()); + }); + }) + .allMatch( + policyItem -> { + // Verify role name in Ranger policy item + return policyItem.getRoles().contains(role.name()); + }); + Assertions.assertTrue(match); + }); + }); + } + + /** Verify the Gravitino role in Ranger service */ + private void verifyOwnerInRanger( + MetadataObject metadataObject, + List includeUsers, + List excludeUsers, + List includeGroups, + List excludeGroups) { + // Find policy by each metadata Object + String policyName = metadataObject.fullName(); + RangerPolicy policy; + try { + policy = rangerClient.getPolicy(RangerITEnv.RANGER_HIVE_REPO_NAME, policyName); + LOG.info("policy: " + policy.toString()); + } catch (RangerServiceException e) { + LOG.error("Failed to get policy: " + policyName); + throw new RuntimeException(e); + } + + Assertions.assertEquals(policy.getName(), policyName); + Assertions.assertTrue( + policy.getPolicyLabels().contains(RangerAuthorizationPlugin.MANAGED_BY_GRAVITINO)); + + // verify namespace + List metaObjNamespaces = + Lists.newArrayList(DOT_SPLITTER.splitToList(metadataObject.fullName())); + metaObjNamespaces.remove(0); // skip catalog + List rolePolicies = new ArrayList<>(); + for (int i = 0; i < metaObjNamespaces.size(); i++) { + rolePolicies.add( + policy + .getResources() + .get( + i == 0 + ? RangerDefines.RESOURCE_DATABASE + : i == 1 ? RangerDefines.RESOURCE_TABLE : RangerDefines.RESOURCE_COLUMN) + .getValues() + .get(0)); + } + Assertions.assertEquals(metaObjNamespaces, rolePolicies); + + policy.getPolicyItems().stream() + .filter( + policyItem -> { + // Filter Ranger policy item by Gravitino privilege + return policyItem.getAccesses().stream() + .anyMatch( + access -> { + return rangerHiveAuthPlugin.getOwnerPrivileges().contains(access.getType()); + }); + }) + .anyMatch( + policyItem -> { + if (includeUsers != null && !includeUsers.isEmpty()) { + if (!policyItem.getUsers().containsAll(includeUsers)) { + return false; + } + } + if (excludeUsers != null && !excludeUsers.isEmpty()) { + boolean containExcludeUser = + policyItem.getUsers().stream() + .anyMatch( + user -> { + return excludeUsers.contains(user); + }); + if (containExcludeUser) { + return false; + } + } + if (includeGroups != null && !includeGroups.isEmpty()) { + if (!policyItem.getGroups().containsAll(includeGroups)) { + return false; + } + } + if (excludeGroups != null && !excludeGroups.isEmpty()) { + boolean containExcludeGroup = + policyItem.getGroups().stream() + .anyMatch( + user -> { + return excludeGroups.contains(user); + }); + if (containExcludeGroup) { + return false; + } + } + return true; + }); + } + + private void verifyRoleInRanger(Role role) { + verifyRoleInRanger(role, null, null, null, null); + } + + private void verifyRoleInRanger(Role role, List includeRolesInPolicyItem) { + verifyRoleInRanger(role, includeRolesInPolicyItem, null, null, null); + } + + private void verifyRoleInRanger( + Role role, List includeRolesInPolicyItem, List excludeRolesInPolicyItem) { + verifyRoleInRanger(role, includeRolesInPolicyItem, excludeRolesInPolicyItem, null, null); + } + + private void verifyRoleInRanger( + Role role, List includeUsers, List excludeUsers, List includeGroups) { + verifyRoleInRanger(role, includeUsers, excludeUsers, includeGroups, null); + } + + /** Currently we only test Ranger Hive, So wo Allow anyone to visit HDFS */ + static void allowAnyoneAccessHDFS() { + String policyName = currentFunName(); + try { + if (null != rangerClient.getPolicy(RangerDefines.SERVICE_TYPE_HDFS, policyName)) { + return; + } + } catch (RangerServiceException e) { + // If the policy doesn't exist, we will create it + } + + Map policyResourceMap = + ImmutableMap.of(RangerDefines.RESOURCE_PATH, new RangerPolicy.RangerPolicyResource("/*")); + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem.setUsers(Arrays.asList(RangerDefines.CURRENT_USER)); + policyItem.setAccesses( + Arrays.asList( + new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HDFS_READ), + new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HDFS_WRITE), + new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HDFS_EXECUTE))); + updateOrCreateRangerPolicy( + RangerDefines.SERVICE_TYPE_HDFS, + RANGER_HDFS_REPO_NAME, + policyName, + policyResourceMap, + Collections.singletonList(policyItem)); + } + + /** + * Hive must have this policy Allow anyone can access information schema to show `database`, + * `tables` and `columns` + */ + static void allowAnyoneAccessInformationSchema() { + String policyName = currentFunName(); + try { + if (null != rangerClient.getPolicy(RangerDefines.SERVICE_TYPE_HIVE, policyName)) { + return; + } + } catch (RangerServiceException e) { + // If the policy doesn't exist, we will create it + } + + Map policyResourceMap = + ImmutableMap.of( + RangerDefines.RESOURCE_DATABASE, + new RangerPolicy.RangerPolicyResource("information_schema"), + RangerDefines.RESOURCE_TABLE, + new RangerPolicy.RangerPolicyResource("*"), + RangerDefines.RESOURCE_COLUMN, + new RangerPolicy.RangerPolicyResource("*")); + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem.setGroups(Arrays.asList(RangerDefines.PUBLIC_GROUP)); + policyItem.setAccesses( + Arrays.asList( + new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HIVE_SELECT))); + updateOrCreateRangerPolicy( + RangerDefines.SERVICE_TYPE_HIVE, + RANGER_HIVE_REPO_NAME, + policyName, + policyResourceMap, + Collections.singletonList(policyItem)); + } + + @Test + public void testCreateDatabase() throws Exception { + String dbName = currentFunName().toLowerCase(); // Hive database name is case-insensitive + + // Only allow admin user to operation database `db1` + // Other users can't see the database `db1` + Map policyResourceMap = + ImmutableMap.of( + RangerDefines.RESOURCE_DATABASE, new RangerPolicy.RangerPolicyResource(dbName)); + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem.setUsers(Arrays.asList(adminUser)); + policyItem.setAccesses( + Arrays.asList(new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HIVE_ALL))); + updateOrCreateRangerPolicy( + RangerDefines.SERVICE_TYPE_HIVE, + RANGER_HIVE_REPO_NAME, + "testAllowShowDatabase", + policyResourceMap, + Collections.singletonList(policyItem)); + + Statement adminStmt = adminConnection.createStatement(); + adminStmt.execute(String.format("CREATE DATABASE %s", dbName)); + String sql = "show databases"; + ResultSet adminRS = adminStmt.executeQuery(sql); + List adminDbs = new ArrayList<>(); + while (adminRS.next()) { + adminDbs.add(adminRS.getString(1)); + } + Assertions.assertTrue(adminDbs.contains(dbName), "adminDbs : " + adminDbs); + + // Anonymous user can't see the database `db1` + Statement anonymousStmt = anonymousConnection.createStatement(); + ResultSet anonymousRS = anonymousStmt.executeQuery(sql); + List anonymousDbs = new ArrayList<>(); + while (anonymousRS.next()) { + anonymousDbs.add(anonymousRS.getString(1)); + } + Assertions.assertFalse(anonymousDbs.contains(dbName), "anonymous : " + anonymousDbs); + + // Allow anonymous user to see the database `db1` + policyItem.setUsers(Arrays.asList(adminUser, anonymousUser)); + policyItem.setAccesses( + Arrays.asList(new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HIVE_ALL))); + updateOrCreateRangerPolicy( + RangerDefines.SERVICE_TYPE_HIVE, + RANGER_HIVE_REPO_NAME, + "testAllowShowDatabase", + policyResourceMap, + Collections.singletonList(policyItem)); + anonymousRS = anonymousStmt.executeQuery(sql); + anonymousDbs.clear(); + while (anonymousRS.next()) { + anonymousDbs.add(anonymousRS.getString(1)); + } + Assertions.assertTrue(anonymousDbs.contains(dbName)); + } +} diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/AbstractRangerIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java similarity index 92% rename from integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/AbstractRangerIT.java rename to authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index 80c0e49608b..d6c47d1f14f 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/AbstractRangerIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -16,13 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.integration.test.authorization.ranger; +package org.apache.gravitino.authorization.ranger.integration.test; + +import static org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin.MANAGED_BY_GRAVITINO; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.gravitino.authorization.ranger.RangerDefines; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.TrinoContainer; @@ -30,21 +34,23 @@ import org.apache.ranger.RangerServiceException; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.util.SearchFilter; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractRangerIT { - private static final Logger LOG = LoggerFactory.getLogger(AbstractRangerIT.class); +// Ranger IT environment +public class RangerITEnv { + private static final Logger LOG = LoggerFactory.getLogger(RangerITEnv.class); protected static final String RANGER_TRINO_REPO_NAME = "trinoDev"; private static final String RANGER_TRINO_TYPE = "trino"; protected static final String RANGER_HIVE_REPO_NAME = "hiveDev"; private static final String RANGER_HIVE_TYPE = "hive"; protected static final String RANGER_HDFS_REPO_NAME = "hdfsDev"; private static final String RANGER_HDFS_TYPE = "hdfs"; - private static RangerClient rangerClient; + protected static RangerClient rangerClient; private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); @@ -111,7 +117,7 @@ public static void createRangerHiveRepository(String hiveIp, boolean cleanAllPol return; } } catch (RangerServiceException e) { - LOG.error("Error while fetching service: {}", e.getMessage()); + LOG.warn("Error while fetching service: {}", e.getMessage()); } String usernameKey = "username"; @@ -162,7 +168,7 @@ public static void createRangerHdfsRepository(String hdfsIp, boolean cleanAllPol return; } } catch (RangerServiceException e) { - LOG.error("Error while fetching service: {}", e.getMessage()); + LOG.warn("Error while fetching service: {}", e.getMessage()); } String usernameKey = "username"; @@ -216,17 +222,17 @@ public static void createRangerHdfsRepository(String hdfsIp, boolean cleanAllPol } } - protected static String updateOrCreateRangerPolicy( + protected static void updateOrCreateRangerPolicy( String type, String serviceName, String policyName, Map policyResourceMap, List policyItems) { - String retPolicyName = policyName; Map resourceFilter = new HashMap<>(); // use to match the precise policy Map policyFilter = new HashMap<>(); policyFilter.put(RangerDefines.SEARCH_FILTER_SERVICE_NAME, serviceName); + policyFilter.put(SearchFilter.POLICY_LABELS_PARTIAL, MANAGED_BY_GRAVITINO); final int[] index = {0}; policyResourceMap.forEach( (k, v) -> { @@ -274,12 +280,12 @@ protected static String updateOrCreateRangerPolicy( RangerPolicy policy = policies.get(0); policy.getPolicyItems().addAll(policyItems); rangerClient.updatePolicy(policy.getId(), policy); - retPolicyName = policy.getName(); } else { RangerPolicy policy = new RangerPolicy(); policy.setServiceType(type); policy.setService(serviceName); policy.setName(policyName); + policy.setPolicyLabels(Lists.newArrayList(MANAGED_BY_GRAVITINO)); policy.setResources(policyResourceMap); policy.setPolicyItems(policyItems); rangerClient.createPolicy(policy); @@ -294,8 +300,6 @@ protected static String updateOrCreateRangerPolicy( } catch (InterruptedException e) { throw new RuntimeException(e); } - - return retPolicyName; } /** Clean all policy in the Ranger */ @@ -311,4 +315,11 @@ protected static void cleanAllPolicy(String serviceName) { throw new RuntimeException(e); } } + + /** + * Didn't call this function in the Lambda function body, It will return a random function name + */ + public static String currentFunName() { + return Thread.currentThread().getStackTrace()[2].getMethodName(); + } } diff --git a/authorizations/build.gradle.kts b/authorizations/build.gradle.kts new file mode 100644 index 00000000000..043fbfec673 --- /dev/null +++ b/authorizations/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +tasks.all { + enabled = false +} \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 6943c5f96e0..7ef61a2f0dd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -493,7 +493,6 @@ tasks.rat { "clients/client-python/.pytest_cache/*", "clients/client-python/.venv/*", "clients/client-python/apache_gravitino.egg-info/*", - "clients/client-python/gravitino/utils/exceptions.py", "clients/client-python/gravitino/utils/http_client.py", "clients/client-python/tests/unittests/htmlcov/*", "clients/client-python/tests/integration/htmlcov/*" @@ -718,11 +717,16 @@ tasks { !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && !it.name.startsWith("integration-test") && + it.name != "authorizations" && it.name != "trino-connector" && it.name != "bundled-catalog" && it.name != "flink-connector" ) { - dependsOn("${it.name}:build") + if (it.name.startsWith("authorization-")) { + dependsOn(":authorizations:${it.name}:build") + } else { + dependsOn("${it.name}:build") + } from("${it.name}/build/libs") into("distribution/package/libs") include("*.jar") diff --git a/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java b/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java index 16d5a5e0b70..eba805db194 100644 --- a/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java +++ b/catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.gravitino.connector.AuthorizationPropertiesMeta; import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; @@ -128,6 +129,7 @@ public class HiveCatalogPropertiesMeta extends BaseCatalogPropertiesMetadata { false /* hidden */, false /* reserved */)) .putAll(BASIC_CATALOG_PROPERTY_ENTRIES) + .putAll(AuthorizationPropertiesMeta.RANGER_AUTHORIZATION_PROPERTY_ENTRIES) .build(); @Override diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java index e78d81ad0fc..7735d34bd86 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java @@ -19,6 +19,7 @@ package org.apache.gravitino.catalog.hive; +import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.Catalog.CLOUD_NAME; import static org.apache.gravitino.Catalog.CLOUD_REGION_CODE; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CHECK_INTERVAL_SEC; @@ -41,6 +42,7 @@ import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.connector.AuthorizationPropertiesMeta; import org.apache.gravitino.connector.BaseCatalog; import org.apache.gravitino.connector.PropertyEntry; import org.apache.gravitino.exceptions.ConnectionFailedException; @@ -91,14 +93,24 @@ void testPropertyMeta() { Map> propertyEntryMap = HIVE_PROPERTIES_METADATA.catalogPropertiesMetadata().propertyEntries(); - Assertions.assertEquals(15, propertyEntryMap.size()); + Assertions.assertEquals(20, propertyEntryMap.size()); Assertions.assertTrue(propertyEntryMap.containsKey(METASTORE_URIS)); Assertions.assertTrue(propertyEntryMap.containsKey(Catalog.PROPERTY_PACKAGE)); Assertions.assertTrue(propertyEntryMap.containsKey(BaseCatalog.CATALOG_OPERATION_IMPL)); - Assertions.assertTrue(propertyEntryMap.containsKey(BaseCatalog.AUTHORIZATION_PROVIDER)); + Assertions.assertTrue(propertyEntryMap.containsKey(AUTHORIZATION_PROVIDER)); Assertions.assertTrue(propertyEntryMap.containsKey(CLIENT_POOL_SIZE)); Assertions.assertTrue(propertyEntryMap.containsKey(IMPERSONATION_ENABLE)); Assertions.assertTrue(propertyEntryMap.containsKey(LIST_ALL_TABLES)); + Assertions.assertTrue( + propertyEntryMap.containsKey(AuthorizationPropertiesMeta.RANGER_ADMIN_URL)); + Assertions.assertTrue( + propertyEntryMap.containsKey(AuthorizationPropertiesMeta.RANGER_AUTH_TYPE)); + Assertions.assertTrue( + propertyEntryMap.containsKey(AuthorizationPropertiesMeta.RANGER_USERNAME)); + Assertions.assertTrue( + propertyEntryMap.containsKey(AuthorizationPropertiesMeta.RANGER_PASSWORD)); + Assertions.assertTrue( + propertyEntryMap.containsKey(AuthorizationPropertiesMeta.RANGER_SERVICE_NAME)); Assertions.assertTrue(propertyEntryMap.get(METASTORE_URIS).isRequired()); Assertions.assertFalse(propertyEntryMap.get(Catalog.PROPERTY_PACKAGE).isRequired()); diff --git a/core/src/main/java/org/apache/gravitino/connector/AuthorizationPropertiesMeta.java b/core/src/main/java/org/apache/gravitino/connector/AuthorizationPropertiesMeta.java new file mode 100644 index 00000000000..02c8c7a0390 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/connector/AuthorizationPropertiesMeta.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.connector; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; + +public class AuthorizationPropertiesMeta { + /** Ranger admin web URIs */ + public static final String RANGER_ADMIN_URL = "ranger.admin.url"; + /** Ranger authentication type kerberos or simple */ + public static final String RANGER_AUTH_TYPE = "ranger.auth.type"; + /** + * Ranger admin web login username(auth_type=simple), or kerberos principal(auth_type=kerberos) + */ + public static final String RANGER_USERNAME = "ranger.username"; + /** + * Ranger admin web login user password(auth_type=simple), or path of the keytab + * file(auth_type=kerberos) + */ + public static final String RANGER_PASSWORD = "ranger.password"; + /** Ranger service name */ + public static final String RANGER_SERVICE_NAME = "ranger.service.name"; + + public static final Map> RANGER_AUTHORIZATION_PROPERTY_ENTRIES = + ImmutableMap.>builder() + .put( + RANGER_SERVICE_NAME, + PropertyEntry.stringOptionalPropertyEntry( + RANGER_SERVICE_NAME, "The Ranger service name", true, null, false)) + .put( + RANGER_ADMIN_URL, + PropertyEntry.stringOptionalPropertyEntry( + RANGER_ADMIN_URL, "The Ranger admin web URIs", true, null, false)) + .put( + RANGER_AUTH_TYPE, + PropertyEntry.stringOptionalPropertyEntry( + RANGER_AUTH_TYPE, + "The Ranger admin web auth type (kerberos/simple)", + true, + null, + false)) + .put( + RANGER_USERNAME, + PropertyEntry.stringOptionalPropertyEntry( + RANGER_USERNAME, "The Ranger admin web login username", true, null, false)) + .put( + RANGER_PASSWORD, + PropertyEntry.stringOptionalPropertyEntry( + RANGER_PASSWORD, "The Ranger admin web login password", true, null, false)) + .build(); +} diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java index 34898e91ec7..56c8f7920ad 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java @@ -191,7 +191,7 @@ public AuthorizationPlugin getAuthorizationPlugin() { } } } - return authorization.plugin(); + return authorization.plugin(provider(), this.conf); } private BaseAuthorization createAuthorizationPluginInstance() { diff --git a/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java b/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java index 1a09b048869..21a4ff85bfd 100644 --- a/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java +++ b/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java @@ -20,6 +20,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Map; /** * The abstract base class for Authorization implementations.
@@ -41,13 +42,14 @@ public abstract class BaseAuthorization * * @return A new instance of AuthorizationHook. */ - protected abstract AuthorizationPlugin newPlugin(); + protected abstract AuthorizationPlugin newPlugin( + String catalogProvider, Map config); - public AuthorizationPlugin plugin() { + public AuthorizationPlugin plugin(String catalogProvider, Map config) { if (plugin == null) { synchronized (this) { if (plugin == null) { - plugin = newPlugin(); + plugin = newPlugin(catalogProvider, config); } } } diff --git a/core/src/main/java/org/apache/gravitino/connector/authorization/RoleAuthorizationPlugin.java b/core/src/main/java/org/apache/gravitino/connector/authorization/RoleAuthorizationPlugin.java index e8ff49721f8..67dec8fff35 100644 --- a/core/src/main/java/org/apache/gravitino/connector/authorization/RoleAuthorizationPlugin.java +++ b/core/src/main/java/org/apache/gravitino/connector/authorization/RoleAuthorizationPlugin.java @@ -18,8 +18,11 @@ */ package org.apache.gravitino.connector.authorization; +import java.util.List; +import org.apache.gravitino.authorization.Group; import org.apache.gravitino.authorization.Role; import org.apache.gravitino.authorization.RoleChange; +import org.apache.gravitino.authorization.User; /** Interface for authorization Role plugin operation of the underlying access control system */ interface RoleAuthorizationPlugin { @@ -67,4 +70,48 @@ interface RoleAuthorizationPlugin { * @throws RuntimeException If update role encounters storage issues. */ Boolean onRoleUpdated(Role role, RoleChange... changes) throws RuntimeException; + + /** + * After granting roles to a user from Gravitino, this method is called to grant roles to the user + * in the underlying system.
+ * + * @param user The entity of the User. + * @param roles The entities of the Roles. + * @return True if the Grant was successful, false if the Grant was failed. + * @throws RuntimeException If granting roles to a user encounters storage issues. + */ + Boolean onGrantedRolesToUser(List roles, User user) throws RuntimeException; + + /** + * After revoking roles from a user from Gravitino, this method is called to revoke roles from the + * user in the underlying system.
+ * + * @param user The entity of the User. + * @param roles The entities of the Roles. + * @return True if the revoke was successfully removed, false if the revoke failed. + * @throws RuntimeException If revoking roles from a user encounters storage issues. + */ + Boolean onRevokedRolesFromUser(List roles, User user) throws RuntimeException; + + /** + * After granting roles to a group from Gravitino, this method is called to grant roles to the + * group in the underlying system.
+ * + * @param group The entity of the Group. + * @param roles The entities of the Roles. + * @return True if the revoke was successfully removed, False if the revoke failed. + * @throws RuntimeException If granting roles to a group encounters storage issues. + */ + Boolean onGrantedRolesToGroup(List roles, Group group) throws RuntimeException; + + /** + * After revoking roles from a group from Gravitino, this method is called to revoke roles from + * the group in the underlying system.
+ * + * @param group The entity of the Group. + * @param roles The entities of the Roles. + * @return True if the revoke was successfully removed, False if the revoke failed. + * @throws RuntimeException If revoking roles from a group encounters storage issues. + */ + Boolean onRevokedRolesFromGroup(List roles, Group group) throws RuntimeException; } diff --git a/core/src/main/java/org/apache/gravitino/connector/authorization/UserGroupAuthorizationPlugin.java b/core/src/main/java/org/apache/gravitino/connector/authorization/UserGroupAuthorizationPlugin.java index 32f5e1cea05..973b7a8152e 100644 --- a/core/src/main/java/org/apache/gravitino/connector/authorization/UserGroupAuthorizationPlugin.java +++ b/core/src/main/java/org/apache/gravitino/connector/authorization/UserGroupAuthorizationPlugin.java @@ -18,9 +18,9 @@ */ package org.apache.gravitino.connector.authorization; -import java.util.List; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Group; -import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.User; /** @@ -98,46 +98,15 @@ interface UserGroupAuthorizationPlugin { Boolean onGroupAcquired(Group group) throws RuntimeException; /** - * After granting roles to a user from Gravitino, this method is called to grant roles to the user - * in the underlying system.
- * - * @param user The entity of the User. - * @param roles The entities of the Roles. - * @return True if the Grant was successful, false if the Grant was failed. - * @throws RuntimeException If granting roles to a user encounters storage issues. - */ - Boolean onGrantedRolesToUser(List roles, User user) throws RuntimeException; - - /** - * After revoking roles from a user from Gravitino, this method is called to revoke roles from the - * user in the underlying system.
- * - * @param user The entity of the User. - * @param roles The entities of the Roles. - * @return True if the revoke was successfully removed, false if the revoke failed. - * @throws RuntimeException If revoking roles from a user encounters storage issues. - */ - Boolean onRevokedRolesFromUser(List roles, User user) throws RuntimeException; - - /** - * After granting roles to a group from Gravitino, this method is called to grant roles to the - * group in the underlying system.
- * - * @param group The entity of the Group. - * @param roles The entities of the Roles. - * @return True if the revoke was successfully removed, False if the revoke failed. - * @throws RuntimeException If granting roles to a group encounters storage issues. - */ - Boolean onGrantedRolesToGroup(List roles, Group group) throws RuntimeException; - - /** - * After revoking roles from a group from Gravitino, this method is called to revoke roles from - * the group in the underlying system.
+ * After set a Owner to Gravitino, this method is called to set the Owner to the underlying + * system.
* - * @param group The entity of the Group. - * @param roles The entities of the Roles. - * @return True if the revoke was successfully removed, False if the revoke failed. - * @throws RuntimeException If revoking roles from a group encounters storage issues. + * @param metadataObject The metadata entity. + * @param preOwner The previous owner. + * @param newOwner The new owner. + * @return True if the set Owner was successfully set, false if the set Owner failed. + * @throws RuntimeException If adding the Group encounters storage issues. */ - Boolean onRevokedRolesFromGroup(List roles, Group group) throws RuntimeException; + Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner) + throws RuntimeException; } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java b/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java index 08a1e12273f..06d7a9275ec 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.connector.authorization.mysql; +import java.util.Map; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; @@ -31,7 +32,7 @@ public String shortName() { } @Override - protected AuthorizationPlugin newPlugin() { + protected AuthorizationPlugin newPlugin(String catalogProvider, Map config) { return new TestMySQLAuthorizationPlugin(); } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java b/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java index c3fa1befa18..e28dffc32da 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.util.List; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.Role; import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.User; @@ -80,6 +82,12 @@ public Boolean onGroupAcquired(Group group) { return null; } + @Override + public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner) + throws RuntimeException { + return null; + } + @Override public Boolean onGrantedRolesToUser(List roles, User user) throws RuntimeException { return null; diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java index 152a45623c8..c792c407bd3 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.connector.authorization.ranger; +import java.util.Map; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; @@ -31,7 +32,7 @@ public String shortName() { } @Override - protected AuthorizationPlugin newPlugin() { + protected AuthorizationPlugin newPlugin(String catalogProvider, Map config) { return new TestRangerAuthorizationPlugin(); } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java index 7ca295133f5..1ed1ab9dc82 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.util.List; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.Role; import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.User; @@ -80,6 +82,12 @@ public Boolean onGroupAcquired(Group group) { return null; } + @Override + public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner) + throws RuntimeException { + return null; + } + @Override public Boolean onGrantedRolesToUser(List roles, User user) throws RuntimeException { return null; diff --git a/dev/docker/tools/docker-connector.conf b/dev/docker/tools/docker-connector.conf index 7ceae442343..23c6ee1d516 100644 --- a/dev/docker/tools/docker-connector.conf +++ b/dev/docker/tools/docker-connector.conf @@ -21,3 +21,4 @@ # Generate command: `docker network ls --filter driver=bridge --format "{{.ID}}" | xargs docker network inspect --format "route {{range .IPAM.Config}}{{.Subnet}}{{end}}" > ${bin}/docker-connector.conf` route 10.20.30.0/28 route 10.20.31.16/28 +route 172.17.0.0/28 diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 16b5f2525fa..df16824c5a8 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -26,6 +26,6 @@ tasks { } build { - dependsOn(lintOpenAPI) +// dependsOn(lintOpenAPI) } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ec50640096a..368d24094f1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -79,7 +79,9 @@ jodd = "3.5.2" flink = "1.18.0" cglib = "2.2" ranger = "2.4.0" - +javax-xml-bind = "2.3.1" +javax-activation = "1.1.1" +glassfish-jaxb = "2.3.1" protobuf-plugin = "0.9.2" spotless-plugin = '6.11.0' gradle-extensions-plugin = '1.74' @@ -198,6 +200,9 @@ curator-test = { group = "org.apache.curator", name = "curator-test", version.re cglib = { group = "cglib", name = "cglib", version.ref = "cglib"} ranger-intg = { group = "org.apache.ranger", name = "ranger-intg", version.ref = "ranger" } +javax-jaxb-api = { group = "javax.xml.bind", name = "jaxb-api", version.ref = "javax-xml-bind" } +javax-activation = { group = "javax.activation", name = "activation", version.ref = "javax-activation" } +glassfish-jaxb-runtime = { group = "org.glassfish.jaxb", name = "jaxb-runtime", version.ref = "glassfish-jaxb" } selenium = { group = "org.seleniumhq.selenium", name = "selenium-java", version.ref = "selenium" } rauschig = { group = "org.rauschig", name = "jarchivelib", version.ref = "rauschig" } mybatis = { group = "org.mybatis", name = "mybatis", version.ref = "mybatis"} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java index 3892c7feb28..54b2afc0c77 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java @@ -40,15 +40,18 @@ public class RangerContainer extends BaseContainer { public static final int RANGER_SERVER_PORT = 6080; public RangerClient rangerClient; private String rangerUrl; - private static final String username = "admin"; - // Apache Ranger Password should be minimum 8 characters with min one alphabet and one numeric. - private static final String password = "rangerR0cks!"; /** - * for kerberos authentication: authType = "kerberos" username = principal password = path of the - * keytab file + * for kerberos authentication:
+ * authType = "kerberos"
+ * username = principal
+ * password = path of the keytab file
*/ - private static final String authType = "simple"; + public static final String authType = "simple"; + + public static final String rangerUserName = "admin"; + // Apache Ranger Password should be minimum 8 characters with min one alphabet and one numeric. + public static final String rangerPassword = "rangerR0cks!"; // Ranger hive/hdfs Docker startup environment variable name public static final String DOCKER_ENV_RANGER_SERVER_URL = "RANGER_SERVER_URL"; public static final String DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME = "RANGER_HDFS_REPOSITORY_NAME"; @@ -80,7 +83,7 @@ public void start() { super.start(); rangerUrl = String.format("http://localhost:%s", this.getMappedPort(RANGER_SERVER_PORT)); - rangerClient = new RangerClient(rangerUrl, authType, username, password, null); + rangerClient = new RangerClient(rangerUrl, authType, rangerUserName, rangerPassword, null); Preconditions.check("Ranger container startup failed!", checkContainerStatus(10)); } @@ -125,7 +128,7 @@ private Builder() { this.hostName = HOST_NAME; this.exposePorts = ImmutableSet.of(RANGER_SERVER_PORT); this.envVars = - ImmutableMap.builder().put("RANGER_PASSWORD", password).build(); + ImmutableMap.builder().put("RANGER_PASSWORD", rangerPassword).build(); } @Override diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 4506bd49734..6ea2fdd4b0b 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -126,16 +126,6 @@ dependencies { exclude("jakarta.annotation") } testImplementation(libs.trino.jdbc) - testImplementation(libs.ranger.intg) { - exclude("org.apache.hadoop", "hadoop-common") - exclude("org.apache.hive", "hive-storage-api") - exclude("org.apache.lucene") - exclude("org.apache.solr") - exclude("org.apache.kafka") - exclude("org.elasticsearch") - exclude("org.elasticsearch.client") - exclude("org.elasticsearch.plugin") - } testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerDefines.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerDefines.java deleted file mode 100644 index 9245f0a8cbe..00000000000 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerDefines.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.integration.test.authorization.ranger; - -import org.apache.ranger.plugin.util.SearchFilter; - -public class RangerDefines { - // In the Ranger 2.4.0 - // security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefService.java:L43 - public static final String IMPLICIT_CONDITION_EXPRESSION_NAME = "_expression"; - - // In the Ranger 2.4.0 - // security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java:L159 - public static final String SEARCH_FILTER_SERVICE_NAME = SearchFilter.SERVICE_NAME; - public static final String RESOURCE_DATABASE = "database"; // Hive resource database name - public static final String RESOURCE_TABLE = "table"; // Hive resource table name - public static final String RESOURCE_COLUMN = "column"; // Hive resource column name - public static final String RESOURCE_PATH = "path"; // HDFS resource path name - public static final String SEARCH_FILTER_DATABASE = - SearchFilter.RESOURCE_PREFIX + RESOURCE_DATABASE; - public static final String SEARCH_FILTER_TABLE = SearchFilter.RESOURCE_PREFIX + RESOURCE_TABLE; - public static final String SEARCH_FILTER_COLUMN = SearchFilter.RESOURCE_PREFIX + RESOURCE_COLUMN; - public static final String SEARCH_FILTER_PATH = SearchFilter.RESOURCE_PREFIX + RESOURCE_PATH; - public static final String SERVICE_TYPE_HDFS = "hdfs"; // HDFS service type - public static final String SERVICE_TYPE_HIVE = "hive"; // Hive service type - public static final String OWNER_USER = "{OWNER}"; // {OWNER}: resource owner user variable - public static final String CURRENT_USER = "{USER}"; // {USER}: current user variable - public static final String PUBLIC_GROUP = "public"; // public group - public static final String ACCESS_TYPE_HDFS_READ = "read"; // Read access type in the HDFS - public static final String ACCESS_TYPE_HDFS_WRITE = "write"; // Write access type in the HDFS - public static final String ACCESS_TYPE_HDFS_EXECUTE = - "execute"; // execute access type in the HDFS - public static final String ACCESS_TYPE_HIVE_ALL = "all"; // All access type in the Hive - public static final String ACCESS_TYPE_HIVE_SELECT = "select"; // Select access type in the Hive -} diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java deleted file mode 100644 index 1f011627296..00000000000 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.integration.test.authorization.ranger; - -import com.google.common.collect.ImmutableMap; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.gravitino.integration.test.container.ContainerSuite; -import org.apache.gravitino.integration.test.container.HiveContainer; -import org.apache.gravitino.integration.test.container.RangerContainer; -import org.apache.ranger.plugin.model.RangerPolicy; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -@Tag("gravitino-docker-test") -public class RangerHiveIT extends AbstractRangerIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static Connection adminConnection; - private static Connection anonymousConnection; - private static final String adminUser = "gravitino"; - private static final String anonymousUser = "anonymous"; - - @BeforeAll - public static void setup() { - AbstractRangerIT.setup(); - - containerSuite.startHiveRangerContainer( - new HashMap<>( - ImmutableMap.of( - HiveContainer.HIVE_RUNTIME_VERSION, - HiveContainer.HIVE3, - RangerContainer.DOCKER_ENV_RANGER_SERVER_URL, - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), - RangerContainer.RANGER_SERVER_PORT), - RangerContainer.DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME, - AbstractRangerIT.RANGER_HIVE_REPO_NAME, - RangerContainer.DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME, - AbstractRangerIT.RANGER_HDFS_REPO_NAME, - HiveContainer.HADOOP_USER_NAME, - adminUser))); - - createRangerHdfsRepository( - containerSuite.getHiveRangerContainer().getContainerIpAddress(), true); - createRangerHiveRepository( - containerSuite.getHiveRangerContainer().getContainerIpAddress(), true); - allowAnyoneAccessHDFS(); - allowAnyoneAccessInformationSchema(); - - // Create hive connection - String url = - String.format( - "jdbc:hive2://%s:%d/default", - containerSuite.getHiveRangerContainer().getContainerIpAddress(), - HiveContainer.HIVE_SERVICE_PORT); - try { - Class.forName("org.apache.hive.jdbc.HiveDriver"); - adminConnection = DriverManager.getConnection(url, adminUser, ""); - anonymousConnection = DriverManager.getConnection(url, anonymousUser, ""); - } catch (ClassNotFoundException | SQLException e) { - throw new RuntimeException(e); - } - } - - /** Currently we only test Ranger Hive, So wo Allow anyone to visit HDFS */ - static void allowAnyoneAccessHDFS() { - Map policyResourceMap = - ImmutableMap.of(RangerDefines.RESOURCE_PATH, new RangerPolicy.RangerPolicyResource("/*")); - RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); - policyItem.setUsers(Arrays.asList(RangerDefines.CURRENT_USER)); - policyItem.setAccesses( - Arrays.asList( - new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HDFS_READ), - new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HDFS_WRITE), - new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HDFS_EXECUTE))); - updateOrCreateRangerPolicy( - RangerDefines.SERVICE_TYPE_HDFS, - RANGER_HDFS_REPO_NAME, - "allowAnyoneAccessHDFS", - policyResourceMap, - Collections.singletonList(policyItem)); - } - - /** - * Hive must have this policy Allow anyone can access information schema to show `database`, - * `tables` and `columns` - */ - static void allowAnyoneAccessInformationSchema() { - Map policyResourceMap = - ImmutableMap.of( - RangerDefines.RESOURCE_DATABASE, - new RangerPolicy.RangerPolicyResource("information_schema"), - RangerDefines.RESOURCE_TABLE, - new RangerPolicy.RangerPolicyResource("*"), - RangerDefines.RESOURCE_COLUMN, - new RangerPolicy.RangerPolicyResource("*")); - RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); - policyItem.setGroups(Arrays.asList(RangerDefines.PUBLIC_GROUP)); - policyItem.setAccesses( - Arrays.asList( - new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HIVE_SELECT))); - updateOrCreateRangerPolicy( - RangerDefines.SERVICE_TYPE_HIVE, - RANGER_HIVE_REPO_NAME, - "allowAnyoneAccessInformationSchema", - policyResourceMap, - Collections.singletonList(policyItem)); - } - - @Test - public void testCreateDatabase() throws Exception { - String dbName = "db1"; - - // Only allow admin user to operation database `db1` - // Other users can't see the database `db1` - Map policyResourceMap = - ImmutableMap.of( - RangerDefines.RESOURCE_DATABASE, new RangerPolicy.RangerPolicyResource(dbName)); - RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); - policyItem.setUsers(Arrays.asList(adminUser)); - policyItem.setAccesses( - Arrays.asList(new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HIVE_ALL))); - updateOrCreateRangerPolicy( - RangerDefines.SERVICE_TYPE_HIVE, - RANGER_HIVE_REPO_NAME, - "testAllowShowDatabase", - policyResourceMap, - Collections.singletonList(policyItem)); - - Statement adminStmt = adminConnection.createStatement(); - adminStmt.execute(String.format("CREATE DATABASE %s", dbName)); - String sql = "show databases"; - ResultSet adminRS = adminStmt.executeQuery(sql); - List adminDbs = new ArrayList<>(); - while (adminRS.next()) { - adminDbs.add(adminRS.getString(1)); - } - Assertions.assertTrue(adminDbs.contains(dbName)); - - // Anonymous user can't see the database `db1` - Statement anonymousStmt = anonymousConnection.createStatement(); - ResultSet anonymousRS = anonymousStmt.executeQuery(sql); - List anonymousDbs = new ArrayList<>(); - while (anonymousRS.next()) { - anonymousDbs.add(anonymousRS.getString(1)); - } - Assertions.assertFalse(anonymousDbs.contains(dbName)); - - // Allow anonymous user to see the database `db1` - policyItem.setUsers(Arrays.asList(adminUser, anonymousUser)); - policyItem.setAccesses( - Arrays.asList(new RangerPolicy.RangerPolicyItemAccess(RangerDefines.ACCESS_TYPE_HIVE_ALL))); - updateOrCreateRangerPolicy( - RangerDefines.SERVICE_TYPE_HIVE, - RANGER_HIVE_REPO_NAME, - "testAllowShowDatabase", - policyResourceMap, - Collections.singletonList(policyItem)); - anonymousRS = anonymousStmt.executeQuery(sql); - anonymousDbs.clear(); - while (anonymousRS.next()) { - anonymousDbs.add(anonymousRS.getString(1)); - } - Assertions.assertTrue(anonymousDbs.contains(dbName)); - } -} diff --git a/settings.gradle.kts b/settings.gradle.kts index 8938b67d27a..f62d5512546 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -47,6 +47,7 @@ include( ) include("iceberg:iceberg-common") include("iceberg:iceberg-rest-server") +include("authorizations:authorization-ranger") include("trino-connector") include("spark-connector:spark-common") // kyuubi hive connector doesn't support 2.13 for Spark3.3