[apache#4662] improve(IT): Add ranger authorization Hive E2E test

.github/workflows/access-control-integration-test.yml

@@ -0,0 +1,96 @@
+name: Access Control Integration Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main", "branch-*" ]
+  pull_request:
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            source_changes:
+              - api/**
+              - authorizations/**
+              - catalogs/**
+              - clients/client-java/**
+              - clients/client-java-runtime/**
+              - common/**
+              - core/**
+              - integration-test-common/**
+              - server/**
+              - server-common/**
+    outputs:
+      source_changes: ${{ steps.filter.outputs.source_changes }}
+
+  # Integration test for AMD64 architecture
+  test-amd64-arch:
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        # Integration test for AMD64 architecture
+        architecture: [linux/amd64]
+        java-version: [ 17 ]
+        test-mode: [ embedded, deploy ]
+        include:
+          - test-mode: 'embedded'
+            backend: 'h2'
+          - test-mode: 'deploy'
+            backend: 'mysql'
+
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        if : ${{ matrix.test-mode == 'deploy' }}
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Authorization Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+        id: integrationTest
+        run: |
+          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdbcBackend=${{ matrix.backend }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**"
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: authorizations-integrate-test-reports-${{ matrix.java-version }}
+          path: |
+            build/reports
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log

.github/workflows/backend-integration-test.yml

@@ -90,13 +90,84 @@ jobs:
         run: |
           dev/ci/util_free_space.sh
 
-      - name: Backend Integration Test(JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+      - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
         id: integrationTest
         run: >
           ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
           -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+          -x :authorizations:authorization-ranger:test
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
+          path: |
+            build/reports
+            iceberg/iceberg-rest-server/build/*.log
+            integration-test/build/*.log
+            integration-test/build/*.tar
+            integration-test/build/trino-ci-container-log
+            distribution/package/logs/*.out
+            distribution/package/logs/*.log
+            catalogs/**/*.log
+            catalogs/**/*.tar
+            distribution/**/*.log
+
+  test-on-pr:
+    needs: changes
+    if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        # Integration test for AMD64 architecture
+        architecture: [ linux/amd64 ]
+        java-version: [ 17 ]
+        test-mode: [ embedded, deploy ]
+        include:
+          - test-mode: 'embedded'
+            backend: 'h2'
+          - test-mode: 'deploy'
+            backend: 'mysql'
+
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        if: ${{ matrix.test-mode == 'deploy' }}
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
+        id: integrationTest
+        run: >
+          ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
+          -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
+          -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
+          -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+          -x :authorizations:authorization-ranger:test
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

LICENSE.bin

@@ -310,6 +310,8 @@
    Apache Yetus - Audience Annotations
    Apache Kerby
    Apache Kyuubi
+   Apache Ranger
+   Apache Ranger intg
    Jackson JSON processor
    DataNucleus
    Modernizer Maven Plugin
@@ -364,7 +366,7 @@
 
    This product bundles various third-party components also under the
    BSD license
-   
+
    JSR305
    LevelDB JNI
    RocksDB JNI
@@ -446,6 +448,7 @@
    Jakarta RESTful WS API
    Jakarta XML Binding API
    JavaServer Pages(TM) API
+   Javax WS RS API
    HK2 API Module
    HK2 Service Locator
    HK2 Utils

NOTICE.bin

@@ -22,6 +22,9 @@ Copyright 2008-2023 The Apache Software Foundation
 Apache Zeppelin
 Copyright 2016-2023 The Apache Software Foundation
 
+Apache Ranger
+Copyright 2014-2024 The Apache Software Foundation
+
 Apache Hadoop
 Copyright 2006 and onwards The Apache Software Foundation.
 

authorizations/authorization-ranger/build.gradle.kts

@@ -51,11 +51,17 @@ dependencies {
     exclude("org.apache.ranger", "ranger-plugins-audit")
     exclude("org.apache.ranger", "ranger-plugins-cred")
     exclude("org.apache.ranger", "ranger-plugins-classloader")
+    exclude("javax.ws.rs")
   }
+  implementation(libs.javax.ws.rs.api)
   implementation(libs.javax.jaxb.api) {
     exclude("*")
   }
 
+  testImplementation(project(":common"))
+  testImplementation(project(":clients:client-java"))
+  testImplementation(project(":server"))
+  testImplementation(project(":catalogs:catalog-common"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.mockito.core)
@@ -69,24 +75,33 @@ dependencies {
     exclude("org.elasticsearch")
     exclude("org.elasticsearch.client")
     exclude("org.elasticsearch.plugin")
+    exclude("javax.ws.rs")
   }
   testImplementation(libs.hive2.jdbc) {
     exclude("org.slf4j")
   }
+  testImplementation(libs.mysql.driver)
 }
 
 tasks {
   val runtimeJars by registering(Copy::class) {
     from(configurations.runtimeClasspath)
     into("build/libs")
   }
-}
 
-tasks.build {
-  dependsOn("runtimeJars", "javadoc")
+  val copyAuthorizationLibs by registering(Copy::class) {
+    dependsOn("jar", "runtimeJars")
+    from("build/libs")
+    into("$rootDir/distribution/package/authorizations/ranger/libs")
+  }
+
+  register("copyLibAndConfig", Copy::class) {
+    dependsOn(copyAuthorizationLibs)
+  }
 }
 
 tasks.test {
+  dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars")
   val skipUTs = project.hasProperty("skipTests")
   if (skipUTs) {
     // Only run integration tests
@@ -101,7 +116,3 @@ tasks.test {
     dependsOn(tasks.jar)
   }
 }
-
-tasks.getByName("generateMetadataFileForMavenJavaPublication") {
-  dependsOn("runtimeJars")
-}