Optimize.

apache · Oct 29, 2024 · 27393e5 · 27393e5
1 parent 268831d
commit 27393e5
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 39 deletions.
diff --git a/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.gravitino.Catalog;
+import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
 import org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
 import org.apache.gravitino.storage.GCSProperties;
@@ -120,19 +121,15 @@ public void tearDown() throws IOException {
    */
   protected Configuration convertGvfsConfigToRealFileSystemConfig(Configuration gvfsConf) {
     Configuration gcsConf = new Configuration();
-    gvfsConf.forEach(
-        entry -> {
-          if (entry.getKey().startsWith("gravitino.bypass.")) {
-            gcsConf.set(entry.getKey().replace("gravitino.bypass.", ""), entry.getValue());
-          } else if (GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY.containsKey(
-              entry.getKey())) {
-            gcsConf.set(
-                GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY.get(entry.getKey()),
-                entry.getValue());
-          } else {
-            gcsConf.set(entry.getKey(), entry.getValue());
-          }
-        });
+    Map<String, String> map = Maps.newHashMap();
+
+    gvfsConf.forEach(entry -> map.put(entry.getKey(), entry.getValue()));
+
+    Map<String, String> hadoopConfMap =
+        FileSystemUtils.toHadoopConfigMap(
+            map, GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY);
+
+    hadoopConfMap.forEach(gcsConf::set);
 
     return gcsConf;
   }

diff --git a/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java b/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java
@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.Map;
 import org.apache.gravitino.Catalog;
+import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
 import org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
 import org.apache.gravitino.storage.OSSProperties;
@@ -128,19 +129,15 @@ public void tearDown() throws IOException {
    */
   protected Configuration convertGvfsConfigToRealFileSystemConfig(Configuration gvfsConf) {
     Configuration ossConf = new Configuration();
-    gvfsConf.forEach(
-        entry -> {
-          if (entry.getKey().startsWith("gravitino.bypass.")) {
-            ossConf.set(entry.getKey().replace("gravitino.bypass.", ""), entry.getValue());
-          } else if (GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY.containsKey(
-              entry.getKey())) {
-            ossConf.set(
-                GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY.get(entry.getKey()),
-                entry.getValue());
-          } else {
-            ossConf.set(entry.getKey(), entry.getValue());
-          }
-        });
+    Map<String, String> map = Maps.newHashMap();
+
+    gvfsConf.forEach(entry -> map.put(entry.getKey(), entry.getValue()));
+
+    Map<String, String> hadoopConfMap =
+        FileSystemUtils.toHadoopConfigMap(
+            map, GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY);
+
+    hadoopConfMap.forEach(ossConf::set);
 
     return ossConf;
   }

diff --git a/...g/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java b/...g/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java
@@ -29,6 +29,7 @@
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import org.apache.gravitino.Catalog;
+import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
 import org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration;
 import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
@@ -185,19 +186,15 @@ public void tearDown() throws IOException {
    */
   protected Configuration convertGvfsConfigToRealFileSystemConfig(Configuration gvfsConf) {
     Configuration s3Conf = new Configuration();
-    gvfsConf.forEach(
-        entry -> {
-          if (entry.getKey().startsWith("gravitino.bypass.")) {
-            s3Conf.set(entry.getKey().replace("gravitino.bypass.", ""), entry.getValue());
-          } else if (GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY.containsKey(
-              entry.getKey())) {
-            s3Conf.set(
-                GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY.get(entry.getKey()),
-                entry.getValue());
-          } else {
-            s3Conf.set(entry.getKey(), entry.getValue());
-          }
-        });
+    Map<String, String> map = Maps.newHashMap();
+
+    gvfsConf.forEach(entry -> map.put(entry.getKey(), entry.getValue()));
+
+    Map<String, String> hadoopConfMap =
+        FileSystemUtils.toHadoopConfigMap(
+            map, GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY);
+
+    hadoopConfMap.forEach(s3Conf::set);
 
     return s3Conf;
   }