[Improve][E2E] update doris image to official version (#7773)

apache · Sep 30, 2024 · a89c9fc · a89c9fc
1 parent 35b1750
commit a89c9fc
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 17 deletions.
diff --git a/...tor-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java b/...tor-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java
@@ -30,8 +30,11 @@
 import com.google.common.collect.Lists;
 import lombok.extern.slf4j.Slf4j;
 
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.sql.Connection;
-import java.sql.DriverManager;
+import java.sql.Driver;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
@@ -47,7 +50,7 @@
 public abstract class AbstractDorisIT extends TestSuiteBase implements TestResource {
 
     protected GenericContainer<?> container;
-    private static final String DOCKER_IMAGE = "bingquanzhao/doris:2.0.3";
+    private static final String DOCKER_IMAGE = "apache/doris:doris-all-in-one-2.1.0";
     protected static final String HOST = "doris_e2e";
     protected static final int QUERY_PORT = 9030;
     protected static final int HTTP_PORT = 8030;
@@ -58,8 +61,15 @@ public abstract class AbstractDorisIT extends TestSuiteBase implements TestResou
     protected Connection jdbcConnection;
     private static final String SET_SQL =
             "ADMIN SET FRONTEND CONFIG (\"enable_batch_delete_by_default\" = \"true\")";
+    private static final String SET_CONNECTIONS =
+            "SET PROPERTY FOR 'root' 'max_user_connections' = '10000'";
+    private static final String SHOW_FE = "SHOW FRONTENDS";
     private static final String SHOW_BE = "SHOW BACKENDS";
+    private static final String DROP_BE = "ALTER SYSTEM DROPP BACKEND \"127.0.0.1:9050\"";
+    private static final String ADD_BE = "ALTER SYSTEM ADD BACKEND \"%s:9050\"";
     protected static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver";
+    protected static final String DRIVER_JAR =
+            "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar";
 
     @BeforeAll
     @Override
@@ -76,24 +86,29 @@ public void startUp() {
                         String.format("%s:%s", QUERY_PORT, QUERY_PORT),
                         String.format("%s:%s", HTTP_PORT, HTTP_PORT),
                         String.format("%s:%s", BE_HTTP_PORT, BE_HTTP_PORT)));
-
         Startables.deepStart(Stream.of(container)).join();
         log.info("doris container started");
         given().ignoreExceptions()
                 .await()
-                .atMost(10000, TimeUnit.SECONDS)
+                .atMost(360, TimeUnit.SECONDS)
                 .untilAsserted(this::initializeJdbcConnection);
     }
 
-    protected void initializeJdbcConnection() throws SQLException {
+    protected void initializeJdbcConnection()
+            throws SQLException, ClassNotFoundException, MalformedURLException,
+                    InstantiationException, IllegalAccessException {
+        URLClassLoader urlClassLoader =
+                new URLClassLoader(new URL[] {new URL(DRIVER_JAR)}, DorisIT.class.getClassLoader());
+        Thread.currentThread().setContextClassLoader(urlClassLoader);
+        Driver driver = (Driver) urlClassLoader.loadClass(DRIVER_CLASS).newInstance();
         Properties props = new Properties();
         props.put("user", USERNAME);
         props.put("password", PASSWORD);
-
-        jdbcConnection =
-                DriverManager.getConnection(String.format(URL, container.getHost()), props);
+        jdbcConnection = driver.connect(String.format(URL, container.getHost()), props);
+        initializeBE();
         try (Statement statement = jdbcConnection.createStatement()) {
             statement.execute(SET_SQL);
+            statement.execute(SET_CONNECTIONS);
             ResultSet resultSet = null;
             do {
                 if (resultSet != null) {
@@ -104,6 +119,23 @@ protected void initializeJdbcConnection() throws SQLException {
         }
     }
 
+    // The Host of the official image [apache/doris:doris-all-in-one-2.1.0] BE is 127.0.0.1, causing
+    // cross-container access failure. Delete the BE and add it again
+    private void initializeBE() {
+        try (Statement statement = jdbcConnection.createStatement()) {
+            ResultSet resultSet = statement.executeQuery(SHOW_FE);
+            String feIp = null;
+            while (resultSet.next()) {
+                feIp = resultSet.getString("Host");
+            }
+            statement.execute(DROP_BE);
+            statement.execute(String.format(ADD_BE, feIp));
+
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private boolean isBeReady(ResultSet rs, Duration duration) throws SQLException {
         if (rs.next()) {
             String isAlive = rs.getString("Alive").trim();

diff --git a/...ctor-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java b/...ctor-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java
@@ -94,7 +94,6 @@ private void initCatalogFactory() {
     private void initCatalog() {
         String catalogName = "doris";
         String frontEndNodes = container.getHost() + ":" + HTTP_PORT;
-
         factory = new DorisCatalogFactory();
 
         Map<String, Object> map = new HashMap<>();
@@ -106,6 +105,7 @@ private void initCatalog() {
         catalog = (DorisCatalog) factory.createCatalog(catalogName, ReadonlyConfig.fromMap(map));
 
         catalog.open();
+        catalog.createDatabase(tablePath, false);
     }
 
     @Test

diff --git a/...nector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java b/...nector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java
@@ -17,6 +17,7 @@
 
 package org.apache.seatunnel.e2e.connector.doris;
 
+import org.apache.seatunnel.connectors.doris.exception.DorisConnectorErrorCode;
 import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
 import org.apache.seatunnel.e2e.common.container.EngineType;
 import org.apache.seatunnel.e2e.common.container.TestContainer;
@@ -41,8 +42,6 @@
 @Slf4j
 public class DorisErrorIT extends AbstractDorisIT {
     private static final String TABLE = "doris_e2e_table";
-    private static final String DRIVER_JAR =
-            "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar";
 
     private static final String sinkDB = "e2e_sink";
 
@@ -82,8 +81,7 @@ public void testDoris(TestContainer container) throws InterruptedException, Exec
         Assertions.assertTrue(
                 future.get()
                         .getStderr()
-                        .contains(
-                                "Caused by: org.apache.seatunnel.connectors.doris.exception.DorisConnectorException: ErrorCode:[Doris-01], ErrorDescription:[stream load error]"));
+                        .contains(DorisConnectorErrorCode.STREAM_LOAD_FAILED.getCode()));
         Assertions.assertTrue(
                 future.get()
                         .getStderr()
@@ -94,7 +92,7 @@ public void testDoris(TestContainer container) throws InterruptedException, Exec
         // wait for the container to restart
         given().ignoreExceptions()
                 .await()
-                .atMost(10000, TimeUnit.SECONDS)
+                .atMost(360, TimeUnit.SECONDS)
                 .untilAsserted(this::initializeJdbcConnection);
     }
 

diff --git a/...e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java b/...e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java
@@ -61,9 +61,6 @@
 public class DorisIT extends AbstractDorisIT {
     private static final String UNIQUE_TABLE = "doris_e2e_unique_table";
     private static final String DUPLICATE_TABLE = "doris_duplicate_table";
-    private static final String DRIVER_JAR =
-            "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar";
-
     private static final String sourceDB = "e2e_source";
     private static final String sinkDB = "e2e_sink";
     private Connection conn;