From a89c9fca7905d70892fb7082df248abe576f5f37 Mon Sep 17 00:00:00 2001 From: zhangdonghao <39961809+hawk9821@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:11:39 +0800 Subject: [PATCH] [Improve][E2E] update doris image to official version (#7773) --- .../e2e/connector/doris/AbstractDorisIT.java | 48 +++++++++++++++---- .../e2e/connector/doris/DorisCatalogIT.java | 2 +- .../e2e/connector/doris/DorisErrorIT.java | 8 ++-- .../e2e/connector/doris/DorisIT.java | 3 -- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java index 8392f9ae336..53b254426a9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/AbstractDorisIT.java @@ -30,8 +30,11 @@ import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; import java.sql.Connection; -import java.sql.DriverManager; +import java.sql.Driver; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -47,7 +50,7 @@ public abstract class AbstractDorisIT extends TestSuiteBase implements TestResource { protected GenericContainer container; - private static final String DOCKER_IMAGE = "bingquanzhao/doris:2.0.3"; + private static final String DOCKER_IMAGE = "apache/doris:doris-all-in-one-2.1.0"; protected static final String HOST = "doris_e2e"; protected static final int QUERY_PORT = 9030; protected static final int HTTP_PORT = 8030; @@ -58,8 +61,15 @@ public abstract class AbstractDorisIT extends TestSuiteBase implements TestResou protected Connection jdbcConnection; private static final String SET_SQL = "ADMIN SET FRONTEND CONFIG (\"enable_batch_delete_by_default\" = \"true\")"; + private static final String SET_CONNECTIONS = + "SET PROPERTY FOR 'root' 'max_user_connections' = '10000'"; + private static final String SHOW_FE = "SHOW FRONTENDS"; private static final String SHOW_BE = "SHOW BACKENDS"; + private static final String DROP_BE = "ALTER SYSTEM DROPP BACKEND \"127.0.0.1:9050\""; + private static final String ADD_BE = "ALTER SYSTEM ADD BACKEND \"%s:9050\""; protected static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + protected static final String DRIVER_JAR = + "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; @BeforeAll @Override @@ -76,24 +86,29 @@ public void startUp() { String.format("%s:%s", QUERY_PORT, QUERY_PORT), String.format("%s:%s", HTTP_PORT, HTTP_PORT), String.format("%s:%s", BE_HTTP_PORT, BE_HTTP_PORT))); - Startables.deepStart(Stream.of(container)).join(); log.info("doris container started"); given().ignoreExceptions() .await() - .atMost(10000, TimeUnit.SECONDS) + .atMost(360, TimeUnit.SECONDS) .untilAsserted(this::initializeJdbcConnection); } - protected void initializeJdbcConnection() throws SQLException { + protected void initializeJdbcConnection() + throws SQLException, ClassNotFoundException, MalformedURLException, + InstantiationException, IllegalAccessException { + URLClassLoader urlClassLoader = + new URLClassLoader(new URL[] {new URL(DRIVER_JAR)}, DorisIT.class.getClassLoader()); + Thread.currentThread().setContextClassLoader(urlClassLoader); + Driver driver = (Driver) urlClassLoader.loadClass(DRIVER_CLASS).newInstance(); Properties props = new Properties(); props.put("user", USERNAME); props.put("password", PASSWORD); - - jdbcConnection = - DriverManager.getConnection(String.format(URL, container.getHost()), props); + jdbcConnection = driver.connect(String.format(URL, container.getHost()), props); + initializeBE(); try (Statement statement = jdbcConnection.createStatement()) { statement.execute(SET_SQL); + statement.execute(SET_CONNECTIONS); ResultSet resultSet = null; do { if (resultSet != null) { @@ -104,6 +119,23 @@ protected void initializeJdbcConnection() throws SQLException { } } + // The Host of the official image [apache/doris:doris-all-in-one-2.1.0] BE is 127.0.0.1, causing + // cross-container access failure. Delete the BE and add it again + private void initializeBE() { + try (Statement statement = jdbcConnection.createStatement()) { + ResultSet resultSet = statement.executeQuery(SHOW_FE); + String feIp = null; + while (resultSet.next()) { + feIp = resultSet.getString("Host"); + } + statement.execute(DROP_BE); + statement.execute(String.format(ADD_BE, feIp)); + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + private boolean isBeReady(ResultSet rs, Duration duration) throws SQLException { if (rs.next()) { String isAlive = rs.getString("Alive").trim(); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java index 4ece65fda89..b1d299004f3 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java @@ -94,7 +94,6 @@ private void initCatalogFactory() { private void initCatalog() { String catalogName = "doris"; String frontEndNodes = container.getHost() + ":" + HTTP_PORT; - factory = new DorisCatalogFactory(); Map map = new HashMap<>(); @@ -106,6 +105,7 @@ private void initCatalog() { catalog = (DorisCatalog) factory.createCatalog(catalogName, ReadonlyConfig.fromMap(map)); catalog.open(); + catalog.createDatabase(tablePath, false); } @Test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java index be0416d79e9..d255c917c3c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisErrorIT.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.e2e.connector.doris; +import org.apache.seatunnel.connectors.doris.exception.DorisConnectorErrorCode; import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; import org.apache.seatunnel.e2e.common.container.EngineType; import org.apache.seatunnel.e2e.common.container.TestContainer; @@ -41,8 +42,6 @@ @Slf4j public class DorisErrorIT extends AbstractDorisIT { private static final String TABLE = "doris_e2e_table"; - private static final String DRIVER_JAR = - "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; private static final String sinkDB = "e2e_sink"; @@ -82,8 +81,7 @@ public void testDoris(TestContainer container) throws InterruptedException, Exec Assertions.assertTrue( future.get() .getStderr() - .contains( - "Caused by: org.apache.seatunnel.connectors.doris.exception.DorisConnectorException: ErrorCode:[Doris-01], ErrorDescription:[stream load error]")); + .contains(DorisConnectorErrorCode.STREAM_LOAD_FAILED.getCode())); Assertions.assertTrue( future.get() .getStderr() @@ -94,7 +92,7 @@ public void testDoris(TestContainer container) throws InterruptedException, Exec // wait for the container to restart given().ignoreExceptions() .await() - .atMost(10000, TimeUnit.SECONDS) + .atMost(360, TimeUnit.SECONDS) .untilAsserted(this::initializeJdbcConnection); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java index e57349bbbed..3d9b0572db4 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisIT.java @@ -61,9 +61,6 @@ public class DorisIT extends AbstractDorisIT { private static final String UNIQUE_TABLE = "doris_e2e_unique_table"; private static final String DUPLICATE_TABLE = "doris_duplicate_table"; - private static final String DRIVER_JAR = - "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; - private static final String sourceDB = "e2e_source"; private static final String sinkDB = "e2e_sink"; private Connection conn;