From 5a3951f8c55f1c760c9c31656f76a88501e7b6a1 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 4 Feb 2026 16:47:34 +0800 Subject: [PATCH 01/10] add include_table_list --- .../org/apache/doris/catalog/Resource.java | 2 ++ .../doris/datasource/ExternalCatalog.java | 27 +++++++++++++++++++ .../datasource/hive/HMSExternalCatalog.java | 10 ++++++- .../hive/ThriftHMSCachedClient.java | 1 + 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index 5db82e3875ae33..ecddc6c03e4ecf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -51,6 +51,8 @@ public abstract class Resource implements Writable, GsonPostProcessable { public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; public static final String META_NAMES_MAPPING = "meta_names_mapping"; + // db1.tbl1,db2.tbl2,... + public static final String INCLUDE_TABLE_LIST = "include_table_list"; public enum ResourceType { UNKNOWN, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 194543cc407fe9..e0d2ae7e677708 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1068,6 +1068,33 @@ protected Map getExcludeDatabaseMap() { return getSpecifiedDatabaseMap(Resource.EXCLUDE_DATABASE_LIST); } + protected Map> getIncludeTableMap() { + Map> includeTableMap = Maps.newHashMap(); + String tableList = catalogProperty.getOrDefault(Resource.INCLUDE_TABLE_LIST, ""); + if (Strings.isNullOrEmpty(tableList)) { + return includeTableMap; + } + String[] parts = tableList.split(","); + for (String part : parts) { + String dbTbl = part.trim(); + String[] splits = dbTbl.split("\\."); + if (splits.length != 2) { + LOG.warn("debug invalid include table list: {}, ignore", part); + continue; + } + String db = splits[0]; + String tbl = splits[1]; + List tbls = includeTableMap.get(db); + if (tbls == null) { + includeTableMap.put(db, Lists.newArrayList()); + tbls = includeTableMap.get(db); + } + tbls.add(tbl); + } + LOG.info("debug get include table map: {}", includeTableMap); + return includeTableMap; + } + private Map getSpecifiedDatabaseMap(String catalogPropertyKey) { String specifiedDatabaseList = catalogProperty.getOrDefault(catalogPropertyKey, ""); Map specifiedDatabaseMap = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 7c4636d3a7e3d2..2bf24cb88cabf6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.hive; +import com.google.common.collect.Lists; import org.apache.doris.catalog.Env; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.DdlException; @@ -178,7 +179,14 @@ public void onClose() { @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); - return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); + Map> includeTableMap = getIncludeTableMap(); + if (includeTableMap.containsKey(dbName) && !includeTableMap.get(dbName).isEmpty()) { + LOG.info("debug get table list from include map. catalog: {}, db: {}, tables: {}", + name, dbName, includeTableMap.get(dbName)); + return includeTableMap.get(dbName); + } else { + return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java index 840ad765587e34..1c71dc08b98f5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java @@ -133,6 +133,7 @@ public List getAllDatabases() { @Override public List getAllTables(String dbName) { + LOG.info("debug still call getAllTables from db: {}", dbName, new Exception()); try (ThriftHMSClient client = getClient()) { try { return ugiDoAs(() -> client.client.getAllTables(dbName)); From 0ffe092064d1a2b06d1b33d6a3ed79ae18a4ad55 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 6 Feb 2026 20:38:58 +0800 Subject: [PATCH 02/10] 2 --- .../org/apache/doris/datasource/hive/HMSExternalCatalog.java | 5 ++--- .../apache/doris/datasource/hive/ThriftHMSCachedClient.java | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 2bf24cb88cabf6..b8a16901e27ee6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.hive; -import com.google.common.collect.Lists; import org.apache.doris.catalog.Env; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.DdlException; @@ -181,8 +180,8 @@ public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); Map> includeTableMap = getIncludeTableMap(); if (includeTableMap.containsKey(dbName) && !includeTableMap.get(dbName).isEmpty()) { - LOG.info("debug get table list from include map. catalog: {}, db: {}, tables: {}", - name, dbName, includeTableMap.get(dbName)); + LOG.debug("get table list from include map. catalog: {}, db: {}, tables: {}", + name, dbName, includeTableMap.get(dbName)); return includeTableMap.get(dbName); } else { return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java index 1c71dc08b98f5e..840ad765587e34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java @@ -133,7 +133,6 @@ public List getAllDatabases() { @Override public List getAllTables(String dbName) { - LOG.info("debug still call getAllTables from db: {}", dbName, new Exception()); try (ThriftHMSClient client = getClient()) { try { return ugiDoAs(() -> client.client.getAllTables(dbName)); From 2fdae9d2eae621e45c537a1f2b436f943bc21018 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 6 Feb 2026 21:00:11 +0800 Subject: [PATCH 03/10] modify name place --- .../apache/doris/catalog/JdbcResource.java | 16 +- .../org/apache/doris/catalog/Resource.java | 6 - .../doris/datasource/ExternalCatalog.java | 18 ++- .../datasource/jdbc/JdbcExternalCatalog.java | 2 +- .../jdbc/client/JdbcClientConfig.java | 5 +- .../doris/datasource/ExternalCatalogTest.java | 139 ++++++++++++++++++ .../jdbc/JdbcExternalCatalogTest.java | 7 +- 7 files changed, 167 insertions(+), 26 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 819e4d21a7da21..f1e312faebeb2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -124,10 +124,10 @@ public class JdbcResource extends Resource { TYPE, CREATE_TIME, ONLY_SPECIFIED_DATABASE, - LOWER_CASE_META_NAMES, - META_NAMES_MAPPING, - INCLUDE_DATABASE_LIST, - EXCLUDE_DATABASE_LIST, + ExternalCatalog.LOWER_CASE_META_NAMES, + ExternalCatalog.META_NAMES_MAPPING, + ExternalCatalog.INCLUDE_DATABASE_LIST, + ExternalCatalog.EXCLUDE_DATABASE_LIST, CONNECTION_POOL_MIN_SIZE, CONNECTION_POOL_MAX_SIZE, CONNECTION_POOL_MAX_LIFE_TIME, @@ -145,10 +145,10 @@ public class JdbcResource extends Resource { static { OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ONLY_SPECIFIED_DATABASE, "false"); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(LOWER_CASE_META_NAMES, "false"); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(META_NAMES_MAPPING, ""); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(INCLUDE_DATABASE_LIST, ""); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(EXCLUDE_DATABASE_LIST, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.LOWER_CASE_META_NAMES, "false"); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.META_NAMES_MAPPING, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.INCLUDE_DATABASE_LIST, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.EXCLUDE_DATABASE_LIST, ""); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MIN_SIZE, "1"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_SIZE, "30"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_LIFE_TIME, "1800000"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index ecddc6c03e4ecf..de0c5542b3eaab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -47,12 +47,6 @@ public abstract class Resource implements Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(OdbcCatalogResource.class); public static final String REFERENCE_SPLIT = "@"; - public static final String INCLUDE_DATABASE_LIST = "include_database_list"; - public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; - public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; - public static final String META_NAMES_MAPPING = "meta_names_mapping"; - // db1.tbl1,db2.tbl2,... - public static final String INCLUDE_TABLE_LIST = "include_table_list"; public enum ResourceType { UNKNOWN, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index e0d2ae7e677708..f7936eb681969e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -23,7 +23,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.MysqlDb; -import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.TableIf; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; @@ -135,6 +134,13 @@ public abstract class ExternalCatalog public static final String TEST_CONNECTION = "test_connection"; public static final boolean DEFAULT_TEST_CONNECTION = false; + public static final String INCLUDE_DATABASE_LIST = "include_database_list"; + public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; + public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; + public static final String META_NAMES_MAPPING = "meta_names_mapping"; + // db1.tbl1,db2.tbl2,... + public static final String INCLUDE_TABLE_LIST = "include_table_list"; + // Unique id of this catalog, will be assigned after catalog is loaded. @SerializedName(value = "id") protected long id; @@ -1061,16 +1067,16 @@ public void registerDatabase(long dbId, String dbName) { } protected Map getIncludeDatabaseMap() { - return getSpecifiedDatabaseMap(Resource.INCLUDE_DATABASE_LIST); + return getSpecifiedDatabaseMap(ExternalCatalog.INCLUDE_DATABASE_LIST); } protected Map getExcludeDatabaseMap() { - return getSpecifiedDatabaseMap(Resource.EXCLUDE_DATABASE_LIST); + return getSpecifiedDatabaseMap(ExternalCatalog.EXCLUDE_DATABASE_LIST); } protected Map> getIncludeTableMap() { Map> includeTableMap = Maps.newHashMap(); - String tableList = catalogProperty.getOrDefault(Resource.INCLUDE_TABLE_LIST, ""); + String tableList = catalogProperty.getOrDefault(ExternalCatalog.INCLUDE_TABLE_LIST, ""); if (Strings.isNullOrEmpty(tableList)) { return includeTableMap; } @@ -1114,7 +1120,7 @@ private Map getSpecifiedDatabaseMap(String catalogPropertyKey) public String getLowerCaseMetaNames() { - return catalogProperty.getOrDefault(Resource.LOWER_CASE_META_NAMES, "false"); + return catalogProperty.getOrDefault(LOWER_CASE_META_NAMES, "false"); } public int getOnlyTestLowerCaseTableNames() { @@ -1122,7 +1128,7 @@ public int getOnlyTestLowerCaseTableNames() { } public String getMetaNamesMapping() { - return catalogProperty.getOrDefault(Resource.META_NAMES_MAPPING, ""); + return catalogProperty.getOrDefault(ExternalCatalog.META_NAMES_MAPPING, ""); } public String bindBrokerName() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 0d23bb1cb43745..b3362249791491 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -105,7 +105,7 @@ public void checkProperties() throws DdlException { } JdbcResource.checkBooleanProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, getOnlySpecifiedDatabase()); - JdbcResource.checkBooleanProperty(JdbcResource.LOWER_CASE_META_NAMES, getLowerCaseMetaNames()); + JdbcResource.checkBooleanProperty(ExternalCatalog.LOWER_CASE_META_NAMES, getLowerCaseMetaNames()); JdbcResource.checkBooleanProperty(JdbcResource.CONNECTION_POOL_KEEP_ALIVE, String.valueOf(isConnectionPoolKeepAlive())); JdbcResource.checkBooleanProperty(JdbcResource.TEST_CONNECTION, String.valueOf(isTestConnection())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java index 2d13fbc964b6e3..a35f908ed4b709 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.JdbcResource; import org.apache.doris.datasource.CatalogProperty; +import org.apache.doris.datasource.ExternalCatalog; import com.google.common.collect.Maps; @@ -52,8 +53,8 @@ public class JdbcClientConfig implements Cloneable { public JdbcClientConfig() { this.onlySpecifiedDatabase = JdbcResource.getDefaultPropertyValue(JdbcResource.ONLY_SPECIFIED_DATABASE); - this.isLowerCaseMetaNames = JdbcResource.getDefaultPropertyValue(JdbcResource.LOWER_CASE_META_NAMES); - this.metaNamesMapping = JdbcResource.getDefaultPropertyValue(JdbcResource.META_NAMES_MAPPING); + this.isLowerCaseMetaNames = JdbcResource.getDefaultPropertyValue(ExternalCatalog.LOWER_CASE_META_NAMES); + this.metaNamesMapping = JdbcResource.getDefaultPropertyValue(ExternalCatalog.META_NAMES_MAPPING); this.connectionPoolMinSize = Integer.parseInt( JdbcResource.getDefaultPropertyValue(JdbcResource.CONNECTION_POOL_MIN_SIZE)); this.connectionPoolMaxSize = Integer.parseInt( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java index e75925f5203251..68ffe7221bcb3d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java @@ -215,6 +215,145 @@ public void testExternalCatalogFilteredDatabase() throws Exception { Assertions.assertEquals(MysqlStateType.OK, rootCtx.getState().getStateType()); } + @Test + public void testGetIncludeTableMap() throws Exception { + NereidsParser nereidsParser = new NereidsParser(); + + // Test 1: Empty include_table_list + String createStmt = "create catalog test_include_table_empty properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\"\n" + + ");"; + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + TestExternalCatalog ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_empty"); + Map> includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertTrue(includeTableMap.isEmpty()); + + // Test 2: Single table + createStmt = "create catalog test_include_table_single properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_single"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(1, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertEquals(1, includeTableMap.get("db1").size()); + Assertions.assertEquals("tbl1", includeTableMap.get("db1").get(0)); + + // Test 3: Multiple tables in same database + createStmt = "create catalog test_include_table_same_db properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db1.tbl2,db1.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_same_db"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(1, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertEquals(3, includeTableMap.get("db1").size()); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl1")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl2")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl3")); + + // Test 4: Multiple tables in different databases + createStmt = "create catalog test_include_table_diff_db properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db2.tbl2,db3.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_diff_db"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(3, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertTrue(includeTableMap.containsKey("db3")); + Assertions.assertEquals(1, includeTableMap.get("db1").size()); + Assertions.assertEquals(1, includeTableMap.get("db2").size()); + Assertions.assertEquals(1, includeTableMap.get("db3").size()); + Assertions.assertEquals("tbl1", includeTableMap.get("db1").get(0)); + Assertions.assertEquals("tbl2", includeTableMap.get("db2").get(0)); + Assertions.assertEquals("tbl3", includeTableMap.get("db3").get(0)); + + // Test 5: Invalid format (should be ignored) + createStmt = "create catalog test_include_table_invalid properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,invalid_format,db2.tbl2,too.many.dots\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_invalid"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertFalse(includeTableMap.containsKey("invalid_format")); + Assertions.assertFalse(includeTableMap.containsKey("too")); + + // Test 6: With whitespace (should be trimmed) + createStmt = "create catalog test_include_table_whitespace properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \" db1.tbl1 , db2.tbl2 \"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_whitespace"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + + // Test 7: Mixed valid and invalid with multiple tables in same db + createStmt = "create catalog test_include_table_mixed properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db1.tbl2,invalid,db2.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_mixed"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertEquals(2, includeTableMap.get("db1").size()); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl1")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl2")); + Assertions.assertEquals(1, includeTableMap.get("db2").size()); + Assertions.assertTrue(includeTableMap.get("db2").contains("tbl3")); + } + public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider { public static final Map>> MOCKED_META; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java index 5a7379b2e842ec..c63aebfcb35b0a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java @@ -21,6 +21,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogFactory; +import org.apache.doris.datasource.ExternalCatalog; import com.google.common.collect.Maps; import org.junit.Assert; @@ -96,14 +97,14 @@ public void checkPropertiesTest() { exception1.getMessage()); jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, "true"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.LOWER_CASE_META_NAMES, "1"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.LOWER_CASE_META_NAMES, "1"); Exception exception2 = Assert.assertThrows(DdlException.class, () -> jdbcExternalCatalog.checkProperties()); Assert.assertEquals("errCode = 2, detailMessage = lower_case_meta_names must be true or false", exception2.getMessage()); jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, "false"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.LOWER_CASE_META_NAMES, "false"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.INCLUDE_DATABASE_LIST, "db1,db2"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.LOWER_CASE_META_NAMES, "false"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.INCLUDE_DATABASE_LIST, "db1,db2"); DdlException exceptione3 = Assert.assertThrows(DdlException.class, () -> jdbcExternalCatalog.checkProperties()); Assert.assertEquals( "errCode = 2, detailMessage = include_database_list and exclude_database_list cannot be set when only_specified_database is false", From 6c2ee6662a6003695eeb87d48f3224b5ed616a87 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 6 Feb 2026 23:40:06 +0800 Subject: [PATCH 04/10] 2 --- .../doris/datasource/ExternalCatalog.java | 80 +++++++++- ...meComparedLowercaseMetaCacheFalseTest.java | 134 +++++++++++++++++ ...ameComparedLowercaseMetaCacheTrueTest.java | 134 +++++++++++++++++ ...NameStoredLowercaseMetaCacheFalseTest.java | 138 ++++++++++++++++++ ...eNameStoredLowercaseMetaCacheTrueTest.java | 138 ++++++++++++++++++ 5 files changed, 622 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index f7936eb681969e..66410960b4c76d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -112,7 +112,11 @@ public abstract class ExternalCatalog public static final boolean DEFAULT_USE_META_CACHE = true; public static final String FOUND_CONFLICTING = "Found conflicting"; + @Deprecated + // use LOWER_CASE_TABLE_NAMES instead public static final String ONLY_TEST_LOWER_CASE_TABLE_NAMES = "only_test_lower_case_table_names"; + public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; + public static final String LOWER_CASE_DATABASE_NAMES = "lower_case_database_names"; // https://help.aliyun.com/zh/emr/emr-on-ecs/user-guide/use-rootpolicy-to-access-oss-hdfs?spm=a2c4g.11186623.help-menu-search-28066.d_0 public static final String OOS_ROOT_POLICY = "oss.root_policy"; @@ -176,6 +180,8 @@ public abstract class ExternalCatalog protected MetaCache> metaCache; protected ExecutionAuthenticator executionAuthenticator; protected ThreadPoolExecutor threadPoolWithPreAuth; + // Map lowercase database names to actual remote database names for case-insensitive lookup + private Map lowerCaseToDatabaseName = Maps.newConcurrentMap(); private volatile Configuration cachedConf = null; private byte[] confLock = new byte[0]; @@ -474,6 +480,7 @@ private List> getFilteredDatabaseNames() { Map includeDatabaseMap = getIncludeDatabaseMap(); Map excludeDatabaseMap = getExcludeDatabaseMap(); + lowerCaseToDatabaseName.clear(); List> remoteToLocalPairs = Lists.newArrayList(); allDatabases = allDatabases.stream().filter(dbName -> { @@ -491,6 +498,13 @@ private List> getFilteredDatabaseNames() { for (String remoteDbName : allDatabases) { String localDbName = fromRemoteDatabaseName(remoteDbName); + // Populate lowercase mapping for case-insensitive lookups + lowerCaseToDatabaseName.put(remoteDbName.toLowerCase(), remoteDbName); + // Apply lower_case_database_names mode to local name + int dbNameMode = getLowerCaseDatabaseNames(); + if (dbNameMode == 1) { + localDbName = localDbName.toLowerCase(); + } remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); } @@ -546,6 +560,7 @@ public synchronized void resetToUninitialized(boolean invalidCache) { synchronized (this.confLock) { this.cachedConf = null; } + this.lowerCaseToDatabaseName.clear(); onClose(); onRefreshCache(invalidCache); } @@ -666,6 +681,12 @@ public ExternalDatabase getDbNullable(String dbName) { realDbName = InfoSchemaDb.DATABASE_NAME; } else if (realDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) { realDbName = MysqlDb.DATABASE_NAME; + } else { + // Apply case-insensitive lookup for non-system databases + String localDbName = getLocalDatabaseName(realDbName, false); + if (localDbName != null) { + realDbName = localDbName; + } } // must use full qualified name to generate id. @@ -773,7 +794,14 @@ public Optional> getDbForReplay(String if (!isInitialized()) { return Optional.empty(); } - return metaCache.tryGetMetaObj(dbName); + + // Apply case-insensitive lookup with isReplay=true (no remote calls) + String localDbName = getLocalDatabaseName(dbName, true); + if (localDbName == null) { + localDbName = dbName; // Fallback to original name + } + + return metaCache.tryGetMetaObj(localDbName); } /** @@ -896,6 +924,9 @@ public void gsonPostProcess() throws IOException { if (tableAutoAnalyzePolicy == null) { tableAutoAnalyzePolicy = Maps.newHashMap(); } + if (this.lowerCaseToDatabaseName == null) { + this.lowerCaseToDatabaseName = Maps.newConcurrentMap(); + } } public void addDatabaseForTest(ExternalDatabase db) { @@ -1124,13 +1155,58 @@ public String getLowerCaseMetaNames() { } public int getOnlyTestLowerCaseTableNames() { - return Integer.parseInt(catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0")); + return Integer.parseInt(catalogProperty.getOrDefault(LOWER_CASE_TABLE_NAMES, + catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0"))); + } + + /** + * Get the lower_case_database_names configuration value. + * Returns the mode for database name case handling: + * - 0: Case-sensitive (default) + * - 1: Database names are stored as lowercase + * - 2: Database name comparison is case-insensitive + */ + public int getLowerCaseDatabaseNames() { + return Integer.parseInt(catalogProperty.getOrDefault(LOWER_CASE_DATABASE_NAMES, "0")); } public String getMetaNamesMapping() { return catalogProperty.getOrDefault(ExternalCatalog.META_NAMES_MAPPING, ""); } + /** + * Get the local database name based on the lower_case_database_names mode. + * Handles case-insensitive database lookup similar to ExternalDatabase.getLocalTableName(). + */ + @Nullable + private String getLocalDatabaseName(String dbName, boolean isReplay) { + String finalName = dbName; + int mode = getLowerCaseDatabaseNames(); + + if (mode == 1) { + // Mode 1: Store as lowercase + finalName = dbName.toLowerCase(); + } else if (mode == 2) { + // Mode 2: Case-insensitive comparison + finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase()); + if (finalName == null && !isReplay) { + // Refresh database list and try again + try { + getFilteredDatabaseNames(); + finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase()); + } catch (Exception e) { + LOG.warn("Failed to refresh database list for catalog {}", getName(), e); + } + } + if (finalName == null && LOG.isDebugEnabled()) { + LOG.debug("Failed to get database name from: {}.{}, isReplay={}", + getName(), dbName, isReplay); + } + } + + return finalName; + } + public String bindBrokerName() { return catalogProperty.getProperties().get(HMSExternalCatalog.BIND_BROKER_NAME); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000000..33eae7ff31251d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 2 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest$ExternalDatabaseNameComparedLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"2\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(2, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with lowercase, should retrieve original case + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("database1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("DATABASE1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DataBase2"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("DATABASE2", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("DATABASE1")); // Original case preserved + Assertions.assertTrue(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with uppercase names that preserve case + MOCKED_META.put("DATABASE1", tables); + MOCKED_META.put("DATABASE2", tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java new file mode 100644 index 00000000000000..299bdb48cc653a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 2 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest$ExternalDatabaseNameComparedLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"2\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(2, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with lowercase, should retrieve original case + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("database1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("DATABASE1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DataBase2"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("DATABASE2", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("DATABASE1")); // Original case preserved + Assertions.assertTrue(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with uppercase names that preserve case + MOCKED_META.put("DATABASE1", tables); + MOCKED_META.put("DATABASE2", tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000000..626f2494035dd4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 1 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest$ExternalDatabaseNameStoredLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"1\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(1, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with uppercase, should retrieve lowercase + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DATABASE1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("database1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("Database1"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("database1", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("database1")); + Assertions.assertTrue(dbNames.contains("database2")); + Assertions.assertTrue(dbNames.contains("database3")); + Assertions.assertFalse(dbNames.contains("Database1")); + Assertions.assertFalse(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with mixed case in remote system + MOCKED_META.put("Database1", tables); + MOCKED_META.put("DATABASE2", tables); + MOCKED_META.put("database3", tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java new file mode 100644 index 00000000000000..83268e5704ec5d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 1 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest$ExternalDatabaseNameStoredLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"1\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(1, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with uppercase, should retrieve lowercase + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DATABASE1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("database1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("Database1"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("database1", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("database1")); + Assertions.assertTrue(dbNames.contains("database2")); + Assertions.assertTrue(dbNames.contains("database3")); + Assertions.assertFalse(dbNames.contains("Database1")); + Assertions.assertFalse(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with mixed case in remote system + MOCKED_META.put("Database1", tables); + MOCKED_META.put("DATABASE2", tables); + MOCKED_META.put("database3", tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} From 0fd8a8572931a4019d252114b6b06201ced9dd81 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 7 Feb 2026 00:47:54 +0800 Subject: [PATCH 05/10] 2 --- .../doris/datasource/ExternalCatalog.java | 24 +- .../doris/RemoteDorisExternalCatalog.java | 3 +- .../datasource/es/EsExternalCatalog.java | 3 +- .../datasource/hive/HMSExternalCatalog.java | 12 +- .../iceberg/IcebergExternalCatalog.java | 3 +- .../datasource/jdbc/JdbcExternalCatalog.java | 3 +- .../lakesoul/LakeSoulExternalCatalog.java | 2 +- .../maxcompute/MaxComputeExternalCatalog.java | 3 +- .../paimon/PaimonExternalCatalog.java | 3 +- .../datasource/test/TestExternalCatalog.java | 3 +- .../TrinoConnectorExternalCatalog.java | 3 +- .../datasource/IncludeTableListTest.java | 424 ++++++++++++++++++ 12 files changed, 457 insertions(+), 29 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 66410960b4c76d..432486069e9af0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -295,9 +295,29 @@ public void checkWhenCreating() throws DdlException { /** * @param dbName - * @return names of tables in specified database + * @return names of tables in specified database, filtered by include_table_list if configured */ - public abstract List listTableNames(SessionContext ctx, String dbName); + public final List listTableNames(SessionContext ctx, String dbName) { + makeSureInitialized(); + Map> includeTableMap = getIncludeTableMap(); + if (includeTableMap.containsKey(dbName) && !includeTableMap.get(dbName).isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("get table list from include map. catalog: {}, db: {}, tables: {}", + name, dbName, includeTableMap.get(dbName)); + } + return includeTableMap.get(dbName); + } + return listTableNamesFromRemote(ctx, dbName); + } + + /** + * Subclasses implement this method to list table names from the remote data source. + * + * @param ctx session context + * @param dbName database name + * @return names of tables in the specified database from the remote source + */ + protected abstract List listTableNamesFromRemote(SessionContext ctx, String dbName); /** * check if the specified table exist. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java index 76d42b70b4260b..40dec9e5795877 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java @@ -197,8 +197,7 @@ protected List listDatabaseNames() { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return dorisRestClient.getTablesNameList(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java index 8e9c6b08d72151..60371365c50536 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java @@ -134,8 +134,7 @@ protected void initLocalObjectsImpl() { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return esRestClient.listTable(enableIncludeHiddenIndex()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index b8a16901e27ee6..a0bad3a4516400 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -176,16 +176,8 @@ public void onClose() { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); - Map> includeTableMap = getIncludeTableMap(); - if (includeTableMap.containsKey(dbName) && !includeTableMap.get(dbName).isEmpty()) { - LOG.debug("get table list from include map. catalog: {}, db: {}, tables: {}", - name, dbName, includeTableMap.get(dbName)); - return includeTableMap.get(dbName); - } else { - return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); - } + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { + return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 6175800a18f9b1..ae2379333789b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -172,8 +172,7 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { // On the Doris side, the result of SHOW TABLES for Iceberg external tables includes both tables and views, // so the combined set of tables and views is used here. List tableNames = metadataOps.listTableNames(dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index b3362249791491..c849d779217e7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -280,8 +280,7 @@ public String fromRemoteDatabaseName(String remoteDatabaseName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return jdbcClient.getTablesNameList(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java index 0d44e41dce9d1f..465f2c78382212 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java @@ -60,7 +60,7 @@ protected List listDatabaseNames() { } @Override - public List listTableNames(SessionContext ctx, String dbName) { + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { // makeSureInitialized(); // List tifs = lakesoulMetadataManager.getTableInfosByNamespace(dbName); // List tableNames = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java index 4990e48ab3a98f..2abe07c78a2130 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java @@ -291,8 +291,7 @@ public List listPartitionNames(String dbName, String tbl, long skip, lon } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return mcStructureHelper.listTableNames(getClient(), dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index f15309ea0e96bd..87f606dfcca451 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -89,8 +89,7 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return metadataOps.listTableNames(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java index 1fd8e3721f3908..57a483f28c7c58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java @@ -85,8 +85,7 @@ public List mockedSchema(String dbName, String tblName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return mockedTableNames(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java index 3627aff636db77..e2db35707ddb7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java @@ -184,8 +184,7 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { QualifiedTablePrefix qualifiedTablePrefix = new QualifiedTablePrefix(trinoCatalogHandle.getCatalogName(), dbName); List tables = trinoListTables(qualifiedTablePrefix); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java new file mode 100644 index 00000000000000..2f8e5d5177c00a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java @@ -0,0 +1,424 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class IncludeTableListTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + private void createCatalog(String catalogName, String providerClass, String includeTableList) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("create catalog ").append(catalogName).append(" properties(\n"); + sb.append(" \"type\" = \"test\",\n"); + sb.append(" \"catalog_provider.class\" = \"").append(providerClass).append("\""); + if (includeTableList != null) { + sb.append(",\n \"").append(ExternalCatalog.INCLUDE_TABLE_LIST) + .append("\" = \"").append(includeTableList).append("\""); + } + sb.append("\n);"); + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(sb.toString()); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + private void dropCatalog(String catalogName) throws Exception { + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog " + catalogName); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + private void refreshCatalog(String catalogName) { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand(catalogName, null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + } + + private static final String PROVIDER_CLASS = + "org.apache.doris.datasource.IncludeTableListTest$IncludeTableListProvider"; + + // ==================== Basic filtering tests ==================== + + /** + * When include_table_list is not configured, all tables should be visible. + */ + @Test + public void testNoIncludeTableList() throws Exception { + String catalogName = "test_no_include"; + createCatalog(catalogName, PROVIDER_CLASS, null); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + // All 3 tables in db1 should be visible + Assertions.assertEquals(3, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2TableNames = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2TableNames.size()); + Assertions.assertTrue(db2TableNames.contains("tbl_a")); + Assertions.assertTrue(db2TableNames.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies a single table in one db, only that table should be visible + * in that db; other dbs should show all tables. + */ + @Test + public void testSingleTableInclude() throws Exception { + String catalogName = "test_single_include"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + // Only tbl1 should be visible in db1 + Assertions.assertEquals(1, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertFalse(tableNames.contains("tbl2")); + Assertions.assertFalse(tableNames.contains("tbl3")); + + // db2 should still show all tables (not in include_table_list) + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2TableNames = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2TableNames.size()); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies multiple tables in same db. + */ + @Test + public void testMultipleTablesInSameDb() throws Exception { + String catalogName = "test_multi_same_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,db1.tbl3"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + // Only tbl1 and tbl3 should be visible + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertFalse(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies tables across multiple dbs. + */ + @Test + public void testMultipleTablesAcrossDbs() throws Exception { + String catalogName = "test_multi_cross_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl2,db2.tbl_a"); + try { + refreshCatalog(catalogName); + + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set db1Tables = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl2")); + + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(1, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + } finally { + dropCatalog(catalogName); + } + } + + // ==================== Error / edge case tests ==================== + + /** + * When include_table_list specifies a table that does NOT exist in the remote source, + * the table name should still appear in listTableNames but getTableNullable should return null. + */ + @Test + public void testNonExistentTableInIncludeList() throws Exception { + String catalogName = "test_nonexist_tbl"; + // "nonexistent_table" does not exist in the provider's db1 + createCatalog(catalogName, PROVIDER_CLASS, "db1.nonexistent_table"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + // The include list overrides remote listing, so it reports "nonexistent_table" + Set tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, tableNames.size()); + Assertions.assertTrue(tableNames.contains("nonexistent_table")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * Mix of existing and non-existing tables in include_table_list. + * Existing table should be accessible; non-existing table should return null. + */ + @Test + public void testMixExistentAndNonExistentTables() throws Exception { + String catalogName = "test_mix_exist"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,db1.no_such_table"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("no_such_table")); + + // Existing table should be accessible + Assertions.assertNotNull(db1.getTableNullable("tbl1")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list refers to a non-existent database, that db entry + * in the include map is simply ignored (the db won't appear). + */ + @Test + public void testNonExistentDbInIncludeList() throws Exception { + String catalogName = "test_nonexist_db"; + // "no_such_db" does not exist in the provider + createCatalog(catalogName, PROVIDER_CLASS, "no_such_db.tbl1,db1.tbl1"); + try { + refreshCatalog(catalogName); + // db1 should still work with filtered table + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set db1Tables = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl1")); + + // The non-existent db should return null + Assertions.assertNull( + env.getCatalogMgr().getCatalog(catalogName).getDbNullable("no_such_db")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list contains entries with invalid format (no dot separator), + * those entries are silently ignored. + */ + @Test + public void testInvalidFormatInIncludeList() throws Exception { + String catalogName = "test_invalid_fmt"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,bad_format,db2.tbl_a,too.many.dots"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set db1Tables = db1.getTableNamesWithLock(); + // Only "db1.tbl1" is a valid entry for db1 + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl1")); + + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(1, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list entries have extra whitespace, they should be trimmed properly. + */ + @Test + public void testWhitespaceInIncludeList() throws Exception { + String catalogName = "test_whitespace"; + createCatalog(catalogName, PROVIDER_CLASS, " db1.tbl1 , db1.tbl2 "); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("tbl2")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list is set but all entries are for one db, + * another db should show all its tables (unaffected). + */ + @Test + public void testUnaffectedDbShowsAllTables() throws Exception { + String catalogName = "test_unaffected_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1"); + try { + refreshCatalog(catalogName); + // db2 is not mentioned in include_table_list, so all tables should be visible + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + Assertions.assertTrue(db2Tables.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * Test that listTableNames (the catalog-level API) returns the included list directly + * when include_table_list is configured for that db. + */ + @Test + public void testListTableNamesAPI() throws Exception { + String catalogName = "test_api_list"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl2,db1.tbl3"); + try { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog(catalogName); + List tableNames = catalog.listTableNames(null, "db1"); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + Assertions.assertFalse(tableNames.contains("tbl1")); + + // db2 not in include map → returns all remote tables + List db2Names = catalog.listTableNames(null, "db2"); + Assertions.assertEquals(2, db2Names.size()); + Assertions.assertTrue(db2Names.contains("tbl_a")); + Assertions.assertTrue(db2Names.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + // ==================== Mock data provider ==================== + + public static class IncludeTableListProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + + // db1 with 3 tables + Map> db1Tables = Maps.newHashMap(); + db1Tables.put("tbl1", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("name", PrimitiveType.VARCHAR))); + db1Tables.put("tbl2", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("value", PrimitiveType.BIGINT))); + db1Tables.put("tbl3", Lists.newArrayList( + new Column("key", PrimitiveType.VARCHAR), + new Column("data", PrimitiveType.STRING))); + MOCKED_META.put("db1", db1Tables); + + // db2 with 2 tables + Map> db2Tables = Maps.newHashMap(); + db2Tables.put("tbl_a", Lists.newArrayList( + new Column("col1", PrimitiveType.INT), + new Column("col2", PrimitiveType.FLOAT))); + db2Tables.put("tbl_b", Lists.newArrayList( + new Column("x", PrimitiveType.BIGINT), + new Column("y", PrimitiveType.DOUBLE))); + MOCKED_META.put("db2", db2Tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} From aeb6ff034f20e03d4f6c0c890a05d04c75631b9c Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 10 Feb 2026 18:21:19 +0800 Subject: [PATCH 06/10] fix lower case where --- .../AbstractBackupTableRefClause.java | 8 +- .../java/org/apache/doris/catalog/Env.java | 16 +++ .../apache/doris/datasource/CatalogIf.java | 11 ++ .../doris/datasource/ExternalCatalog.java | 9 +- .../doris/datasource/ExternalDatabase.java | 8 +- .../httpv2/rest/TableQueryPlanAction.java | 2 +- .../org/apache/doris/nereids/CTEContext.java | 15 ++- .../doris/nereids/StatementContext.java | 22 +++- .../rules/analysis/ExpressionAnalyzer.java | 118 +++++++++++++----- .../trees/plans/commands/BackupCommand.java | 3 +- .../trees/plans/commands/RestoreCommand.java | 3 +- .../plans/commands/ShowTableCommand.java | 3 +- .../trees/plans/commands/UpdateCommand.java | 10 +- .../plans/logical/LogicalSubQueryAlias.java | 8 +- 14 files changed, 183 insertions(+), 53 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AbstractBackupTableRefClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AbstractBackupTableRefClause.java index 6a5f27c5073e3a..3e68185f57b7af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AbstractBackupTableRefClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AbstractBackupTableRefClause.java @@ -20,6 +20,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.info.TableRefInfo; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.GlobalVariable; import com.google.common.base.Joiner; @@ -46,7 +47,12 @@ public void analyze() throws UserException { // normalize // table name => table ref Map tblPartsMap; - if (GlobalVariable.lowerCaseTableNames == 0) { + int lctNames = GlobalVariable.lowerCaseTableNames; + ConnectContext ctx = ConnectContext.get(); + if (ctx != null && ctx.getCurrentCatalog() != null) { + lctNames = ctx.getCurrentCatalog().getLowerCaseTableNames(); + } + if (lctNames == 0) { // comparisons case sensitive tblPartsMap = Maps.newTreeMap(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index e82cd015a76e7f..8ab962734c61bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -7079,6 +7079,22 @@ public static boolean isTableNamesCaseSensitive() { return GlobalVariable.lowerCaseTableNames == 0; } + public static int getLowerCaseTableNames(String catalogName) { + if (catalogName == null) { + return GlobalVariable.lowerCaseTableNames; + } + CatalogIf catalog = getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + return catalog != null ? catalog.getLowerCaseTableNames() : GlobalVariable.lowerCaseTableNames; + } + + public static int getLowerCaseDatabaseNames(String catalogName) { + if (catalogName == null) { + return 0; // InternalCatalog default: case-sensitive + } + CatalogIf catalog = getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + return catalog != null ? catalog.getLowerCaseDatabaseNames() : 0; + } + private static void getTableMeta(OlapTable olapTable, TGetMetaDBMeta dbMeta) { if (LOG.isDebugEnabled()) { LOG.debug("get table meta. table: {}", olapTable.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 9553c9f260442b..538efde8f2cbe4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo; import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo; +import org.apache.doris.qe.GlobalVariable; import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; @@ -208,6 +209,16 @@ void truncateTable(String dbName, String tableName, PartitionNamesInfo partition String rawTruncateSql) throws DdlException; + /** 0=case-sensitive, 1=stored lowercase, 2=case-insensitive comparison */ + default int getLowerCaseTableNames() { + return GlobalVariable.lowerCaseTableNames; + } + + /** For InternalCatalog, DB names are always case-sensitive (return 0). */ + default int getLowerCaseDatabaseNames() { + return 0; + } + // Convert from remote database name to local database name, overridden by subclass if necessary default String fromRemoteDatabaseName(String remoteDatabaseName) { return remoteDatabaseName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 432486069e9af0..564c381ec9ab2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -66,6 +66,7 @@ import org.apache.doris.persist.TableBranchOrTagInfo; import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.transaction.TransactionManager; import com.google.common.base.Objects; @@ -1169,14 +1170,15 @@ private Map getSpecifiedDatabaseMap(String catalogPropertyKey) return specifiedDatabaseMap; } - public String getLowerCaseMetaNames() { return catalogProperty.getOrDefault(LOWER_CASE_META_NAMES, "false"); } - public int getOnlyTestLowerCaseTableNames() { + @Override + public int getLowerCaseTableNames() { return Integer.parseInt(catalogProperty.getOrDefault(LOWER_CASE_TABLE_NAMES, - catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0"))); + catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, + String.valueOf(GlobalVariable.lowerCaseTableNames)))); } /** @@ -1186,6 +1188,7 @@ public int getOnlyTestLowerCaseTableNames() { * - 1: Database names are stored as lowercase * - 2: Database name comparison is case-insensitive */ + @Override public int getLowerCaseDatabaseNames() { return Integer.parseInt(catalogProperty.getOrDefault(LOWER_CASE_DATABASE_NAMES, "0")); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 76925a42921bfa..5dfd68c20aac88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -595,15 +595,11 @@ public boolean registerTable(TableIf tableIf) { } private boolean isStoredTableNamesLowerCase() { - // Because we have added a test configuration item, - // it needs to be judged together with Env.isStoredTableNamesLowerCase() - return Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1; + return extCatalog.getLowerCaseTableNames() == 1; } private boolean isTableNamesCaseInsensitive() { - // Because we have added a test configuration item, - // it needs to be judged together with Env.isTableNamesCaseInsensitive() - return Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2; + return extCatalog.getLowerCaseTableNames() == 2; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java index c7b2b9e81b436e..87d1eedaa9541d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java @@ -232,7 +232,7 @@ private void handleQuery(ConnectContext context, String requestDb, String reques String dbName = tableQualifier.get(1); String tableName = tableQualifier.get(2); - if (GlobalVariable.lowerCaseTableNames == 0) { + if (Env.getLowerCaseTableNames(InternalCatalog.INTERNAL_CATALOG_NAME) == 0) { if (!(dbName.equals(requestDb) && tableName.equals(requestTable))) { throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "requested database and table must consistent with sql: request [ " diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java index 87d57a2c6b4819..999de72582174b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.GlobalVariable; import com.google.common.collect.ImmutableList; @@ -47,6 +48,14 @@ public class CTEContext { private final Map cteContextMap; + private static int currentLowerCaseTableNames() { + ConnectContext ctx = ConnectContext.get(); + if (ctx != null && ctx.getCurrentCatalog() != null) { + return ctx.getCurrentCatalog().getLowerCaseTableNames(); + } + return GlobalVariable.lowerCaseTableNames; + } + /* build head CTEContext */ public CTEContext() { this(CTEId.DEFAULT, null, (CTEContext) null); @@ -60,7 +69,7 @@ public CTEContext(CTEId cteId, @Nullable LogicalSubQueryAlias parsedPlan, if ((parsedPlan == null && previousCteContext != null) || (parsedPlan != null && previousCteContext == null)) { throw new AnalysisException("Only first CteContext can contains null cte plan or previousCteContext"); } - this.name = parsedPlan == null ? null : GlobalVariable.lowerCaseTableNames != 0 + this.name = parsedPlan == null ? null : currentLowerCaseTableNames() != 0 ? parsedPlan.getAlias().toLowerCase(Locale.ROOT) : parsedPlan.getAlias(); this.cteContextMap = previousCteContext == null ? ImmutableMap.of() @@ -78,7 +87,7 @@ public CTEContext(CTEId cteId, @Nullable LogicalSubQueryAlias parsedPlan, */ public CTEContext(CTEId cteId, String cteName, List recursiveCteOutputs) { this.cteId = cteId; - this.name = GlobalVariable.lowerCaseTableNames != 0 ? cteName.toLowerCase(Locale.ROOT) : cteName; + this.name = currentLowerCaseTableNames() != 0 ? cteName.toLowerCase(Locale.ROOT) : cteName; this.recursiveCteOutputs = recursiveCteOutputs != null ? ImmutableList.copyOf(recursiveCteOutputs) : ImmutableList.of(); this.cteContextMap = ImmutableMap.of(name, this); @@ -110,7 +119,7 @@ public Optional getAnalyzedCTEPlan(String cteName) { * findCTEContext */ public Optional findCTEContext(String cteName) { - if (GlobalVariable.lowerCaseTableNames != 0) { + if (currentLowerCaseTableNames() != 0) { cteName = cteName.toLowerCase(Locale.ROOT); } if (cteName.equals(name)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 92b7282a0ef1b9..dd2b1a6122694c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.Partition; @@ -57,6 +58,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.ShortCircuitQueryContext; @@ -315,10 +317,14 @@ public enum TableFrom { private final Set mustInlineCTE = new HashSet<>(); + private final Map lowerCaseTableNamesCache = Maps.newHashMap(); + private final Map lowerCaseDatabaseNamesCache = Maps.newHashMap(); + public StatementContext() { this(ConnectContext.get(), null, 0); } + // For StatementScopeIdGenerator public StatementContext(int initialId) { this(ConnectContext.get(), null, initialId); } @@ -330,7 +336,7 @@ public StatementContext(ConnectContext connectContext, OriginStatement originSta /** * StatementContext */ - public StatementContext(ConnectContext connectContext, OriginStatement originStatement, int initialId) { + private StatementContext(ConnectContext connectContext, OriginStatement originStatement, int initialId) { this.connectContext = connectContext; this.originStatement = originStatement; exprIdGenerator = ExprId.createGenerator(initialId); @@ -1195,4 +1201,18 @@ public void addToMustLineCTEs(CTEId cteId) { public Set getMustInlineCTEs() { return mustInlineCTE; } + + public int getLowerCaseTableNames(String catalogName) { + if (catalogName == null) { + return GlobalVariable.lowerCaseTableNames; + } + return lowerCaseTableNamesCache.computeIfAbsent(catalogName, Env::getLowerCaseTableNames); + } + + public int getLowerCaseDatabaseNames(String catalogName) { + if (catalogName == null) { + return 0; + } + return lowerCaseDatabaseNamesCache.computeIfAbsent(catalogName, Env::getLowerCaseDatabaseNames); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index 3ef9941946d67a..4ddbe908dece93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -106,7 +106,6 @@ import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.VariableMgr; import org.apache.doris.qe.VariableVarConverters; @@ -1005,12 +1004,21 @@ private BoundStar bindQualifiedStar(List qualifierStar, List bound // bound slot is `column` and no qualified case 0: return false; - case 1: // bound slot is `table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)); - case 2:// bound slot is `db`.`table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(1)); - case 3:// bound slot is `catalog`.`db`.`table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(2)); + case 1: { // bound slot is `table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + return sameTableName(qualifierStar.get(0), boundSlotQualifier.get(0), lct); + } + case 2: { // bound slot is `db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + return sameTableName(qualifierStar.get(0), boundSlotQualifier.get(1), lct); + } + case 3: { // bound slot is `catalog`.`db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + return sameTableName(qualifierStar.get(0), boundSlotQualifier.get(2), lct); + } default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifierStar, ".")); @@ -1022,12 +1030,22 @@ private BoundStar bindQualifiedStar(List qualifierStar, List bound case 0: case 1: // bound slot is `table`.`column` return false; - case 2:// bound slot is `db`.`table`.`column` - return compareDbNameIgnoreClusterName(qualifierStar.get(0), boundSlotQualifier.get(0)) - && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(1)); - case 3:// bound slot is `catalog`.`db`.`table`.`column` - return compareDbNameIgnoreClusterName(qualifierStar.get(0), boundSlotQualifier.get(1)) - && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(2)); + case 2: { // bound slot is `db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + int lcdb = resolveLowerCaseDatabaseNames(catName); + return compareDbNameIgnoreClusterName( + qualifierStar.get(0), boundSlotQualifier.get(0), lcdb) + && sameTableName(qualifierStar.get(1), boundSlotQualifier.get(1), lct); + } + case 3: { // bound slot is `catalog`.`db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + int lcdb = resolveLowerCaseDatabaseNames(catName); + return compareDbNameIgnoreClusterName( + qualifierStar.get(0), boundSlotQualifier.get(1), lcdb) + && sameTableName(qualifierStar.get(1), boundSlotQualifier.get(2), lct); + } default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifierStar, ".") + ".*"); @@ -1040,10 +1058,15 @@ private BoundStar bindQualifiedStar(List qualifierStar, List bound case 1: // bound slot is `table`.`column` case 2: // bound slot is `db`.`table`.`column` return false; - case 3:// bound slot is `catalog`.`db`.`table`.`column` + case 3: { // bound slot is `catalog`.`db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + int lcdb = resolveLowerCaseDatabaseNames(catName); return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)) - && compareDbNameIgnoreClusterName(qualifierStar.get(1), boundSlotQualifier.get(1)) - && qualifierStar.get(2).equalsIgnoreCase(boundSlotQualifier.get(2)); + && compareDbNameIgnoreClusterName( + qualifierStar.get(1), boundSlotQualifier.get(1), lcdb) + && sameTableName(qualifierStar.get(2), boundSlotQualifier.get(2), lct); + } default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifierStar, ".") + ".*"); @@ -1222,14 +1245,39 @@ private Optional bindNestedFields(UnboundSlot unboundSlot, Slot slot return Optional.of(new Alias(expression, unboundSlot.getName(), slot.getQualifier())); } - public static boolean sameTableName(String boundSlot, String unboundSlot) { - if (GlobalVariable.lowerCaseTableNames != 1) { + public static boolean sameTableName(String boundSlot, String unboundSlot, int lowerCaseTableNames) { + if (lowerCaseTableNames != 1) { return boundSlot.equals(unboundSlot); } else { return boundSlot.equalsIgnoreCase(unboundSlot); } } + /** Extract catalog name from bound slot qualifier, fallback to current catalog */ + private String extractCatalogName(List boundSlotQualifier) { + if (boundSlotQualifier.size() >= 3) { + return boundSlotQualifier.get(boundSlotQualifier.size() - 3); + } + ConnectContext ctx = ConnectContext.get(); + return (ctx != null) ? ctx.getDefaultCatalog() : null; + } + + private int resolveLowerCaseTableNames(String catalogName) { + CascadesContext cascadesCtx = getCascadesContext(); + if (cascadesCtx != null) { + return cascadesCtx.getStatementContext().getLowerCaseTableNames(catalogName); + } + return Env.getLowerCaseTableNames(catalogName); + } + + private int resolveLowerCaseDatabaseNames(String catalogName) { + CascadesContext cascadesCtx = getCascadesContext(); + if (cascadesCtx != null) { + return cascadesCtx.getStatementContext().getLowerCaseDatabaseNames(catalogName); + } + return Env.getLowerCaseDatabaseNames(catalogName); + } + private boolean shouldBindSlotBy(int namePartSize, Slot boundSlot) { return namePartSize <= boundSlot.getQualifier().size() + 1; } @@ -1256,7 +1304,9 @@ private List bindSingleSlotByTable(String table, String name, Scope scope) } List boundSlotQualifier = boundSlot.getQualifier(); String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); - if (!sameTableName(boundSlotTable, table)) { + String catalogName = extractCatalogName(boundSlotQualifier); + int lctNames = resolveLowerCaseTableNames(catalogName); + if (!sameTableName(boundSlotTable, table, lctNames)) { continue; } // set sql case as alias @@ -1275,7 +1325,11 @@ private List bindSingleSlotByDb(String db, String table, String name, Scop List boundSlotQualifier = boundSlot.getQualifier(); String boundSlotDb = boundSlotQualifier.get(boundSlotQualifier.size() - 2); String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); - if (!compareDbNameIgnoreClusterName(boundSlotDb, db) || !sameTableName(boundSlotTable, table)) { + String catalogName = extractCatalogName(boundSlotQualifier); + int lctNames = resolveLowerCaseTableNames(catalogName); + int lcdbNames = resolveLowerCaseDatabaseNames(catalogName); + if (!compareDbNameIgnoreClusterName(boundSlotDb, db, lcdbNames) + || !sameTableName(boundSlotTable, table, lctNames)) { continue; } // set sql case as alias @@ -1295,9 +1349,12 @@ private List bindSingleSlotByCatalog(String catalog, String db, String tab String boundSlotCatalog = boundSlotQualifier.get(boundSlotQualifier.size() - 3); String boundSlotDb = boundSlotQualifier.get(boundSlotQualifier.size() - 2); String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); + String catalogName = extractCatalogName(boundSlotQualifier); + int lctNames = resolveLowerCaseTableNames(catalogName); + int lcdbNames = resolveLowerCaseDatabaseNames(catalogName); if (!boundSlotCatalog.equalsIgnoreCase(catalog) - || !compareDbNameIgnoreClusterName(boundSlotDb, db) - || !sameTableName(boundSlotTable, table)) { + || !compareDbNameIgnoreClusterName(boundSlotDb, db, lcdbNames) + || !sameTableName(boundSlotTable, table, lctNames)) { continue; } // set sql case as alias @@ -1307,20 +1364,23 @@ private List bindSingleSlotByCatalog(String catalog, String db, String tab } /**compareDbNameIgnoreClusterName.*/ - public static boolean compareDbNameIgnoreClusterName(String name1, String name2) { - if (name1.equalsIgnoreCase(name2)) { + public static boolean compareDbNameIgnoreClusterName(String name1, String name2, + int lowerCaseDatabaseNames) { + boolean ignoreCase = (lowerCaseDatabaseNames != 0); + if (ignoreCase ? name1.equalsIgnoreCase(name2) : name1.equals(name2)) { return true; } - String ignoreClusterName1 = name1; + // Strip cluster namespace prefix (before ':') + String stripped1 = name1; int idx1 = name1.indexOf(":"); if (idx1 > -1) { - ignoreClusterName1 = name1.substring(idx1 + 1); + stripped1 = name1.substring(idx1 + 1); } - String ignoreClusterName2 = name2; + String stripped2 = name2; int idx2 = name2.indexOf(":"); if (idx2 > -1) { - ignoreClusterName2 = name2.substring(idx2 + 1); + stripped2 = name2.substring(idx2 + 1); } - return ignoreClusterName1.equalsIgnoreCase(ignoreClusterName2); + return ignoreCase ? stripped1.equalsIgnoreCase(stripped2) : stripped1.equals(stripped2); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java index 5e9f376a5d56bb..a46f645e6d0bcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java @@ -31,7 +31,6 @@ import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.StmtExecutor; import com.google.common.base.Joiner; @@ -149,7 +148,7 @@ private void checkTableRefWithoutDatabase() throws AnalysisException { private void updateTableRefInfos() throws AnalysisException { Map tblPartsMap; - if (GlobalVariable.lowerCaseTableNames == 0) { + if (Env.getLowerCaseTableNames(InternalCatalog.INTERNAL_CATALOG_NAME) == 0) { // comparisons case sensitive tblPartsMap = Maps.newTreeMap(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RestoreCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RestoreCommand.java index 2773c2d6064e83..79c47f94d74193 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RestoreCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RestoreCommand.java @@ -38,7 +38,6 @@ import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.StmtExecutor; import com.google.common.base.Joiner; @@ -184,7 +183,7 @@ private void checkTableRefWithoutDatabase() throws AnalysisException { private void updateTableRefInfos() throws AnalysisException { Map tblPartsMap; - if (GlobalVariable.lowerCaseTableNames == 0) { + if (Env.getLowerCaseTableNames(InternalCatalog.INTERNAL_CATALOG_NAME) == 0) { // comparisons case sensitive tblPartsMap = Maps.newTreeMap(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java index aff66182ebb138..2e1247906cf9ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java @@ -39,7 +39,6 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.ShowResultSet; import org.apache.doris.qe.ShowResultSetMetaData; import org.apache.doris.qe.StmtExecutor; @@ -107,7 +106,7 @@ public void validate(ConnectContext ctx) throws AnalysisException { * isShowTablesCaseSensitive */ public boolean isShowTablesCaseSensitive() { - if (GlobalVariable.lowerCaseTableNames == 0) { + if (Env.getLowerCaseTableNames(catalog) == 0) { return CaseSensibility.TABLE.getCaseSensibility(); } return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java index 133122ad91cd86..9e40034cef6657 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.StmtType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; @@ -224,9 +225,14 @@ public static void checkAssignmentColumn(ConnectContext ctx, List column throw new AnalysisException("column in assignment list is invalid, " + String.join(".", columnNameParts)); } List tableQualifier = RelationUtil.getQualifierName(ctx, tableNameParts); - if (!ExpressionAnalyzer.sameTableName(tableAlias == null ? tableQualifier.get(2) : tableAlias, tableName) + String catalogName = tableQualifier.get(0); + int lctNames = Env.getLowerCaseTableNames(catalogName); + int lcdbNames = Env.getLowerCaseDatabaseNames(catalogName); + if (!ExpressionAnalyzer.sameTableName( + tableAlias == null ? tableQualifier.get(2) : tableAlias, tableName, lctNames) || (dbName != null - && !ExpressionAnalyzer.compareDbNameIgnoreClusterName(tableQualifier.get(1), dbName))) { + && !ExpressionAnalyzer.compareDbNameIgnoreClusterName( + tableQualifier.get(1), dbName, lcdbNames))) { throw new AnalysisException("column in assignment list is invalid, " + String.join(".", columnNameParts)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 4aff4dda9900f0..7d3b11de7626f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.LazyCompute; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.GlobalVariable; import com.google.common.base.Preconditions; @@ -141,7 +142,12 @@ private Supplier computeIsRecursiveCte() { if (nameParts.size() == 1) { String aliasName = getAlias(); String tablename = nameParts.get(0); - if (GlobalVariable.lowerCaseTableNames != 0) { + int lctNames = 0; + ConnectContext ctx = ConnectContext.get(); + if (ctx != null && ctx.getCurrentCatalog() != null) { + lctNames = ctx.getCurrentCatalog().getLowerCaseTableNames(); + } + if (lctNames != 0) { aliasName = aliasName.toLowerCase(Locale.ROOT); tablename = tablename.toLowerCase(Locale.ROOT); } From 6c9c6bdeb369b7996a4336c9888f57ca2ebc1a45 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 10 Feb 2026 18:46:38 +0800 Subject: [PATCH 07/10] fix =2 --- .../java/org/apache/doris/datasource/ExternalCatalog.java | 7 +++++-- .../java/org/apache/doris/datasource/ExternalDatabase.java | 7 +++---- .../doris/nereids/rules/analysis/ExpressionAnalyzer.java | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 564c381ec9ab2c..7b4bf151b08797 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -525,12 +525,15 @@ private List> getFilteredDatabaseNames() { int dbNameMode = getLowerCaseDatabaseNames(); if (dbNameMode == 1) { localDbName = localDbName.toLowerCase(); + } else if (dbNameMode == 2) { + // Mode 2: preserve original remote case for display + localDbName = remoteDbName; } remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); } - // Check for conflicts when lower_case_meta_names = true - if (Boolean.parseBoolean(getLowerCaseMetaNames())) { + // Check for conflicts when lower_case_meta_names = true or lower_case_database_names = 2 + if (Boolean.parseBoolean(getLowerCaseMetaNames()) || getLowerCaseDatabaseNames() == 2) { // Map to track lowercase local names and their corresponding remote names Map> lowerCaseToRemoteNames = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 5dfd68c20aac88..2c20daec0bfe1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -199,6 +199,9 @@ private List> listTableNames() { String localTableName = extCatalog.fromRemoteTableName(remoteName, tableName); if (this.isStoredTableNamesLowerCase()) { localTableName = localTableName.toLowerCase(); + } else if (this.isTableNamesCaseInsensitive()) { + // Mode 2: preserve original remote case for display + localTableName = tableName; } lowerCaseToTableName.put(tableName.toLowerCase(), tableName); return Pair.of(tableName, localTableName); @@ -513,10 +516,6 @@ private String getLocalTableName(String tableName, boolean isReplay) { } } } - if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true") - && (this.isTableNamesCaseInsensitive())) { - finalName = tableName.toLowerCase(); - } if (LOG.isDebugEnabled()) { LOG.debug("get table {} from database: {}.{}, final name is: {}, catalog id: {}", tableName, getCatalog().getName(), getFullName(), finalName, getCatalog().getId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index 4ddbe908dece93..528df148ea88f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -1246,7 +1246,7 @@ private Optional bindNestedFields(UnboundSlot unboundSlot, Slot slot } public static boolean sameTableName(String boundSlot, String unboundSlot, int lowerCaseTableNames) { - if (lowerCaseTableNames != 1) { + if (lowerCaseTableNames == 0) { return boundSlot.equals(unboundSlot); } else { return boundSlot.equalsIgnoreCase(unboundSlot); From f9c68c845cde10811fedabda2cdf4856c4f92e13 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 10 Feb 2026 23:49:45 +0800 Subject: [PATCH 08/10] fix 3 --- .../org/apache/doris/nereids/CTEContext.java | 16 +- .../plans/logical/LogicalSubQueryAlias.java | 1 - .../lowercase/ShowAndSelectLowercaseTest.java | 599 ++++++++++++++++++ 3 files changed, 607 insertions(+), 9 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ShowAndSelectLowercaseTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java index 999de72582174b..c54ca35137e838 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java @@ -48,14 +48,6 @@ public class CTEContext { private final Map cteContextMap; - private static int currentLowerCaseTableNames() { - ConnectContext ctx = ConnectContext.get(); - if (ctx != null && ctx.getCurrentCatalog() != null) { - return ctx.getCurrentCatalog().getLowerCaseTableNames(); - } - return GlobalVariable.lowerCaseTableNames; - } - /* build head CTEContext */ public CTEContext() { this(CTEId.DEFAULT, null, (CTEContext) null); @@ -93,6 +85,14 @@ public CTEContext(CTEId cteId, String cteName, List recursiveCteOutputs) { this.cteContextMap = ImmutableMap.of(name, this); } + private static int currentLowerCaseTableNames() { + ConnectContext ctx = ConnectContext.get(); + if (ctx != null && ctx.getCurrentCatalog() != null) { + return ctx.getCurrentCatalog().getLowerCaseTableNames(); + } + return GlobalVariable.lowerCaseTableNames; + } + public void setRecursiveCteOutputs(List recursiveCteOutputs) { this.recursiveCteOutputs = recursiveCteOutputs; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 7d3b11de7626f7..a6f076733d3ace 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -31,7 +31,6 @@ import org.apache.doris.nereids.util.LazyCompute; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ShowAndSelectLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ShowAndSelectLowercaseTest.java new file mode 100644 index 00000000000000..6f7dc84a544421 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ShowAndSelectLowercaseTest.java @@ -0,0 +1,599 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowDatabasesCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowTableCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * SQL-level tests for SHOW TABLES/DATABASES and SELECT query resolution + * under different per-catalog lower_case_table_names / lower_case_database_names modes (0/1/2). + * + * Mode 0: case-sensitive (exact match required) + * Mode 1: stored as lowercase (remote already lowercase; any-case input lowered for lookup) + * Mode 2: case-insensitive comparison (original case preserved; any-case input matches) + */ +public class ShowAndSelectLowercaseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 0; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // Mode 0: mixed-case remote names, case-sensitive + createCatalog("test_mode0", + "org.apache.doris.datasource.lowercase.ShowAndSelectLowercaseTest$MixedCaseProvider", + 0, 0); + // Mode 1: lowercase remote names (real-world: remote stores lowercase) + createCatalog("test_mode1", + "org.apache.doris.datasource.lowercase.ShowAndSelectLowercaseTest$LowercaseProvider", + 1, 1); + // Mode 2: mixed-case remote names, case-insensitive comparison + createCatalog("test_mode2", + "org.apache.doris.datasource.lowercase.ShowAndSelectLowercaseTest$MixedCaseProvider", + 2, 2); + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + dropCatalog("test_mode0"); + dropCatalog("test_mode1"); + dropCatalog("test_mode2"); + } + + // ==================== Helper methods ==================== + + private void createCatalog(String name, String providerClass, + int lowerCaseTableNames, int lowerCaseDatabaseNames) throws Exception { + String createStmt = "create catalog " + name + " properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" = \"" + providerClass + "\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_TABLE_NAMES + "\" = \"" + lowerCaseTableNames + "\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"" + lowerCaseDatabaseNames + "\"\n" + + ");"; + NereidsParser parser = new NereidsParser(); + LogicalPlan plan = parser.parseSingle(createStmt); + if (plan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) plan).run(rootCtx, null); + } + } + + private void dropCatalog(String name) throws Exception { + NereidsParser parser = new NereidsParser(); + LogicalPlan plan = parser.parseSingle("drop catalog " + name); + if (plan instanceof DropCatalogCommand) { + ((DropCatalogCommand) plan).run(rootCtx, null); + } + } + + private List getShowDatabaseNames(String catalog) throws Exception { + rootCtx.setThreadLocalInfo(); + ShowDatabasesCommand cmd = new ShowDatabasesCommand(catalog, null, null); + ShowResultSet resultSet = cmd.doRun(rootCtx, new StmtExecutor(rootCtx, "")); + return resultSet.getResultRows().stream() + .map(row -> row.get(0)) + .filter(n -> !n.equalsIgnoreCase("information_schema") && !n.equalsIgnoreCase("mysql")) + .collect(Collectors.toList()); + } + + private List getShowTableNames(String db, String catalog) throws Exception { + rootCtx.setThreadLocalInfo(); + ShowTableCommand cmd = new ShowTableCommand(db, catalog, false, PlanType.SHOW_TABLES); + ShowResultSet resultSet = cmd.doRun(rootCtx, new StmtExecutor(rootCtx, "")); + return resultSet.getResultRows().stream() + .map(row -> row.get(0)) + .collect(Collectors.toList()); + } + + private List getShowTableNamesLike(String db, String catalog, String pattern) throws Exception { + rootCtx.setThreadLocalInfo(); + ShowTableCommand cmd = new ShowTableCommand(db, catalog, false, pattern, null, PlanType.SHOW_TABLES); + ShowResultSet resultSet = cmd.doRun(rootCtx, new StmtExecutor(rootCtx, "")); + return resultSet.getResultRows().stream() + .map(row -> row.get(0)) + .collect(Collectors.toList()); + } + + private String getPlanOrError(String sql) throws Exception { + rootCtx.setThreadLocalInfo(); + return getSQLPlanOrErrorMsg(rootCtx, "EXPLAIN " + sql, false); + } + + private boolean hasNameResolutionError(String result) { + return result.contains("does not exist") + || result.contains("Unknown column") + || result.contains("unknown qualifier") + || result.contains("unknown database"); + } + + private void assertSqlSuccess(String sql) throws Exception { + String result = getPlanOrError(sql); + Assertions.assertFalse(hasNameResolutionError(result), + "Expected success but got name resolution error: " + result); + } + + private void assertSqlError(String sql) throws Exception { + String result = getPlanOrError(sql); + Assertions.assertTrue(hasNameResolutionError(result), + "Expected name resolution error but got: " + result); + } + + // ==================== SHOW DATABASES tests ==================== + + @Test + public void testShowDatabasesMode0() throws Exception { + List dbs = getShowDatabaseNames("test_mode0"); + Assertions.assertTrue(dbs.contains("DB_UPPER"), "Expected DB_UPPER, got: " + dbs); + Assertions.assertTrue(dbs.contains("MixedDb"), "Expected MixedDb, got: " + dbs); + } + + @Test + public void testShowDatabasesMode1() throws Exception { + List dbs = getShowDatabaseNames("test_mode1"); + Assertions.assertTrue(dbs.contains("db_upper"), "Expected db_upper, got: " + dbs); + Assertions.assertTrue(dbs.contains("mixeddb"), "Expected mixeddb, got: " + dbs); + } + + @Test + public void testShowDatabasesMode2() throws Exception { + List dbs = getShowDatabaseNames("test_mode2"); + Assertions.assertTrue(dbs.contains("DB_UPPER"), "Expected DB_UPPER, got: " + dbs); + Assertions.assertTrue(dbs.contains("MixedDb"), "Expected MixedDb, got: " + dbs); + } + + // ==================== SHOW TABLES tests ==================== + + @Test + public void testShowTablesMode0() throws Exception { + List tables = getShowTableNames("DB_UPPER", "test_mode0"); + Assertions.assertTrue(tables.contains("TBL_UPPER"), "Expected TBL_UPPER, got: " + tables); + Assertions.assertTrue(tables.contains("MixedTbl"), "Expected MixedTbl, got: " + tables); + } + + @Test + public void testShowTablesMode1() throws Exception { + List tables = getShowTableNames("db_upper", "test_mode1"); + Assertions.assertTrue(tables.contains("tbl_upper"), "Expected tbl_upper, got: " + tables); + Assertions.assertTrue(tables.contains("mixedtbl"), "Expected mixedtbl, got: " + tables); + } + + @Test + public void testShowTablesMode2() throws Exception { + List tables = getShowTableNames("db_upper", "test_mode2"); + Assertions.assertTrue(tables.contains("TBL_UPPER"), "Expected TBL_UPPER, got: " + tables); + Assertions.assertTrue(tables.contains("MixedTbl"), "Expected MixedTbl, got: " + tables); + } + + // ==================== SHOW TABLES DB case-sensitivity tests ==================== + + @Test + public void testShowTablesMode0WrongDbCase() { + Assertions.assertThrows(Exception.class, () -> { + getShowTableNames("db_upper", "test_mode0"); + }); + } + + @Test + public void testShowTablesMode1AnyDbCase() throws Exception { + // Mode 1 lowercases input, so DB_UPPER -> db_upper which exists + List tables = getShowTableNames("DB_UPPER", "test_mode1"); + Assertions.assertTrue(tables.contains("tbl_upper"), "Expected tbl_upper, got: " + tables); + Assertions.assertTrue(tables.contains("mixedtbl"), "Expected mixedtbl, got: " + tables); + } + + @Test + public void testShowTablesMode2AnyDbCase() throws Exception { + // Mode 2 does case-insensitive lookup, so Db_Upper finds DB_UPPER + List tables = getShowTableNames("Db_Upper", "test_mode2"); + Assertions.assertTrue(tables.contains("TBL_UPPER"), "Expected TBL_UPPER, got: " + tables); + Assertions.assertTrue(tables.contains("MixedTbl"), "Expected MixedTbl, got: " + tables); + } + + // ==================== SHOW TABLES LIKE tests ==================== + + @Test + public void testShowTablesLikeMode0CaseSensitive() throws Exception { + // Mode 0: LIKE is case-sensitive, "tbl%" should NOT match "TBL_UPPER" + List tables = getShowTableNamesLike("DB_UPPER", "test_mode0", "tbl%"); + Assertions.assertEquals(0, tables.size(), "Mode 0 LIKE should be case-sensitive, got: " + tables); + } + + @Test + public void testShowTablesLikeMode0CaseSensitiveMatch() throws Exception { + List tables = getShowTableNamesLike("DB_UPPER", "test_mode0", "TBL%"); + Assertions.assertEquals(1, tables.size(), "Expected 1 match for TBL%, got: " + tables); + Assertions.assertTrue(tables.contains("TBL_UPPER")); + } + + @Test + public void testShowTablesLikeMode1CaseInsensitive() throws Exception { + // Mode 1: LIKE is case-insensitive, "TBL%" matches "tbl_upper" + List tables = getShowTableNamesLike("db_upper", "test_mode1", "TBL%"); + Assertions.assertEquals(1, tables.size(), "Expected 1 match for TBL%, got: " + tables); + Assertions.assertTrue(tables.contains("tbl_upper")); + } + + @Test + public void testShowTablesLikeMode2CaseInsensitive() throws Exception { + // Mode 2: LIKE is case-insensitive, "tbl%" matches "TBL_UPPER" + List tables = getShowTableNamesLike("db_upper", "test_mode2", "tbl%"); + Assertions.assertEquals(1, tables.size(), "Expected 1 match for tbl%, got: " + tables); + Assertions.assertTrue(tables.contains("TBL_UPPER")); + } + + // ==================== SELECT tests — FROM clause ==================== + + @Test + public void testSelectExactCaseMode0() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectWrongTableCaseMode0() throws Exception { + assertSqlError("SELECT * FROM test_mode0.DB_UPPER.tbl_upper"); + } + + @Test + public void testSelectWrongDbCaseMode0() throws Exception { + assertSqlError("SELECT * FROM test_mode0.db_upper.TBL_UPPER"); + } + + @Test + public void testSelectAnyCaseMode1() throws Exception { + // Mode 1: DB_UPPER -> db_upper, TBL_UPPER -> tbl_upper (both exist in lowercase provider) + assertSqlSuccess("SELECT * FROM test_mode1.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectLowerCaseMode1() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode1.db_upper.tbl_upper"); + } + + @Test + public void testSelectAnyCaseMode2() throws Exception { + // Mode 2: db_upper finds DB_UPPER, tbl_upper finds TBL_UPPER + assertSqlSuccess("SELECT * FROM test_mode2.db_upper.tbl_upper"); + } + + @Test + public void testSelectMixedCaseMode2() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode2.Db_Upper.Tbl_Upper"); + } + + // ==================== A. SELECT list — qualified column references ==================== + + @Test + public void testSelectListTableQualMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.id, TBL_UPPER.name FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectListTableQualMode0WrongCase() throws Exception { + assertSqlError("SELECT tbl_upper.id FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectListTableQualMode1() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.id FROM test_mode1.db_upper.tbl_upper"); + } + + @Test + public void testSelectListTableQualMode2() throws Exception { + assertSqlSuccess("SELECT tbl_upper.id FROM test_mode2.db_upper.TBL_UPPER"); + } + + // ==================== B. SELECT list — qualified star ==================== + + @Test + public void testSelectStarTableQualMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.* FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectStarTableQualMode0WrongCase() throws Exception { + assertSqlError("SELECT tbl_upper.* FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectStarTableQualMode1() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.* FROM test_mode1.db_upper.tbl_upper"); + } + + @Test + public void testSelectStarDbTableQualMode1() throws Exception { + assertSqlSuccess("SELECT DB_UPPER.TBL_UPPER.* FROM test_mode1.db_upper.tbl_upper"); + } + + @Test + public void testSelectStarTableQualMode2() throws Exception { + assertSqlSuccess("SELECT tbl_upper.* FROM test_mode2.db_upper.TBL_UPPER"); + } + + // ==================== C. WHERE clause — filter predicates ==================== + + @Test + public void testWhereTableQualMode0() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER WHERE TBL_UPPER.id = 1"); + } + + @Test + public void testWhereTableQualMode0WrongCase() throws Exception { + assertSqlError("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER WHERE tbl_upper.id = 1"); + } + + @Test + public void testWhereTableQualMode1() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode1.db_upper.tbl_upper WHERE TBL_UPPER.id = 1"); + } + + @Test + public void testWhereTableQualMode2() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode2.db_upper.TBL_UPPER WHERE tbl_upper.id = 1"); + } + + @Test + public void testWhereDbTableQualMode0() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER WHERE DB_UPPER.TBL_UPPER.id = 1"); + } + + @Test + public void testWhereDbTableQualMode0WrongCase() throws Exception { + assertSqlError("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER WHERE db_upper.tbl_upper.id = 1"); + } + + @Test + public void testWhereDbTableQualMode1() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode1.db_upper.tbl_upper WHERE DB_UPPER.TBL_UPPER.id = 1"); + } + + @Test + public void testWhereDbTableQualMode2() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode2.db_upper.TBL_UPPER WHERE db_upper.tbl_upper.id = 1"); + } + + // ==================== D. JOIN — table reference + ON clause qualifiers ==================== + + @Test + public void testJoinMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.id FROM test_mode0.DB_UPPER.TBL_UPPER " + + "JOIN test_mode0.DB_UPPER.MixedTbl ON TBL_UPPER.id = MixedTbl.k1"); + } + + @Test + public void testJoinMode0WrongCase() throws Exception { + assertSqlError("SELECT tbl_upper.id FROM test_mode0.DB_UPPER.TBL_UPPER " + + "JOIN test_mode0.DB_UPPER.MixedTbl ON tbl_upper.id = mixedtbl.k1"); + } + + @Test + public void testJoinMode1() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.id FROM test_mode1.db_upper.tbl_upper " + + "JOIN test_mode1.db_upper.mixedtbl ON TBL_UPPER.id = MixedTbl.k1"); + } + + @Test + public void testJoinMode2() throws Exception { + assertSqlSuccess("SELECT tbl_upper.id FROM test_mode2.db_upper.TBL_UPPER " + + "JOIN test_mode2.db_upper.MixedTbl ON tbl_upper.id = mixedtbl.k1"); + } + + // ==================== E. GROUP BY — qualified column reference ==================== + + @Test + public void testGroupByMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.name, count(*) FROM test_mode0.DB_UPPER.TBL_UPPER " + + "GROUP BY TBL_UPPER.name"); + } + + @Test + public void testGroupByMode0WrongCase() throws Exception { + assertSqlError("SELECT TBL_UPPER.name, count(*) FROM test_mode0.DB_UPPER.TBL_UPPER " + + "GROUP BY tbl_upper.name"); + } + + @Test + public void testGroupByMode1() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.name, count(*) FROM test_mode1.db_upper.tbl_upper " + + "GROUP BY TBL_UPPER.name"); + } + + @Test + public void testGroupByMode2() throws Exception { + assertSqlSuccess("SELECT tbl_upper.name, count(*) FROM test_mode2.db_upper.TBL_UPPER " + + "GROUP BY tbl_upper.name"); + } + + // ==================== F. ORDER BY — qualified column reference ==================== + + @Test + public void testOrderByMode0() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER ORDER BY TBL_UPPER.id"); + } + + @Test + public void testOrderByMode0WrongCase() throws Exception { + assertSqlError("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER ORDER BY tbl_upper.id"); + } + + @Test + public void testOrderByMode1() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode1.db_upper.tbl_upper ORDER BY TBL_UPPER.id"); + } + + @Test + public void testOrderByMode2() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode2.db_upper.TBL_UPPER ORDER BY tbl_upper.id"); + } + + // ==================== G. HAVING — qualified column in aggregate filter ==================== + + @Test + public void testHavingMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.name FROM test_mode0.DB_UPPER.TBL_UPPER " + + "GROUP BY TBL_UPPER.name HAVING count(TBL_UPPER.id) > 0"); + } + + @Test + public void testHavingMode0WrongCase() throws Exception { + assertSqlError("SELECT TBL_UPPER.name FROM test_mode0.DB_UPPER.TBL_UPPER " + + "GROUP BY TBL_UPPER.name HAVING count(tbl_upper.id) > 0"); + } + + @Test + public void testHavingMode1() throws Exception { + assertSqlSuccess("SELECT tbl_upper.name FROM test_mode1.db_upper.tbl_upper " + + "GROUP BY tbl_upper.name HAVING count(TBL_UPPER.id) > 0"); + } + + // ==================== H. Subquery — table reference inside subquery ==================== + + @Test + public void testSubqueryMode0() throws Exception { + assertSqlSuccess("SELECT * FROM (SELECT TBL_UPPER.id FROM test_mode0.DB_UPPER.TBL_UPPER) t"); + } + + @Test + public void testSubqueryMode0WrongCase() throws Exception { + assertSqlError("SELECT * FROM (SELECT id FROM test_mode0.DB_UPPER.tbl_upper) t"); + } + + @Test + public void testSubqueryMode1() throws Exception { + assertSqlSuccess("SELECT * FROM (SELECT id FROM test_mode1.DB_UPPER.TBL_UPPER) t"); + } + + @Test + public void testSubqueryMode2() throws Exception { + assertSqlSuccess("SELECT * FROM (SELECT id FROM test_mode2.Db_Upper.Tbl_Upper) t"); + } + + // ==================== I. Cross-database JOIN ==================== + + @Test + public void testCrossDbJoinMode1() throws Exception { + assertSqlSuccess("SELECT a.id FROM test_mode1.DB_UPPER.TBL_UPPER a " + + "JOIN test_mode1.MIXEDDB.another_tbl b ON a.id = b.x"); + } + + @Test + public void testCrossDbJoinMode2() throws Exception { + assertSqlSuccess("SELECT a.id FROM test_mode2.db_upper.tbl_upper a " + + "JOIN test_mode2.mixeddb.another_tbl b ON a.id = b.x"); + } + + // ==================== Mock data providers ==================== + + /** + * Mixed-case provider for mode 0 (case-sensitive) and mode 2 (case-insensitive comparison). + * Remote names preserve original casing. + */ + public static class MixedCaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + + Map> dbUpperTables = Maps.newHashMap(); + dbUpperTables.put("TBL_UPPER", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("name", PrimitiveType.VARCHAR))); + dbUpperTables.put("MixedTbl", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR))); + MOCKED_META.put("DB_UPPER", dbUpperTables); + + Map> mixedDbTables = Maps.newHashMap(); + mixedDbTables.put("another_tbl", Lists.newArrayList( + new Column("x", PrimitiveType.INT), + new Column("y", PrimitiveType.VARCHAR))); + MOCKED_META.put("MixedDb", mixedDbTables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } + + /** + * Lowercase provider for mode 1 (stored as lowercase). + * In real-world mode 1, the remote system stores names in lowercase. + * The test verifies that any-case SQL input gets lowered and resolves correctly. + */ + public static class LowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + + Map> dbUpperTables = Maps.newHashMap(); + dbUpperTables.put("tbl_upper", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("name", PrimitiveType.VARCHAR))); + dbUpperTables.put("mixedtbl", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR))); + MOCKED_META.put("db_upper", dbUpperTables); + + Map> mixedDbTables = Maps.newHashMap(); + mixedDbTables.put("another_tbl", Lists.newArrayList( + new Column("x", PrimitiveType.INT), + new Column("y", PrimitiveType.VARCHAR))); + MOCKED_META.put("mixeddb", mixedDbTables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} From 8fa146dedc7e907ed4cf3a154529103cf5bdb626 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 11 Feb 2026 11:14:46 +0800 Subject: [PATCH 09/10] fix cases --- .../external_table_p0/hive/test_hive_case_sensibility.out | 2 ++ .../iceberg/test_iceberg_hadoop_case_sensibility.out | 2 ++ .../iceberg/test_iceberg_hms_case_sensibility.out | 2 ++ .../iceberg/test_iceberg_rest_case_sensibility.out | 2 ++ .../hive/test_hive_case_sensibility.groovy | 2 +- .../iceberg/test_iceberg_hms_case_sensibility.groovy | 2 +- ..._case_meta_with_lower_table_conf_show_and_select.groovy | 7 ++++--- 7 files changed, 14 insertions(+), 5 deletions(-) diff --git a/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out b/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out index 5ce72113334eb8..32628a0578083f 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out +++ b/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out @@ -101,6 +101,7 @@ case_db1 -- !sql7 -- -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 @@ -195,6 +196,7 @@ case_db1 -- !sql7 -- -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out index 953e628f309658..87b98358ad1205 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out @@ -106,6 +106,7 @@ iceberg_hadoop_case_db1 -- !sql7 -- -- !sqlx -- +case_tbl11 -- !sql8 -- @@ -205,6 +206,7 @@ iceberg_hadoop_case_db1 -- !sql7 -- -- !sqlx -- +CASE_TBL11 -- !sql8 -- CASE_TBL14 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.out index a3a1d68e7e2aa2..d70c1966c14e24 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.out @@ -90,6 +90,7 @@ iceberg_hms_case_db1 -- !sql7 -- -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 @@ -173,6 +174,7 @@ iceberg_hms_case_db1 -- !sql7 -- -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out index c6379666096100..3ab57467fc5947 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out @@ -106,6 +106,7 @@ iceberg_rest_case_db1 -- !sql7 -- -- !sqlx -- +case_tbl11 -- !sql8 -- @@ -205,6 +206,7 @@ iceberg_rest_case_db1 -- !sql7 -- -- !sqlx -- +CASE_TBL11 -- !sql8 -- CASE_TBL14 diff --git a/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy b/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy index bdf8ee094ed6e5..1d00df13ee2dbf 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy @@ -130,7 +130,7 @@ suite("test_hive_case_sensibility", "p0,external,doris,external_docker,external_ sql """create table case_tbl13 (k1 int);""" sql """create table CASE_TBL14 (k1 int);""" - qt_sql8 """show tables like "%CASE_TBL14%"""" // empty + qt_sql8 """show tables like "%CASE_TBL14%"""" // not empty, because lower=1 qt_sql9 """show tables like "%case_tbl14%"""" qt_sql10 """show tables like "%case_tbl13%"""" diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.groovy index 01a10c22ebfad1..9e87b119926925 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.groovy @@ -131,7 +131,7 @@ suite("test_iceberg_hms_case_sensibility", "p0,external,doris,external_docker,ex sql """create table case_tbl13 (k1 int);""" sql """create table CASE_TBL14 (k1 int);""" - qt_sql8 """show tables like "%CASE_TBL14%"""" // empty + qt_sql8 """show tables like "%CASE_TBL14%"""" // not empty, because lower=1 qt_sql9 """show tables like "%case_tbl14%"""" qt_sql10 """show tables like "%case_tbl13%"""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy index 1c218457e6f2b4..682d6a469c24b6 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -26,7 +26,8 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external String jdbcPassword = "C123_567p" String s3_endpoint = getS3Endpoint() String bucket = getS3BucketName() - String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + // String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + String driver_url = "mysql-connector-j-8.4.0.jar" def wait_table_sync = { String db -> Awaitility.await().atMost(10, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until{ @@ -414,7 +415,7 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external // Verification results include lower and UPPER check { result, ex, startTime, endTime -> - def expectedTables = ["lower_with_conf", "upper_with_conf"] + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] expectedTables.each { tableName -> assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") } @@ -519,7 +520,7 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external // Verification results include lower and UPPER check { result, ex, startTime, endTime -> - def expectedTables = ["lower_with_conf", "upper_with_conf"] + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] expectedTables.each { tableName -> assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") } From ebb8b086fb5a437a35178ef5e6f8f38124226b81 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 13 Feb 2026 13:06:49 +0800 Subject: [PATCH 10/10] fix bug --- .../iceberg/test_iceberg_hadoop_case_sensibility.out | 1 + .../iceberg/test_iceberg_rest_case_sensibility.out | 1 + ...wer_case_meta_with_lower_table_conf_show_and_select.groovy | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out index 87b98358ad1205..e2320f66df5cad 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out @@ -109,6 +109,7 @@ iceberg_hadoop_case_db1 case_tbl11 -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out index 3ab57467fc5947..7f6b8eda420cca 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out @@ -109,6 +109,7 @@ iceberg_rest_case_db1 case_tbl11 -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy index 682d6a469c24b6..87a0bf27e50665 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -26,8 +26,8 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external String jdbcPassword = "C123_567p" String s3_endpoint = getS3Endpoint() String bucket = getS3BucketName() - // String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" - String driver_url = "mysql-connector-j-8.4.0.jar" + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + // String driver_url = "mysql-connector-j-8.4.0.jar" def wait_table_sync = { String db -> Awaitility.await().atMost(10, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until{