From 0872a858290e058cd4f2b841ccb4e2e0fc4a1187 Mon Sep 17 00:00:00 2001 From: wenzhenghu Date: Fri, 6 Feb 2026 22:40:05 +0800 Subject: [PATCH 1/7] [feature](file-cache) Separate file cache control for OLAP tables and external catalogs --- be/src/exec/rowid_fetcher.cpp | 2 +- be/src/io/file_factory.cpp | 4 +- be/src/io/io_common.h | 1 + be/src/olap/rowset/beta_rowset_reader.cpp | 4 +- be/src/runtime/query_context.cpp | 1 - be/src/vec/exec/scan/file_scanner.cpp | 4 +- .../doris/datasource/ExternalScanNode.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 9 ++-- .../apache/doris/qe/NereidsCoordinator.java | 4 +- .../org/apache/doris/qe/SessionVariable.java | 51 +++++++++++-------- .../doris/statistics/util/StatisticsUtil.java | 3 +- .../nereids/parser/NereidsParserTest.java | 2 - gensrc/thrift/PaloInternalService.thrift | 9 ++-- .../cloud_p0/cache/test_load_cache.groovy | 16 +++--- .../warm_up_select/test_warmup_select.groovy | 4 +- .../cache/test_hive_warmup_select.groovy | 15 ++---- 16 files changed, 63 insertions(+), 68 deletions(-) diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index 079f3c292a7e6f..82efca71747cd0 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -422,7 +422,7 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequest& request, << ", row_size:" << row_size; *response->add_row_locs() = row_loc; }); - // TODO: supoort session variable enable_page_cache and disable_file_cache if necessary. + // TODO: supoort session variable enable_page_cache and enable_file_cache_olap_tables if necessary. SegmentCacheHandle segment_cache; RETURN_IF_ERROR(scope_timer_run( [&]() { diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 7b1821c0fa80f5..cfb52536987991 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -66,8 +66,8 @@ io::FileReaderOptions FileFactory::get_reader_options(RuntimeState* state, .mtime = fd.mtime, }; if (config::enable_file_cache && state != nullptr && - state->query_options().__isset.enable_file_cache && - state->query_options().enable_file_cache) { + state->query_options().__isset.enable_file_cache_external_catalogs && + state->query_options().enable_file_cache_external_catalogs) { opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; } if (state != nullptr && state->query_options().__isset.file_cache_base_path && diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index 311fa1b01ded08..cfc090d7cddedc 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -79,6 +79,7 @@ struct IOContext { // FIXME(plat1ko): Seems `is_disposable` can be inferred from the `reader_type`? bool is_disposable = false; bool is_index_data = false; + // have not been used in Doris bool read_file_cache = true; // TODO(lightman): use following member variables to control file cache bool is_persistent = false; diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 26b8d360449cd9..f17046b4f8d09d 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -220,10 +220,8 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context _read_options.target_cast_type_for_variants = _read_context->target_cast_type_for_variants; if (_read_context->runtime_state != nullptr) { _read_options.io_ctx.query_id = &_read_context->runtime_state->query_id(); - _read_options.io_ctx.read_file_cache = - _read_context->runtime_state->query_options().enable_file_cache; _read_options.io_ctx.is_disposable = - _read_context->runtime_state->query_options().disable_file_cache; + _read_context->runtime_state->query_options().enable_file_cache_olap_tables; } if (_read_context->condition_cache_digest) { diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp index a0c271efd3973a..4d17bebb9f306b 100644 --- a/be/src/runtime/query_context.cpp +++ b/be/src/runtime/query_context.cpp @@ -113,7 +113,6 @@ QueryContext::QueryContext(TUniqueId query_id, ExecEnv* exec_env, bool initialize_context_holder = config::enable_file_cache && config::enable_file_cache_query_limit && - query_options.__isset.enable_file_cache && query_options.enable_file_cache && query_options.__isset.file_cache_query_limit_percent && query_options.file_cache_query_limit_percent < 100; diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 63a9ca8abb3358..c84ede50c7816d 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -169,7 +169,7 @@ Status FileScanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts RETURN_IF_ERROR(_init_io_ctx()); _io_ctx->file_cache_stats = _file_cache_statistics.get(); _io_ctx->file_reader_stats = _file_reader_stats.get(); - _io_ctx->is_disposable = _state->query_options().disable_file_cache; + _io_ctx->is_disposable = !_state->query_options().enable_file_cache_olap_tables; if (_is_load) { _src_row_desc.reset(new RowDescriptor(_state->desc_tbl(), @@ -1812,7 +1812,7 @@ void FileScanner::update_realtime_counters() { void FileScanner::_collect_profile_before_close() { Scanner::_collect_profile_before_close(); - if (config::enable_file_cache && _state->query_options().enable_file_cache && + if (config::enable_file_cache && _state->query_options().enable_file_cache_external_catalogs && _profile != nullptr) { io::FileCacheProfileReporter cache_profile(_profile); cache_profile.update(_file_cache_statistics.get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java index 141a3753a22a96..62489be2373ed2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java @@ -44,7 +44,7 @@ public abstract class ExternalScanNode extends ScanNode { protected boolean needCheckColumnPriv; protected final FederationBackendPolicy backendPolicy = (ConnectContext.get() != null - && (ConnectContext.get().getSessionVariable().enableFileCache + && (ConnectContext.get().getSessionVariable().enableFileCacheExternalCatalogs || ConnectContext.get().getSessionVariable().getUseConsistentHashForExternalScan())) ? new FederationBackendPolicy(NodeSelectionStrategy.CONSISTENT_HASHING) : new FederationBackendPolicy(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 82314c719a0dbc..e3331762fb3abf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -9135,14 +9135,15 @@ public LogicalPlan visitWarmUpSelect(DorisParser.WarmUpSelectContext ctx) { LogicalProject project = new LogicalProject(projectList, filter); - if (Config.isNotCloudMode() && (!ConnectContext.get().getSessionVariable().isEnableFileCache())) { + if (Config.isNotCloudMode() && (!ConnectContext.get().getSessionVariable().isEnableFileCacheOlapTables() + && !ConnectContext.get().getSessionVariable().isEnableFileCacheExternalCatalogs())) { throw new AnalysisException("WARM UP SELECT requires session variable" - + " enable_file_cache=true"); + + " enable_file_cache_olap_tables=true or enable_file_cache_external_catalogs=true"); } - if (Config.isCloudMode() && ConnectContext.get().getSessionVariable().isDisableFileCache()) { + if (Config.isCloudMode() && !ConnectContext.get().getSessionVariable().isEnableFileCacheOlapTables()) { throw new AnalysisException("WARM UP SELECT requires session variable" - + " disable_file_cache=false in cloud mode"); + + " enable_file_cache_olap_tables=true in cloud mode"); } UnboundBlackholeSink sink = new UnboundBlackholeSink<>(project, diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index 4084e4a84e8a2a..866ad98047dfdf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -130,7 +130,7 @@ public NereidsCoordinator(Long jobId, TUniqueId queryId, DescriptorTable descTab descTable, timezone, loadZeroTolerance, enableProfile ); // same reason in `setForInsert` - this.coordinatorContext.queryOptions.setDisableFileCache(true); + this.coordinatorContext.queryOptions.setEnableFileCacheOlapTables(false); this.needEnqueue = false; Preconditions.checkState(!fragments.isEmpty() @@ -490,7 +490,7 @@ private void setForInsert(long jobId) { JobProcessor jobProc = new LoadProcessor(this.coordinatorContext, jobId); this.coordinatorContext.setJobProcessor(jobProc); // Set this field to true to avoid data entering the normal cache LRU queue - this.coordinatorContext.queryOptions.setDisableFileCache(true); + this.coordinatorContext.queryOptions.setEnableFileCacheOlapTables(false); this.coordinatorContext.queryOptions.setNewVersionUnixTimestamp(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index e30207880b18b7..67bf97706a71e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -471,10 +471,8 @@ public class SessionVariable implements Serializable, Writable { public static final String TOPN_OPT_LIMIT_THRESHOLD = "topn_opt_limit_threshold"; public static final String TOPN_FILTER_RATIO = "topn_filter_ratio"; public static final String ENABLE_SNAPSHOT_POINT_QUERY = "enable_snapshot_point_query"; - - public static final String ENABLE_FILE_CACHE = "enable_file_cache"; - - public static final String DISABLE_FILE_CACHE = "disable_file_cache"; + public static final String ENABLE_FILE_CACHE_OLAP_TABLES = "enable_file_cache_olap_tables"; + public static final String ENABLE_FILE_CACHE_EXTERNAL_CATALOGS = "enable_file_cache_external_catalogs"; public static final String FILE_CACHE_QUERY_LIMIT_PERCENT = "file_cache_query_limit_percent"; @@ -2131,18 +2129,19 @@ public boolean isEnableHboNonStrictMatchingMode() { @VariableMgr.VarAttr(name = GROUP_BY_AND_HAVING_USE_ALIAS_FIRST, varType = VariableAnnotation.DEPRECATED) public boolean groupByAndHavingUseAliasFirst = false; - // Whether disable block file cache. Block cache only works when FE's query options sets disableFileCache false - // along with BE's config `enable_file_cache` true - @VariableMgr.VarAttr(name = DISABLE_FILE_CACHE, needForward = true) - public boolean disableFileCache = false; + @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_OLAP_TABLES, needForward = true, description = { + "是否启用 olap table file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", + "Set wether to use olap table file cache. This variable takes effect only if the BE config enable_file_cache=true. " + + "The cache is not used when BE config enable_file_cache=false."}) + public boolean enableFileCacheOlapTables = true; - // Whether enable block file cache. Only take effect when BE config item enable_file_cache is true. - @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE, needForward = true, description = { - "是否启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_EXTERNAL_CATALOGS, needForward = true, description = { + "是否启用 external catalog file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", - "Set wether to use file cache. This variable takes effect only if the BE config enable_file_cache=true. " + "Set wether to use external catalog file cache. This variable takes effect only if the BE config enable_file_cache=true. " + "The cache is not used when BE config enable_file_cache=false."}) - public boolean enableFileCache = false; + public boolean enableFileCacheExternalCatalogs = false; // Specify base path for file cache, or chose a random path. @VariableMgr.VarAttr(name = FILE_CACHE_BASE_PATH, needForward = true, description = { @@ -4026,11 +4025,11 @@ public void setResourceGroup(String resourceGroup) { } public boolean isDisableFileCache() { - return Config.isCloudMode() ? disableFileCache : false; + return Config.isCloudMode() ? !enableFileCacheOlapTables : false; } public void setDisableFileCache(boolean disableFileCache) { - this.disableFileCache = disableFileCache; + this.enableFileCacheOlapTables = !disableFileCache; } public boolean isDisableColocatePlan() { @@ -4868,12 +4867,20 @@ public long convertBoolToLong(Boolean val) { return val ? 1 : 0; } - public boolean isEnableFileCache() { - return enableFileCache; + public boolean isEnableFileCacheOlapTables() { + return enableFileCacheOlapTables; + } + + public void setEnableFileCacheOlapTables(boolean enableFileCacheOlapTables) { + this.enableFileCacheOlapTables = enableFileCacheOlapTables; + } + + public boolean isEnableFileCacheExternalCatalogs() { + return enableFileCacheExternalCatalogs; } - public void setEnableFileCache(boolean enableFileCache) { - this.enableFileCache = enableFileCache; + public void setEnableFileCacheExternalCatalogs(boolean enableFileCacheExternalCatalogs) { + this.enableFileCacheExternalCatalogs = enableFileCacheExternalCatalogs; } public String getFileCacheBasePath() { @@ -5062,8 +5069,6 @@ public TQueryOptions toThrift() { tResult.setSkipDeleteBitmap(skipDeleteBitmap); - tResult.setEnableFileCache(enableFileCache); - tResult.setEnablePageCache(enablePageCache); tResult.setEnableParquetFilePageCache(enableParquetFilePageCache); @@ -5106,7 +5111,9 @@ public TQueryOptions toThrift() { tResult.setParallelScanMinRowsPerScanner(parallelScanMinRowsPerScanner); tResult.setOptimizeIndexScanParallelism(optimizeIndexScanParallelism); tResult.setSkipBadTablet(skipBadTablet); - tResult.setDisableFileCache(disableFileCache); + + tResult.setEnableFileCacheOlapTables(enableFileCacheOlapTables); + tResult.setEnableFileCacheExternalCatalogs(enableFileCacheExternalCatalogs); tResult.setEnablePreferCachedRowset(getEnablePreferCachedRowset()); tResult.setQueryFreshnessToleranceMs(getQueryFreshnessToleranceMs()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 014c9ace70f887..40d3188b2ed724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -223,7 +223,6 @@ public static AutoCloseConnectContext buildConnectContext(boolean useFileCacheFo sessionVariable.parallelPipelineTaskNum = Config.statistics_sql_parallel_exec_instance_num; sessionVariable.setQueryTimeoutS(StatisticsUtil.getAnalyzeTimeout()); sessionVariable.insertTimeoutS = StatisticsUtil.getAnalyzeTimeout(); - sessionVariable.enableFileCache = false; sessionVariable.forbidUnknownColStats = false; sessionVariable.enablePushDownMinMaxOnUnique = true; sessionVariable.enablePushDownStringMinMax = true; @@ -243,7 +242,7 @@ public static AutoCloseConnectContext buildConnectContext(boolean useFileCacheFo LOG.warn("failed to connect to cloud cluster", e); return ctx; } - sessionVariable.disableFileCache = !useFileCacheForStat; + sessionVariable.enableFileCacheOlapTables = useFileCacheForStat; return ctx; } else { return new AutoCloseConnectContext(connectContext); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 1263cc7e95d877..64dfe258854cf1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -1393,8 +1393,6 @@ public void testAdminRotateTdeRootKey() { @Test public void testWarmUpSelect() { ConnectContext ctx = ConnectContext.get(); - ctx.getSessionVariable().setEnableFileCache(true); - ctx.getSessionVariable().setDisableFileCache(false); NereidsParser nereidsParser = new NereidsParser(); // Test basic warm up select statement diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index e76b2471ddef29..5be6639a76ad7f 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -201,7 +201,7 @@ struct TQueryOptions { // deprecated 60: optional i32 partitioned_hash_agg_rows_threshold = 0 // deprecated - 61: optional bool enable_file_cache = false + 61: optional bool enable_file_cache_deprecated = false // deprecated 62: optional i32 insert_timeout = 14400 @@ -423,14 +423,13 @@ struct TQueryOptions { // runtime profiling to choose the most efficient algorithm for the data pattern 183: optional bool enable_use_hybrid_sort = false; 184: optional i32 cte_max_recursion_depth; - - 185: optional bool enable_parquet_file_page_cache = true; - // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. - 1000: optional bool disable_file_cache = false + 186: optional bool enable_file_cache_olap_tables = true; + 187: optional bool enable_file_cache_external_catalogs = false; + 1001: optional i32 file_cache_query_limit_percent = -1 } diff --git a/regression-test/suites/cloud_p0/cache/test_load_cache.groovy b/regression-test/suites/cloud_p0/cache/test_load_cache.groovy index 7fe5c5173822c8..b7b0a31a9015e5 100644 --- a/regression-test/suites/cloud_p0/cache/test_load_cache.groovy +++ b/regression-test/suites/cloud_p0/cache/test_load_cache.groovy @@ -21,14 +21,14 @@ import groovy.json.JsonSlurper /* Test Description: -1. When disable_file_cache = true and enable_file_cache = true, it is expected that the S3 TVF load (import phase) will NOT enter the cache, while the query +1. When enable_file_cache_olap_tables = false, it is expected that the S3 TVF load (import phase) will NOT enter the cache, while the query phase will enter the Disposable queue. Specifically: Normal queue size should be 0, Disposable queue size should be 91163 bytes. -2. When disable_file_cache = false and enable_file_cache = true, it is expected that the S3 TVF load (import phase) will enter the Normal queue, and the query +2. When enable_file_cache_olap_tables = true, it is expected that the S3 TVF load (import phase) will enter the Normal queue, and the query phase will still enter the Disposable queue. Specifically: Normal queue size should be 236988 bytes, Disposable queue size should still be 91163 bytes. -Explanation: The query phase caches the compressed file, so the Disposable queue size is checked for an exact value; for the import phase cache, since future +Explanation: The query phase caches the compressed file, so the Disposable queue size is checked for an exact value; for the import phase cache, since future changes to statistics are possible, only a reasonable range is required. */ @@ -173,15 +173,14 @@ suite('test_load_cache', 'docker') { def s3_tvf_uri = "s3://${s3_bucket}/regression/tpch/sf0.01/customer.csv.gz" // ============================================================================ - // SCENARIO 1: disable_file_cache = true + // SCENARIO 1: enable_file_cache_olap_tables = false // ============================================================================ // Clear file cache before test clearFileCacheOnAllBackends() // Set session variables for Scenario 1 - sql "set disable_file_cache = true;" - sql "set enable_file_cache = true;" + sql "set enable_file_cache_olap_tables = false;" // Create test table sql """DROP TABLE IF EXISTS load_test_table""" @@ -264,15 +263,14 @@ suite('test_load_cache', 'docker') { sleep(3000) // ============================================================================ - // SCENARIO 2: disable_file_cache = false + // SCENARIO 2: enable_file_cache_olap_tables = true // ============================================================================ // Clear file cache before test clearFileCacheOnAllBackends() // Set session variables for Scenario 2 - sql "set disable_file_cache = false;" - sql "set enable_file_cache = true;" + sql "set enable_file_cache_olap_tables = true;" // Create test table sql """DROP TABLE IF EXISTS load_test_table""" diff --git a/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy b/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy index 2b0b39affc2cc0..e0738cde834296 100644 --- a/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy +++ b/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy @@ -61,7 +61,7 @@ suite("test_warmup_select") { def test_basic_warmup = { // Enable file cache for warm up functionality - sql "set disable_file_cache=false" + sql "set enable_file_cache_olap_tables=true" sql "WARM UP SELECT * FROM lineitem" @@ -72,7 +72,7 @@ suite("test_warmup_select") { def test_warmup_negative_cases = { // Enable file cache for warm up functionality - sql "set disable_file_cache=false" + sql "set enable_file_cache_olap_tables=true" // These should fail as warm up select doesn't support these operations try { diff --git a/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy b/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy index d1adc2b6b84bfb..0e892a7f9d4cda 100644 --- a/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy +++ b/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy @@ -24,8 +24,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock def test_basic_warmup = { // Enable file cache for warm up functionality - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalogs=true" sql "WARM UP SELECT * FROM lineitem" @@ -36,8 +35,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock def test_warmup_negative_cases = { // Enable file cache for warm up functionality - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalogs=true" // These should fail as warm up select doesn't support these operations try { @@ -90,8 +88,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock // Test: user with only SELECT privilege should be able to run WARM UP SELECT connect(user1, "${pwd}", url) { - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalogs=true" // This should succeed - only SELECT privilege is needed sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" @@ -107,8 +104,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock sql """REVOKE SELECT_PRIV ON ${catalog_name}.tpch1_parquet.lineitem FROM ${user1}""" connect(user1, "${pwd}", url) { - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalogs=true" test { sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" exception "denied" @@ -119,8 +115,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock sql """GRANT LOAD_PRIV ON ${catalog_name}.tpch1_parquet.lineitem TO ${user1}""" connect(user1, "${pwd}", url) { - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalogs=true" test { sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" exception "denied" From 73c18162cd4b4953fff5331e9e244d0bafbb1abd Mon Sep 17 00:00:00 2001 From: Wen Zhenghu Date: Mon, 9 Feb 2026 08:42:48 +0800 Subject: [PATCH 2/7] Invert file cache option in beta_rowset_reader --- be/src/olap/rowset/beta_rowset_reader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index f17046b4f8d09d..1ea7be18b92f5c 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -221,7 +221,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context if (_read_context->runtime_state != nullptr) { _read_options.io_ctx.query_id = &_read_context->runtime_state->query_id(); _read_options.io_ctx.is_disposable = - _read_context->runtime_state->query_options().enable_file_cache_olap_tables; + !_read_context->runtime_state->query_options().enable_file_cache_olap_tables; } if (_read_context->condition_cache_digest) { From c123e3d0148e0ae1d556c753e6d23fba87161fe6 Mon Sep 17 00:00:00 2001 From: Wen Zhenghu Date: Mon, 9 Feb 2026 08:47:13 +0800 Subject: [PATCH 3/7] Change disposable flag to use external catalogs --- be/src/vec/exec/scan/file_scanner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index c84ede50c7816d..e433a1ea7ac3a3 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -169,7 +169,7 @@ Status FileScanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts RETURN_IF_ERROR(_init_io_ctx()); _io_ctx->file_cache_stats = _file_cache_statistics.get(); _io_ctx->file_reader_stats = _file_reader_stats.get(); - _io_ctx->is_disposable = !_state->query_options().enable_file_cache_olap_tables; + _io_ctx->is_disposable = !_state->query_options().enable_file_cache_external_catalogs; if (_is_load) { _src_row_desc.reset(new RowDescriptor(_state->desc_tbl(), From 72f04db03289721079ab4fa31046e6ff9b058cdb Mon Sep 17 00:00:00 2001 From: xuchenhao <419062425@qq.com> Date: Wed, 11 Feb 2026 13:46:57 +0800 Subject: [PATCH 4/7] Update and refine code framework. --- be/src/exec/rowid_fetcher.cpp | 2 +- be/src/io/file_factory.cpp | 4 +- be/src/io/io_common.h | 1 - be/src/olap/rowset/beta_rowset_reader.cpp | 2 +- be/src/runtime/query_context.cpp | 10 ++- be/src/vec/exec/scan/file_scanner.cpp | 4 +- .../doris/datasource/ExternalScanNode.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 9 +-- .../apache/doris/planner/OlapTableSink.java | 2 +- .../apache/doris/qe/NereidsCoordinator.java | 4 +- .../org/apache/doris/qe/SessionVariable.java | 74 +++++++++++++------ .../doris/statistics/util/StatisticsUtil.java | 3 +- .../nereids/parser/NereidsParserTest.java | 1 + gensrc/thrift/PaloInternalService.thrift | 11 ++- 14 files changed, 82 insertions(+), 47 deletions(-) diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index 82efca71747cd0..079f3c292a7e6f 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -422,7 +422,7 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequest& request, << ", row_size:" << row_size; *response->add_row_locs() = row_loc; }); - // TODO: supoort session variable enable_page_cache and enable_file_cache_olap_tables if necessary. + // TODO: supoort session variable enable_page_cache and disable_file_cache if necessary. SegmentCacheHandle segment_cache; RETURN_IF_ERROR(scope_timer_run( [&]() { diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index cfb52536987991..7cba22bcead157 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -66,8 +66,8 @@ io::FileReaderOptions FileFactory::get_reader_options(RuntimeState* state, .mtime = fd.mtime, }; if (config::enable_file_cache && state != nullptr && - state->query_options().__isset.enable_file_cache_external_catalogs && - state->query_options().enable_file_cache_external_catalogs) { + state->query_options().__isset.enable_file_cache_external_catalog && + state->query_options().enable_file_cache_external_catalog) { opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; } if (state != nullptr && state->query_options().__isset.file_cache_base_path && diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index cfc090d7cddedc..311fa1b01ded08 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -79,7 +79,6 @@ struct IOContext { // FIXME(plat1ko): Seems `is_disposable` can be inferred from the `reader_type`? bool is_disposable = false; bool is_index_data = false; - // have not been used in Doris bool read_file_cache = true; // TODO(lightman): use following member variables to control file cache bool is_persistent = false; diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 1ea7be18b92f5c..d87bb2a7ffcd66 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -221,7 +221,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context if (_read_context->runtime_state != nullptr) { _read_options.io_ctx.query_id = &_read_context->runtime_state->query_id(); _read_options.io_ctx.is_disposable = - !_read_context->runtime_state->query_options().enable_file_cache_olap_tables; + !_read_context->runtime_state->query_options().enable_file_cache_olap_table; } if (_read_context->condition_cache_digest) { diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp index 4d17bebb9f306b..bdc83fc412a726 100644 --- a/be/src/runtime/query_context.cpp +++ b/be/src/runtime/query_context.cpp @@ -111,10 +111,12 @@ QueryContext::QueryContext(TUniqueId query_id, ExecEnv* exec_env, _timeout_second = query_options.execution_timeout; - bool initialize_context_holder = - config::enable_file_cache && config::enable_file_cache_query_limit && - query_options.__isset.file_cache_query_limit_percent && - query_options.file_cache_query_limit_percent < 100; + bool initialize_context_holder = config::enable_file_cache && + config::enable_file_cache_query_limit && + !(query_options.query_type == TQueryType::EXTERNAL && + !query_options.enable_file_cache_external_catalog) && + query_options.__isset.file_cache_query_limit_percent && + query_options.file_cache_query_limit_percent < 100; // Initialize file cache context holders if (initialize_context_holder) { diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index e433a1ea7ac3a3..15f3d1991b313e 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -169,7 +169,7 @@ Status FileScanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts RETURN_IF_ERROR(_init_io_ctx()); _io_ctx->file_cache_stats = _file_cache_statistics.get(); _io_ctx->file_reader_stats = _file_reader_stats.get(); - _io_ctx->is_disposable = !_state->query_options().enable_file_cache_external_catalogs; + _io_ctx->is_disposable = !_state->query_options().enable_file_cache_external_catalog; if (_is_load) { _src_row_desc.reset(new RowDescriptor(_state->desc_tbl(), @@ -1812,7 +1812,7 @@ void FileScanner::update_realtime_counters() { void FileScanner::_collect_profile_before_close() { Scanner::_collect_profile_before_close(); - if (config::enable_file_cache && _state->query_options().enable_file_cache_external_catalogs && + if (config::enable_file_cache && _state->query_options().enable_file_cache_external_catalog && _profile != nullptr) { io::FileCacheProfileReporter cache_profile(_profile); cache_profile.update(_file_cache_statistics.get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java index 62489be2373ed2..beb4fd3656e52a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java @@ -44,7 +44,7 @@ public abstract class ExternalScanNode extends ScanNode { protected boolean needCheckColumnPriv; protected final FederationBackendPolicy backendPolicy = (ConnectContext.get() != null - && (ConnectContext.get().getSessionVariable().enableFileCacheExternalCatalogs + && (ConnectContext.get().getSessionVariable().enableFileCacheExternalCatalog || ConnectContext.get().getSessionVariable().getUseConsistentHashForExternalScan())) ? new FederationBackendPolicy(NodeSelectionStrategy.CONSISTENT_HASHING) : new FederationBackendPolicy(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index e3331762fb3abf..ab80bdf132ac07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -9135,15 +9135,14 @@ public LogicalPlan visitWarmUpSelect(DorisParser.WarmUpSelectContext ctx) { LogicalProject project = new LogicalProject(projectList, filter); - if (Config.isNotCloudMode() && (!ConnectContext.get().getSessionVariable().isEnableFileCacheOlapTables() - && !ConnectContext.get().getSessionVariable().isEnableFileCacheExternalCatalogs())) { + if (Config.isNotCloudMode() && !ConnectContext.get().getSessionVariable().isEnableFileCacheExternalCatalog()) { throw new AnalysisException("WARM UP SELECT requires session variable" - + " enable_file_cache_olap_tables=true or enable_file_cache_external_catalogs=true"); + + " enable_file_cache_external_catalog=true"); } - if (Config.isCloudMode() && !ConnectContext.get().getSessionVariable().isEnableFileCacheOlapTables()) { + if (Config.isCloudMode() && !ConnectContext.get().getSessionVariable().isEnableFileCacheOlapTable()) { throw new AnalysisException("WARM UP SELECT requires session variable" - + " enable_file_cache_olap_tables=true in cloud mode"); + + " enable_file_cache_olap_table=true in cloud mode"); } UnboundBlackholeSink sink = new UnboundBlackholeSink<>(project, diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index a84c14083cc012..7dc29f34fd01fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -156,7 +156,7 @@ public void init(TUniqueId loadId, long txnId, long dbId, long loadChannelTimeou tSink.setLoadChannelTimeoutS(loadChannelTimeoutS); tSink.setSendBatchParallelism(sendBatchParallelism); tSink.setWriteFileCache(ConnectContext.get() != null - ? !ConnectContext.get().getSessionVariable().isDisableFileCache() + ? ConnectContext.get().getSessionVariable().isEnableFileCacheOlapTable() : false); this.isStrictMode = isStrictMode; this.txnId = txnId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index 866ad98047dfdf..612ceb7ba902c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -130,7 +130,7 @@ public NereidsCoordinator(Long jobId, TUniqueId queryId, DescriptorTable descTab descTable, timezone, loadZeroTolerance, enableProfile ); // same reason in `setForInsert` - this.coordinatorContext.queryOptions.setEnableFileCacheOlapTables(false); + this.coordinatorContext.queryOptions.setEnableFileCacheOlapTable(false); this.needEnqueue = false; Preconditions.checkState(!fragments.isEmpty() @@ -490,7 +490,7 @@ private void setForInsert(long jobId) { JobProcessor jobProc = new LoadProcessor(this.coordinatorContext, jobId); this.coordinatorContext.setJobProcessor(jobProc); // Set this field to true to avoid data entering the normal cache LRU queue - this.coordinatorContext.queryOptions.setEnableFileCacheOlapTables(false); + this.coordinatorContext.queryOptions.setEnableFileCacheOlapTable(false); this.coordinatorContext.queryOptions.setNewVersionUnixTimestamp(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 3ae1c09b9208a5..07d7de43798c59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -473,8 +473,15 @@ public class SessionVariable implements Serializable, Writable { public static final String TOPN_OPT_LIMIT_THRESHOLD = "topn_opt_limit_threshold"; public static final String TOPN_FILTER_RATIO = "topn_filter_ratio"; public static final String ENABLE_SNAPSHOT_POINT_QUERY = "enable_snapshot_point_query"; - public static final String ENABLE_FILE_CACHE_OLAP_TABLES = "enable_file_cache_olap_tables"; - public static final String ENABLE_FILE_CACHE_EXTERNAL_CATALOGS = "enable_file_cache_external_catalogs"; + + // deprecated + public static final String ENABLE_FILE_CACHE = "enable_file_cache"; + + // deprecated + public static final String DISABLE_FILE_CACHE = "disable_file_cache"; + + public static final String ENABLE_FILE_CACHE_OLAP_TABLE = "enable_file_cache_olap_table"; + public static final String ENABLE_FILE_CACHE_EXTERNAL_CATALOG = "enable_file_cache_external_catalog"; public static final String FILE_CACHE_QUERY_LIMIT_PERCENT = "file_cache_query_limit_percent"; @@ -2134,19 +2141,32 @@ public boolean isEnableHboNonStrictMatchingMode() { @VariableMgr.VarAttr(name = GROUP_BY_AND_HAVING_USE_ALIAS_FIRST, varType = VariableAnnotation.DEPRECATED) public boolean groupByAndHavingUseAliasFirst = false; - @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_OLAP_TABLES, needForward = true, description = { + // Whether disable block file cache. Block cache only works when FE's query options sets disableFileCache false + // along with BE's config `enable_file_cache` true + @VariableMgr.VarAttr(name = DISABLE_FILE_CACHE, needForward = true, varType = VariableAnnotation.DEPRECATED) + public boolean disableFileCache = false; + + // Whether enable block file cache. Only take effect when BE config item enable_file_cache is true. + @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE, needForward = true, varType = VariableAnnotation.DEPRECATED, + description = {"是否启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", + "Set whether to use file cache. This variable takes effect only if the BE config " + + "enable_file_cache=true. The cache is not used when BE config enable_file_cache=false."}) + public boolean enableFileCache = false; + + @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_OLAP_TABLE, needForward = true, description = { "是否启用 olap table file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", - "Set wether to use olap table file cache. This variable takes effect only if the BE config enable_file_cache=true. " - + "The cache is not used when BE config enable_file_cache=false."}) - public boolean enableFileCacheOlapTables = true; + "Set whether to use olap table file cache. This variable takes effect only if the BE config " + + "enable_file_cache=true. The cache is not used when BE config enable_file_cache=false."}) + public boolean enableFileCacheOlapTable = true; - @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_EXTERNAL_CATALOGS, needForward = true, description = { + @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_EXTERNAL_CATALOG, needForward = true, description = { "是否启用 external catalog file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", - "Set wether to use external catalog file cache. This variable takes effect only if the BE config enable_file_cache=true. " - + "The cache is not used when BE config enable_file_cache=false."}) - public boolean enableFileCacheExternalCatalogs = false; + "Set whether to use external catalog file cache. This variable takes effect only if the BE config " + + "enable_file_cache=true. The cache is not used when BE config enable_file_cache=false."}) + public boolean enableFileCacheExternalCatalog = false; // Specify base path for file cache, or chose a random path. @VariableMgr.VarAttr(name = FILE_CACHE_BASE_PATH, needForward = true, description = { @@ -4030,11 +4050,11 @@ public void setResourceGroup(String resourceGroup) { } public boolean isDisableFileCache() { - return Config.isCloudMode() ? !enableFileCacheOlapTables : false; + return Config.isCloudMode() ? disableFileCache : false; } public void setDisableFileCache(boolean disableFileCache) { - this.enableFileCacheOlapTables = !disableFileCache; + this.disableFileCache = disableFileCache; } public boolean isDisableColocatePlan() { @@ -4872,20 +4892,28 @@ public long convertBoolToLong(Boolean val) { return val ? 1 : 0; } - public boolean isEnableFileCacheOlapTables() { - return enableFileCacheOlapTables; + public boolean isEnableFileCache() { + return enableFileCache; } - public void setEnableFileCacheOlapTables(boolean enableFileCacheOlapTables) { - this.enableFileCacheOlapTables = enableFileCacheOlapTables; + public void setEnableFileCache(boolean enableFileCache) { + this.enableFileCache = enableFileCache; } - public boolean isEnableFileCacheExternalCatalogs() { - return enableFileCacheExternalCatalogs; + public boolean isEnableFileCacheOlapTable() { + return enableFileCacheOlapTable; } - public void setEnableFileCacheExternalCatalogs(boolean enableFileCacheExternalCatalogs) { - this.enableFileCacheExternalCatalogs = enableFileCacheExternalCatalogs; + public void setEnableFileCacheOlapTable(boolean enableFileCacheOlapTable) { + this.enableFileCacheOlapTable = enableFileCacheOlapTable; + } + + public boolean isEnableFileCacheExternalCatalog() { + return enableFileCache; + } + + public void setEnableFileCacheExternalCatalog(boolean enableFileCacheExternalCatalog) { + this.enableFileCacheExternalCatalog = enableFileCacheExternalCatalog; } public String getFileCacheBasePath() { @@ -5074,6 +5102,8 @@ public TQueryOptions toThrift() { tResult.setSkipDeleteBitmap(skipDeleteBitmap); + tResult.setEnableFileCache(enableFileCache); + tResult.setEnablePageCache(enablePageCache); tResult.setEnableParquetFilePageCache(enableParquetFilePageCache); @@ -5117,9 +5147,7 @@ public TQueryOptions toThrift() { tResult.setParallelScanMinRowsPerScanner(parallelScanMinRowsPerScanner); tResult.setOptimizeIndexScanParallelism(optimizeIndexScanParallelism); tResult.setSkipBadTablet(skipBadTablet); - - tResult.setEnableFileCacheOlapTables(enableFileCacheOlapTables); - tResult.setEnableFileCacheExternalCatalogs(enableFileCacheExternalCatalogs); + tResult.setDisableFileCache(disableFileCache); tResult.setEnablePreferCachedRowset(getEnablePreferCachedRowset()); tResult.setQueryFreshnessToleranceMs(getQueryFreshnessToleranceMs()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 40d3188b2ed724..569ca52b8709d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -223,6 +223,7 @@ public static AutoCloseConnectContext buildConnectContext(boolean useFileCacheFo sessionVariable.parallelPipelineTaskNum = Config.statistics_sql_parallel_exec_instance_num; sessionVariable.setQueryTimeoutS(StatisticsUtil.getAnalyzeTimeout()); sessionVariable.insertTimeoutS = StatisticsUtil.getAnalyzeTimeout(); + sessionVariable.enableFileCache = false; sessionVariable.forbidUnknownColStats = false; sessionVariable.enablePushDownMinMaxOnUnique = true; sessionVariable.enablePushDownStringMinMax = true; @@ -242,7 +243,7 @@ public static AutoCloseConnectContext buildConnectContext(boolean useFileCacheFo LOG.warn("failed to connect to cloud cluster", e); return ctx; } - sessionVariable.enableFileCacheOlapTables = useFileCacheForStat; + sessionVariable.enableFileCacheOlapTable = useFileCacheForStat; return ctx; } else { return new AutoCloseConnectContext(connectContext); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 64dfe258854cf1..f69e2c19daa303 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -1393,6 +1393,7 @@ public void testAdminRotateTdeRootKey() { @Test public void testWarmUpSelect() { ConnectContext ctx = ConnectContext.get(); + ctx.getSessionVariable().setEnableFileCacheOlapTable(true); NereidsParser nereidsParser = new NereidsParser(); // Test basic warm up select statement diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 0c20808011a17d..fcf35067ee127a 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -201,7 +201,8 @@ struct TQueryOptions { // deprecated 60: optional i32 partitioned_hash_agg_rows_threshold = 0 // deprecated - 61: optional bool enable_file_cache_deprecated = false // deprecated + // deprecated + 61: optional bool enable_file_cache = false 62: optional i32 insert_timeout = 14400 @@ -424,6 +425,7 @@ struct TQueryOptions { 183: optional bool enable_use_hybrid_sort = false; 184: optional i32 cte_max_recursion_depth; + 185: optional bool enable_parquet_file_page_cache = true; 195: optional bool enable_left_semi_direct_return_opt; @@ -431,10 +433,13 @@ struct TQueryOptions { // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. - 186: optional bool enable_file_cache_olap_tables = true; - 187: optional bool enable_file_cache_external_catalogs = false; + // deprecated + 1000: optional bool disable_file_cache = false 1001: optional i32 file_cache_query_limit_percent = -1 + + 1002: optional bool enable_file_cache_olap_table = true; + 1003: optional bool enable_file_cache_external_catalog = false; } From be884ad45820178bff22e427e45ee5d9d5e6699d Mon Sep 17 00:00:00 2001 From: xuchenhao <419062425@qq.com> Date: Wed, 11 Feb 2026 15:54:47 +0800 Subject: [PATCH 5/7] Fix UT and RT cases. --- .../java/org/apache/doris/qe/SessionVariable.java | 2 +- .../doris/nereids/parser/NereidsParserTest.java | 1 + .../suites/cloud_p0/cache/test_load_cache.groovy | 14 +++++++------- .../warm_up_select/test_warmup_select.groovy | 4 ++-- .../cache/test_hive_warmup_select.groovy | 10 +++++----- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 07d7de43798c59..0ff65715589d17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -4909,7 +4909,7 @@ public void setEnableFileCacheOlapTable(boolean enableFileCacheOlapTable) { } public boolean isEnableFileCacheExternalCatalog() { - return enableFileCache; + return enableFileCacheExternalCatalog; } public void setEnableFileCacheExternalCatalog(boolean enableFileCacheExternalCatalog) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index f69e2c19daa303..d31532d57dc5b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -1394,6 +1394,7 @@ public void testAdminRotateTdeRootKey() { public void testWarmUpSelect() { ConnectContext ctx = ConnectContext.get(); ctx.getSessionVariable().setEnableFileCacheOlapTable(true); + ctx.getSessionVariable().setEnableFileCacheExternalCatalog(true); NereidsParser nereidsParser = new NereidsParser(); // Test basic warm up select statement diff --git a/regression-test/suites/cloud_p0/cache/test_load_cache.groovy b/regression-test/suites/cloud_p0/cache/test_load_cache.groovy index b7b0a31a9015e5..52fdd7251c5db1 100644 --- a/regression-test/suites/cloud_p0/cache/test_load_cache.groovy +++ b/regression-test/suites/cloud_p0/cache/test_load_cache.groovy @@ -21,11 +21,11 @@ import groovy.json.JsonSlurper /* Test Description: -1. When enable_file_cache_olap_tables = false, it is expected that the S3 TVF load (import phase) will NOT enter the cache, while the query +1. When enable_file_cache_olap_table = false, it is expected that the S3 TVF load (import phase) will NOT enter the cache, while the query phase will enter the Disposable queue. Specifically: Normal queue size should be 0, Disposable queue size should be 91163 bytes. -2. When enable_file_cache_olap_tables = true, it is expected that the S3 TVF load (import phase) will enter the Normal queue, and the query - phase will still enter the Disposable queue. +2. When enable_file_cache_olap_table = true, it is expected that the S3 TVF load (import phase) will enter the Normal queue, and the query + phase will enter the Normal queue. Specifically: Normal queue size should be 236988 bytes, Disposable queue size should still be 91163 bytes. Explanation: The query phase caches the compressed file, so the Disposable queue size is checked for an exact value; for the import phase cache, since future @@ -173,14 +173,14 @@ suite('test_load_cache', 'docker') { def s3_tvf_uri = "s3://${s3_bucket}/regression/tpch/sf0.01/customer.csv.gz" // ============================================================================ - // SCENARIO 1: enable_file_cache_olap_tables = false + // SCENARIO 1: enable_file_cache_olap_table = false // ============================================================================ // Clear file cache before test clearFileCacheOnAllBackends() // Set session variables for Scenario 1 - sql "set enable_file_cache_olap_tables = false;" + sql "set enable_file_cache_olap_table = false;" // Create test table sql """DROP TABLE IF EXISTS load_test_table""" @@ -263,14 +263,14 @@ suite('test_load_cache', 'docker') { sleep(3000) // ============================================================================ - // SCENARIO 2: enable_file_cache_olap_tables = true + // SCENARIO 2: enable_file_cache_olap_table = true // ============================================================================ // Clear file cache before test clearFileCacheOnAllBackends() // Set session variables for Scenario 2 - sql "set enable_file_cache_olap_tables = true;" + sql "set enable_file_cache_olap_table = true;" // Create test table sql """DROP TABLE IF EXISTS load_test_table""" diff --git a/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy b/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy index e0738cde834296..a4cb549d08d4f7 100644 --- a/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy +++ b/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy @@ -61,7 +61,7 @@ suite("test_warmup_select") { def test_basic_warmup = { // Enable file cache for warm up functionality - sql "set enable_file_cache_olap_tables=true" + sql "set enable_file_cache_olap_table=true" sql "WARM UP SELECT * FROM lineitem" @@ -72,7 +72,7 @@ suite("test_warmup_select") { def test_warmup_negative_cases = { // Enable file cache for warm up functionality - sql "set enable_file_cache_olap_tables=true" + sql "set enable_file_cache_olap_table=true" // These should fail as warm up select doesn't support these operations try { diff --git a/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy b/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy index 0e892a7f9d4cda..3e86084d09d59d 100644 --- a/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy +++ b/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy @@ -24,7 +24,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock def test_basic_warmup = { // Enable file cache for warm up functionality - sql "set enable_file_cache_external_catalogs=true" + sql "set enable_file_cache_external_catalog=true" sql "WARM UP SELECT * FROM lineitem" @@ -35,7 +35,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock def test_warmup_negative_cases = { // Enable file cache for warm up functionality - sql "set enable_file_cache_external_catalogs=true" + sql "set enable_file_cache_external_catalog=true" // These should fail as warm up select doesn't support these operations try { @@ -88,7 +88,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock // Test: user with only SELECT privilege should be able to run WARM UP SELECT connect(user1, "${pwd}", url) { - sql "set enable_file_cache_external_catalogs=true" + sql "set enable_file_cache_external_catalog=true" // This should succeed - only SELECT privilege is needed sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" @@ -104,7 +104,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock sql """REVOKE SELECT_PRIV ON ${catalog_name}.tpch1_parquet.lineitem FROM ${user1}""" connect(user1, "${pwd}", url) { - sql "set enable_file_cache_external_catalogs=true" + sql "set enable_file_cache_external_catalog=true" test { sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" exception "denied" @@ -115,7 +115,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock sql """GRANT LOAD_PRIV ON ${catalog_name}.tpch1_parquet.lineitem TO ${user1}""" connect(user1, "${pwd}", url) { - sql "set enable_file_cache_external_catalogs=true" + sql "set enable_file_cache_external_catalog=true" test { sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" exception "denied" From 3b59a3bc88b62728bd1b58199a075a23823e07c9 Mon Sep 17 00:00:00 2001 From: xuchenhao <419062425@qq.com> Date: Wed, 11 Feb 2026 16:13:11 +0800 Subject: [PATCH 6/7] Fix RT cases. --- ...m_up_with_compaction_use_peer_cache.groovy | 2 +- .../cache/test_file_cache_features.groovy | 4 +- .../cache/test_file_cache_query_limit.groovy | 2 +- .../cache/test_file_cache_statistics.groovy | 4 +- .../hive/hive_tpch_sf1_orc.groovy | 52 +++++++++---------- .../hive/hive_tpch_sf1_parquet.groovy | 52 +++++++++---------- .../paimon/test_paimon_catalog.groovy | 2 +- .../hive/test_trino_hive_tpch_sf1_orc.groovy | 52 +++++++++---------- .../test_trino_hive_tpch_sf1_parquet.groovy | 52 +++++++++---------- 9 files changed, 111 insertions(+), 111 deletions(-) diff --git a/regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy b/regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy index 3730a8f603a66f..06767023fdb383 100644 --- a/regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy +++ b/regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy @@ -77,7 +77,7 @@ suite('test_balance_warm_up_with_compaction_use_peer_cache', 'docker') { def testCase = { table -> def ms = cluster.getAllMetaservices().get(0) def msHttpPort = ms.host + ":" + ms.httpPort - sql """set enable_file_cache=true""" + sql """set enable_file_cache_olap_table=true""" sql """CREATE TABLE $table ( `id` BIGINT, `deleted` TINYINT, diff --git a/regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy b/regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy index aad831030235a7..90371699848868 100644 --- a/regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy +++ b/regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy @@ -62,7 +62,7 @@ suite("test_file_cache_features", "external_docker,hive,external_docker_hive,p0, String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") - sql """set global enable_file_cache=true""" + sql """set global enable_file_cache_external_catalog=true""" sql """drop catalog if exists ${catalog_name} """ sql """CREATE CATALOG ${catalog_name} PROPERTIES ( @@ -236,6 +236,6 @@ suite("test_file_cache_features", "external_docker,hive,external_docker_hive,p0, } // ===== End File Cache Features Metrics Check ===== - sql """set global enable_file_cache=false""" + sql """set global enable_file_cache_external_catalog=false""" return true } diff --git a/regression-test/suites/external_table_p0/cache/test_file_cache_query_limit.groovy b/regression-test/suites/external_table_p0/cache/test_file_cache_query_limit.groovy index a6ccf96c38c72a..d94cb0219cb8bf 100644 --- a/regression-test/suites/external_table_p0/cache/test_file_cache_query_limit.groovy +++ b/regression-test/suites/external_table_p0/cache/test_file_cache_query_limit.groovy @@ -50,7 +50,7 @@ suite("test_file_cache_query_limit", "external_docker,hive,external_docker_hive, return } - sql """set enable_file_cache=true""" + sql """set enable_file_cache_external_catalog=true""" // Check backend configuration prerequisites // Note: This test case assumes a single backend scenario. Testing with single backend is logically equivalent diff --git a/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy b/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy index 965dfdbcaa9a74..2e7790eea78be9 100644 --- a/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy +++ b/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy @@ -73,7 +73,7 @@ suite("test_file_cache_statistics", "external_docker,hive,external_docker_hive,p String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") - sql """set global enable_file_cache=true""" + sql """set global enable_file_cache_external_catalog=true""" sql """drop catalog if exists ${catalog_name} """ sql """CREATE CATALOG ${catalog_name} PROPERTIES ( @@ -253,7 +253,7 @@ suite("test_file_cache_statistics", "external_docker,hive,external_docker_hive,p assertTrue(false, TOTAL_READ_COUNTS_DID_NOT_INCREASE_MSG) } // ===== End Hit and Read Counts Metrics Check ===== - sql """set global enable_file_cache=false""" + sql """set global enable_file_cache_external_catalog=false""" return true } diff --git a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy index 9834ab5550ab62..5d052b2e807c5a 100644 --- a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy +++ b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy @@ -17,11 +17,11 @@ suite("test_catalog_hive_orc", "p0,external,hive,external_docker,external_docker_hive") { - String enable_file_cache = "false" + String enable_file_cache_external_catalog = "false" def q01 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -52,7 +52,7 @@ order by def q02 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -102,7 +102,7 @@ limit 100; def q03 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -136,7 +136,7 @@ limit 10; def q04 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -166,7 +166,7 @@ order by def q05 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -188,7 +188,7 @@ order by revenue desc; def q06 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -207,7 +207,7 @@ where def q07 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -248,7 +248,7 @@ order by def q08 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -295,7 +295,7 @@ order by def q09 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -328,7 +328,7 @@ order by def q10 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -371,7 +371,7 @@ limit 20; def q11 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -408,7 +408,7 @@ order by def q12 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -445,7 +445,7 @@ order by def q13 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -475,7 +475,7 @@ order by def q14 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -498,7 +498,7 @@ where def q15 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -538,7 +538,7 @@ order by def q16 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -578,7 +578,7 @@ order by def q17 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -607,7 +607,7 @@ where def q18 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -655,7 +655,7 @@ limit 100; def q19 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -700,7 +700,7 @@ where def q20 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -735,7 +735,7 @@ order by s_name; def q21 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -776,7 +776,7 @@ limit 100; def q22 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -868,19 +868,19 @@ order by sql """use `tpch1_orc`""" // without file cache - enable_file_cache = "false" + enable_file_cache_external_catalog = "false" def startTime = System.currentTimeMillis() run_tpch() def without_cache_time = System.currentTimeMillis() - startTime // with file cache, run the first time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_first_time = System.currentTimeMillis() - startTime // with file cache, run the second time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_second_time = System.currentTimeMillis() - startTime diff --git a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy index d167a51353d9ca..a63c9dacaed23f 100644 --- a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy +++ b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy @@ -17,11 +17,11 @@ suite("test_catalog_hive_parquet", "p0,external,hive,external_docker,external_docker_hive") { - String enable_file_cache = "false" + String enable_file_cache_external_catalog = "false" def q01 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -52,7 +52,7 @@ order by def q02 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -102,7 +102,7 @@ limit 100; def q03 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -136,7 +136,7 @@ limit 10; def q04 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -166,7 +166,7 @@ order by def q05 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -188,7 +188,7 @@ order by revenue desc; def q06 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -207,7 +207,7 @@ where def q07 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -248,7 +248,7 @@ order by def q08 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -295,7 +295,7 @@ order by def q09 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -328,7 +328,7 @@ order by def q10 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -371,7 +371,7 @@ limit 20; def q11 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -408,7 +408,7 @@ order by def q12 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -445,7 +445,7 @@ order by def q13 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -475,7 +475,7 @@ order by def q14 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -498,7 +498,7 @@ where def q15 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -538,7 +538,7 @@ order by def q16 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -578,7 +578,7 @@ order by def q17 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -607,7 +607,7 @@ where def q18 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -655,7 +655,7 @@ limit 100; def q19 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -700,7 +700,7 @@ where def q20 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -735,7 +735,7 @@ order by s_name; def q21 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -776,7 +776,7 @@ limit 100; def q22 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -869,19 +869,19 @@ order by sql """use `tpch1_parquet`""" // without file cache - enable_file_cache = "false" + enable_file_cache_external_catalog = "false" def startTime = System.currentTimeMillis() run_tpch() def without_cache_time = System.currentTimeMillis() - startTime // with file cache, run the first time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_first_time = System.currentTimeMillis() - startTime // with file cache, run the second time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_second_time = System.currentTimeMillis() - startTime diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy index b5ca34e5a18a48..7c017910222104 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy @@ -195,7 +195,7 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ def test_cases = { String force, String cache -> sql """ set force_jni_scanner=${force} """ - sql """ set enable_file_cache=${cache} """ + sql """ set enable_file_cache_external_catalog=${cache} """ qt_all_type("all_table") qt_all_type("all_table_with_parquet") diff --git a/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy b/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy index c35c1fea2e689f..0915b2b15fd0a6 100644 --- a/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy +++ b/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy @@ -17,10 +17,10 @@ suite("test_trino_hive_tpch_sf1_orc", "p2,external,hive,external_docker,external_docker_hive") { - String enable_file_cache = "false" + String enable_file_cache_external_catalog = "false" def q01 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -51,7 +51,7 @@ order by def q02 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -101,7 +101,7 @@ limit 100; def q03 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -135,7 +135,7 @@ limit 10; def q04 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -165,7 +165,7 @@ order by def q05 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -187,7 +187,7 @@ order by revenue desc; def q06 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -206,7 +206,7 @@ where def q07 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -247,7 +247,7 @@ order by def q08 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -294,7 +294,7 @@ order by def q09 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -327,7 +327,7 @@ order by def q10 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -370,7 +370,7 @@ limit 20; def q11 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -407,7 +407,7 @@ order by def q12 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -444,7 +444,7 @@ order by def q13 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -474,7 +474,7 @@ order by def q14 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -497,7 +497,7 @@ where def q15 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -537,7 +537,7 @@ order by def q16 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -577,7 +577,7 @@ order by def q17 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -606,7 +606,7 @@ where def q18 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -654,7 +654,7 @@ limit 100; def q19 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -699,7 +699,7 @@ where def q20 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -734,7 +734,7 @@ order by s_name; def q21 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -775,7 +775,7 @@ limit 100; def q22 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -871,19 +871,19 @@ order by sql """use `tpch1_orc`""" // without file cache - enable_file_cache = "false" + enable_file_cache_external_catalog = "false" def startTime = System.currentTimeMillis() run_tpch() def without_cache_time = System.currentTimeMillis() - startTime // with file cache, run the first time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_first_time = System.currentTimeMillis() - startTime // with file cache, run the second time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_second_time = System.currentTimeMillis() - startTime diff --git a/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy b/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy index ae30fac410b3fa..e23cb8498d7887 100644 --- a/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy +++ b/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy @@ -17,10 +17,10 @@ suite("test_trino_hive_tpch_sf1_parquet", "p0,external,hive,external_docker,external_docker_hive") { - String enable_file_cache = "false" + String enable_file_cache_external_catalog = "false" def q01 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -51,7 +51,7 @@ order by def q02 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -101,7 +101,7 @@ limit 100; def q03 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -135,7 +135,7 @@ limit 10; def q04 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -165,7 +165,7 @@ order by def q05 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -187,7 +187,7 @@ order by revenue desc; def q06 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -206,7 +206,7 @@ where def q07 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -247,7 +247,7 @@ order by def q08 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -294,7 +294,7 @@ order by def q09 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -327,7 +327,7 @@ order by def q10 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -370,7 +370,7 @@ limit 20; def q11 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -407,7 +407,7 @@ order by def q12 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -444,7 +444,7 @@ order by def q13 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -474,7 +474,7 @@ order by def q14 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -497,7 +497,7 @@ where def q15 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -537,7 +537,7 @@ order by def q16 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -577,7 +577,7 @@ order by def q17 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -606,7 +606,7 @@ where def q18 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -654,7 +654,7 @@ limit 100; def q19 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -699,7 +699,7 @@ where def q20 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -734,7 +734,7 @@ order by s_name; def q21 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -775,7 +775,7 @@ limit 100; def q22 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -873,19 +873,19 @@ order by sql """use `tpch1_parquet`""" // without file cache - enable_file_cache = "false" + enable_file_cache_external_catalog = "false" def startTime = System.currentTimeMillis() run_tpch() def without_cache_time = System.currentTimeMillis() - startTime // with file cache, run the first time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_first_time = System.currentTimeMillis() - startTime // with file cache, run the second time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_second_time = System.currentTimeMillis() - startTime From bca47d1a9c51fcb1b1f25610ca90db251d04361b Mon Sep 17 00:00:00 2001 From: xuchenhao <419062425@qq.com> Date: Thu, 12 Feb 2026 09:04:42 +0800 Subject: [PATCH 7/7] Improve the comments. --- .../java/org/apache/doris/qe/SessionVariable.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index fc8d41178c916f..b9de0b337b0135 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2166,17 +2166,19 @@ public boolean isEnableHboNonStrictMatchingMode() { public boolean enableFileCache = false; @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_OLAP_TABLE, needForward = true, description = { - "是否启用 olap table file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + "是否在存算分离场景下启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", - "Set whether to use olap table file cache. This variable takes effect only if the BE config " - + "enable_file_cache=true. The cache is not used when BE config enable_file_cache=false."}) + "Set whether to enable file cache for OLAP tables in cloud mode. " + + "This variable takes effect only if the BE config enable_file_cache=true. " + + "The cache is not used when BE config enable_file_cache=false."}) public boolean enableFileCacheOlapTable = true; @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_EXTERNAL_CATALOG, needForward = true, description = { - "是否启用 external catalog file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + "是否在湖仓一体场景下启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", - "Set whether to use external catalog file cache. This variable takes effect only if the BE config " - + "enable_file_cache=true. The cache is not used when BE config enable_file_cache=false."}) + "Set whether to enable file cache for external catalogs in lakehouse scenarios. " + + "This variable takes effect only if the BE config enable_file_cache=true. " + + "The cache is not used when BE config enable_file_cache=false."}) public boolean enableFileCacheExternalCatalog = false; // Specify base path for file cache, or chose a random path.