diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 7b1821c0fa80f5..7cba22bcead157 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -66,8 +66,8 @@ io::FileReaderOptions FileFactory::get_reader_options(RuntimeState* state, .mtime = fd.mtime, }; if (config::enable_file_cache && state != nullptr && - state->query_options().__isset.enable_file_cache && - state->query_options().enable_file_cache) { + state->query_options().__isset.enable_file_cache_external_catalog && + state->query_options().enable_file_cache_external_catalog) { opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE; } if (state != nullptr && state->query_options().__isset.file_cache_base_path && diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 26b8d360449cd9..d87bb2a7ffcd66 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -220,10 +220,8 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context _read_options.target_cast_type_for_variants = _read_context->target_cast_type_for_variants; if (_read_context->runtime_state != nullptr) { _read_options.io_ctx.query_id = &_read_context->runtime_state->query_id(); - _read_options.io_ctx.read_file_cache = - _read_context->runtime_state->query_options().enable_file_cache; _read_options.io_ctx.is_disposable = - _read_context->runtime_state->query_options().disable_file_cache; + !_read_context->runtime_state->query_options().enable_file_cache_olap_table; } if (_read_context->condition_cache_digest) { diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp index a0c271efd3973a..bdc83fc412a726 100644 --- a/be/src/runtime/query_context.cpp +++ b/be/src/runtime/query_context.cpp @@ -111,11 +111,12 @@ QueryContext::QueryContext(TUniqueId query_id, ExecEnv* exec_env, _timeout_second = query_options.execution_timeout; - bool initialize_context_holder = - config::enable_file_cache && config::enable_file_cache_query_limit && - query_options.__isset.enable_file_cache && query_options.enable_file_cache && - query_options.__isset.file_cache_query_limit_percent && - query_options.file_cache_query_limit_percent < 100; + bool initialize_context_holder = config::enable_file_cache && + config::enable_file_cache_query_limit && + !(query_options.query_type == TQueryType::EXTERNAL && + !query_options.enable_file_cache_external_catalog) && + query_options.__isset.file_cache_query_limit_percent && + query_options.file_cache_query_limit_percent < 100; // Initialize file cache context holders if (initialize_context_holder) { diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 17170df68f79e6..cd83714824e4cc 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -169,7 +169,7 @@ Status FileScanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts RETURN_IF_ERROR(_init_io_ctx()); _io_ctx->file_cache_stats = _file_cache_statistics.get(); _io_ctx->file_reader_stats = _file_reader_stats.get(); - _io_ctx->is_disposable = _state->query_options().disable_file_cache; + _io_ctx->is_disposable = !_state->query_options().enable_file_cache_external_catalog; if (_is_load) { _src_row_desc.reset(new RowDescriptor(_state->desc_tbl(), @@ -1808,7 +1808,7 @@ void FileScanner::update_realtime_counters() { void FileScanner::_collect_profile_before_close() { Scanner::_collect_profile_before_close(); - if (config::enable_file_cache && _state->query_options().enable_file_cache && + if (config::enable_file_cache && _state->query_options().enable_file_cache_external_catalog && _profile != nullptr) { io::FileCacheProfileReporter cache_profile(_profile); cache_profile.update(_file_cache_statistics.get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java index 141a3753a22a96..beb4fd3656e52a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalScanNode.java @@ -44,7 +44,7 @@ public abstract class ExternalScanNode extends ScanNode { protected boolean needCheckColumnPriv; protected final FederationBackendPolicy backendPolicy = (ConnectContext.get() != null - && (ConnectContext.get().getSessionVariable().enableFileCache + && (ConnectContext.get().getSessionVariable().enableFileCacheExternalCatalog || ConnectContext.get().getSessionVariable().getUseConsistentHashForExternalScan())) ? new FederationBackendPolicy(NodeSelectionStrategy.CONSISTENT_HASHING) : new FederationBackendPolicy(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index af11f75b4daf0d..754289f1db3e5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -9134,14 +9134,14 @@ public LogicalPlan visitWarmUpSelect(DorisParser.WarmUpSelectContext ctx) { LogicalProject project = new LogicalProject(projectList, filter); - if (Config.isNotCloudMode() && (!ConnectContext.get().getSessionVariable().isEnableFileCache())) { + if (Config.isNotCloudMode() && !ConnectContext.get().getSessionVariable().isEnableFileCacheExternalCatalog()) { throw new AnalysisException("WARM UP SELECT requires session variable" - + " enable_file_cache=true"); + + " enable_file_cache_external_catalog=true"); } - if (Config.isCloudMode() && ConnectContext.get().getSessionVariable().isDisableFileCache()) { + if (Config.isCloudMode() && !ConnectContext.get().getSessionVariable().isEnableFileCacheOlapTable()) { throw new AnalysisException("WARM UP SELECT requires session variable" - + " disable_file_cache=false in cloud mode"); + + " enable_file_cache_olap_table=true in cloud mode"); } UnboundBlackholeSink sink = new UnboundBlackholeSink<>(project, diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index a84c14083cc012..7dc29f34fd01fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -156,7 +156,7 @@ public void init(TUniqueId loadId, long txnId, long dbId, long loadChannelTimeou tSink.setLoadChannelTimeoutS(loadChannelTimeoutS); tSink.setSendBatchParallelism(sendBatchParallelism); tSink.setWriteFileCache(ConnectContext.get() != null - ? !ConnectContext.get().getSessionVariable().isDisableFileCache() + ? ConnectContext.get().getSessionVariable().isEnableFileCacheOlapTable() : false); this.isStrictMode = isStrictMode; this.txnId = txnId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index 4084e4a84e8a2a..612ceb7ba902c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -130,7 +130,7 @@ public NereidsCoordinator(Long jobId, TUniqueId queryId, DescriptorTable descTab descTable, timezone, loadZeroTolerance, enableProfile ); // same reason in `setForInsert` - this.coordinatorContext.queryOptions.setDisableFileCache(true); + this.coordinatorContext.queryOptions.setEnableFileCacheOlapTable(false); this.needEnqueue = false; Preconditions.checkState(!fragments.isEmpty() @@ -490,7 +490,7 @@ private void setForInsert(long jobId) { JobProcessor jobProc = new LoadProcessor(this.coordinatorContext, jobId); this.coordinatorContext.setJobProcessor(jobProc); // Set this field to true to avoid data entering the normal cache LRU queue - this.coordinatorContext.queryOptions.setDisableFileCache(true); + this.coordinatorContext.queryOptions.setEnableFileCacheOlapTable(false); this.coordinatorContext.queryOptions.setNewVersionUnixTimestamp(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 7aa82a3e0ee0d3..4a3233073c3c92 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -482,10 +482,15 @@ public class SessionVariable implements Serializable, Writable { public static final String TOPN_FILTER_RATIO = "topn_filter_ratio"; public static final String ENABLE_SNAPSHOT_POINT_QUERY = "enable_snapshot_point_query"; + // deprecated public static final String ENABLE_FILE_CACHE = "enable_file_cache"; + // deprecated public static final String DISABLE_FILE_CACHE = "disable_file_cache"; + public static final String ENABLE_FILE_CACHE_OLAP_TABLE = "enable_file_cache_olap_table"; + public static final String ENABLE_FILE_CACHE_EXTERNAL_CATALOG = "enable_file_cache_external_catalog"; + public static final String FILE_CACHE_QUERY_LIMIT_PERCENT = "file_cache_query_limit_percent"; public static final String FILE_CACHE_BASE_PATH = "file_cache_base_path"; @@ -2171,17 +2176,33 @@ public boolean isEnableHboNonStrictMatchingMode() { // Whether disable block file cache. Block cache only works when FE's query options sets disableFileCache false // along with BE's config `enable_file_cache` true - @VariableMgr.VarAttr(name = DISABLE_FILE_CACHE, needForward = true) + @VariableMgr.VarAttr(name = DISABLE_FILE_CACHE, needForward = true, varType = VariableAnnotation.DEPRECATED) public boolean disableFileCache = false; // Whether enable block file cache. Only take effect when BE config item enable_file_cache is true. - @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE, needForward = true, description = { - "是否启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE, needForward = true, varType = VariableAnnotation.DEPRECATED, + description = {"是否启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", - "Set wether to use file cache. This variable takes effect only if the BE config enable_file_cache=true. " - + "The cache is not used when BE config enable_file_cache=false."}) + "Set whether to use file cache. This variable takes effect only if the BE config " + + "enable_file_cache=true. The cache is not used when BE config enable_file_cache=false."}) public boolean enableFileCache = false; + @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_OLAP_TABLE, needForward = true, description = { + "是否在存算分离场景下启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", + "Set whether to enable file cache for OLAP tables in cloud mode. " + + "This variable takes effect only if the BE config enable_file_cache=true. " + + "The cache is not used when BE config enable_file_cache=false."}) + public boolean enableFileCacheOlapTable = true; + + @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE_EXTERNAL_CATALOG, needForward = true, description = { + "是否在湖仓一体场景下启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" + + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", + "Set whether to enable file cache for external catalogs in lakehouse scenarios. " + + "This variable takes effect only if the BE config enable_file_cache=true. " + + "The cache is not used when BE config enable_file_cache=false."}) + public boolean enableFileCacheExternalCatalog = false; + // Specify base path for file cache, or chose a random path. @VariableMgr.VarAttr(name = FILE_CACHE_BASE_PATH, needForward = true, description = { "指定 block file cache 在 BE 上的存储路径，默认 'random'，随机选择 BE 配置的存储路径。", @@ -4971,6 +4992,22 @@ public void setEnableFileCache(boolean enableFileCache) { this.enableFileCache = enableFileCache; } + public boolean isEnableFileCacheOlapTable() { + return enableFileCacheOlapTable; + } + + public void setEnableFileCacheOlapTable(boolean enableFileCacheOlapTable) { + this.enableFileCacheOlapTable = enableFileCacheOlapTable; + } + + public boolean isEnableFileCacheExternalCatalog() { + return enableFileCacheExternalCatalog; + } + + public void setEnableFileCacheExternalCatalog(boolean enableFileCacheExternalCatalog) { + this.enableFileCacheExternalCatalog = enableFileCacheExternalCatalog; + } + public String getFileCacheBasePath() { return fileCacheBasePath; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 02d275795dedda..41465c348843da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -244,7 +244,7 @@ public static AutoCloseConnectContext buildConnectContext(boolean useFileCacheFo LOG.warn("failed to connect to cloud cluster", e); return ctx; } - sessionVariable.disableFileCache = !useFileCacheForStat; + sessionVariable.enableFileCacheOlapTable = useFileCacheForStat; return ctx; } else { return new AutoCloseConnectContext(connectContext); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 1263cc7e95d877..d31532d57dc5b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -1393,8 +1393,8 @@ public void testAdminRotateTdeRootKey() { @Test public void testWarmUpSelect() { ConnectContext ctx = ConnectContext.get(); - ctx.getSessionVariable().setEnableFileCache(true); - ctx.getSessionVariable().setDisableFileCache(false); + ctx.getSessionVariable().setEnableFileCacheOlapTable(true); + ctx.getSessionVariable().setEnableFileCacheExternalCatalog(true); NereidsParser nereidsParser = new NereidsParser(); // Test basic warm up select statement diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index a1b9e237c6cda1..fa61fa5da2047d 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -201,6 +201,7 @@ struct TQueryOptions { // deprecated 60: optional i32 partitioned_hash_agg_rows_threshold = 0 // deprecated + // deprecated 61: optional bool enable_file_cache = false 62: optional i32 insert_timeout = 14400 @@ -424,6 +425,7 @@ struct TQueryOptions { 183: optional bool enable_use_hybrid_sort = false; 184: optional i32 cte_max_recursion_depth; + 185: optional bool enable_parquet_file_page_cache = true; 186: optional bool enable_streaming_agg_hash_join_force_passthrough; @@ -441,8 +443,13 @@ struct TQueryOptions { // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. + + // deprecated 1000: optional bool disable_file_cache = false 1001: optional i32 file_cache_query_limit_percent = -1 + + 1002: optional bool enable_file_cache_olap_table = true; + 1003: optional bool enable_file_cache_external_catalog = false; } diff --git a/regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy b/regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy index 3730a8f603a66f..06767023fdb383 100644 --- a/regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy +++ b/regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy @@ -77,7 +77,7 @@ suite('test_balance_warm_up_with_compaction_use_peer_cache', 'docker') { def testCase = { table -> def ms = cluster.getAllMetaservices().get(0) def msHttpPort = ms.host + ":" + ms.httpPort - sql """set enable_file_cache=true""" + sql """set enable_file_cache_olap_table=true""" sql """CREATE TABLE $table ( `id` BIGINT, `deleted` TINYINT, diff --git a/regression-test/suites/cloud_p0/cache/test_load_cache.groovy b/regression-test/suites/cloud_p0/cache/test_load_cache.groovy index 7fe5c5173822c8..52fdd7251c5db1 100644 --- a/regression-test/suites/cloud_p0/cache/test_load_cache.groovy +++ b/regression-test/suites/cloud_p0/cache/test_load_cache.groovy @@ -21,14 +21,14 @@ import groovy.json.JsonSlurper /* Test Description: -1. When disable_file_cache = true and enable_file_cache = true, it is expected that the S3 TVF load (import phase) will NOT enter the cache, while the query +1. When enable_file_cache_olap_table = false, it is expected that the S3 TVF load (import phase) will NOT enter the cache, while the query phase will enter the Disposable queue. Specifically: Normal queue size should be 0, Disposable queue size should be 91163 bytes. -2. When disable_file_cache = false and enable_file_cache = true, it is expected that the S3 TVF load (import phase) will enter the Normal queue, and the query - phase will still enter the Disposable queue. +2. When enable_file_cache_olap_table = true, it is expected that the S3 TVF load (import phase) will enter the Normal queue, and the query + phase will enter the Normal queue. Specifically: Normal queue size should be 236988 bytes, Disposable queue size should still be 91163 bytes. -Explanation: The query phase caches the compressed file, so the Disposable queue size is checked for an exact value; for the import phase cache, since future +Explanation: The query phase caches the compressed file, so the Disposable queue size is checked for an exact value; for the import phase cache, since future changes to statistics are possible, only a reasonable range is required. */ @@ -173,15 +173,14 @@ suite('test_load_cache', 'docker') { def s3_tvf_uri = "s3://${s3_bucket}/regression/tpch/sf0.01/customer.csv.gz" // ============================================================================ - // SCENARIO 1: disable_file_cache = true + // SCENARIO 1: enable_file_cache_olap_table = false // ============================================================================ // Clear file cache before test clearFileCacheOnAllBackends() // Set session variables for Scenario 1 - sql "set disable_file_cache = true;" - sql "set enable_file_cache = true;" + sql "set enable_file_cache_olap_table = false;" // Create test table sql """DROP TABLE IF EXISTS load_test_table""" @@ -264,15 +263,14 @@ suite('test_load_cache', 'docker') { sleep(3000) // ============================================================================ - // SCENARIO 2: disable_file_cache = false + // SCENARIO 2: enable_file_cache_olap_table = true // ============================================================================ // Clear file cache before test clearFileCacheOnAllBackends() // Set session variables for Scenario 2 - sql "set disable_file_cache = false;" - sql "set enable_file_cache = true;" + sql "set enable_file_cache_olap_table = true;" // Create test table sql """DROP TABLE IF EXISTS load_test_table""" diff --git a/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy b/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy index 2b0b39affc2cc0..a4cb549d08d4f7 100644 --- a/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy +++ b/regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy @@ -61,7 +61,7 @@ suite("test_warmup_select") { def test_basic_warmup = { // Enable file cache for warm up functionality - sql "set disable_file_cache=false" + sql "set enable_file_cache_olap_table=true" sql "WARM UP SELECT * FROM lineitem" @@ -72,7 +72,7 @@ suite("test_warmup_select") { def test_warmup_negative_cases = { // Enable file cache for warm up functionality - sql "set disable_file_cache=false" + sql "set enable_file_cache_olap_table=true" // These should fail as warm up select doesn't support these operations try { diff --git a/regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy b/regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy index aad831030235a7..90371699848868 100644 --- a/regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy +++ b/regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy @@ -62,7 +62,7 @@ suite("test_file_cache_features", "external_docker,hive,external_docker_hive,p0, String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") - sql """set global enable_file_cache=true""" + sql """set global enable_file_cache_external_catalog=true""" sql """drop catalog if exists ${catalog_name} """ sql """CREATE CATALOG ${catalog_name} PROPERTIES ( @@ -236,6 +236,6 @@ suite("test_file_cache_features", "external_docker,hive,external_docker_hive,p0, } // ===== End File Cache Features Metrics Check ===== - sql """set global enable_file_cache=false""" + sql """set global enable_file_cache_external_catalog=false""" return true } diff --git a/regression-test/suites/external_table_p0/cache/test_file_cache_query_limit.groovy b/regression-test/suites/external_table_p0/cache/test_file_cache_query_limit.groovy index a6ccf96c38c72a..d94cb0219cb8bf 100644 --- a/regression-test/suites/external_table_p0/cache/test_file_cache_query_limit.groovy +++ b/regression-test/suites/external_table_p0/cache/test_file_cache_query_limit.groovy @@ -50,7 +50,7 @@ suite("test_file_cache_query_limit", "external_docker,hive,external_docker_hive, return } - sql """set enable_file_cache=true""" + sql """set enable_file_cache_external_catalog=true""" // Check backend configuration prerequisites // Note: This test case assumes a single backend scenario. Testing with single backend is logically equivalent diff --git a/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy b/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy index 965dfdbcaa9a74..2e7790eea78be9 100644 --- a/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy +++ b/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy @@ -73,7 +73,7 @@ suite("test_file_cache_statistics", "external_docker,hive,external_docker_hive,p String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") - sql """set global enable_file_cache=true""" + sql """set global enable_file_cache_external_catalog=true""" sql """drop catalog if exists ${catalog_name} """ sql """CREATE CATALOG ${catalog_name} PROPERTIES ( @@ -253,7 +253,7 @@ suite("test_file_cache_statistics", "external_docker,hive,external_docker_hive,p assertTrue(false, TOTAL_READ_COUNTS_DID_NOT_INCREASE_MSG) } // ===== End Hit and Read Counts Metrics Check ===== - sql """set global enable_file_cache=false""" + sql """set global enable_file_cache_external_catalog=false""" return true } diff --git a/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy b/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy index d1adc2b6b84bfb..3e86084d09d59d 100644 --- a/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy +++ b/regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy @@ -24,8 +24,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock def test_basic_warmup = { // Enable file cache for warm up functionality - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalog=true" sql "WARM UP SELECT * FROM lineitem" @@ -36,8 +35,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock def test_warmup_negative_cases = { // Enable file cache for warm up functionality - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalog=true" // These should fail as warm up select doesn't support these operations try { @@ -90,8 +88,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock // Test: user with only SELECT privilege should be able to run WARM UP SELECT connect(user1, "${pwd}", url) { - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalog=true" // This should succeed - only SELECT privilege is needed sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" @@ -107,8 +104,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock sql """REVOKE SELECT_PRIV ON ${catalog_name}.tpch1_parquet.lineitem FROM ${user1}""" connect(user1, "${pwd}", url) { - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalog=true" test { sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" exception "denied" @@ -119,8 +115,7 @@ suite("test_hive_warmup_select", "p0,external,hive,external_docker,external_dock sql """GRANT LOAD_PRIV ON ${catalog_name}.tpch1_parquet.lineitem TO ${user1}""" connect(user1, "${pwd}", url) { - sql "set enable_file_cache=true" - sql "set disable_file_cache=false" + sql "set enable_file_cache_external_catalog=true" test { sql "WARM UP SELECT * FROM ${catalog_name}.tpch1_parquet.lineitem" exception "denied" diff --git a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy index 9834ab5550ab62..5d052b2e807c5a 100644 --- a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy +++ b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy @@ -17,11 +17,11 @@ suite("test_catalog_hive_orc", "p0,external,hive,external_docker,external_docker_hive") { - String enable_file_cache = "false" + String enable_file_cache_external_catalog = "false" def q01 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -52,7 +52,7 @@ order by def q02 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -102,7 +102,7 @@ limit 100; def q03 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -136,7 +136,7 @@ limit 10; def q04 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -166,7 +166,7 @@ order by def q05 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -188,7 +188,7 @@ order by revenue desc; def q06 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -207,7 +207,7 @@ where def q07 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -248,7 +248,7 @@ order by def q08 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -295,7 +295,7 @@ order by def q09 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -328,7 +328,7 @@ order by def q10 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -371,7 +371,7 @@ limit 20; def q11 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -408,7 +408,7 @@ order by def q12 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -445,7 +445,7 @@ order by def q13 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -475,7 +475,7 @@ order by def q14 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -498,7 +498,7 @@ where def q15 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -538,7 +538,7 @@ order by def q16 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -578,7 +578,7 @@ order by def q17 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -607,7 +607,7 @@ where def q18 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -655,7 +655,7 @@ limit 100; def q19 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -700,7 +700,7 @@ where def q20 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -735,7 +735,7 @@ order by s_name; def q21 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -776,7 +776,7 @@ limit 100; def q22 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -868,19 +868,19 @@ order by sql """use `tpch1_orc`""" // without file cache - enable_file_cache = "false" + enable_file_cache_external_catalog = "false" def startTime = System.currentTimeMillis() run_tpch() def without_cache_time = System.currentTimeMillis() - startTime // with file cache, run the first time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_first_time = System.currentTimeMillis() - startTime // with file cache, run the second time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_second_time = System.currentTimeMillis() - startTime diff --git a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy index d167a51353d9ca..a63c9dacaed23f 100644 --- a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy +++ b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy @@ -17,11 +17,11 @@ suite("test_catalog_hive_parquet", "p0,external,hive,external_docker,external_docker_hive") { - String enable_file_cache = "false" + String enable_file_cache_external_catalog = "false" def q01 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -52,7 +52,7 @@ order by def q02 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -102,7 +102,7 @@ limit 100; def q03 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -136,7 +136,7 @@ limit 10; def q04 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -166,7 +166,7 @@ order by def q05 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -188,7 +188,7 @@ order by revenue desc; def q06 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -207,7 +207,7 @@ where def q07 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -248,7 +248,7 @@ order by def q08 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -295,7 +295,7 @@ order by def q09 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -328,7 +328,7 @@ order by def q10 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -371,7 +371,7 @@ limit 20; def q11 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -408,7 +408,7 @@ order by def q12 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -445,7 +445,7 @@ order by def q13 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -475,7 +475,7 @@ order by def q14 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -498,7 +498,7 @@ where def q15 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -538,7 +538,7 @@ order by def q16 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -578,7 +578,7 @@ order by def q17 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -607,7 +607,7 @@ where def q18 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -655,7 +655,7 @@ limit 100; def q19 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -700,7 +700,7 @@ where def q20 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -735,7 +735,7 @@ order by s_name; def q21 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -776,7 +776,7 @@ limit 100; def q22 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -869,19 +869,19 @@ order by sql """use `tpch1_parquet`""" // without file cache - enable_file_cache = "false" + enable_file_cache_external_catalog = "false" def startTime = System.currentTimeMillis() run_tpch() def without_cache_time = System.currentTimeMillis() - startTime // with file cache, run the first time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_first_time = System.currentTimeMillis() - startTime // with file cache, run the second time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_second_time = System.currentTimeMillis() - startTime diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy index b5ca34e5a18a48..7c017910222104 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy @@ -195,7 +195,7 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ def test_cases = { String force, String cache -> sql """ set force_jni_scanner=${force} """ - sql """ set enable_file_cache=${cache} """ + sql """ set enable_file_cache_external_catalog=${cache} """ qt_all_type("all_table") qt_all_type("all_table_with_parquet") diff --git a/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy b/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy index c35c1fea2e689f..0915b2b15fd0a6 100644 --- a/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy +++ b/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy @@ -17,10 +17,10 @@ suite("test_trino_hive_tpch_sf1_orc", "p2,external,hive,external_docker,external_docker_hive") { - String enable_file_cache = "false" + String enable_file_cache_external_catalog = "false" def q01 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -51,7 +51,7 @@ order by def q02 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -101,7 +101,7 @@ limit 100; def q03 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -135,7 +135,7 @@ limit 10; def q04 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -165,7 +165,7 @@ order by def q05 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -187,7 +187,7 @@ order by revenue desc; def q06 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -206,7 +206,7 @@ where def q07 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -247,7 +247,7 @@ order by def q08 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -294,7 +294,7 @@ order by def q09 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -327,7 +327,7 @@ order by def q10 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -370,7 +370,7 @@ limit 20; def q11 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -407,7 +407,7 @@ order by def q12 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -444,7 +444,7 @@ order by def q13 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -474,7 +474,7 @@ order by def q14 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -497,7 +497,7 @@ where def q15 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -537,7 +537,7 @@ order by def q16 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -577,7 +577,7 @@ order by def q17 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -606,7 +606,7 @@ where def q18 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -654,7 +654,7 @@ limit 100; def q19 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -699,7 +699,7 @@ where def q20 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -734,7 +734,7 @@ order by s_name; def q21 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -775,7 +775,7 @@ limit 100; def q22 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -871,19 +871,19 @@ order by sql """use `tpch1_orc`""" // without file cache - enable_file_cache = "false" + enable_file_cache_external_catalog = "false" def startTime = System.currentTimeMillis() run_tpch() def without_cache_time = System.currentTimeMillis() - startTime // with file cache, run the first time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_first_time = System.currentTimeMillis() - startTime // with file cache, run the second time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_second_time = System.currentTimeMillis() - startTime diff --git a/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy b/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy index ae30fac410b3fa..e23cb8498d7887 100644 --- a/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy +++ b/regression-test/suites/external_table_p2/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy @@ -17,10 +17,10 @@ suite("test_trino_hive_tpch_sf1_parquet", "p0,external,hive,external_docker,external_docker_hive") { - String enable_file_cache = "false" + String enable_file_cache_external_catalog = "false" def q01 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -51,7 +51,7 @@ order by def q02 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -101,7 +101,7 @@ limit 100; def q03 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -135,7 +135,7 @@ limit 10; def q04 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -165,7 +165,7 @@ order by def q05 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -187,7 +187,7 @@ order by revenue desc; def q06 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -206,7 +206,7 @@ where def q07 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -247,7 +247,7 @@ order by def q08 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -294,7 +294,7 @@ order by def q09 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -327,7 +327,7 @@ order by def q10 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -370,7 +370,7 @@ limit 20; def q11 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -407,7 +407,7 @@ order by def q12 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=2""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -444,7 +444,7 @@ order by def q13 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=4""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -474,7 +474,7 @@ order by def q14 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -497,7 +497,7 @@ where def q15 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -537,7 +537,7 @@ order by def q16 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -577,7 +577,7 @@ order by def q17 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=1""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -606,7 +606,7 @@ where def q18 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=false""" @@ -654,7 +654,7 @@ limit 100; def q19 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=false""" @@ -699,7 +699,7 @@ where def q20 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -734,7 +734,7 @@ order by s_name; def q21 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=true""" sql """set enable_cost_based_join_reorder=true""" @@ -775,7 +775,7 @@ limit 100; def q22 = { sql """set exec_mem_limit=8589934592""" - sql """set enable_file_cache=${enable_file_cache}""" + sql """set enable_file_cache_external_catalog=${enable_file_cache_external_catalog}""" sql """set parallel_pipeline_task_num=8""" sql """set disable_join_reorder=false""" sql """set enable_cost_based_join_reorder=true""" @@ -873,19 +873,19 @@ order by sql """use `tpch1_parquet`""" // without file cache - enable_file_cache = "false" + enable_file_cache_external_catalog = "false" def startTime = System.currentTimeMillis() run_tpch() def without_cache_time = System.currentTimeMillis() - startTime // with file cache, run the first time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_first_time = System.currentTimeMillis() - startTime // with file cache, run the second time - enable_file_cache = "true" + enable_file_cache_external_catalog = "true" startTime = System.currentTimeMillis() run_tpch() def with_cache_second_time = System.currentTimeMillis() - startTime