From 76cb1053fe1a9fed0cebaddec71a14d0705b3f7f Mon Sep 17 00:00:00 2001 From: zapdos26 Date: Mon, 16 Feb 2026 22:47:48 -0500 Subject: [PATCH 1/4] perf: prune sibling directories during module search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a user requests a specific deep module path (e.g. tools/cat15/pkg10/3.0), the directory walker no longer enters sibling directories that cannot match. This avoids scanning the entire subtree under a modulepath root. For a 3000-modulefile tree (tools/category/pkg/version layout): - module load tools/cat15/pkg10/3.0: 13,007 → 157 syscalls (98.8% reduction) - module avail tools/cat15/pkg10: 13,005 → 155 syscalls (98.8% reduction) - module avail tools/cat15: 13,005 → 535 syscalls (95.9% reduction) Broad queries (module avail, wildcards, --contains) are unaffected. Signed-off-by: zapdos26 --- tcl/modfind.tcl.in | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/tcl/modfind.tcl.in b/tcl/modfind.tcl.in index ec54eb4d2..0e864b772 100644 --- a/tcl/modfind.tcl.in +++ b/tcl/modfind.tcl.in @@ -2955,7 +2955,7 @@ proc findModulesInMemCache {searchid} { # Walk through provided list of directories and files to find modules proc findModulesFromDirsAndFiles {dir full_list depthlvl fetch_mtime\ res_arrname {indir_arrname {}} {hidden_listname {}} {fknown_arrname {}}\ - {dknown_arrname {}}} { + {dknown_arrname {}} {prunespec {}}} { # link to variables/arrays from upper context upvar $res_arrname mod_list if {$indir_arrname ne {}} { @@ -2971,6 +2971,15 @@ proc findModulesFromDirsAndFiles {dir full_list depthlvl fetch_mtime\ upvar $dknown_arrname dknown_arr } + # build pruning path components from spec (e.g. tools/cat15/pkg10/3.0 -> + # {tools cat15 pkg10 3.0}) to skip non-matching directories during walk + if {$prunespec ne {}} { + set pruneparts [file split $prunespec] + set prunedepth [llength $pruneparts] + } else { + set prunedepth 0 + } + foreach igndir [getConf ignored_dirs] { set ignored_dirs($igndir) 1 } @@ -2986,6 +2995,12 @@ proc findModulesFromDirsAndFiles {dir full_list depthlvl fetch_mtime\ if {[info exists dknown_arr($modulename)] || (![info exists\ fknown_arr($modulename)] && [file isdirectory $element])} { if {![info exists ignored_dirs($tail)]} { + # prune directories that cannot match the query path: skip if + # this dir does not match the corresponding query path component + if {$prunedepth > 0 && $moddepthlvl <= $prunedepth && [lindex\ + $pruneparts [expr {$moddepthlvl - 1}]] ne $tail} { + continue + } if {[catch { set elt_list [getFilesInDirectory $element 1] } errMsg]} { @@ -3051,12 +3066,12 @@ proc findModulesFromDirsAndFiles {dir full_list depthlvl fetch_mtime\ } # finds all module-related files matching mod in the module path dir -proc findModules {dir mod depthlvl fetch_mtime} { +proc findModules {dir mod depthlvl fetch_mtime {prunespec {}}} { reportDebug "finding '$mod' in $dir (depthlvl=$depthlvl,\ fetch_mtime=$fetch_mtime)" # generated search id (for cache search/save) by compacting given args - set searchid $dir:$mod:$depthlvl:$fetch_mtime + set searchid $dir:$mod:$depthlvl:$fetch_mtime:$prunespec # look at memory cache for a compatible result lassign [findModulesInMemCache $searchid] cache_searchid cache_list @@ -3095,7 +3110,8 @@ proc findModules {dir mod depthlvl fetch_mtime} { } # walk through list of dirs and files to find modules - findModulesFromDirsAndFiles $dir $full_list $depthlvl $fetch_mtime mod_list + findModulesFromDirsAndFiles $dir $full_list $depthlvl $fetch_mtime \ + mod_list {} {} {} {} $prunespec reportDebug "found [array names mod_list]" @@ -3201,7 +3217,18 @@ proc getModules {dir {mod {}} {fetch_mtime 0} {search {}} {filter {}}} { # unless EMS need to be performed (findModules should fetch everything) set depthlvl [expr {$indepth || $ems_required ? 0 : $querydepth + 1}] - array set found_list [findModules $dir $findmod $depthlvl $fetch_mtime] + # enable directory pruning when query is a specific deep path without + # wildcards (e.g. tools/cat15/pkg10/3.0) to avoid scanning sibling + # directories that cannot match + if {$hasmoddir && !$find_all && !$contains && $modqe eq\ + [string map {* {} ? {}} $modqe]} { + set prunespec $modqe + } else { + set prunespec {} + } + + array set found_list [findModules $dir $findmod $depthlvl $fetch_mtime\ + $prunespec] } # Phase #1: consolidate every kind of entries (directory, modulefile, From bc79e0e4d0de608ff78b510f75f6f463689969a8 Mon Sep 17 00:00:00 2001 From: zapdos26 Date: Mon, 16 Feb 2026 23:55:10 -0500 Subject: [PATCH 2/4] test: add directory pruning correctness tests Verify that deep module queries return correct results when directory pruning skips non-matching sibling directories. Tests cover: - 3-level and 2-level specific path queries (pruned) - top-level broad queries (not pruned) - exact version queries (pruned) - load through deep path (pruned) - wildcard queries (not pruned, returns all matches) Signed-off-by: zapdos26 --- .../modulefiles.deep/prunetest/cat1/pkg1/1.0 | 2 + .../modulefiles.deep/prunetest/cat1/pkg1/2.0 | 2 + .../modulefiles.deep/prunetest/cat1/pkg2/1.0 | 2 + .../modulefiles.deep/prunetest/cat1/pkg2/2.0 | 2 + .../modulefiles.deep/prunetest/cat2/pkg1/1.0 | 2 + .../modulefiles.deep/prunetest/cat2/pkg1/2.0 | 2 + .../modulefiles.deep/prunetest/cat2/pkg2/1.0 | 2 + .../modulefiles.deep/prunetest/cat2/pkg2/2.0 | 2 + .../modules.80-deep/026-pruning-deep.exp | 55 +++++++++++++++++++ 9 files changed, 71 insertions(+) create mode 100644 testsuite/modulefiles.deep/prunetest/cat1/pkg1/1.0 create mode 100644 testsuite/modulefiles.deep/prunetest/cat1/pkg1/2.0 create mode 100644 testsuite/modulefiles.deep/prunetest/cat1/pkg2/1.0 create mode 100644 testsuite/modulefiles.deep/prunetest/cat1/pkg2/2.0 create mode 100644 testsuite/modulefiles.deep/prunetest/cat2/pkg1/1.0 create mode 100644 testsuite/modulefiles.deep/prunetest/cat2/pkg1/2.0 create mode 100644 testsuite/modulefiles.deep/prunetest/cat2/pkg2/1.0 create mode 100644 testsuite/modulefiles.deep/prunetest/cat2/pkg2/2.0 create mode 100644 testsuite/modules.80-deep/026-pruning-deep.exp diff --git a/testsuite/modulefiles.deep/prunetest/cat1/pkg1/1.0 b/testsuite/modulefiles.deep/prunetest/cat1/pkg1/1.0 new file mode 100644 index 000000000..b909bf08b --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat1/pkg1/1.0 @@ -0,0 +1,2 @@ +#%Module1.0 +module-whatis "prunetest/$d/$v" diff --git a/testsuite/modulefiles.deep/prunetest/cat1/pkg1/2.0 b/testsuite/modulefiles.deep/prunetest/cat1/pkg1/2.0 new file mode 100644 index 000000000..b909bf08b --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat1/pkg1/2.0 @@ -0,0 +1,2 @@ +#%Module1.0 +module-whatis "prunetest/$d/$v" diff --git a/testsuite/modulefiles.deep/prunetest/cat1/pkg2/1.0 b/testsuite/modulefiles.deep/prunetest/cat1/pkg2/1.0 new file mode 100644 index 000000000..b909bf08b --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat1/pkg2/1.0 @@ -0,0 +1,2 @@ +#%Module1.0 +module-whatis "prunetest/$d/$v" diff --git a/testsuite/modulefiles.deep/prunetest/cat1/pkg2/2.0 b/testsuite/modulefiles.deep/prunetest/cat1/pkg2/2.0 new file mode 100644 index 000000000..b909bf08b --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat1/pkg2/2.0 @@ -0,0 +1,2 @@ +#%Module1.0 +module-whatis "prunetest/$d/$v" diff --git a/testsuite/modulefiles.deep/prunetest/cat2/pkg1/1.0 b/testsuite/modulefiles.deep/prunetest/cat2/pkg1/1.0 new file mode 100644 index 000000000..b909bf08b --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat2/pkg1/1.0 @@ -0,0 +1,2 @@ +#%Module1.0 +module-whatis "prunetest/$d/$v" diff --git a/testsuite/modulefiles.deep/prunetest/cat2/pkg1/2.0 b/testsuite/modulefiles.deep/prunetest/cat2/pkg1/2.0 new file mode 100644 index 000000000..b909bf08b --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat2/pkg1/2.0 @@ -0,0 +1,2 @@ +#%Module1.0 +module-whatis "prunetest/$d/$v" diff --git a/testsuite/modulefiles.deep/prunetest/cat2/pkg2/1.0 b/testsuite/modulefiles.deep/prunetest/cat2/pkg2/1.0 new file mode 100644 index 000000000..b909bf08b --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat2/pkg2/1.0 @@ -0,0 +1,2 @@ +#%Module1.0 +module-whatis "prunetest/$d/$v" diff --git a/testsuite/modulefiles.deep/prunetest/cat2/pkg2/2.0 b/testsuite/modulefiles.deep/prunetest/cat2/pkg2/2.0 new file mode 100644 index 000000000..b909bf08b --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat2/pkg2/2.0 @@ -0,0 +1,2 @@ +#%Module1.0 +module-whatis "prunetest/$d/$v" diff --git a/testsuite/modules.80-deep/026-pruning-deep.exp b/testsuite/modules.80-deep/026-pruning-deep.exp new file mode 100644 index 000000000..a7a83123d --- /dev/null +++ b/testsuite/modules.80-deep/026-pruning-deep.exp @@ -0,0 +1,55 @@ +############################################################################## +# Modules Revision 3.0 +# Providing a flexible user environment +# +# Description: Directory pruning for deep module queries +# Command: avail, load +# Modulefiles: prunetest/cat{1,2}/pkg{1,2}/{1.0,2.0} +# +# Test that specific deep queries correctly return matching modules +# when directory pruning skips non-matching sibling directories. +# Ensures pruning does not omit valid results and broad queries +# still return everything. +# +############################################################################## + +# use deep modulefiles path for these tests +set deepmodpath "$env(TESTSUITEDIR)/modulefiles.deep" +setenv_path_var MODULEPATH $deepmodpath + +# ensure auto symbolic versions are not set for these tests +setenv_var MODULES_ADVANCED_VERSION_SPEC 0 + +# avail on specific 3-level deep path should return only that pkg +set ts_sh "$deepmodpath:\nprunetest/cat1/pkg1/1.0\nprunetest/cat1/pkg1/2.0" +testouterr_cmd "sh" "avail -t prunetest/cat1/pkg1" "OK" $ts_sh + +# avail on 2-level deep path returns all pkgs under that category +set ts_sh "$deepmodpath:\nprunetest/cat1/pkg1/1.0\nprunetest/cat1/pkg1/2.0\nprunetest/cat1/pkg2/1.0\nprunetest/cat1/pkg2/2.0" +testouterr_cmd "sh" "avail -t prunetest/cat1" "OK" $ts_sh + +skip_if_quick_mode + +# avail on the top level returns all modules (no pruning) +set ts_sh "$deepmodpath:\nprunetest/cat1/pkg1/1.0\nprunetest/cat1/pkg1/2.0\nprunetest/cat1/pkg2/1.0\nprunetest/cat1/pkg2/2.0\nprunetest/cat2/pkg1/1.0\nprunetest/cat2/pkg1/2.0\nprunetest/cat2/pkg2/1.0\nprunetest/cat2/pkg2/2.0" +testouterr_cmd "sh" "avail -t prunetest" "OK" $ts_sh + +# avail on exact version path +set ts_sh "$deepmodpath:\nprunetest/cat2/pkg2/2.0" +testouterr_cmd "sh" "avail -t prunetest/cat2/pkg2/2.0" "OK" $ts_sh + +# load a specific deep module +set ans [list] +lappend ans [list set _LMFILES_ \ + "$deepmodpath/prunetest/cat1/pkg2/1.0"] +lappend ans [list set LOADEDMODULES "prunetest/cat1/pkg2/1.0"] +test_cmd_re "sh" "load prunetest/cat1/pkg2/1.0" $ans + +# wildcard query should not be pruned and return all matches +set ts_sh "$deepmodpath:\nprunetest/cat1/pkg1/1.0\nprunetest/cat1/pkg1/2.0\nprunetest/cat2/pkg1/1.0\nprunetest/cat2/pkg1/2.0" +testouterr_cmd_re "sh" "avail -t prunetest/*/pkg1" "OK" $ts_sh + +# cleanup +setenv_path_var MODULEPATH $modpath +unsetenv_var MODULES_ADVANCED_VERSION_SPEC +unset ts_sh ans deepmodpath From b324ada9065432d52a2bb00c2ce473d17211b2ec Mon Sep 17 00:00:00 2001 From: zapdos26 Date: Tue, 17 Feb 2026 00:35:02 -0500 Subject: [PATCH 3/4] fix: preserve .modulerc in pruned sibling directories When pruning non-matching sibling directories during a deep module query, still read the directory to pick up .modulerc and .version files. This ensures cross-directory aliases and symbol definitions are not lost when their containing directory is pruned. Adds a test for a cross-directory alias defined in cat1/.modulerc that references a module in cat2/. Signed-off-by: zapdos26 --- tcl/modfind.tcl.in | 15 +++++++++++++-- .../modulefiles.deep/prunetest/cat1/.modulerc | 2 ++ testsuite/modules.80-deep/026-pruning-deep.exp | 5 +++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 testsuite/modulefiles.deep/prunetest/cat1/.modulerc diff --git a/tcl/modfind.tcl.in b/tcl/modfind.tcl.in index 0e864b772..b01cc7e23 100644 --- a/tcl/modfind.tcl.in +++ b/tcl/modfind.tcl.in @@ -2995,10 +2995,21 @@ proc findModulesFromDirsAndFiles {dir full_list depthlvl fetch_mtime\ if {[info exists dknown_arr($modulename)] || (![info exists\ fknown_arr($modulename)] && [file isdirectory $element])} { if {![info exists ignored_dirs($tail)]} { - # prune directories that cannot match the query path: skip if - # this dir does not match the corresponding query path component + # prune directories that cannot match the query path: read + # dir for .modulerc/.version but skip subdirs and files if {$prunedepth > 0 && $moddepthlvl <= $prunedepth && [lindex\ $pruneparts [expr {$moddepthlvl - 1}]] ne $tail} { + if {![catch { + set elt_list [getFilesInDirectory $element 1] + }]} { + foreach {fpelt hid} $elt_list { + set fptail [file tail $fpelt] + if {$fptail eq {.modulerc} || $fptail eq\ + {.version}} { + lappend full_list $fpelt + } + } + } continue } if {[catch { diff --git a/testsuite/modulefiles.deep/prunetest/cat1/.modulerc b/testsuite/modulefiles.deep/prunetest/cat1/.modulerc new file mode 100644 index 000000000..59548db53 --- /dev/null +++ b/testsuite/modulefiles.deep/prunetest/cat1/.modulerc @@ -0,0 +1,2 @@ +#%Module1.0 +module-alias prunetest/shortcut prunetest/cat2/pkg1/1.0 diff --git a/testsuite/modules.80-deep/026-pruning-deep.exp b/testsuite/modules.80-deep/026-pruning-deep.exp index a7a83123d..f9c9c12db 100644 --- a/testsuite/modules.80-deep/026-pruning-deep.exp +++ b/testsuite/modules.80-deep/026-pruning-deep.exp @@ -49,6 +49,11 @@ test_cmd_re "sh" "load prunetest/cat1/pkg2/1.0" $ans set ts_sh "$deepmodpath:\nprunetest/cat1/pkg1/1.0\nprunetest/cat1/pkg1/2.0\nprunetest/cat2/pkg1/1.0\nprunetest/cat2/pkg1/2.0" testouterr_cmd_re "sh" "avail -t prunetest/*/pkg1" "OK" $ts_sh +# cross-directory alias: cat1/.modulerc defines prunetest/shortcut +# pointing to cat2/pkg1/1.0; pruning cat1 must still find the alias +set ts_sh "$deepmodpath:\nprunetest/shortcut" +testouterr_cmd "sh" "avail -t prunetest/shortcut" "OK" $ts_sh + # cleanup setenv_path_var MODULEPATH $modpath unsetenv_var MODULES_ADVANCED_VERSION_SPEC From b00aac42a0ca627f67ac6237657385f5741641bf Mon Sep 17 00:00:00 2001 From: zapdos26 Date: Tue, 17 Feb 2026 01:56:08 -0500 Subject: [PATCH 4/4] fix: disable directory pruning for virtual module names When the query path contains virtual names (symbols/aliases like 'fld', 'sfld', 'dadj') that don't correspond to real directories, multi-depth pruning incorrectly skipped sibling directories needed for cross-directory alias resolution defined in .modulerc files. Gate pruning on filesystem existence: verify every path component in the query is a real directory before enabling prunespec. Virtual names fail the isdirectory check, so pruning is safely disabled. Real deep paths still get full multi-depth pruning for lstat savings. Signed-off-by: zapdos26 --- tcl/modfind.tcl.in | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tcl/modfind.tcl.in b/tcl/modfind.tcl.in index b01cc7e23..9cddd8400 100644 --- a/tcl/modfind.tcl.in +++ b/tcl/modfind.tcl.in @@ -3230,12 +3230,23 @@ proc getModules {dir {mod {}} {fetch_mtime 0} {search {}} {filter {}}} { # enable directory pruning when query is a specific deep path without # wildcards (e.g. tools/cat15/pkg10/3.0) to avoid scanning sibling - # directories that cannot match + # directories that cannot match; verify every path component exists + # on disk so virtual names (symbols/aliases) do not trigger pruning + set prunespec {} if {$hasmoddir && !$find_all && !$contains && $modqe eq\ [string map {* {} ? {}} $modqe]} { - set prunespec $modqe - } else { - set prunespec {} + set _canprune 1 + set _checkpath $dir + foreach _qp [file split $modqe] { + set _checkpath [file join $_checkpath $_qp] + if {![file isdirectory $_checkpath]} { + set _canprune 0 + break + } + } + if {$_canprune} { + set prunespec $modqe + } } array set found_list [findModules $dir $findmod $depthlvl $fetch_mtime\