diff --git a/package.json b/package.json index 9c7b41e4..b92c6288 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,24 @@ "description": "%python-envs.terminal.useEnvFile.description%", "default": false, "scope": "resource" + }, + "python-env.globalSearchPaths": { + "type": "array", + "description": "%python-env.globalSearchPaths.description%", + "default": [], + "scope": "application", + "items": { + "type": "string" + } + }, + "python-env.workspaceSearchPaths": { + "type": "array", + "description": "%python-env.workspaceSearchPaths.description%", + "default": [], + "scope": "resource", + "items": { + "type": "string" + } } } }, diff --git a/package.nls.json b/package.nls.json index 0885ff5b..f6830057 100644 --- a/package.nls.json +++ b/package.nls.json @@ -11,6 +11,8 @@ "python-envs.terminal.autoActivationType.shellStartup": "Activation by modifying the terminal shell startup script. To use this feature we will need to modify your shell startup scripts.", "python-envs.terminal.autoActivationType.off": "No automatic activation of environments.", "python-envs.terminal.useEnvFile.description": "Controls whether environment variables from .env files and python.envFile setting are injected into terminals.", + "python-env.globalSearchPaths.description": "Global search paths for Python environments. Absolute directory paths that are searched at the user level.", + "python-env.workspaceSearchPaths.description": "Workspace search paths for Python environments. Can be relative directory paths or regex patterns searched within the workspace.", "python-envs.terminal.revertStartupScriptChanges.title": "Revert Shell Startup Script Changes", "python-envs.reportIssue.title": "Report Issue", "python-envs.setEnvManager.title": "Set Environment Manager", diff --git a/src/managers/builtin/venvManager.ts b/src/managers/builtin/venvManager.ts index cf5027a9..80bbd637 100644 --- a/src/managers/builtin/venvManager.ts +++ b/src/managers/builtin/venvManager.ts @@ -317,7 +317,7 @@ export class VenvManager implements EnvironmentManager { this.api, this.log, this, - scope ? [scope] : this.api.getPythonProjects().map((p) => p.uri), + scope ? [scope] : undefined, ); await this.loadEnvMap(); diff --git a/src/managers/common/nativePythonFinder.ts b/src/managers/common/nativePythonFinder.ts index 4a1306af..c42a4b03 100644 --- a/src/managers/common/nativePythonFinder.ts +++ b/src/managers/common/nativePythonFinder.ts @@ -2,12 +2,12 @@ import * as ch from 'child_process'; import * as fs from 'fs-extra'; import * as path from 'path'; import { PassThrough } from 'stream'; -import { Disposable, ExtensionContext, LogOutputChannel, Uri } from 'vscode'; +import { Disposable, ExtensionContext, LogOutputChannel, Uri, workspace } from 'vscode'; import * as rpc from 'vscode-jsonrpc/node'; import { PythonProjectApi } from '../../api'; import { ENVS_EXTENSION_ID, PYTHON_EXTENSION_ID } from '../../common/constants'; import { getExtension } from '../../common/extension.apis'; -import { traceVerbose } from '../../common/logging'; +import { traceLog, traceVerbose } from '../../common/logging'; import { untildify } from '../../common/utils/pathUtils'; import { isWindows } from '../../common/utils/platformUtils'; import { createRunningWorkerPool, WorkerPool } from '../../common/utils/workerPool'; @@ -326,10 +326,14 @@ class NativePythonFinderImpl implements NativePythonFinder { * Must be invoked when ever there are changes to any data related to the configuration details. */ private async configure() { + // Get all extra search paths including legacy settings and new searchPaths + const extraSearchPaths = await getAllExtraSearchPaths(); + + traceLog('Final environment directories:', extraSearchPaths); + const options: ConfigurationOptions = { workspaceDirectories: this.api.getPythonProjects().map((item) => item.uri.fsPath), - // We do not want to mix this with `search_paths` - environmentDirectories: getCustomVirtualEnvDirs(), + environmentDirectories: extraSearchPaths, condaExecutable: getPythonSettingAndUntildify('condaPath'), poetryExecutable: getPythonSettingAndUntildify('poetryPath'), cacheDirectory: this.cacheDirectory?.fsPath, @@ -357,9 +361,9 @@ type ConfigurationOptions = { cacheDirectory?: string; }; /** - * Gets all custom virtual environment locations to look for environments. + * Gets all custom virtual environment locations to look for environments from the legacy python settings (venvPath, venvFolders). */ -function getCustomVirtualEnvDirs(): string[] { +function getCustomVirtualEnvDirsLegacy(): string[] { const venvDirs: string[] = []; const venvPath = getPythonSettingAndUntildify('venvPath'); if (venvPath) { @@ -380,6 +384,272 @@ function getPythonSettingAndUntildify(name: string, scope?: Uri): T | undefin return value; } +/** + * Checks if a search path is a regex pattern. + * A path is considered a regex pattern if it contains regex special characters + * but is not a Windows path (which can contain backslashes). + * @param searchPath The search path to check + * @returns true if the path is a regex pattern, false otherwise + */ +function isRegexSearchPattern(searchPath: string): boolean { + // Check if it's a regex pattern (contains regex special characters) + // Note: Windows paths contain backslashes, so we need to be more careful + const regexChars = /[*?[\]{}()^$+|\\]/; + const hasBackslash = searchPath.includes('\\'); + const isWindowsPath = hasBackslash && (searchPath.match(/^[A-Za-z]:\\/) || searchPath.match(/^\\\\[^\\]+\\/)); + return regexChars.test(searchPath) && !isWindowsPath; +} + +/** + * Extracts the environment directory from a Python executable path. + * This follows the pattern: executable -> bin -> env -> search directory + * @param executablePath Path to Python executable + * @returns The environment directory path, or undefined if not found + */ +function extractEnvironmentDirectory(executablePath: string): string | undefined { + try { + // TODO: This logic may need to be adjusted for Windows paths (esp with Conda as doesn't use Scripts folder?) + const environmentDir = path.dirname(path.dirname(path.dirname(executablePath))); + if (environmentDir && environmentDir !== path.dirname(environmentDir)) { + traceLog('Extracted environment directory:', environmentDir, 'from executable:', executablePath); + return environmentDir; + } else { + traceLog( + 'Warning: identified executable python at', + executablePath, + 'not configured in correct folder structure, skipping', + ); + return undefined; + } + } catch (error) { + traceLog('Error extracting environment directory from:', executablePath, 'Error:', error); + return undefined; + } +} + +/** + * Gets all extra environment search paths from various configuration sources. + * Combines legacy python settings (with migration), globalSearchPaths, and workspaceSearchPaths. + * @returns Array of search directory paths + */ +async function getAllExtraSearchPaths(): Promise { + const searchDirectories: string[] = []; + + // Handle migration from legacy python settings to new search paths settings + const legacyPathsCovered = await handleLegacyPythonSettingsMigration(); + + // Only get legacy custom venv directories if they haven't been migrated to globalSearchPaths correctly + if (!legacyPathsCovered) { + const customVenvDirs = getCustomVirtualEnvDirsLegacy(); + searchDirectories.push(...customVenvDirs); + traceLog('Added legacy custom venv directories (not covered by globalSearchPaths):', customVenvDirs); + } else { + traceLog('Skipping legacy custom venv directories - they are covered by globalSearchPaths'); + } + + // Get globalSearchPaths (absolute paths, no regex) + const globalSearchPaths = getGlobalSearchPaths(); + traceLog('Retrieved globalSearchPaths:', globalSearchPaths); + for (const globalPath of globalSearchPaths) { + try { + if (!globalPath || globalPath.trim() === '') { + continue; + } + const trimmedPath = globalPath.trim(); + traceLog('Processing global search path:', trimmedPath); + // Simply add the trimmed global path + searchDirectories.push(trimmedPath); + } catch (error) { + traceLog('Error processing global search path:', globalPath, 'Error:', error); + } + } + + // Get workspaceSearchPaths (can include regex patterns) + const workspaceSearchPaths = getWorkspaceSearchPaths(); + traceLog('Retrieved workspaceSearchPaths:', workspaceSearchPaths); + for (const searchPath of workspaceSearchPaths) { + try { + if (!searchPath || searchPath.trim() === '') { + continue; + } + + const trimmedPath = searchPath.trim(); + const isRegexPattern = isRegexSearchPattern(trimmedPath); + + if (isRegexPattern) { + // Search for Python executables using the regex pattern + // Look for common Python executable names within the pattern + const pythonExecutablePatterns = isWindows() + ? [`${trimmedPath}/**/python.exe`, `${trimmedPath}/**/python3.exe`] + : [`${trimmedPath}/**/python`, `${trimmedPath}/**/python3`]; + + traceLog('Searching for Python executables with patterns:', pythonExecutablePatterns); + for (const pattern of pythonExecutablePatterns) { + try { + const foundFiles = await workspace.findFiles(pattern, null); + traceLog( + 'Python executable search found', + foundFiles.length, + 'files matching pattern:', + pattern, + ); + + for (const file of foundFiles) { + // given the executable path, extract and save the environment directory + const environmentDir = extractEnvironmentDirectory(file.fsPath); + if (environmentDir) { + searchDirectories.push(environmentDir); + } + } + } catch (error) { + traceLog('Error searching for Python executables with pattern:', pattern, 'Error:', error); + } + } + } else { + // If it's not a regex, treat it as a normal directory path and just add it + searchDirectories.push(trimmedPath); + } + } catch (error) { + traceLog('Error processing workspace search path:', searchPath, 'Error:', error); + } + } + + // Remove duplicates and return + const uniquePaths = Array.from(new Set(searchDirectories)); + traceLog( + 'getAllExtraSearchPaths completed. Total unique search directories:', + uniquePaths.length, + 'Paths:', + uniquePaths, + ); + return uniquePaths; +} + +/** + * Gets globalSearchPaths setting with proper validation. + * Only gets user-level (global) setting since this setting is application-scoped. + */ +function getGlobalSearchPaths(): string[] { + try { + const envConfig = getConfiguration('python-env'); + const inspection = envConfig.inspect('globalSearchPaths'); + + const globalPaths = inspection?.globalValue || []; + traceLog('Retrieved globalSearchPaths:', globalPaths); + return untildifyArray(globalPaths); + } catch (error) { + traceLog('Error getting globalSearchPaths:', error); + return []; + } +} + +/** + * Gets workspaceSearchPaths setting with workspace precedence. + * Gets the most specific workspace-level setting available. + */ +function getWorkspaceSearchPaths(): string[] { + try { + const envConfig = getConfiguration('python-env'); + const inspection = envConfig.inspect('workspaceSearchPaths'); + + // For workspace settings, prefer workspaceFolder > workspace + if (inspection?.workspaceFolderValue) { + traceLog('Using workspaceFolder level workspaceSearchPaths setting'); + return inspection.workspaceFolderValue; + } + + if (inspection?.workspaceValue) { + traceLog('Using workspace level workspaceSearchPaths setting'); + return inspection.workspaceValue; + } + + // Default empty array (don't use global value for workspace settings) + traceLog('No workspaceSearchPaths setting found at workspace level, using empty array'); + return []; + } catch (error) { + traceLog('Error getting workspaceSearchPaths:', error); + return []; + } +} + +/** + * Applies untildify to an array of paths + * @param paths Array of potentially tilde-containing paths + * @returns Array of expanded paths + */ +function untildifyArray(paths: string[]): string[] { + return paths.map((p) => untildify(p)); +} + +/** + * Handles migration from legacy python settings to the new globalSearchPaths setting. + * Legacy settings (venvPath, venvFolders) are User-scoped only, so they all migrate to globalSearchPaths. + * Does NOT delete the old settings, only adds them to the new settings. + * @returns true if legacy paths are covered by globalSearchPaths (either already there or just migrated), false if legacy paths should be included separately + */ +async function handleLegacyPythonSettingsMigration(): Promise { + try { + const pythonConfig = getConfiguration('python'); + const envConfig = getConfiguration('python-env'); + + // Get legacy settings at global level only (they were User-scoped) + const venvPathInspection = pythonConfig.inspect('venvPath'); + const venvFoldersInspection = pythonConfig.inspect('venvFolders'); + + // Collect global (user-level) legacy paths for globalSearchPaths + const globalLegacyPaths: string[] = []; + if (venvPathInspection?.globalValue) { + globalLegacyPaths.push(venvPathInspection.globalValue); + } + if (venvFoldersInspection?.globalValue) { + globalLegacyPaths.push(...venvFoldersInspection.globalValue); + } + + if (globalLegacyPaths.length === 0) { + // No legacy settings exist, so they're "covered" (nothing to worry about) + traceLog('No legacy python settings found'); + return true; + } + + traceLog('Found legacy python settings - global paths:', globalLegacyPaths); + + // Check if legacy paths are already in globalSearchPaths + const globalSearchPathsInspection = envConfig.inspect('globalSearchPaths'); + const currentGlobalSearchPaths = globalSearchPathsInspection?.globalValue || []; + + // Check if all legacy paths are already covered by globalSearchPaths + const legacyPathsAlreadyCovered = globalLegacyPaths.every((legacyPath) => + currentGlobalSearchPaths.includes(legacyPath), + ); + + if (legacyPathsAlreadyCovered) { + traceLog('All legacy paths are already in globalSearchPaths, no migration needed'); + return true; // Legacy paths are covered + } + + // Need to migrate - add legacy paths to globalSearchPaths + const combinedGlobalPaths = Array.from(new Set([...currentGlobalSearchPaths, ...globalLegacyPaths])); + await envConfig.update('globalSearchPaths', combinedGlobalPaths, true); // true = global/user level + traceLog('Migrated legacy global python settings to globalSearchPaths. Combined paths:', combinedGlobalPaths); + + // Show notification to user about migration + if (!migrationNotificationShown) { + migrationNotificationShown = true; + traceLog( + 'User notification: Automatically migrated legacy python settings to python-env.globalSearchPaths.', + ); + } + + return true; // Legacy paths are now covered by globalSearchPaths + } catch (error) { + traceLog('Error during legacy python settings migration:', error); + return false; // On error, include legacy paths separately to be safe + } +} + +// Module-level variable to track migration notification +let migrationNotificationShown = false; + export function getCacheDirectory(context: ExtensionContext): Uri { return Uri.joinPath(context.globalStorageUri, 'pythonLocator'); } diff --git a/src/test/managers/nativePythonFinder.unit.test.ts b/src/test/managers/nativePythonFinder.unit.test.ts new file mode 100644 index 00000000..fca2b89a --- /dev/null +++ b/src/test/managers/nativePythonFinder.unit.test.ts @@ -0,0 +1,248 @@ +import assert from 'node:assert'; +import * as path from 'path'; +import * as sinon from 'sinon'; + +// Tests for the updated searchPaths functionality +suite('NativePythonFinder SearchPaths Tests', () => { + teardown(() => { + sinon.restore(); + }); + + suite('Configuration reading', () => { + test('should handle python-env configuration namespace', () => { + // Test that we can distinguish between python and python-env namespaces + assert.strictEqual('python-env', 'python-env'); + assert.notStrictEqual('python-env', 'python'); + }); + + test('should handle empty search paths array', () => { + const searchPaths: string[] = []; + assert.deepStrictEqual(searchPaths, []); + assert.strictEqual(searchPaths.length, 0); + }); + + test('should handle populated search paths array', () => { + const searchPaths = ['/home/user/.virtualenvs', '**/bin/python*']; + assert.strictEqual(searchPaths.length, 2); + assert.deepStrictEqual(searchPaths, ['/home/user/.virtualenvs', '**/bin/python*']); + }); + }); + + suite('Regex pattern detection', () => { + test('should correctly identify regex patterns', () => { + const regexPatterns = [ + '**/bin/python*', + '**/*.py', + 'python[0-9]*', + 'python{3,4}', + 'python+', + 'python?', + 'python.*', + '[Pp]ython' + ]; + + const regexChars = /[*?[\]{}()^$+|]/; + regexPatterns.forEach(pattern => { + assert.ok(regexChars.test(pattern), `Pattern ${pattern} should be detected as regex`); + }); + }); + + test('should not identify regular directory paths as regex', () => { + const regularPaths = [ + '/usr/local/python', + '/home/user/.virtualenvs', + '/opt/python3.9' + ]; + + const regexChars = /[*?[\]{}()^$+|]/; + regularPaths.forEach(testPath => { + assert.ok(!regexChars.test(testPath), `Path ${testPath} should not be detected as regex`); + }); + }); + + test('should handle Windows paths specially', () => { + const windowsPath = 'C:\\Users\\user\\envs'; + const regexChars = /[*?[\]{}()^$+|\\]/; // Added backslash to match implementation + + // Windows paths contain backslashes which are regex characters + // Our implementation should handle this case by checking for valid Windows path patterns + assert.ok(regexChars.test(windowsPath), 'Windows paths contain regex chars'); + + // Test that we can identify valid Windows paths and NOT treat them as regex + const hasBackslash = windowsPath.includes('\\'); + const isWindowsPath = hasBackslash && (windowsPath.match(/^[A-Za-z]:\\/) || windowsPath.match(/^\\\\[^\\]+\\/)); + const isRegexPattern = regexChars.test(windowsPath) && !isWindowsPath; + + assert.ok(isWindowsPath, 'Should recognize Windows path pattern'); + assert.ok(!isRegexPattern, 'Should not treat Windows path as regex pattern'); + }); + }); + + suite('Environment directory path extraction', () => { + test('should extract correct environment directory from executable path', () => { + const executablePath = '/home/user/.virtualenvs/myenv/bin/python'; + const expected = '/home/user/.virtualenvs'; + + // Test path manipulation logic + const environmentDir = path.dirname(path.dirname(path.dirname(executablePath))); + assert.strictEqual(environmentDir, expected); + }); + + test('should handle deep nested paths', () => { + const executablePath = '/very/deep/nested/path/to/env/bin/python'; + const expected = '/very/deep/nested/path/to'; + + const environmentDir = path.dirname(path.dirname(path.dirname(executablePath))); + assert.strictEqual(environmentDir, expected); + }); + + test('should handle shallow paths gracefully', () => { + const executablePath = '/bin/python'; + + const greatGrandParent = path.dirname(path.dirname(path.dirname(executablePath))); + // This should result in root + assert.ok(greatGrandParent); + assert.strictEqual(greatGrandParent, '/'); + }); + + test('should handle Windows style paths', function () { + // Skip this test on non-Windows systems since path.dirname behaves differently + if (process.platform !== 'win32') { + this.skip(); + return; + } + + const executablePath = 'C:\\Users\\user\\envs\\myenv\\Scripts\\python.exe'; + + const greatGrandParent = path.dirname(path.dirname(path.dirname(executablePath))); + const expected = 'C:\\Users\\user\\envs'; + assert.strictEqual(greatGrandParent, expected); + }); + }); + + suite('Array deduplication logic', () => { + test('should remove duplicate paths', () => { + const paths = ['/path1', '/path2', '/path1', '/path3', '/path2']; + const unique = Array.from(new Set(paths)); + + assert.strictEqual(unique.length, 3); + assert.deepStrictEqual(unique, ['/path1', '/path2', '/path3']); + }); + + test('should handle empty arrays', () => { + const paths: string[] = []; + const unique = Array.from(new Set(paths)); + + assert.strictEqual(unique.length, 0); + assert.deepStrictEqual(unique, []); + }); + + test('should handle single item arrays', () => { + const paths = ['/single/path']; + const unique = Array.from(new Set(paths)); + + assert.strictEqual(unique.length, 1); + assert.deepStrictEqual(unique, ['/single/path']); + }); + }); + + suite('String trimming and validation', () => { + test('should handle empty and whitespace-only strings', () => { + const testStrings = ['', ' ', '\t\n', 'valid']; + const filtered = testStrings.filter(s => s && s.trim() !== ''); + + assert.deepStrictEqual(filtered, ['valid']); + }); + + test('should trim whitespace from paths', () => { + const pathWithWhitespace = ' /path/to/directory '; + const trimmed = pathWithWhitespace.trim(); + + assert.strictEqual(trimmed, '/path/to/directory'); + }); + }); + + suite('Python executable detection', () => { + test('should identify python-like filenames', () => { + const filenames = [ + 'python', + 'python3', + 'python3.9', + 'python.exe', + 'Python.exe', + 'python3.11.exe' + ]; + + filenames.forEach(filename => { + const lowerFilename = filename.toLowerCase(); + assert.ok( + lowerFilename.includes('python') || path.basename(lowerFilename).startsWith('python'), + `${filename} should be identified as python executable` + ); + }); + }); + + test('should not identify non-python files', () => { + const filenames = [ + 'node', + 'npm', + 'pip', + 'bash', + 'zsh' + ]; + + filenames.forEach(filename => { + const lowerFilename = filename.toLowerCase(); + const isPython = lowerFilename.includes('python') || path.basename(lowerFilename).startsWith('python'); + assert.ok(!isPython, `${filename} should not be identified as python executable`); + }); + }); + }); + + suite('Settings precedence logic', () => { + test('should handle array equality comparison', () => { + const array1 = ['path1', 'path2']; + const array2 = ['path1', 'path2']; + const array3 = ['path1', 'path3']; + + // Arrays with same content should be equal + assert.strictEqual(array1.length, array2.length); + assert.ok(array1.every((val, index) => val === array2[index])); + + // Arrays with different content should not be equal + assert.ok(!array1.every((val, index) => val === array3[index])); + }); + + test('should handle empty arrays in comparison', () => { + const emptyArray1: string[] = []; + const emptyArray2: string[] = []; + const nonEmptyArray = ['path1']; + + // Empty arrays should be equal (every element matches) + assert.ok(emptyArray1.every((val, index) => val === emptyArray2[index])); + + // Empty array should not match non-empty array (lengths differ) + assert.ok(emptyArray1.length !== nonEmptyArray.length); + }); + }); + + suite('Path type detection', () => { + test('should detect directory paths correctly', () => { + const directoryPaths = [ + '/home/user/.virtualenvs', + '/opt/python/envs', + 'C:\\Users\\user\\envs' + ]; + + // These are all valid directory-style paths + directoryPaths.forEach(dirPath => { + assert.ok(typeof dirPath === 'string' && dirPath.length > 0); + // Should not contain regex characters (except Windows backslashes) + if (!dirPath.includes('\\')) { + const regexChars = /[*?[\]{}()^$+|]/; + assert.ok(!regexChars.test(dirPath), `${dirPath} should be a plain directory path`); + } + }); + }); + }); +}); \ No newline at end of file