Fixes to handle file names in module resolution watching and createGetCanonicalFileName (#36106)

* Add test case to verify directory casing preservation when watching

* Fix unicode file name handling when watching failed lookup locations

* Add special file name lower conversion routine and use that instead of toLowerCase
Fixes #31819 and #35559

* Remove unicode from code

* Replace toLocaleLowerCase on filenames with ts.toFileNameLowerCase

* Make the intent of using toFileNameLowerCase more clear and why we make the restriction on turkish I with dot on top of it

* Update baselines for newly added tests in master

Author: sheetalkamat

src/compiler/commandLineParser.ts

@@ -2618,7 +2618,7 @@ namespace ts {
         errors: Push<Diagnostic>,
         extendedConfigCache?: Map<ExtendedConfigCacheEntry>
     ): ParsedTsconfig | undefined {
-        const path = host.useCaseSensitiveFileNames ? extendedConfigPath : toLowerCase(extendedConfigPath);
+        const path = host.useCaseSensitiveFileNames ? extendedConfigPath : toFileNameLowerCase(extendedConfigPath);
         let value: ExtendedConfigCacheEntry | undefined;
         let extendedResult: TsConfigSourceFile;
         let extendedConfig: ParsedTsconfig | undefined;
@@ -2922,7 +2922,7 @@ namespace ts {
     export function getFileNamesFromConfigSpecs(spec: ConfigFileSpecs, basePath: string, options: CompilerOptions, host: ParseConfigHost, extraFileExtensions: readonly FileExtensionInfo[] = []): ExpandResult {
         basePath = normalizePath(basePath);
 
-        const keyMapper = host.useCaseSensitiveFileNames ? identity : toLowerCase;
+        const keyMapper = createGetCanonicalFileName(host.useCaseSensitiveFileNames);
 
         // Literal file names (provided via the "files" array in tsconfig.json) are stored in a
         // file map with a possibly case insensitive key. We use this map later when when including
@@ -3091,7 +3091,7 @@ namespace ts {
         const match = wildcardDirectoryPattern.exec(spec);
         if (match) {
             return {
-                key: useCaseSensitiveFileNames ? match[0] : match[0].toLowerCase(),
+                key: useCaseSensitiveFileNames ? match[0] : toFileNameLowerCase(match[0]),
                 flags: watchRecursivePattern.test(spec) ? WatchDirectoryFlags.Recursive : WatchDirectoryFlags.None
             };
         }

src/compiler/core.ts

@@ -1404,6 +1404,46 @@ namespace ts {
     /** Returns lower case string */
     export function toLowerCase(x: string) { return x.toLowerCase(); }
 
+    // We convert the file names to lower case as key for file name on case insensitive file system
+    // While doing so we need to handle special characters (eg \u0130) to ensure that we dont convert
+    // it to lower case, fileName with its lowercase form can exist along side it.
+    // Handle special characters and make those case sensitive instead
+    //
+    // |-#--|-Unicode--|-Char code-|-Desc-------------------------------------------------------------------|
+    // | 1. | i        | 105       | Ascii i                                                                |
+    // | 2. | I        | 73        | Ascii I                                                                |
+    // |-------- Special characters ------------------------------------------------------------------------|
+    // | 3. | \u0130   | 304       | Uppper case I with dot above                                           |
+    // | 4. | i,\u0307 | 105,775   | i, followed by 775: Lower case of (3rd item)                           |
+    // | 5. | I,\u0307 | 73,775    | I, followed by 775: Upper case of (4th item), lower case is (4th item) |
+    // | 6. | \u0131   | 305       | Lower case i without dot, upper case is I (2nd item)                   |
+    // | 7. | \u00DF   | 223       | Lower case sharp s                                                     |
+    //
+    // Because item 3 is special where in its lowercase character has its own
+    // upper case form we cant convert its case.
+    // Rest special characters are either already in lower case format or
+    // they have corresponding upper case character so they dont need special handling
+    //
+    // But to avoid having to do string building for most common cases, also ignore
+    // a-z, 0-9, \u0131, \u00DF, \, /, ., : and space
+    const fileNameLowerCaseRegExp = /[^\u0130\u0131\u00DFa-z0-9\\/:\-_\. ]+/g;
+    /**
+     * Case insensitive file systems have descripencies in how they handle some characters (eg. turkish Upper case I with dot on top - \u0130)
+     * This function is used in places where we want to make file name as a key on these systems
+     * It is possible on mac to be able to refer to file name with I with dot on top as a fileName with its lower case form
+     * But on windows we cannot. Windows can have fileName with I with dot on top next to its lower case and they can not each be referred with the lowercase forms
+     * Technically we would want this function to be platform sepcific as well but
+     * our api has till now only taken caseSensitive as the only input and just for some characters we dont want to update API and ensure all customers use those api
+     * We could use upper case and we would still need to deal with the descripencies but
+     * we want to continue using lower case since in most cases filenames are lowercasewe and wont need any case changes and avoid having to store another string for the key
+     * So for this function purpose, we go ahead and assume character I with dot on top it as case sensitive since its very unlikely to use lower case form of that special character
+     */
+    export function toFileNameLowerCase(x: string) {
+        return fileNameLowerCaseRegExp.test(x) ?
+            x.replace(fileNameLowerCaseRegExp, toLowerCase) :
+            x;
+    }
+
     /** Throws an error because a function is not implemented. */
     export function notImplemented(): never {
         throw new Error("Not implemented");
@@ -1865,7 +1905,7 @@ namespace ts {
 
     export type GetCanonicalFileName = (fileName: string) => string;
     export function createGetCanonicalFileName(useCaseSensitiveFileNames: boolean): GetCanonicalFileName {
-        return useCaseSensitiveFileNames ? identity : toLowerCase;
+        return useCaseSensitiveFileNames ? identity : toFileNameLowerCase;
     }
 
     /** Represents a "prefix*suffix" pattern. */

src/compiler/program.ts

@@ -1397,7 +1397,7 @@ namespace ts {
                 }
                 if (resolveTypeReferenceDirectiveNamesWorker) {
                     // We lower-case all type references because npm automatically lowercases all packages. See GH#9824.
-                    const typesReferenceDirectives = map(newSourceFile.typeReferenceDirectives, ref => ref.fileName.toLocaleLowerCase());
+                    const typesReferenceDirectives = map(newSourceFile.typeReferenceDirectives, ref => toFileNameLowerCase(ref.fileName));
                     const resolutions = resolveTypeReferenceDirectiveNamesWorker(typesReferenceDirectives, newSourceFilePath, getResolvedProjectReferenceToRedirect(newSourceFile.originalFileName));
                     // ensure that types resolutions are still correct
                     const resolutionsChanged = hasChangesInResolutions(typesReferenceDirectives, resolutions, oldSourceFile.resolvedTypeReferenceDirectiveNames, typeDirectiveIsEqualTo);
@@ -2190,7 +2190,7 @@ namespace ts {
         }
 
         function getLibFileFromReference(ref: FileReference) {
-            const libName = ref.fileName.toLocaleLowerCase();
+            const libName = toFileNameLowerCase(ref.fileName);
             const libFileName = libMap.get(libName);
             if (libFileName) {
                 return getSourceFile(combinePaths(defaultLibraryPath, libFileName));
@@ -2438,7 +2438,7 @@ namespace ts {
                 addFileToRefFileMap(fileName, file, refFile);
 
                 if (host.useCaseSensitiveFileNames()) {
-                    const pathLowerCase = path.toLowerCase();
+                    const pathLowerCase = toFileNameLowerCase(path);
                     // for case-sensitive file systems check if we've already seen some file with similar filename ignoring case
                     const existingFile = filesByNameIgnoreCase!.get(pathLowerCase);
                     if (existingFile) {
@@ -2650,7 +2650,7 @@ namespace ts {
 
         function processTypeReferenceDirectives(file: SourceFile) {
             // We lower-case all type references because npm automatically lowercases all packages. See GH#9824.
-            const typeDirectives = map(file.typeReferenceDirectives, ref => ref.fileName.toLocaleLowerCase());
+            const typeDirectives = map(file.typeReferenceDirectives, ref => toFileNameLowerCase(ref.fileName));
             if (!typeDirectives) {
                 return;
             }
@@ -2661,7 +2661,7 @@ namespace ts {
                 const ref = file.typeReferenceDirectives[i];
                 const resolvedTypeReferenceDirective = resolutions[i];
                 // store resolved type directive on the file
-                const fileName = ref.fileName.toLocaleLowerCase();
+                const fileName = toFileNameLowerCase(ref.fileName);
                 setResolvedTypeReferenceDirective(file, fileName, resolvedTypeReferenceDirective);
                 processTypeReferenceDirective(
                     fileName,
@@ -2753,7 +2753,7 @@ namespace ts {
 
         function processLibReferenceDirectives(file: SourceFile) {
             forEach(file.libReferenceDirectives, libReference => {
-                const libName = libReference.fileName.toLocaleLowerCase();
+                const libName = toFileNameLowerCase(libReference.fileName);
                 const libFileName = libMap.get(libName);
                 if (libFileName) {
                     // we ignore any 'no-default-lib' reference set on this file.
@@ -3200,7 +3200,7 @@ namespace ts {
                         blockEmittingOfFile(emitFileName, createCompilerDiagnosticFromMessageChain(chain));
                     }
 
-                    const emitFileKey = !host.useCaseSensitiveFileNames() ? emitFilePath.toLocaleLowerCase() : emitFilePath;
+                    const emitFileKey = !host.useCaseSensitiveFileNames() ? toFileNameLowerCase(emitFilePath) : emitFilePath;
                     // Report error if multiple files write into same file
                     if (emitFilesSeen.has(emitFileKey)) {
                         // Already seen the same emit file - report error

src/compiler/resolutionCache.ts

@@ -176,6 +176,7 @@ namespace ts {
         const directoryWatchesOfFailedLookups = createMap<DirectoryWatchesOfFailedLookup>();
         const rootDir = rootDirForResolution && removeTrailingDirectorySeparator(getNormalizedAbsolutePath(rootDirForResolution, getCurrentDirectory()));
         const rootPath = (rootDir && resolutionHost.toPath(rootDir)) as Path; // TODO: GH#18217
+        const rootSplitLength = rootPath !== undefined ? rootPath.split(directorySeparator).length : 0;
 
         // TypeRoot watches for the types that get added as part of getAutomaticTypeDirectiveNames
         const typeRootsWatches = createMap<FileWatcher>();
@@ -439,15 +440,23 @@ namespace ts {
             if (isInDirectoryPath(rootPath, failedLookupLocationPath)) {
                 // Ensure failed look up is normalized path
                 failedLookupLocation = isRootedDiskPath(failedLookupLocation) ? normalizePath(failedLookupLocation) : getNormalizedAbsolutePath(failedLookupLocation, getCurrentDirectory());
-                Debug.assert(failedLookupLocation.length === failedLookupLocationPath.length, `FailedLookup: ${failedLookupLocation} failedLookupLocationPath: ${failedLookupLocationPath}`);
-                const subDirectoryInRoot = failedLookupLocationPath.indexOf(directorySeparator, rootPath.length + 1);
-                if (subDirectoryInRoot !== -1) {
+                const failedLookupPathSplit = failedLookupLocationPath.split(directorySeparator);
+                const failedLookupSplit = failedLookupLocation.split(directorySeparator);
+                Debug.assert(failedLookupSplit.length === failedLookupPathSplit.length, `FailedLookup: ${failedLookupLocation} failedLookupLocationPath: ${failedLookupLocationPath}`);
+                if (failedLookupPathSplit.length > rootSplitLength + 1) {
                     // Instead of watching root, watch directory in root to avoid watching excluded directories not needed for module resolution
-                    return { dir: failedLookupLocation.substr(0, subDirectoryInRoot), dirPath: failedLookupLocationPath.substr(0, subDirectoryInRoot) as Path };
+                    return {
+                        dir: failedLookupSplit.slice(0, rootSplitLength + 1).join(directorySeparator),
+                        dirPath: failedLookupPathSplit.slice(0, rootSplitLength + 1).join(directorySeparator) as Path
+                    };
                 }
                 else {
                     // Always watch root directory non recursively
-                    return { dir: rootDir!, dirPath: rootPath, nonRecursive: false }; // TODO: GH#18217
+                    return {
+                        dir: rootDir!,
+                        dirPath: rootPath,
+                        nonRecursive: false
+                    };
                 }
             }
 

src/compiler/transformers/declarations.ts

@@ -401,7 +401,7 @@ namespace ts {
             forEach(sourceFile.libReferenceDirectives, ref => {
                 const lib = host.getLibFileFromReference(ref);
                 if (lib) {
-                    ret.set(ref.fileName.toLocaleLowerCase(), true);
+                    ret.set(toFileNameLowerCase(ref.fileName), true);
                 }
             });
             return ret;