From 8ef85a98ab44e9761e07ee2d1b04879c50949c3a Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:49:32 -0800 Subject: [PATCH 1/6] Add concurrent find and install --- src/code/FindHelper.cs | 396 ++++++++++++++++++++++++----------- src/code/InstallHelper.cs | 214 +++++++++++++++---- src/code/Utils.cs | 86 ++++++-- src/code/V2ServerAPICalls.cs | 24 +-- 4 files changed, 530 insertions(+), 190 deletions(-) diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index d8287c689..b1d5f9ddc 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -2,15 +2,18 @@ // Licensed under the MIT License. using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; using NuGet.Versioning; using System; using System.Collections.Generic; using System.Linq; using System.Management.Automation; using System.Net; +using System.Security.Cryptography; using System.Runtime.ExceptionServices; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -35,7 +38,12 @@ internal class FindHelper private bool _includeDependencies = false; private bool _repositoryNameContainsWildcard = true; private NetworkCredential _networkCredential; + + // TODO: Update to be concurrency safe; TryAdd needs to be updates as well + // TODO: look at # of allocations (lists, etc.) private Dictionary> _packagesFound; + private HashSet _knownLatestPkgVersion; + List depPkgsFound; #endregion @@ -49,6 +57,7 @@ public FindHelper(CancellationToken cancellationToken, PSCmdlet cmdletPassedIn, _cmdletPassedIn = cmdletPassedIn; _networkCredential = networkCredential; _packagesFound = new Dictionary>(StringComparer.OrdinalIgnoreCase); + _knownLatestPkgVersion = new HashSet(StringComparer.OrdinalIgnoreCase); } #endregion @@ -1096,67 +1105,96 @@ private string FormatPkgVersionString(PSResourceInfo pkg) #region Internal Client Search Methods - internal IEnumerable FindDependencyPackages( - ServerApiCall currentServer, - ResponseUtil currentResponseUtil, - PSResourceInfo currentPkg, - PSRepositoryInfo repository) + internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { - if (currentPkg.Dependencies.Length > 0) + depPkgsFound = new List(); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); + + foreach (var pkg in depPkgsFound) { - foreach (var dep in currentPkg.Dependencies) + if (!_packagesFound.ContainsKey(pkg.Name)) + { + TryAddToPackagesFound(pkg); + yield return pkg; + } + else { - PSResourceInfo depPkg = null; + List pkgVersions = _packagesFound[pkg.Name]; + // _packagesFound has item.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(pkg))) + { + TryAddToPackagesFound(pkg); + + yield return pkg; + } + } + } + } - if (dep.VersionRange.Equals(VersionRange.All)) + // Method 2 + internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) + { + List errors = new List(); + if (currentPkg.Dependencies.Length > 0) + { + // If installing more than 5 packages, do so concurrently + if (currentPkg.Dependencies.Length > 5) + { + Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, dep => { - FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - yield return null; - continue; - } + // Console.WriteLine($"FindDependencyPackages Processing number: {dep}, Thread ID: {Task.CurrentId}"); + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + }); - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + // todo: write any errors here + } + else + { + foreach (var dep in currentPkg.Dependencies) + { + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } - depPkg = currentResult.returnedObject; + // todo write out errors here + } + } + } - if (!_packagesFound.ContainsKey(depPkg.Name)) + // Method 3 + private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) + { + FindDependencyWithNoUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) + { + FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); + if (errRecord != null) + { + errors = ProcessErrorRecord(errRecord, errors); + } + else + { + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + if (!depPkgsFound.Contains(depPkg)) { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } else { @@ -1164,56 +1202,60 @@ internal IEnumerable FindDependencyPackages( // _packagesFound has depPkg.name in it, but the version is not the same if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } + // // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); + } } } - else - { - FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - yield return null; - continue; - } + } + } + else + { + FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); + if (errRecord != null) + { + errors = ProcessErrorRecord(errRecord, errors); + } - if (responses.IsFindResultsEmpty()) + if (responses.IsFindResultsEmpty()) + { + errors.Add(new ErrorRecord( + new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), + "FindDepPackagesFindVersionGlobbingFailure", + ErrorCategory.InvalidResult, + this)); + } + else + { + foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) + { + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) { - _cmdletPassedIn.WriteError(new ErrorRecord( - new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), - "FindDepPackagesFindVersionGlobbingFailure", - ErrorCategory.InvalidResult, - this)); - yield return null; - continue; + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + + // if (errRecord.Exception is ResourceNotFoundException) + // { + // _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); + // } + // else + // { + // _cmdletPassedIn.WriteError(errRecord); + // } + // yield return null; + // continue; } - - foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) + else { - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - - yield return null; - continue; - } - - // Check to see if version falls within version range + // Check to see if version falls within version range PSResourceInfo foundDep = currentResult.returnedObject; string depVersionStr = $"{foundDep.Version}"; if (foundDep.IsPrerelease) @@ -1225,57 +1267,173 @@ internal IEnumerable FindDependencyPackages( && dep.VersionRange.Satisfies(depVersion)) { depPkg = foundDep; + break; } } + } - if (depPkg == null) + if (depPkg == null) + { + // // Console.WriteLine($"depPkg is null and I don't know what this means"); + } + else + { + if (!depPkgsFound.Contains(depPkg)) { - continue; - } + // // Console.WriteLine($"PackagesFound contains {depPkg.Name}"); - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } else { List pkgVersions = _packagesFound[depPkg.Name] as List; // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } + // // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); } } } } } + } - if (!_packagesFound.ContainsKey(currentPkg.Name)) + // Method 4 + private PSResourceInfo FindDependencyWithNoUpperBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + // _knownLatestPkgVersion will tell us if we already have the latest version available for a particular package. + if (!_knownLatestPkgVersion.Contains(dep.Name)) { - TryAddToPackagesFound(currentPkg); + FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); + if (errRecord != null) + { + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + // todo add error here? + } + + return depPkg; + } - yield return currentPkg; + if (responses == null) + { + // todo error + return depPkg; + } + + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + + return depPkg; + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + + return depPkg; + } + + depPkg = currentResult.returnedObject; + if (!depPkgsFound.Contains(depPkg)) + { + _knownLatestPkgVersion.Add(depPkg.Name); + + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); + + } + } + } + + return depPkg; + } + + + private List ProcessErrorRecord(ErrorRecord errRecord, List errors) + { + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package could not be found: '{errRecord.Exception.Message}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); } else { - List pkgVersions = _packagesFound[currentPkg.Name] as List; - // _packagesFound has currentPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(currentPkg))) - { - TryAddToPackagesFound(currentPkg); + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package could not be found: '{errRecord.Exception.Message}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } - yield return currentPkg; - } + return errors; + } + + private List ProcessPSResourceResult(PSResourceResult currentResult, List errors) + { + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{currentResult.returnedObject.Name}' could not be found in repository '{currentResult.returnedObject.Repository}'."), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{currentResult.returnedObject.Name}' could not be found in repository '{currentResult.returnedObject.Repository}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); } + return errors; } - #endregion + #endregion } -} +} \ No newline at end of file diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index fef419a4f..b1a513b80 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -2,10 +2,12 @@ // Licensed under the MIT License. using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; using NuGet.Versioning; using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.IO.Compression; @@ -15,6 +17,7 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -396,7 +399,7 @@ private void MoveFilesIntoInstallPath( string moduleManifestVersion, string scriptPath) { - _cmdletPassedIn.WriteDebug("In InstallHelper::MoveFilesIntoInstallPath()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::MoveFilesIntoInstallPath()"); // Creating the proper installation path depending on whether pkg is a module or script var newPathParent = isModule ? Path.Combine(installPath, pkgInfo.Name) : installPath; var finalModuleVersionDir = isModule ? Path.Combine(installPath, pkgInfo.Name, moduleManifestVersion) : installPath; @@ -405,32 +408,32 @@ private void MoveFilesIntoInstallPath( var tempModuleVersionDir = (!isModule || isLocalRepo) ? dirNameVersion : Path.Combine(tempInstallPath, pkgInfo.Name, newVersion); - _cmdletPassedIn.WriteVerbose($"Installation source path is: '{tempModuleVersionDir}'"); - _cmdletPassedIn.WriteVerbose($"Installation destination path is: '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Installation source path is: '{tempModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Installation destination path is: '{finalModuleVersionDir}'"); if (isModule) { // If new path does not exist if (!Directory.Exists(newPathParent)) { - _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); Directory.CreateDirectory(newPathParent); Utils.MoveDirectory(tempModuleVersionDir, finalModuleVersionDir); } else { - _cmdletPassedIn.WriteVerbose($"Temporary module version directory is: '{tempModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Temporary module version directory is: '{tempModuleVersionDir}'"); if (Directory.Exists(finalModuleVersionDir)) { // Delete the directory path before replacing it with the new module. // If deletion fails (usually due to binary file in use), then attempt restore so that the currently // installed module is not corrupted. - _cmdletPassedIn.WriteVerbose($"Attempting to delete with restore on failure. '{finalModuleVersionDir}'"); + //_cmdletPassedIn.WriteVerbose($"Attempting to delete with restore on failure. '{finalModuleVersionDir}'"); Utils.DeleteDirectoryWithRestore(finalModuleVersionDir); } - _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); Utils.MoveDirectory(tempModuleVersionDir, finalModuleVersionDir); } } @@ -456,27 +459,27 @@ private void MoveFilesIntoInstallPath( if (!Directory.Exists(scriptInfoFolderPath)) { - _cmdletPassedIn.WriteVerbose($"Created '{scriptInfoFolderPath}' path for scripts"); + // _cmdletPassedIn.WriteVerbose($"Created '{scriptInfoFolderPath}' path for scripts"); Directory.CreateDirectory(scriptInfoFolderPath); } // Need to delete old xml files because there can only be 1 per script - _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: '{1}'", scriptXmlFilePath, File.Exists(scriptXmlFilePath))); + // _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: '{1}'", scriptXmlFilePath, File.Exists(scriptXmlFilePath))); if (File.Exists(scriptXmlFilePath)) { - _cmdletPassedIn.WriteVerbose("Deleting script metadata XML"); + // _cmdletPassedIn.WriteVerbose("Deleting script metadata XML"); File.Delete(Path.Combine(scriptInfoFolderPath, scriptXML)); } - _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML))); + // _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML))); Utils.MoveFiles(Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML)); // Need to delete old script file, if that exists - _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: ", File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)))); + // _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: ", File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)))); if (File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))) { - _cmdletPassedIn.WriteVerbose("Deleting script file"); + // _cmdletPassedIn.WriteVerbose("Deleting script file"); File.Delete(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)); } } @@ -489,7 +492,7 @@ private void MoveFilesIntoInstallPath( } } - _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))); + // _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))); Utils.MoveFiles(scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)); } } @@ -533,10 +536,11 @@ private List InstallPackages( currentServer: currentServer, currentResponseUtil: currentResponseUtil, tempInstallPath: tempInstallPath, + skipDependencyCheck: skipDependencyCheck, packagesHash: new Hashtable(StringComparer.InvariantCultureIgnoreCase), errRecord: out ErrorRecord errRecord); - // At this point parent package is installed to temp path. + // At this point all packages are installed to temp path. if (errRecord != null) { if (errRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) @@ -606,6 +610,7 @@ private List InstallPackages( currentServer: currentServer, currentResponseUtil: currentResponseUtil, tempInstallPath: tempInstallPath, + skipDependencyCheck: skipDependencyCheck, packagesHash: packagesHash, errRecord: out ErrorRecord installPkgErrRecord); @@ -682,6 +687,7 @@ private Hashtable BeginPackageInstall( ServerApiCall currentServer, ResponseUtil currentResponseUtil, string tempInstallPath, + bool skipDependencyCheck, Hashtable packagesHash, out ErrorRecord errRecord) { @@ -689,6 +695,7 @@ private Hashtable BeginPackageInstall( FindResults responses = null; errRecord = null; + // Find the parent package that needs to be installed switch (searchVersionType) { case VersionType.VersionRange: @@ -717,6 +724,7 @@ private Hashtable BeginPackageInstall( default: // VersionType.NoVersion responses = currentServer.FindName(pkgNameToInstall, _prerelease, ResourceType.None, out ErrorRecord findNameErrRecord); + if (findNameErrRecord != null) { errRecord = findNameErrRecord; @@ -726,6 +734,7 @@ private Hashtable BeginPackageInstall( break; } + // Convert parent package to PSResourceInfo PSResourceInfo pkgToInstall = null; foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) { @@ -846,25 +855,150 @@ private Hashtable BeginPackageInstall( } else { - // Download the package. + // Concurrently Updates + // Find all dependencies string pkgName = pkgToInstall.Name; - Stream responseStream = currentServer.InstallPackage(pkgName, pkgVersion, _prerelease, out ErrorRecord installNameErrRecord); - if (installNameErrRecord != null) + if (!skipDependencyCheck) + { + List allDependencies = FindAllDependencies(currentServer, currentResponseUtil, pkgToInstall, repository); + + return InstallParentAndDependencyPackages(pkgToInstall, allDependencies, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); + } + else { + + // TODO: check this version and prerelease combo + Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgToInstall.Version.ToString(), true, out ErrorRecord installNameErrRecord); + + if (installNameErrRecord != null) + { + errRecord = installNameErrRecord; + return packagesHash; + } + + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + if (!installedToTempPathSuccessfully) + { + return packagesHash; + } + } + } + + return updatedPackagesHash; + } + + // Concurrency Updates + private List FindAllDependencies(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo pkgToInstall, PSRepositoryInfo repository) + { + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V3) + { + _cmdletPassedIn.WriteWarning("Installing dependencies is not currently supported for V3 server protocol repositories. The package will be installed without installing dependencies."); + } + + var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); + _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{pkgToInstall.Name}'"); + + // the last package added will be the parent package. + List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + + // allDependencies contains parent package as well + foreach (PSResourceInfo pkg in allDependencies) + { + // Console.WriteLine($"{pkg.Name}: {pkg.Version}"); + } + + return allDependencies; + } + + // Concurrently Updates + private Hashtable InstallParentAndDependencyPackages(PSResourceInfo parentPkg, List allDependencies, ServerApiCall currentServer, string tempInstallPath, Hashtable packagesHash, Hashtable updatedPackagesHash, PSResourceInfo pkgToInstall) + { + List errors = new List(); + // If installing more than 5 packages, do so concurrently + if (allDependencies.Count > 5) + { + // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) + Parallel.ForEach(allDependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, depPkg => { - errRecord = installNameErrRecord; + var depPkgName = depPkg.Name; + var depPkgVersion = depPkg.Version.ToString(); + // Console.WriteLine($"Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); + + Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + errors.Add(installNameErrRecord); + } + + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + + if (!installedToTempPathSuccessfully) + { + errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); + } + }); + + if (errors.Count > 0) + { + // Write out all errors collected from Parallel.ForEach + foreach (var err in errors) + { + _cmdletPassedIn.WriteError(err); + } + return packagesHash; } - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + // Install parent package + Stream responseStream = currentServer.InstallPackage(parentPkg.Name, parentPkg.Version.ToString(), true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + _cmdletPassedIn.WriteError(installNameErrRecord); + return packagesHash; + } + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); if (!installedToTempPathSuccessfully) { + _cmdletPassedIn.WriteError(tempSaveErrRecord ?? tempInstallErrRecord); return packagesHash; } + + return updatedPackagesHash; } + else + { + // Install the good old fashioned way + // Make sure to install dependencies first, then install parent pkg + allDependencies.Add(parentPkg); + foreach (var pkgToBeInstalled in allDependencies) + { + var pkgToInstallName = pkgToBeInstalled.Name; + var pkgToInstallVersion = pkgToBeInstalled.Version.ToString(); + Stream responseStream = currentServer.InstallPackage(pkgToInstallName, pkgToInstallVersion, true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + _cmdletPassedIn.WriteError(installNameErrRecord); + return packagesHash; + } - return updatedPackagesHash; + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + + if (!installedToTempPathSuccessfully) + { + _cmdletPassedIn.WriteError(tempSaveErrRecord ?? tempInstallErrRecord); + return packagesHash; + } + } + + return updatedPackagesHash; + } } /// @@ -930,7 +1064,7 @@ private bool TryInstallToTempPath( out Hashtable updatedPackagesHash, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TryInstallToTempPath()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::TryInstallToTempPath()"); error = null; updatedPackagesHash = packagesHash; try @@ -1031,7 +1165,7 @@ private bool TryInstallToTempPath( { foreach (ErrorRecord parseError in parseScriptFileErrors) { - _cmdletPassedIn.WriteError(parseError); + // _cmdletPassedIn.WriteError(parseError); } error = new ErrorRecord( @@ -1048,7 +1182,7 @@ private bool TryInstallToTempPath( // This package is not a PowerShell package (eg a resource from the NuGet Gallery). installPath = _pathsToInstallPkg.Find(path => path.EndsWith("Modules", StringComparison.InvariantCultureIgnoreCase)); - _cmdletPassedIn.WriteVerbose($"This resource is not a PowerShell package and will be installed to the modules path: {installPath}."); + //_cmdletPassedIn.WriteVerbose($"This resource is not a PowerShell package and will be installed to the modules path: {installPath}."); isModule = true; } @@ -1108,7 +1242,7 @@ private bool TrySaveNupkgToTempPath( out Hashtable updatedPackagesHash, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); + // _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); error = null; updatedPackagesHash = packagesHash; @@ -1230,7 +1364,7 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error /// private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hashtable packagesHash) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TryMoveInstallContent()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::TryMoveInstallContent()"); foreach (string pkgName in packagesHash.Keys) { Hashtable pkgInfo = packagesHash[pkgName] as Hashtable; @@ -1255,7 +1389,7 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash moduleManifestVersion: pkgVersion, scriptPath); - _cmdletPassedIn.WriteVerbose($"Successfully installed package '{pkgName}' to location '{installPath}'"); + //_cmdletPassedIn.WriteVerbose($"Successfully installed package '{pkgName}' to location '{installPath}'"); if (!_savePkg && isScript) { @@ -1274,20 +1408,20 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash if (!String.IsNullOrEmpty(envPATHVarValue) && !envPATHVarValue.Contains(installPath) && !envPATHVarValue.Contains(installPathwithBackSlash)) { - _cmdletPassedIn.WriteWarning(String.Format(ScriptPATHWarning, scope, installPath)); + //_cmdletPassedIn.WriteWarning(String.Format(ScriptPATHWarning, scope, installPath)); } } } - catch (Exception e) + catch (Exception) { - _cmdletPassedIn.WriteError(new ErrorRecord( + /*_cmdletPassedIn.WriteError(new ErrorRecord( new PSInvalidOperationException( message: $"Unable to successfully install package '{pkgName}': '{e.Message}'", innerException: e), "InstallPackageFailed", ErrorCategory.InvalidOperation, _cmdletPassedIn)); - + */ return false; } } @@ -1300,7 +1434,7 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash /// private bool CallAcceptLicense(PSResourceInfo p, string moduleManifest, string tempInstallPath, string newVersion, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::CallAcceptLicense()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::CallAcceptLicense()"); error = null; var requireLicenseAcceptance = false; @@ -1404,7 +1538,7 @@ private bool CallAcceptLicense(PSResourceInfo p, string moduleManifest, string t /// private bool DetectClobber(string pkgName, Hashtable parsedMetadataHashtable, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::DetectClobber()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::DetectClobber()"); error = null; bool foundClobber = false; @@ -1468,7 +1602,7 @@ private bool DetectClobber(string pkgName, Hashtable parsedMetadataHashtable, ou /// private bool CreateMetadataXMLFile(string dirNameVersion, string installPath, PSResourceInfo pkg, bool isModule, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::CreateMetadataXMLFile()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::CreateMetadataXMLFile()"); error = null; bool success = true; // Script will have a metadata file similar to: "TestScript_InstalledScriptInfo.xml" @@ -1498,7 +1632,7 @@ private bool CreateMetadataXMLFile(string dirNameVersion, string installPath, PS /// private void DeleteExtraneousFiles(string packageName, string dirNameVersion) { - _cmdletPassedIn.WriteDebug("In InstallHelper::DeleteExtraneousFiles()"); + // _cmdletPassedIn.WriteDebug("In InstallHelper::DeleteExtraneousFiles()"); // Deleting .nupkg SHA file, .nuspec, and .nupkg after unpacking the module // since we download as .zip for HTTP calls, we shouldn't have .nupkg* files // var nupkgSHAToDelete = Path.Combine(dirNameVersion, pkgIdString + ".nupkg.sha512"); @@ -1511,22 +1645,22 @@ private void DeleteExtraneousFiles(string packageName, string dirNameVersion) if (File.Exists(nuspecToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{nuspecToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{nuspecToDelete}'"); File.Delete(nuspecToDelete); } if (File.Exists(contentTypesToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{contentTypesToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{contentTypesToDelete}'"); File.Delete(contentTypesToDelete); } if (Directory.Exists(relsDirToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{relsDirToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{relsDirToDelete}'"); Utils.DeleteDirectory(relsDirToDelete); } if (Directory.Exists(packageDirToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{packageDirToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{packageDirToDelete}'"); Utils.DeleteDirectory(packageDirToDelete); } } diff --git a/src/code/Utils.cs b/src/code/Utils.cs index 26d3ab25e..8dc6bec57 100644 --- a/src/code/Utils.cs +++ b/src/code/Utils.cs @@ -1342,37 +1342,85 @@ private static bool TryReadPSDataFile( out Hashtable dataFileInfo, out Exception error) { + dataFileInfo = null; + error = null; try { if (filePath is null) { throw new PSArgumentNullException(nameof(filePath)); } - string contents = System.IO.File.ReadAllText(filePath); - var scriptBlock = System.Management.Automation.ScriptBlock.Create(contents); - - // Ensure that the content script block is safe to convert into a PSDataFile Hashtable. - // This will throw for unsafe content. - scriptBlock.CheckRestrictedLanguage( - allowedCommands: allowedCommands, - allowedVariables: allowedVariables, - allowEnvironmentVariables: allowEnvironmentVariables); - - // Convert contents into PSDataFile Hashtable by executing content as script. - object result = scriptBlock.InvokeReturnAsIs(); - if (result is PSObject psObject) + + // Parallel.ForEach calls into this method. + // Each thread needs its own runspace created to provide a separate environment for operations to run independently. + Runspace runspace = RunspaceFactory.CreateRunspace(); + runspace.Open(); + runspace.SessionStateProxy.LanguageMode = PSLanguageMode.ConstrainedLanguage; + + // Set the created runspace as the default for the current thread + Runspace.DefaultRunspace = runspace; + + using (System.Management.Automation.PowerShell pwsh = System.Management.Automation.PowerShell.Create()) { - result = psObject.BaseObject; - } + pwsh.Runspace = runspace; - dataFileInfo = (Hashtable)result; - error = null; - return true; + var cmd = new Command( + command: contents, + isScript: true, + useLocalScope: true); + cmd.MergeMyResults( + myResult: PipelineResultTypes.Error | PipelineResultTypes.Warning | PipelineResultTypes.Verbose | PipelineResultTypes.Debug | PipelineResultTypes.Information, + toResult: PipelineResultTypes.Output); + pwsh.Commands.AddCommand(cmd); + + + try + { + // Invoke the pipeline and retrieve the results + var results = pwsh.Invoke(); + + if (results[0] is PSObject pwshObj) + { + switch (pwshObj.BaseObject) + { + case ErrorRecord err: + //_cmdletPassedIn.WriteError(error); + break; + + case WarningRecord warning: + //cmdlet.WriteWarning(warning.Message); + break; + + case VerboseRecord verbose: + //cmdlet.WriteVerbose(verbose.Message); + break; + + case DebugRecord debug: + //cmdlet.WriteDebug(debug.Message); + break; + + case InformationRecord info: + //cmdlet.WriteInformation(info); + break; + + case Hashtable result: + dataFileInfo = result; + return true; + } + } + } + catch (Exception ex) + { + error = ex; + } + } + runspace.Close(); + + return false; } catch (Exception ex) { - dataFileInfo = null; error = ex; return false; } diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index 94d0b3a0b..4fc499cc5 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -336,7 +336,7 @@ public override FindResults FindCommandOrDscResource(string[] tags, bool include /// public override FindResults FindName(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindName()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindName()"); // Make sure to include quotations around the package name // This should return the latest stable version or the latest prerelease version (respectively) @@ -633,7 +633,7 @@ public override FindResults FindVersionGlobbing(string packageName, VersionRange /// public override FindResults FindVersion(string packageName, string version, ResourceType type, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersion()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersion()"); // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion eq '1.1.0' and substringof('PSModule', Tags) eq true // Quotations around package name and version do not matter, same metadata gets returned. // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. @@ -674,7 +674,7 @@ public override FindResults FindVersion(string packageName, string version, Reso } int count = GetCountFromResponse(response, out errRecord); - _cmdletPassedIn.WriteDebug($"Count from response is '{count}'"); + //_cmdletPassedIn.WriteDebug($"Count from response is '{count}'"); if (errRecord != null) { @@ -788,13 +788,13 @@ public override Stream InstallPackage(string packageName, string packageVersion, /// private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCall()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCall()"); errRecord = null; string response = string.Empty; try { - _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + // _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); response = SendV2RequestAsync(request, _sessionClient).GetAwaiter().GetResult(); @@ -834,7 +834,7 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) if (string.IsNullOrEmpty(response)) { - _cmdletPassedIn.WriteDebug("Response is empty"); + // _cmdletPassedIn.WriteDebug("Response is empty"); } return response; @@ -845,13 +845,13 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) /// private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); + // _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); errRecord = null; HttpContent content = null; try { - _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); content = SendV2RequestForContentAsync(request, _sessionClient).GetAwaiter().GetResult(); @@ -883,7 +883,7 @@ private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorReco if (content == null || string.IsNullOrEmpty(content.ToString())) { - _cmdletPassedIn.WriteDebug("Response is empty"); + //_cmdletPassedIn.WriteDebug("Response is empty"); } return content; @@ -1276,7 +1276,7 @@ private string FindNameGlobbingWithTag(string packageName, string[] tags, Resour /// private string FindVersionGlobbing(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, int skip, bool getOnlyLatest, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion gt '1.0.0' and NormalizedVersion lt '2.2.5' and substringof('PSModule', Tags) eq true //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= NormalizedVersion gt '1.1.1' and NormalizedVersion lt '2.2.5' // NormalizedVersion doesn't include trailing zeroes @@ -1382,7 +1382,7 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange /// private Stream InstallVersion(string packageName, string version, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersion()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersion()"); string requestUrlV2; if (_isADORepo) @@ -1483,7 +1483,7 @@ public int GetCountFromResponse(string httpResponse, out ErrorRecord errRecord) } else { - _cmdletPassedIn.WriteDebug($"Property 'count' and 'd:Id' could not be found in response. This may indicate that the package could not be found"); + //_cmdletPassedIn.WriteDebug($"Property 'count' and 'd:Id' could not be found in response. This may indicate that the package could not be found"); } } } From 4e4741f3dec58be167d4901ab3debd1f615be881 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Tue, 23 Dec 2025 20:35:30 -0800 Subject: [PATCH 2/6] Use threadsafe types and add better use of _knownLatestPkgVersion --- src/code/FindHelper.cs | 179 +++++++++++++++++++++++++++-------------- 1 file changed, 119 insertions(+), 60 deletions(-) diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index b1d5f9ddc..361587adc 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -5,6 +5,7 @@ using NuGet.Protocol.Core.Types; using NuGet.Versioning; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Management.Automation; @@ -41,9 +42,11 @@ internal class FindHelper // TODO: Update to be concurrency safe; TryAdd needs to be updates as well // TODO: look at # of allocations (lists, etc.) - private Dictionary> _packagesFound; - private HashSet _knownLatestPkgVersion; - List depPkgsFound; + private ConcurrentDictionary> _packagesFound; + private ConcurrentDictionary _knownLatestPkgVersion; + // Using ConcurrentDictionary and ignoring values in order to use thread-safe type. + // Only 'key' is used, value is arbitrary value. + ConcurrentDictionary depPkgsFound; #endregion @@ -56,8 +59,8 @@ public FindHelper(CancellationToken cancellationToken, PSCmdlet cmdletPassedIn, _cancellationToken = cancellationToken; _cmdletPassedIn = cmdletPassedIn; _networkCredential = networkCredential; - _packagesFound = new Dictionary>(StringComparer.OrdinalIgnoreCase); - _knownLatestPkgVersion = new HashSet(StringComparer.OrdinalIgnoreCase); + _packagesFound = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); + _knownLatestPkgVersion = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); } #endregion @@ -1068,22 +1071,26 @@ private bool TryAddToPackagesFound(PSResourceInfo foundPkg) if (_packagesFound.ContainsKey(foundPkgName)) { - List pkgVersions = _packagesFound[foundPkgName] as List; + _packagesFound.TryGetValue(foundPkgName, out List pkgVersions); if (!pkgVersions.Contains(foundPkgVersion)) { - pkgVersions.Add(foundPkgVersion); - _packagesFound[foundPkgName] = pkgVersions; + List newPkgVersions = new List(pkgVersions) + { + foundPkgVersion + }; + _packagesFound.TryUpdate(foundPkgName, newPkgVersions, pkgVersions); + addedToHash = true; } } else { - _packagesFound.Add(foundPkg.Name, new List { foundPkgVersion }); + _packagesFound.TryAdd(foundPkg.Name, new List { foundPkgVersion }); addedToHash = true; } - _cmdletPassedIn.WriteDebug($"Found package '{foundPkg.Name}' version '{foundPkg.Version}'"); + _cmdletPassedIn.WriteDebug($"Found package '{foundPkgName}' version '{foundPkgVersion}'"); return addedToHash; } @@ -1107,25 +1114,28 @@ private string FormatPkgVersionString(PSResourceInfo pkg) internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { - depPkgsFound = new List(); + depPkgsFound = new ConcurrentDictionary(); FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); - foreach (var pkg in depPkgsFound) + foreach (KeyValuePair entry in depPkgsFound) { - if (!_packagesFound.ContainsKey(pkg.Name)) + PSResourceInfo depPkg = entry.Key; + if (!_packagesFound.ContainsKey(depPkg.Name)) { - TryAddToPackagesFound(pkg); - yield return pkg; + TryAddToPackagesFound(depPkg); + yield return depPkg; } else { - List pkgVersions = _packagesFound[pkg.Name]; - // _packagesFound has item.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(pkg))) + if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) { - TryAddToPackagesFound(pkg); + // _packagesFound has item.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + TryAddToPackagesFound(depPkg); - yield return pkg; + yield return depPkg; + } } } } @@ -1138,9 +1148,12 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response if (currentPkg.Dependencies.Length > 0) { // If installing more than 5 packages, do so concurrently - if (currentPkg.Dependencies.Length > 5) + // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. + int processorCount = Environment.ProcessorCount; + int maxDegreeOfParallelism = processorCount * 5; + if (currentPkg.Dependencies.Length > processorCount) { - Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, dep => + Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, dep => { // Console.WriteLine($"FindDependencyPackages Processing number: {dep}, Thread ID: {Task.CurrentId}"); FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); @@ -1167,7 +1180,21 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS PSResourceInfo depPkg = null; if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) { - FindDependencyWithNoUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + // For no upper bound, check if we have cached latest version first + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedDepPkg)) + { + //_cmdletPassedIn.WriteDebug($"Using cached latest version for dependency '{dep.Name}': {cachedDepPkg.Version}"); + depPkg = cachedDepPkg; + if (!depPkgsFound.ContainsKey(depPkg)) + { + depPkgsFound.TryAdd(depPkg, true); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + } + else + { + depPkg = FindDependencyWithNoUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } } else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) { @@ -1190,25 +1217,26 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS else { depPkg = currentResult.returnedObject; - if (!depPkgsFound.Contains(depPkg)) + if (!depPkgsFound.ContainsKey(depPkg)) { // add pkg then find dependencies - depPkgsFound.Add(depPkg); + depPkgsFound.TryAdd(depPkg, true); FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } else { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) { - // // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); - - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); - + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); + } } } } @@ -1216,6 +1244,29 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS } else { + // For version ranges, check if we have a cached latest version that might satisfy the range + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedRangePkg)) + { + string cachedVersionStr = $"{cachedRangePkg.Version}"; + if (cachedRangePkg.IsPrerelease) + { + cachedVersionStr += $"-{cachedRangePkg.Prerelease}"; + } + + if (NuGetVersion.TryParse(cachedVersionStr, out NuGetVersion cachedVersion) + && dep.VersionRange.Satisfies(cachedVersion)) + { + //_cmdletPassedIn.WriteDebug($"Using cached version for dependency '{dep.Name}' that satisfies range '{dep.VersionRange}': {cachedRangePkg.Version}"); + depPkg = cachedRangePkg; + if (!depPkgsFound.ContainsKey(depPkg)) + { + depPkgsFound.TryAdd(depPkg, true); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + return; + } + } + FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); if (errRecord != null) { @@ -1278,25 +1329,26 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS } else { - if (!depPkgsFound.Contains(depPkg)) + if (!depPkgsFound.ContainsKey(depPkg)) { // // Console.WriteLine($"PackagesFound contains {depPkg.Name}"); // add pkg then find dependencies - depPkgsFound.Add(depPkg); + depPkgsFound.TryAdd(depPkg, true); FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } else { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) { - // // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + } } } } @@ -1309,7 +1361,12 @@ private PSResourceInfo FindDependencyWithNoUpperBound(Dependency dep, ServerApiC { PSResourceInfo depPkg = null; // _knownLatestPkgVersion will tell us if we already have the latest version available for a particular package. - if (!_knownLatestPkgVersion.Contains(dep.Name)) + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedLatestPkg)) + { + //_cmdletPassedIn.WriteDebug($"Using cached latest version for dependency '{dep.Name}': {cachedLatestPkg.Version}"); + return cachedLatestPkg; + } + else { FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); if (errRecord != null) @@ -1360,27 +1417,29 @@ private PSResourceInfo FindDependencyWithNoUpperBound(Dependency dep, ServerApiC } depPkg = currentResult.returnedObject; - if (!depPkgsFound.Contains(depPkg)) + // Cache the latest version PSResourceInfo for future lookups + _knownLatestPkgVersion.TryAdd(depPkg.Name, depPkg); + + if (!depPkgsFound.ContainsKey(depPkg)) { - _knownLatestPkgVersion.Add(depPkg.Name); - // add pkg then find dependencies - depPkgsFound.Add(depPkg); + depPkgsFound.TryAdd(depPkg, true); FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } else { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) { - // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); - - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); + } } } } @@ -1436,4 +1495,4 @@ private List ProcessPSResourceResult(PSResourceResult currentResult #endregion } -} \ No newline at end of file +} From cb925dca80c6598db40bd1a3abb6f6b9b563aa16 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Sat, 27 Dec 2025 01:48:11 -0800 Subject: [PATCH 3/6] save before changes --- src/code/InstallHelper.cs | 121 ++++++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 30 deletions(-) diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index b1a513b80..c06300479 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -56,6 +56,7 @@ internal class InstallHelper private string _tmpPath; private NetworkCredential _networkCredential; private HashSet _packagesOnMachine; + private static readonly ArrayPool _bufferPool = ArrayPool.Shared; #endregion @@ -565,12 +566,15 @@ private List InstallPackages( if (!skipDependencyCheck) { + Console.WriteLine($"~~~Finding Dependencies for {parentPkgObj.Name}~~~"); // Get the dependencies from the installed package. if (parentPkgObj.Dependencies.Length > 0) { bool depFindFailed = false; + // Get all dependency packages for Az, for each one foreach (PSResourceInfo depPkg in findHelper.FindDependencyPackages(currentServer, currentResponseUtil, parentPkgObj, repository)) { + Console.WriteLine($"~~~ entering foreach for for {depPkg.Name} ~~~"); if (depPkg == null) { depFindFailed = true; @@ -579,6 +583,7 @@ private List InstallPackages( if (String.Equals(depPkg.Name, parentPkgObj.Name, StringComparison.OrdinalIgnoreCase)) { + Console.WriteLine($"~~~ depPkg is the parent pkg ~~~"); continue; } @@ -592,6 +597,7 @@ private List InstallPackages( } string depPkgNameVersion = $"{depPkg.Name}{depPkg.Version.ToString()}"; + Console.WriteLine($"~~~ depPkgNameVersion is ${depPkgNameVersion} ~~~"); if (_packagesOnMachine.Contains(depPkgNameVersion) && !depPkg.IsPrerelease) { // if a dependency package is already installed, do not install it again. @@ -601,6 +607,7 @@ private List InstallPackages( continue; } + Console.WriteLine($"~~~ begin install ${depPkg.Name} ~~~"); packagesHash = BeginPackageInstall( searchVersionType: VersionType.SpecificVersion, specificVersion: depVersion, @@ -798,6 +805,7 @@ private Hashtable BeginPackageInstall( } // Check to see if the pkg is already installed (ie the pkg is installed and the version satisfies the version range provided via param) + // TODO: can use cache for this if (!_reinstall) { string currPkgNameVersion = $"{pkgToInstall.Name}{pkgToInstall.Version}"; @@ -896,11 +904,13 @@ private List FindAllDependencies(ServerApiCall currentServer, Re } var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); - _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{pkgToInstall.Name}'"); + _cmdletPassedIn.WriteDebug($" ** Finding dependency packages for (FindAllDependencies) '{pkgToInstall.Name}'"); // the last package added will be the parent package. List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + _cmdletPassedIn.WriteDebug($"** Retrieved all dep packages (FindAllDependencies) for '{pkgToInstall.Name}'"); + // allDependencies contains parent package as well foreach (PSResourceInfo pkg in allDependencies) { @@ -915,14 +925,19 @@ private Hashtable InstallParentAndDependencyPackages(PSResourceInfo parentPkg, L { List errors = new List(); // If installing more than 5 packages, do so concurrently - if (allDependencies.Count > 5) + int processorCount = Environment.ProcessorCount; + if (allDependencies.Count > processorCount) { // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) - Parallel.ForEach(allDependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, depPkg => + + // If installing more than 5 packages, do so concurrently + // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. + int maxDegreeOfParallelism = processorCount * 2; + Parallel.ForEach(allDependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, depPkg => { var depPkgName = depPkg.Name; var depPkgVersion = depPkg.Version.ToString(); - // Console.WriteLine($"Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); + Console.WriteLine($"+++++++++++++=Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); if (installNameErrRecord != null) @@ -1295,9 +1310,8 @@ private bool TrySaveNupkgToTempPath( } /// - /// Extracts files from .nupkg - /// Similar functionality as System.IO.Compression.ZipFile.ExtractToDirectory, - /// but while ExtractToDirectory cannot overwrite files, this method can. + /// Extracts files from .nupkg with optimized bulk operations + /// Uses buffer pooling and parallel processing for better performance. /// private bool TryExtractToDirectory(string zipPath, string extractPath, out ErrorRecord error) { @@ -1316,32 +1330,33 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error { using (ZipArchive archive = ZipFile.OpenRead(zipPath)) { - foreach (ZipArchiveEntry entry in archive.Entries.Where(entry => entry.CompressedLength > 0)) + // Categorize files by type for optimal processing + var manifestFiles = new List(); + var otherFiles = new List(); + + foreach (var entry in archive.Entries.Where(e => e.CompressedLength > 0)) { - // If a file has one or more parent directories. - if (entry.FullName.Contains(Path.DirectorySeparatorChar) || entry.FullName.Contains(Path.AltDirectorySeparatorChar)) - { - // Create the parent directories if they do not already exist - var lastPathSeparatorIdx = entry.FullName.Contains(Path.DirectorySeparatorChar) ? - entry.FullName.LastIndexOf(Path.DirectorySeparatorChar) : entry.FullName.LastIndexOf(Path.AltDirectorySeparatorChar); - var parentDirs = entry.FullName.Substring(0, lastPathSeparatorIdx); - var destinationDirectory = Path.Combine(extractPath, parentDirs); - if (!Directory.Exists(destinationDirectory)) - { - Directory.CreateDirectory(destinationDirectory); - } - } + string ext = Path.GetExtension(entry.Name).ToLowerInvariant(); + + if (ext == ".psd1" || ext == ".psm1" || ext == ".ps1") + manifestFiles.Add(entry); + else + otherFiles.Add(entry); + } - // Gets the full path to ensure that relative segments are removed. - string destinationPath = Path.GetFullPath(Path.Combine(extractPath, entry.FullName)); + // Extract critical files first (manifests) sequentially + foreach (var entry in manifestFiles) + { + ExtractEntryWithBufferPool(entry, extractPath); + } - // Validate that the resolved output path starts with the resolved destination directory. - // For example, if a zip file contains a file entry ..\sneaky-file, and the zip file is extracted to the directory c:\output, - // then naively combining the paths would result in an output file path of c:\output\..\sneaky-file, which would cause the file to be written to c:\sneaky-file. - if (destinationPath.StartsWith(extractPath, StringComparison.Ordinal)) - { - entry.ExtractToFile(destinationPath, overwrite: true); - } + // Extract other files in parallel + if (otherFiles.Count > 0) + { + Parallel.ForEach(otherFiles, new ParallelOptions + { + MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount - 1) // Reserve one core + }, entry => ExtractEntryWithBufferPool(entry, extractPath)); } } } @@ -1359,6 +1374,52 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error return true; } + /// + /// Extracts a single zip entry using buffer pooling for optimal performance + /// + private void ExtractEntryWithBufferPool(ZipArchiveEntry entry, string extractPath) + { + const int BUFFER_SIZE = 81920; // 80KB buffer + + // If a file has one or more parent directories. + if (entry.FullName.Contains(Path.DirectorySeparatorChar) || entry.FullName.Contains(Path.AltDirectorySeparatorChar)) + { + // Create the parent directories if they do not already exist + var lastPathSeparatorIdx = entry.FullName.Contains(Path.DirectorySeparatorChar) ? + entry.FullName.LastIndexOf(Path.DirectorySeparatorChar) : entry.FullName.LastIndexOf(Path.AltDirectorySeparatorChar); + var parentDirs = entry.FullName.Substring(0, lastPathSeparatorIdx); + var destinationDirectory = Path.Combine(extractPath, parentDirs); + if (!Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + } + + // Gets the full path to ensure that relative segments are removed. + string destinationPath = Path.GetFullPath(Path.Combine(extractPath, entry.FullName)); + + // Security check for directory traversal - validate that the resolved output path starts with the resolved destination directory. + if (!destinationPath.StartsWith(extractPath, StringComparison.Ordinal)) + return; + + byte[] buffer = _bufferPool.Rent(BUFFER_SIZE); + try + { + using var entryStream = entry.Open(); + using var fileStream = File.Create(destinationPath); + + int bytesRead; + while ((bytesRead = entryStream.Read(buffer, 0, buffer.Length)) > 0) + { + fileStream.Write(buffer, 0, bytesRead); + } + } + finally + { + _bufferPool.Return(buffer); + } + } + /// /// Moves package files/directories from the temp install path into the final install path location. /// From 5bf4ceb0195afeeb287c74cda5567e15c1f3f3f6 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Fri, 2 Jan 2026 15:52:38 -0800 Subject: [PATCH 4/6] Add async find and install calls --- src/code/ContainerRegistryServerAPICalls.cs | 10 + src/code/FindHelper.cs | 216 +++++++----- src/code/InstallHelper.cs | 116 ++----- src/code/LocalServerApiCalls.cs | 10 + src/code/NuGetServerAPICalls.cs | 9 + src/code/ServerApiCall.cs | 19 ++ src/code/V2ServerAPICalls.cs | 346 +++++++++++++++++++- src/code/V3ServerAPICalls.cs | 10 + 8 files changed, 576 insertions(+), 160 deletions(-) diff --git a/src/code/ContainerRegistryServerAPICalls.cs b/src/code/ContainerRegistryServerAPICalls.cs index 9c17c0db0..1954ed975 100644 --- a/src/code/ContainerRegistryServerAPICalls.cs +++ b/src/code/ContainerRegistryServerAPICalls.cs @@ -81,6 +81,16 @@ public ContainerRegistryServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmd #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return null; + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } + /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index 361587adc..deb1e2d77 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -896,7 +896,20 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R FindResults responses = null; if (_tag.Length == 0) { - responses = currentServer.FindVersion(pkgName, _nugetVersion.ToNormalizedString(), _type, out errRecord); + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { + string key = $"{pkgName}|{_nugetVersion.ToNormalizedString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(pkgName, _nugetVersion.ToNormalizedString(), _type)); + + responses = response.GetAwaiter().GetResult(); + + } + else { + responses = currentServer.FindVersion(pkgName, _nugetVersion.ToNormalizedString(), _type, out errRecord); + } + + } else { @@ -968,7 +981,22 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R FindResults responses = null; if (_tag.Length == 0) { - responses = currentServer.FindVersionGlobbing(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, out errRecord); + + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { + string key = $"{pkgName}|{_versionRange.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false)); + + responses = response.GetAwaiter().GetResult(); + + } + else { + responses = currentServer.FindVersionGlobbing(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, out errRecord); + + } + + } else { @@ -1035,7 +1063,7 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R { foreach (PSResourceInfo currentPkg in parentPkgs) { - _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{currentPkg.Name}'"); + _cmdletPassedIn.WriteDebug($"Finding dependency packages (SearchByNames) for '{currentPkg.Name}'"); foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) { yield return pkgDep; @@ -1090,7 +1118,7 @@ private bool TryAddToPackagesFound(PSResourceInfo foundPkg) addedToHash = true; } - _cmdletPassedIn.WriteDebug($"Found package '{foundPkgName}' version '{foundPkgVersion}'"); + _cmdletPassedIn.WriteDebug($"Found package -- '{foundPkgName}' version '{foundPkgVersion}'"); return addedToHash; } @@ -1114,11 +1142,15 @@ private string FormatPkgVersionString(PSResourceInfo pkg) internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { + _cmdletPassedIn.WriteDebug("In FindDependencyPackages 1"); depPkgsFound = new ConcurrentDictionary(); + _cmdletPassedIn.WriteDebug($"In FindDependencyPackages {currentPkg.Name}"); FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); + _cmdletPassedIn.WriteDebug("In FindDependencyPackages 2"); foreach (KeyValuePair entry in depPkgsFound) { + _cmdletPassedIn.WriteDebug("In FindDependencyPackages 3"); PSResourceInfo depPkg = entry.Key; if (!_packagesFound.ContainsKey(depPkg.Name)) { @@ -1139,19 +1171,23 @@ internal IEnumerable FindDependencyPackages(ServerApiCall curren } } } + _cmdletPassedIn.WriteDebug("In FindDependencyPackages 4"); // Everything up to here is quick } // Method 2 internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { + //_cmdletPassedIn.WriteDebug("In FindDependencyPackagesHelper ! "); + List errors = new List(); if (currentPkg.Dependencies.Length > 0) { - // If installing more than 5 packages, do so concurrently + // If finding more than 5 packages, do so concurrently // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. - int processorCount = Environment.ProcessorCount; - int maxDegreeOfParallelism = processorCount * 5; - if (currentPkg.Dependencies.Length > processorCount) + const int PARALLEL_THRESHOLD = 3; + int processorCount = Environment.ProcessorCount; // 8 + int maxDegreeOfParallelism = processorCount * 4; + if (currentPkg.Dependencies.Length > PARALLEL_THRESHOLD) { Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, dep => { @@ -1178,6 +1214,7 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) { PSResourceInfo depPkg = null; + ErrorRecord errRecord = null; if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) { // For no upper bound, check if we have cached latest version first @@ -1198,7 +1235,21 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS } else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) { - FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); + + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + FindResults responses = null; + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type)); + + responses = response.GetAwaiter().GetResult(); + + } + else { + responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out errRecord); + } + if (errRecord != null) { errors = ProcessErrorRecord(errRecord, errors); @@ -1267,7 +1318,23 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS } } - FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + FindResults responses = null; + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true)); + + responses = response.GetAwaiter().GetResult(); + + } + else { + responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out errRecord); + } + + + + if (errRecord != null) { errors = ProcessErrorRecord(errRecord, errors); @@ -1360,89 +1427,82 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS private PSResourceInfo FindDependencyWithNoUpperBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) { PSResourceInfo depPkg = null; - // _knownLatestPkgVersion will tell us if we already have the latest version available for a particular package. - if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedLatestPkg)) - { - //_cmdletPassedIn.WriteDebug($"Using cached latest version for dependency '{dep.Name}': {cachedLatestPkg.Version}"); - return cachedLatestPkg; - } - else + + FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); + if (errRecord != null) { - FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); - if (errRecord != null) + if (errRecord.Exception is ResourceNotFoundException) { - if (errRecord.Exception is ResourceNotFoundException) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else - { - // todo add error here? - } - - return depPkg; + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); } - - if (responses == null) + else { - // todo error - return depPkg; + // todo add error here? } - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); + return depPkg; + } - return depPkg; - } - else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); + if (responses == null) + { + // todo error + return depPkg; + } - return depPkg; - } + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); - depPkg = currentResult.returnedObject; - // Cache the latest version PSResourceInfo for future lookups - _knownLatestPkgVersion.TryAdd(depPkg.Name, depPkg); - - if (!depPkgsFound.ContainsKey(depPkg)) - { - // add pkg then find dependencies - depPkgsFound.TryAdd(depPkg, true); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - } - else + return depPkg; + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + + return depPkg; + } + + depPkg = currentResult.returnedObject; + // Cache the latest version PSResourceInfo for future lookups + _knownLatestPkgVersion.TryAdd(depPkg.Name, depPkg); + + if (!depPkgsFound.ContainsKey(depPkg)) + { + // add pkg then find dependencies + depPkgsFound.TryAdd(depPkg, true); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + else + { + if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) { - if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) { - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); + // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); - } + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); } } } + return depPkg; } diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index c06300479..d63a111d6 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -56,7 +56,6 @@ internal class InstallHelper private string _tmpPath; private NetworkCredential _networkCredential; private HashSet _packagesOnMachine; - private static readonly ArrayPool _bufferPool = ArrayPool.Shared; #endregion @@ -571,7 +570,6 @@ private List InstallPackages( if (parentPkgObj.Dependencies.Length > 0) { bool depFindFailed = false; - // Get all dependency packages for Az, for each one foreach (PSResourceInfo depPkg in findHelper.FindDependencyPackages(currentServer, currentResponseUtil, parentPkgObj, repository)) { Console.WriteLine($"~~~ entering foreach for for {depPkg.Name} ~~~"); @@ -809,6 +807,7 @@ private Hashtable BeginPackageInstall( if (!_reinstall) { string currPkgNameVersion = $"{pkgToInstall.Name}{pkgToInstall.Version}"; + // Use HashSet lookup instead of Contains for O(1) performance if (_packagesOnMachine.Contains(currPkgNameVersion)) { _cmdletPassedIn.WriteWarning($"Resource '{pkgToInstall.Name}' with version '{pkgVersion}' is already installed. If you would like to reinstall, please run the cmdlet again with the -Reinstall parameter"); @@ -904,12 +903,12 @@ private List FindAllDependencies(ServerApiCall currentServer, Re } var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); - _cmdletPassedIn.WriteDebug($" ** Finding dependency packages for (FindAllDependencies) '{pkgToInstall.Name}'"); + _cmdletPassedIn.WriteDebug($" **Finding dependency packages for (FindAllDependencies) '{pkgToInstall.Name}'"); // the last package added will be the parent package. List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); - _cmdletPassedIn.WriteDebug($"** Retrieved all dep packages (FindAllDependencies) for '{pkgToInstall.Name}'"); + _cmdletPassedIn.WriteDebug($"~~~ Retrieved all dep packages (FindAllDependencies) for '{pkgToInstall.Name}'"); // allDependencies contains parent package as well foreach (PSResourceInfo pkg in allDependencies) @@ -923,21 +922,22 @@ private List FindAllDependencies(ServerApiCall currentServer, Re // Concurrently Updates private Hashtable InstallParentAndDependencyPackages(PSResourceInfo parentPkg, List allDependencies, ServerApiCall currentServer, string tempInstallPath, Hashtable packagesHash, Hashtable updatedPackagesHash, PSResourceInfo pkgToInstall) { - List errors = new List(); - // If installing more than 5 packages, do so concurrently + List errors = new List(allDependencies.Count); // Pre-allocate based on expected size + // Use a lower threshold for better parallelism - 3 packages instead of processor count + const int PARALLEL_THRESHOLD = 3; int processorCount = Environment.ProcessorCount; - if (allDependencies.Count > processorCount) + if (allDependencies.Count > PARALLEL_THRESHOLD) { // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) // If installing more than 5 packages, do so concurrently // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. - int maxDegreeOfParallelism = processorCount * 2; + int maxDegreeOfParallelism = processorCount * 4; Parallel.ForEach(allDependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, depPkg => { var depPkgName = depPkg.Name; var depPkgVersion = depPkg.Version.ToString(); - Console.WriteLine($"+++++++++++++=Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); + // Console.WriteLine($"Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); if (installNameErrRecord != null) @@ -1310,8 +1310,9 @@ private bool TrySaveNupkgToTempPath( } /// - /// Extracts files from .nupkg with optimized bulk operations - /// Uses buffer pooling and parallel processing for better performance. + /// Extracts files from .nupkg + /// Similar functionality as System.IO.Compression.ZipFile.ExtractToDirectory, + /// but while ExtractToDirectory cannot overwrite files, this method can. /// private bool TryExtractToDirectory(string zipPath, string extractPath, out ErrorRecord error) { @@ -1330,33 +1331,32 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error { using (ZipArchive archive = ZipFile.OpenRead(zipPath)) { - // Categorize files by type for optimal processing - var manifestFiles = new List(); - var otherFiles = new List(); - - foreach (var entry in archive.Entries.Where(e => e.CompressedLength > 0)) + foreach (ZipArchiveEntry entry in archive.Entries.Where(entry => entry.CompressedLength > 0)) { - string ext = Path.GetExtension(entry.Name).ToLowerInvariant(); - - if (ext == ".psd1" || ext == ".psm1" || ext == ".ps1") - manifestFiles.Add(entry); - else - otherFiles.Add(entry); - } + // If a file has one or more parent directories. + if (entry.FullName.Contains(Path.DirectorySeparatorChar) || entry.FullName.Contains(Path.AltDirectorySeparatorChar)) + { + // Create the parent directories if they do not already exist + var lastPathSeparatorIdx = entry.FullName.Contains(Path.DirectorySeparatorChar) ? + entry.FullName.LastIndexOf(Path.DirectorySeparatorChar) : entry.FullName.LastIndexOf(Path.AltDirectorySeparatorChar); + var parentDirs = entry.FullName.Substring(0, lastPathSeparatorIdx); + var destinationDirectory = Path.Combine(extractPath, parentDirs); + if (!Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + } - // Extract critical files first (manifests) sequentially - foreach (var entry in manifestFiles) - { - ExtractEntryWithBufferPool(entry, extractPath); - } + // Gets the full path to ensure that relative segments are removed. + string destinationPath = Path.GetFullPath(Path.Combine(extractPath, entry.FullName)); - // Extract other files in parallel - if (otherFiles.Count > 0) - { - Parallel.ForEach(otherFiles, new ParallelOptions - { - MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount - 1) // Reserve one core - }, entry => ExtractEntryWithBufferPool(entry, extractPath)); + // Validate that the resolved output path starts with the resolved destination directory. + // For example, if a zip file contains a file entry ..\sneaky-file, and the zip file is extracted to the directory c:\output, + // then naively combining the paths would result in an output file path of c:\output\..\sneaky-file, which would cause the file to be written to c:\sneaky-file. + if (destinationPath.StartsWith(extractPath, StringComparison.Ordinal)) + { + entry.ExtractToFile(destinationPath, overwrite: true); + } } } } @@ -1374,52 +1374,6 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error return true; } - /// - /// Extracts a single zip entry using buffer pooling for optimal performance - /// - private void ExtractEntryWithBufferPool(ZipArchiveEntry entry, string extractPath) - { - const int BUFFER_SIZE = 81920; // 80KB buffer - - // If a file has one or more parent directories. - if (entry.FullName.Contains(Path.DirectorySeparatorChar) || entry.FullName.Contains(Path.AltDirectorySeparatorChar)) - { - // Create the parent directories if they do not already exist - var lastPathSeparatorIdx = entry.FullName.Contains(Path.DirectorySeparatorChar) ? - entry.FullName.LastIndexOf(Path.DirectorySeparatorChar) : entry.FullName.LastIndexOf(Path.AltDirectorySeparatorChar); - var parentDirs = entry.FullName.Substring(0, lastPathSeparatorIdx); - var destinationDirectory = Path.Combine(extractPath, parentDirs); - if (!Directory.Exists(destinationDirectory)) - { - Directory.CreateDirectory(destinationDirectory); - } - } - - // Gets the full path to ensure that relative segments are removed. - string destinationPath = Path.GetFullPath(Path.Combine(extractPath, entry.FullName)); - - // Security check for directory traversal - validate that the resolved output path starts with the resolved destination directory. - if (!destinationPath.StartsWith(extractPath, StringComparison.Ordinal)) - return; - - byte[] buffer = _bufferPool.Rent(BUFFER_SIZE); - try - { - using var entryStream = entry.Open(); - using var fileStream = File.Create(destinationPath); - - int bytesRead; - while ((bytesRead = entryStream.Read(buffer, 0, buffer.Length)) > 0) - { - fileStream.Write(buffer, 0, bytesRead); - } - } - finally - { - _bufferPool.Return(buffer); - } - } - /// /// Moves package files/directories from the temp install path into the final install path location. /// diff --git a/src/code/LocalServerApiCalls.cs b/src/code/LocalServerApiCalls.cs index cc43c340d..cac3e6ccb 100644 --- a/src/code/LocalServerApiCalls.cs +++ b/src/code/LocalServerApiCalls.cs @@ -13,6 +13,7 @@ using System.Net; using System.Management.Automation; using System.Runtime.ExceptionServices; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -39,6 +40,15 @@ public LocalServerAPICalls (PSRepositoryInfo repository, PSCmdlet cmdletPassedIn #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return null; + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Examples: Search -Repository PSGallery diff --git a/src/code/NuGetServerAPICalls.cs b/src/code/NuGetServerAPICalls.cs index 1497c83da..10c540b70 100644 --- a/src/code/NuGetServerAPICalls.cs +++ b/src/code/NuGetServerAPICalls.cs @@ -48,6 +48,15 @@ public NuGetServerAPICalls (PSRepositoryInfo repository, PSCmdlet cmdletPassedIn #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return null; + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Examples: Search -Repository MyNuGetServer diff --git a/src/code/ServerApiCall.cs b/src/code/ServerApiCall.cs index 4580e362e..bdb97db42 100644 --- a/src/code/ServerApiCall.cs +++ b/src/code/ServerApiCall.cs @@ -10,6 +10,7 @@ using System.Text; using System.Runtime.ExceptionServices; using System.Management.Automation; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -118,6 +119,24 @@ public ServerApiCall(PSRepositoryInfo repository, NetworkCredential networkCrede /// public abstract FindResults FindVersion(string packageName, string version, ResourceType type, out ErrorRecord errRecord); + /// + /// Find method which allows for searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// Examples: Search "PowerShellGet" "2.2.5" + /// + public abstract Task FindVersionAsync(string packageName, string version, ResourceType type); + + + /// + /// Find method which allows for searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// Examples: Search "PowerShellGet" "2.2.5" + /// + + public abstract Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest); + /// /// Find method which allows for searching for single name with specific version. /// Name: no wildcard support diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index 4fc499cc5..ea93ec1a9 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -694,6 +694,64 @@ public override FindResults FindVersion(string packageName, string version, Reso return new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); } + + public override async Task FindVersionAsync(string packageName, string version, ResourceType type) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionAsync()"); + // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion eq '1.1.0' and substringof('PSModule', Tags) eq true + // Quotations around package name and version do not matter, same metadata gets returned. + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary{ + { "$inlinecount", "allpages" }, + { "id", $"'{packageName}'" }, + }); + var filterBuilder = queryBuilder.FilterBuilder; + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + filterBuilder.AddCriterion($"NormalizedVersion eq '{version}'"); + if (type != ResourceType.None) { + filterBuilder.AddCriterion(GetTypeFilterForRequest(type)); + } + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + string response; + try + { + response = await HttpRequestCallAsync(requestUrlV2); + } + catch (Exception e) + { + // usually this is for errors in calling the V2 server, but for ADO V2 this error will include package not found errors which we want to deliver with a standard message + if (_isADORepo && e is ResourceNotFoundException) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' and version '{version}' could not be found in repository '{Repository.Name}'. For ADO feed, if the package is in an upstream feed make sure you are authenticated to the upstream feed.", e); + } + + throw; + } + + int count = GetCountFromResponse(response, out ErrorRecord errRecord); + //_cmdletPassedIn.WriteDebug($"Count from response is '{count}'"); + + if (errRecord != null) + { + throw errRecord.Exception; + } + + if (count == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}', version '{version}' could not be found in repository '{Repository.Name}'."); + } + + return new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + /// /// Find method which allows for searching for single name with specific version and tag. /// Name: no wildcard support @@ -767,6 +825,7 @@ public override FindResults FindVersionWithTag(string packageName, string versio /// public override Stream InstallPackage(string packageName, string packageVersion, bool includePrerelease, out ErrorRecord errRecord) { + errRecord = null; Stream results = new MemoryStream(); if (string.IsNullOrEmpty(packageVersion)) { @@ -779,7 +838,8 @@ public override Stream InstallPackage(string packageName, string packageVersion, return results; } - results = InstallVersion(packageName, packageVersion, out errRecord); + //results = InstallVersion(packageName, packageVersion, out errRecord); + results = InstallVersionAsync(packageName, packageVersion).GetAwaiter().GetResult(); return results; } @@ -840,9 +900,110 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) return response; } + /// + /// Helper method that makes the HTTP request for the V2 server protocol url passed in for find APIs. + /// + private async Task HttpRequestCallAsync(string requestUrlV2) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCall()"); + string response = string.Empty; + + try + { + // _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); + + response = await SendV2RequestAsync(request, _sessionClient); + } + catch (ResourceNotFoundException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "ResourceNotFound", + // ErrorCategory.InvalidResult, + // this); + } + catch (UnauthorizedException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "UnauthorizedRequest", + // ErrorCategory.InvalidResult, + // this); + } + catch (HttpRequestException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestCallFailure", + // ErrorCategory.ConnectionError, + // this); + } + catch (Exception) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestCallFailure", + // ErrorCategory.ConnectionError, + // this); + } + + if (string.IsNullOrEmpty(response)) + { + // _cmdletPassedIn.WriteDebug("Response is empty"); + } + + return response; + } + /// /// Helper method that makes the HTTP request for the V2 server protocol url passed in for install APIs. /// + private async Task HttpRequestCallForContentAsync(string requestUrlV2) + { + // _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContentAsync()"); + HttpContent content = null; + + try + { + //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); + + content = await SendV2RequestForContentAsync(request, _sessionClient); + } + catch (HttpRequestException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestFailure", + // ErrorCategory.ConnectionError, + // this); + } + catch (ArgumentNullException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestFailure", + // ErrorCategory.InvalidData, + // this); + } + catch (InvalidOperationException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestFailure", + // ErrorCategory.InvalidOperation, + // this); + } + + if (content == null || string.IsNullOrEmpty(content.ToString())) + { + //_cmdletPassedIn.WriteDebug("Response is empty"); + } + + return content; + } + private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorRecord errRecord) { // _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); @@ -1372,6 +1533,157 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange return HttpRequestCall(requestUrlV2, out errRecord); } + + public override async Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + List responses = new List(); + int skip = 0; + + var initialResponse = await FindVersionGlobbingAsync(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest); + // if (errRecord != null) + // { + // return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + // } + + int initialCount = GetCountFromResponse(initialResponse, out ErrorRecord errRecord); + if (errRecord != null) + { + return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + + if (initialCount == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'."); + } + + responses.Add(initialResponse); + + if (!getOnlyLatest) + { + int count = (int)Math.Ceiling((double)(initialCount / 100)); + + while (count > 0) + { + _cmdletPassedIn.WriteDebug($"Count is '{count}'"); + // skip 100 + skip += 100; + var tmpResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest, out errRecord); + if (errRecord != null) + { + return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + responses.Add(tmpResponse); + count--; + } + } + + return new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + + + /// + /// Helper method for string[] FindVersionGlobbing() + /// + private async Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, int skip, bool getOnlyLatest) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion gt '1.0.0' and NormalizedVersion lt '2.2.5' and substringof('PSModule', Tags) eq true + //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= NormalizedVersion gt '1.1.1' and NormalizedVersion lt '2.2.5' + // NormalizedVersion doesn't include trailing zeroes + // Notes: this could allow us to take a version range (i.e (2.0.0, 3.0.0.0]) and deconstruct it and add options to the Filter for Version to describe that range + // will need to filter additionally, if IncludePrerelease=false, by default we get stable + prerelease both back + // Current bug: Find PSGet -Version "2.0.*" -> https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= Version gt '2.0.*' and Version lt '2.1' + // Make sure to include quotations around the package name + + //and IsPrerelease eq false + // ex: + // (2.0.0, 3.0.0) + // $filter= NVersion gt '2.0.0' and NVersion lt '3.0.0' + + // [2.0.0, 3.0.0] + // $filter= NVersion ge '2.0.0' and NVersion le '3.0.0' + + // [2.0.0, 3.0.0) + // $filter= NVersion ge '2.0.0' and NVersion lt '3.0.0' + + // (2.0.0, 3.0.0] + // $filter= NVersion gt '2.0.0' and NVersion le '3.0.0' + + // [, 2.0.0] + // $filter= NVersion le '2.0.0' + + string format = "NormalizedVersion {0} {1}"; + string minPart = String.Empty; + string maxPart = String.Empty; + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary { + {"$inlinecount", "allpages"}, + {"$skip", skip.ToString()}, + {"$orderby", "NormalizedVersion desc"}, + {"id", $"'{packageName}'"} + }); + + var filterBuilder = queryBuilder.FilterBuilder; + + if (versionRange.MinVersion != null) + { + string operation = versionRange.IsMinInclusive ? "ge" : "gt"; + minPart = String.Format(format, operation, $"'{versionRange.MinVersion.ToNormalizedString()}'"); + } + + if (versionRange.MaxVersion != null) + { + string operation = versionRange.IsMaxInclusive ? "le" : "lt"; + // Adding '9' as a digit to the end of the patch portion of the version + // because we want to retrieve all the prerelease versions for the upper end of the range + // and PSGallery views prerelease as higher than its stable. + // eg 3.0.0-prerelease > 3.0.0 + // If looking for versions within '[1.9.9,1.9.9]' including prerelease values, this will change it to search for '[1.9.9,1.9.99]' + // and find any pkg versions that are 1.9.9-prerelease. + string maxString = includePrerelease ? $"{versionRange.MaxVersion.Major}.{versionRange.MaxVersion.Minor}.{versionRange.MaxVersion.Patch.ToString() + "9"}" : + $"{versionRange.MaxVersion.ToNormalizedString()}"; + if (NuGetVersion.TryParse(maxString, out NuGetVersion maxVersion)) + { + maxPart = String.Format(format, operation, $"'{maxVersion.ToNormalizedString()}'"); + } + else { + maxPart = String.Format(format, operation, $"'{versionRange.MaxVersion.ToNormalizedString()}'"); + } + } + + string versionFilterParts = String.Empty; + if (!String.IsNullOrEmpty(minPart)) + { + filterBuilder.AddCriterion(minPart); + } + if (!String.IsNullOrEmpty(maxPart)) + { + filterBuilder.AddCriterion(maxPart); + } + if (!includePrerelease) { + filterBuilder.AddCriterion("IsPrerelease eq false"); + } + + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + if (type == ResourceType.Script) { + filterBuilder.AddCriterion($"substringof('PS{type.ToString()}', Tags) eq true"); + } + + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + + return await HttpRequestCallAsync(requestUrlV2); + } + + /// /// Installs package with specific name and version. /// Name: no wildcard support. @@ -1421,6 +1733,38 @@ private Stream InstallVersion(string packageName, string version, out ErrorRecor return response.ReadAsStreamAsync().Result; } + private async Task InstallVersionAsync(string packageName, string version) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersionAsync()"); + string requestUrlV2; + + if (_isADORepo) + { + // eg: https://pkgs.dev.azure.com///_packaging//nuget/v2?id=test_module&version=5.0.0 + requestUrlV2 = $"{Repository.Uri}?id={packageName.ToLower()}&version={version}"; + } + else if (_isJFrogRepo) + { + // eg: https://.jfrog.io/artifactory/api/nuget//Download/test_module/5.0.0 + requestUrlV2 = $"{Repository.Uri}/Download/{packageName}/{version}"; + } + else + { + requestUrlV2 = $"{Repository.Uri}/package/{packageName}/{version}"; + } + + + var response = await HttpRequestCallForContentAsync(requestUrlV2); + + + if (response is null) + { + throw new Exception($"No content was returned by repository '{Repository.Name}'"); + } + + return await response.ReadAsStreamAsync(); + } + private string GetTypeFilterForRequest(ResourceType type) { string typeFilterPart = string.Empty; if (type == ResourceType.Script) diff --git a/src/code/V3ServerAPICalls.cs b/src/code/V3ServerAPICalls.cs index 903b1da55..97d4c2298 100644 --- a/src/code/V3ServerAPICalls.cs +++ b/src/code/V3ServerAPICalls.cs @@ -88,6 +88,16 @@ public V3ServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmdletPassedIn, Ne #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return Task.FromResult(null); + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } + /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Not supported for V3 repository. From 329eb3968763d1058e2f7dd7ec0f79fb8be4d55a Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Wed, 21 Jan 2026 12:47:17 -0800 Subject: [PATCH 5/6] Add async calls --- src/code/ContainerRegistryServerAPICalls.cs | 6 + src/code/FindHelper.cs | 434 +++++++++----------- src/code/InstallHelper.cs | 148 +------ src/code/LocalServerApiCalls.cs | 5 + src/code/NuGetServerAPICalls.cs | 5 + src/code/ServerApiCall.cs | 7 + src/code/V2ServerAPICalls.cs | 80 +++- src/code/V3ServerAPICalls.cs | 5 + 8 files changed, 312 insertions(+), 378 deletions(-) diff --git a/src/code/ContainerRegistryServerAPICalls.cs b/src/code/ContainerRegistryServerAPICalls.cs index 1954ed975..20f084f10 100644 --- a/src/code/ContainerRegistryServerAPICalls.cs +++ b/src/code/ContainerRegistryServerAPICalls.cs @@ -156,6 +156,12 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[] { }, hashtableResponse: pkgResult.ToArray(), responseType: containerRegistryFindResponseType); } + + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index deb1e2d77..cdc1c1cfe 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -1,20 +1,22 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using Microsoft.PowerShell.PSResourceGet.UtilClasses; -using NuGet.Protocol.Core.Types; -using NuGet.Versioning; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Management.Automation; using System.Net; -using System.Security.Cryptography; using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Azure; +using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -40,13 +42,18 @@ internal class FindHelper private bool _repositoryNameContainsWildcard = true; private NetworkCredential _networkCredential; - // TODO: Update to be concurrency safe; TryAdd needs to be updates as well - // TODO: look at # of allocations (lists, etc.) + // Gets intantiated each time a cmdlet is run. + // If running 'Install-PSResource Az, TestModule, NewTestModule', it will contain one parent and its dependencies. private ConcurrentDictionary> _packagesFound; + + // Creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + private ConcurrentDictionary depPkgsFound; + + // Contains the latest found version of a particular package. private ConcurrentDictionary _knownLatestPkgVersion; - // Using ConcurrentDictionary and ignoring values in order to use thread-safe type. - // Only 'key' is used, value is arbitrary value. - ConcurrentDictionary depPkgsFound; + + ConcurrentDictionary> _cachedNetworkCalls; #endregion @@ -61,6 +68,8 @@ public FindHelper(CancellationToken cancellationToken, PSCmdlet cmdletPassedIn, _networkCredential = networkCredential; _packagesFound = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); _knownLatestPkgVersion = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + _type = ResourceType.None; + _cachedNetworkCalls = new ConcurrentDictionary>(); } #endregion @@ -1090,12 +1099,12 @@ private HashSet GetPackageNamesPopulated(string[] pkgNames) return pkgsToDiscover; } - private bool TryAddToPackagesFound(PSResourceInfo foundPkg) - { + { + // This handles prerelease versions as well. bool addedToHash = false; string foundPkgName = foundPkg.Name; - string foundPkgVersion = Utils.GetNormalizedVersionString(foundPkg.Version.ToString(), foundPkg.Prerelease); + string foundPkgVersion = FormatPkgVersionString(foundPkg); if (_packagesFound.ContainsKey(foundPkgName)) { @@ -1142,48 +1151,22 @@ private string FormatPkgVersionString(PSResourceInfo pkg) internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { - _cmdletPassedIn.WriteDebug("In FindDependencyPackages 1"); depPkgsFound = new ConcurrentDictionary(); _cmdletPassedIn.WriteDebug($"In FindDependencyPackages {currentPkg.Name}"); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); - _cmdletPassedIn.WriteDebug("In FindDependencyPackages 2"); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); - foreach (KeyValuePair entry in depPkgsFound) - { - _cmdletPassedIn.WriteDebug("In FindDependencyPackages 3"); - PSResourceInfo depPkg = entry.Key; - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - TryAddToPackagesFound(depPkg); - yield return depPkg; - } - else - { - if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) - { - // _packagesFound has item.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - TryAddToPackagesFound(depPkg); + _cmdletPassedIn.WriteDebug("In FindDependencyPackages 2"); - yield return depPkg; - } - } - } - } - _cmdletPassedIn.WriteDebug("In FindDependencyPackages 4"); // Everything up to here is quick + return depPkgsFound.Keys.ToList(); } // Method 2 internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { - //_cmdletPassedIn.WriteDebug("In FindDependencyPackagesHelper ! "); - List errors = new List(); if (currentPkg.Dependencies.Length > 0) { - // If finding more than 5 packages, do so concurrently - // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. + // If finding more than 3 packages, do so concurrently const int PARALLEL_THRESHOLD = 3; int processorCount = Environment.ProcessorCount; // 8 int maxDegreeOfParallelism = processorCount * 4; @@ -1214,13 +1197,16 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) { PSResourceInfo depPkg = null; - ErrorRecord errRecord = null; + //ErrorRecord errRecord = null; + if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) { - // For no upper bound, check if we have cached latest version first + // Case 1: No upper bound, eg: "*" or "(1.0.0, )" if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedDepPkg)) { - //_cmdletPassedIn.WriteDebug($"Using cached latest version for dependency '{dep.Name}': {cachedDepPkg.Version}"); + // 1) Check if the latest version is cached + // if a package is already cached, its dependencies will have been cached as well. + // But if for some reason it wasn't added to depPkgs, add it now depPkg = cachedDepPkg; if (!depPkgsFound.ContainsKey(depPkg)) { @@ -1230,284 +1216,244 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS } else { - depPkg = FindDependencyWithNoUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + // 2) Find this version from the server + depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); } + } else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) { - - ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); - FindResults responses = null; - Task response = null; - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { - string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; - response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type)); - - responses = response.GetAwaiter().GetResult(); - - } - else { - responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out errRecord); - } - - if (errRecord != null) - { - errors = ProcessErrorRecord(errRecord, errors); - } - else + // Case 2: Exact package version, eg: "1.0.0" or "[1.0.0, 1.0.0]" + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedDepPkg)) { - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + // 1) Check if the latest version is cached, and if this latest version is the version we're looking for + if (!NuGetVersion.TryParse(cachedDepPkg.Version.ToString(), out NuGetVersion cachedPkgVersion)) { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); + // write error } - else + + if (dep.VersionRange.Satisfies(cachedPkgVersion)) { - depPkg = currentResult.returnedObject; + // if a package is already cached, its dependencies will have been cached as well. + // But if for some reason it wasn't added to depPkgs, add it now + depPkg = cachedDepPkg; if (!depPkgsFound.ContainsKey(depPkg)) { - // add pkg then find dependencies depPkgsFound.TryAdd(depPkg, true); FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } - else - { - if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) - { - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); - - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); - } - } - } } } + else + { + // 2) Find this version from the server + depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } } else { - // For version ranges, check if we have a cached latest version that might satisfy the range + // Case 3: Version range with an upper bound, eg: "(1.0.0, 3.0.0)" if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedRangePkg)) - { - string cachedVersionStr = $"{cachedRangePkg.Version}"; - if (cachedRangePkg.IsPrerelease) + { + // 1) Check if the latest version is cached, and if this latest version is the version we're looking for + if (!NuGetVersion.TryParse(cachedRangePkg.Version.ToString(), out NuGetVersion cachedPkgVersion)) { - cachedVersionStr += $"-{cachedRangePkg.Prerelease}"; + // write error } - - if (NuGetVersion.TryParse(cachedVersionStr, out NuGetVersion cachedVersion) - && dep.VersionRange.Satisfies(cachedVersion)) + + if (dep.VersionRange.Satisfies(cachedPkgVersion)) { - //_cmdletPassedIn.WriteDebug($"Using cached version for dependency '{dep.Name}' that satisfies range '{dep.VersionRange}': {cachedRangePkg.Version}"); + // if a package is already cached, its dependencies will have been cached as well. + // But if for some reason it wasn't added to depPkgs, add it now depPkg = cachedRangePkg; if (!depPkgsFound.ContainsKey(depPkg)) { depPkgsFound.TryAdd(depPkg, true); FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } - return; - } + } } + else + { + depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + } + } - ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); - FindResults responses = null; - Task response = null; - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { - string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; - response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true)); - - responses = response.GetAwaiter().GetResult(); - } - else { - responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out errRecord); - } + // Method 4 + private PSResourceInfo FindDependencyWithSpecificVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + ErrorRecord errRecord = null; + FindResults responses = null; + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + { + // See if the network call we're making is already caced, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type)); - - if (errRecord != null) - { - errors = ProcessErrorRecord(errRecord, errors); - } + responses = response.GetAwaiter().GetResult(); + } + else + { + responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out errRecord); + } + - if (responses.IsFindResultsEmpty()) + // Error handling and Convert to PSResource object + if (errRecord != null) + { + errors = ProcessErrorRecord(errRecord, errors); + } + else + { + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) { errors.Add(new ErrorRecord( - new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), - "FindDepPackagesFindVersionGlobbingFailure", - ErrorCategory.InvalidResult, - this)); + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); } else { - foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) - { - if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - - // if (errRecord.Exception is ResourceNotFoundException) - // { - // _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - // } - // else - // { - // _cmdletPassedIn.WriteError(errRecord); - // } - // yield return null; - // continue; - } - else - { - // Check to see if version falls within version range - PSResourceInfo foundDep = currentResult.returnedObject; - string depVersionStr = $"{foundDep.Version}"; - if (foundDep.IsPrerelease) - { - depVersionStr += $"-{foundDep.Prerelease}"; - } - - if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) - && dep.VersionRange.Satisfies(depVersion)) - { - depPkg = foundDep; - break; - } - } - } - - if (depPkg == null) - { - // // Console.WriteLine($"depPkg is null and I don't know what this means"); - } - else + depPkg = currentResult.returnedObject; + if (!depPkgsFound.ContainsKey(depPkg)) { - if (!depPkgsFound.ContainsKey(depPkg)) - { - // // Console.WriteLine($"PackagesFound contains {depPkg.Name}"); - - // add pkg then find dependencies - depPkgsFound.TryAdd(depPkg, true); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - } - else - { - if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) - { - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - } - } - } + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(depPkg, true); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } } } + + return depPkg; } - // Method 4 - private PSResourceInfo FindDependencyWithNoUpperBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + // Method 5 + private PSResourceInfo FindDependencyWithLowerBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) { PSResourceInfo depPkg = null; + FindResults responses = null; + ErrorRecord errRecord = null; + Task response = null; - FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + { + // See if the network call we're making is already caced, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|*|{_type}"; + response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindNameAsync(dep.Name, includePrerelease: true, _type)); + + responses = response.GetAwaiter().GetResult(); + } + else + { + responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out errRecord); + } + + + // Error handling and Convert to PSResource object if (errRecord != null) { - if (errRecord.Exception is ResourceNotFoundException) + errors = ProcessErrorRecord(errRecord, errors); + } + else + { + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) { errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); } else { - // todo add error here? + depPkg = currentResult.returnedObject; + if (!depPkgsFound.ContainsKey(depPkg)) + { + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(depPkg, true); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } } - - return depPkg; } - if (responses == null) - { - // todo error - return depPkg; - } + return depPkg; + } - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null) + + + + // Method 6 + private PSResourceInfo FindDependencyWithUpperBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + + PSResourceInfo depPkg = null; + ErrorRecord errRecord = null; + FindResults responses = null; + Task response = null; + + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { - // This scenario may occur when the package version requested is unlisted. - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); + // See if the network call we're making is already caced, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true)); + + responses = response.GetAwaiter().GetResult(); - return depPkg; } - else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + else { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - - return depPkg; + responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out errRecord); } - depPkg = currentResult.returnedObject; - // Cache the latest version PSResourceInfo for future lookups - _knownLatestPkgVersion.TryAdd(depPkg.Name, depPkg); - - if (!depPkgsFound.ContainsKey(depPkg)) + + // Error handling and Convert to PSResource object + if (errRecord != null) { - // add pkg then find dependencies - depPkgsFound.TryAdd(depPkg, true); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + errors = ProcessErrorRecord(errRecord, errors); } else { - if (_packagesFound.TryGetValue(depPkg.Name, out List pkgVersions)) + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) { - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + if (!depPkgsFound.ContainsKey(depPkg)) { - // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); - - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(depPkg, true); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } } } - return depPkg; } - private List ProcessErrorRecord(ErrorRecord errRecord, List errors) { if (errRecord.Exception is ResourceNotFoundException) diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index d63a111d6..1f408cc0e 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -56,6 +56,7 @@ internal class InstallHelper private string _tmpPath; private NetworkCredential _networkCredential; private HashSet _packagesOnMachine; + private FindHelper _findHelper; #endregion @@ -67,6 +68,8 @@ public InstallHelper(PSCmdlet cmdletPassedIn, NetworkCredential networkCredentia _cancellationToken = source.Token; _cmdletPassedIn = cmdletPassedIn; _networkCredential = networkCredential; + + _findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); } /// @@ -511,6 +514,7 @@ private List InstallPackages( FindHelper findHelper) { _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackages()"); + List pkgsSuccessfullyInstalled = new(); // Install parent package to the temp directory, @@ -563,76 +567,6 @@ private List InstallPackages( Hashtable parentPkgInfo = packagesHash[parentPackage] as Hashtable; PSResourceInfo parentPkgObj = parentPkgInfo["psResourceInfoPkg"] as PSResourceInfo; - if (!skipDependencyCheck) - { - Console.WriteLine($"~~~Finding Dependencies for {parentPkgObj.Name}~~~"); - // Get the dependencies from the installed package. - if (parentPkgObj.Dependencies.Length > 0) - { - bool depFindFailed = false; - foreach (PSResourceInfo depPkg in findHelper.FindDependencyPackages(currentServer, currentResponseUtil, parentPkgObj, repository)) - { - Console.WriteLine($"~~~ entering foreach for for {depPkg.Name} ~~~"); - if (depPkg == null) - { - depFindFailed = true; - continue; - } - - if (String.Equals(depPkg.Name, parentPkgObj.Name, StringComparison.OrdinalIgnoreCase)) - { - Console.WriteLine($"~~~ depPkg is the parent pkg ~~~"); - continue; - } - - NuGetVersion depVersion = null; - if (depPkg.AdditionalMetadata.ContainsKey("NormalizedVersion")) - { - if (!NuGetVersion.TryParse(depPkg.AdditionalMetadata["NormalizedVersion"] as string, out depVersion)) - { - NuGetVersion.TryParse(depPkg.Version.ToString(), out depVersion); - } - } - - string depPkgNameVersion = $"{depPkg.Name}{depPkg.Version.ToString()}"; - Console.WriteLine($"~~~ depPkgNameVersion is ${depPkgNameVersion} ~~~"); - if (_packagesOnMachine.Contains(depPkgNameVersion) && !depPkg.IsPrerelease) - { - // if a dependency package is already installed, do not install it again. - // to determine if the package version is already installed, _packagesOnMachine is used but it only contains name, version info, not version with prerelease info - // if the dependency package is found to be prerelease, it is safer to install it (and worse case it reinstalls) - _cmdletPassedIn.WriteVerbose($"Dependency '{depPkg.Name}' with version '{depPkg.Version}' is already installed."); - continue; - } - - Console.WriteLine($"~~~ begin install ${depPkg.Name} ~~~"); - packagesHash = BeginPackageInstall( - searchVersionType: VersionType.SpecificVersion, - specificVersion: depVersion, - versionRange: null, - pkgNameToInstall: depPkg.Name, - repository: repository, - currentServer: currentServer, - currentResponseUtil: currentResponseUtil, - tempInstallPath: tempInstallPath, - skipDependencyCheck: skipDependencyCheck, - packagesHash: packagesHash, - errRecord: out ErrorRecord installPkgErrRecord); - - if (installPkgErrRecord != null) - { - _cmdletPassedIn.WriteError(installPkgErrRecord); - continue; - } - } - - if (depFindFailed) - { - continue; - } - } - } - // If -WhatIf is passed in, early out. if (_cmdletPassedIn.MyInvocation.BoundParameters.ContainsKey("WhatIf")) { @@ -864,15 +798,17 @@ private Hashtable BeginPackageInstall( { // Concurrently Updates // Find all dependencies - string pkgName = pkgToInstall.Name; if (!skipDependencyCheck) { - List allDependencies = FindAllDependencies(currentServer, currentResponseUtil, pkgToInstall, repository); - - return InstallParentAndDependencyPackages(pkgToInstall, allDependencies, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); + // concurrency updates + List parentAndDeps = _findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + // List returned only includes dependencies, so we'll add the parent pkg to this list to pass on to installation method + parentAndDeps.Add(pkgToInstall); + + return InstallParentAndDependencyPackages(parentAndDeps, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); } else { - + // If we don't install dependencies, we're only installing the parent pkg so we can short circut and simply install the parent pkg. // TODO: check this version and prerelease combo Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgToInstall.Version.ToString(), true, out ErrorRecord installNameErrRecord); @@ -894,46 +830,21 @@ private Hashtable BeginPackageInstall( return updatedPackagesHash; } - // Concurrency Updates - private List FindAllDependencies(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo pkgToInstall, PSRepositoryInfo repository) - { - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V3) - { - _cmdletPassedIn.WriteWarning("Installing dependencies is not currently supported for V3 server protocol repositories. The package will be installed without installing dependencies."); - } - - var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); - _cmdletPassedIn.WriteDebug($" **Finding dependency packages for (FindAllDependencies) '{pkgToInstall.Name}'"); - - // the last package added will be the parent package. - List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); - - _cmdletPassedIn.WriteDebug($"~~~ Retrieved all dep packages (FindAllDependencies) for '{pkgToInstall.Name}'"); - - // allDependencies contains parent package as well - foreach (PSResourceInfo pkg in allDependencies) - { - // Console.WriteLine($"{pkg.Name}: {pkg.Version}"); - } - - return allDependencies; - } - // Concurrently Updates - private Hashtable InstallParentAndDependencyPackages(PSResourceInfo parentPkg, List allDependencies, ServerApiCall currentServer, string tempInstallPath, Hashtable packagesHash, Hashtable updatedPackagesHash, PSResourceInfo pkgToInstall) + private Hashtable InstallParentAndDependencyPackages(List parentAndDeps, ServerApiCall currentServer, string tempInstallPath, Hashtable packagesHash, Hashtable updatedPackagesHash, PSResourceInfo pkgToInstall) { - List errors = new List(allDependencies.Count); // Pre-allocate based on expected size - // Use a lower threshold for better parallelism - 3 packages instead of processor count - const int PARALLEL_THRESHOLD = 3; + List errors = new List(); + + // TODO: figure out a good threshold and parallel count int processorCount = Environment.ProcessorCount; - if (allDependencies.Count > PARALLEL_THRESHOLD) + if (parentAndDeps.Count > processorCount) { - // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) + // Set the maximum degree of parallelism to 32? (Invoke-Command has default of 32, that's where we got this number from) - // If installing more than 5 packages, do so concurrently + // If installing more than 3 packages, do so concurrently // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. int maxDegreeOfParallelism = processorCount * 4; - Parallel.ForEach(allDependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, depPkg => + Parallel.ForEach(parentAndDeps, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, depPkg => { var depPkgName = depPkg.Name; var depPkgVersion = depPkg.Version.ToString(); @@ -966,31 +877,12 @@ private Hashtable InstallParentAndDependencyPackages(PSResourceInfo parentPkg, L return packagesHash; } - // Install parent package - Stream responseStream = currentServer.InstallPackage(parentPkg.Name, parentPkg.Version.ToString(), true, out ErrorRecord installNameErrRecord); - if (installNameErrRecord != null) - { - _cmdletPassedIn.WriteError(installNameErrRecord); - return packagesHash; - } - - ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); - if (!installedToTempPathSuccessfully) - { - _cmdletPassedIn.WriteError(tempSaveErrRecord ?? tempInstallErrRecord); - return packagesHash; - } - return updatedPackagesHash; } else { // Install the good old fashioned way - // Make sure to install dependencies first, then install parent pkg - allDependencies.Add(parentPkg); - foreach (var pkgToBeInstalled in allDependencies) + foreach (var pkgToBeInstalled in parentAndDeps) { var pkgToInstallName = pkgToBeInstalled.Name; var pkgToInstallVersion = pkgToBeInstalled.Version.ToString(); diff --git a/src/code/LocalServerApiCalls.cs b/src/code/LocalServerApiCalls.cs index cac3e6ccb..1726dda17 100644 --- a/src/code/LocalServerApiCalls.cs +++ b/src/code/LocalServerApiCalls.cs @@ -119,6 +119,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return FindNameHelper(packageName, Utils.EmptyStrArray, includePrerelease, type, out errRecord); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/NuGetServerAPICalls.cs b/src/code/NuGetServerAPICalls.cs index 10c540b70..523844fc8 100644 --- a/src/code/NuGetServerAPICalls.cs +++ b/src/code/NuGetServerAPICalls.cs @@ -192,6 +192,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: FindResponseType); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/ServerApiCall.cs b/src/code/ServerApiCall.cs index bdb97db42..f88f14053 100644 --- a/src/code/ServerApiCall.cs +++ b/src/code/ServerApiCall.cs @@ -82,6 +82,13 @@ public ServerApiCall(PSRepositoryInfo repository, NetworkCredential networkCrede /// public abstract FindResults FindName(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord); + /// + /// Find method which allows for searching for package by single name and returns latest version. + /// Name: no wildcard support + /// Examples: Search "PowerShellGet" + /// + public abstract Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type); + /// /// Find method which allows for searching for package by single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index ea93ec1a9..3aba02b64 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -397,6 +397,72 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); } + /// + /// Find method which allows for searching for single name and returns latest version. + /// Name: no wildcard support + /// Examples: Search "PowerShellGet" + /// API call: + /// - No prerelease: http://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet' + /// - Include prerelease: http://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet' + /// Implementation Note: Need to filter further for latest version (prerelease or non-prerelease depending on user preference) + /// + public override async Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindNameAsync()"); + // Make sure to include quotations around the package name + + // This should return the latest stable version or the latest prerelease version (respectively) + // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet'&$filter=IsLatestVersion and substringof('PSModule', Tags) eq true + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary{ + { "$inlinecount", "allpages" }, + { "id", $"'{packageName}'" }, + }); + var filterBuilder = queryBuilder.FilterBuilder; + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + filterBuilder.AddCriterion(includePrerelease ? "IsAbsoluteLatestVersion eq true" : "IsLatestVersion eq true"); + if (type != ResourceType.None) { + filterBuilder.AddCriterion(GetTypeFilterForRequest(type)); + } + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + string response; + try + { + response = await HttpRequestCallAsync(requestUrlV2); + } + catch (Exception e) + { + // usually this is for errors in calling the V2 server, but for ADO V2 this error will include package not found errors which we want to deliver in a standard message + if (_isADORepo && e is ResourceNotFoundException) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'. For ADO feed, if the package is in an upstream feed make sure you are authenticated to the upstream feed.", e); + } + + throw; + } + + int count = GetCountFromResponse(response, out ErrorRecord errRecord); + if (errRecord != null) + { + throw errRecord.Exception; + } + + if (count == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'."); + } + + return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support @@ -966,10 +1032,10 @@ private async Task HttpRequestCallForContentAsync(string requestUrl try { - //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); - HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); + //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); - content = await SendV2RequestForContentAsync(request, _sessionClient); + content = await SendV2RequestForContentAsync(request, _sessionClient); } catch (HttpRequestException) { @@ -1536,7 +1602,7 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange public override async Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); List responses = new List(); int skip = 0; @@ -1565,7 +1631,7 @@ public override async Task FindVersionGlobbingAsync(string packageN while (count > 0) { - _cmdletPassedIn.WriteDebug($"Count is '{count}'"); + //_cmdletPassedIn.WriteDebug($"Count is '{count}'"); // skip 100 skip += 100; var tmpResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest, out errRecord); @@ -1762,7 +1828,9 @@ private async Task InstallVersionAsync(string packageName, string versio throw new Exception($"No content was returned by repository '{Repository.Name}'"); } - return await response.ReadAsStreamAsync(); + var retResponse = await response.ReadAsStreamAsync(); + + return retResponse; } private string GetTypeFilterForRequest(ResourceType type) { diff --git a/src/code/V3ServerAPICalls.cs b/src/code/V3ServerAPICalls.cs index 97d4c2298..2bd701559 100644 --- a/src/code/V3ServerAPICalls.cs +++ b/src/code/V3ServerAPICalls.cs @@ -165,6 +165,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return FindNameHelper(packageName, tags: Utils.EmptyStrArray, includePrerelease, type, out errRecord); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and specified tag(s) and returns latest version. /// Name: no wildcard support From d65e232fa125f39ea6d74bc3a7c626e8d00541cc Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Thu, 22 Jan 2026 18:01:07 -0800 Subject: [PATCH 6/6] Change packagesHash and updatedPackagesHash to be thread safe ConcurrentDictionary --- src/code/FindHelper.cs | 133 ++++++++++++++++++++++++++------------ src/code/InstallHelper.cs | 39 +++++------ 2 files changed, 111 insertions(+), 61 deletions(-) diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index cdc1c1cfe..d2627d34c 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -48,7 +48,7 @@ internal class FindHelper // Creates a new instance of depPkgsFound each time FindDependencyPackages() is called. // This will eventually return the PSResourceInfo object to the main cmdlet class. - private ConcurrentDictionary depPkgsFound; + private ConcurrentDictionary depPkgsFound; // Contains the latest found version of a particular package. private ConcurrentDictionary _knownLatestPkgVersion; @@ -761,7 +761,7 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R { parentPkgs.Add(foundPkg); TryAddToPackagesFound(foundPkg); - _cmdletPassedIn.WriteDebug($"Found package '{foundPkg.Name}' version '{foundPkg.Version}'"); + // _cmdletPassedIn.WriteDebug($"Found package '{foundPkg.Name}' version '{foundPkg.Version}'"); yield return foundPkg; } @@ -1127,7 +1127,34 @@ private bool TryAddToPackagesFound(PSResourceInfo foundPkg) addedToHash = true; } - _cmdletPassedIn.WriteDebug($"Found package -- '{foundPkgName}' version '{foundPkgVersion}'"); + //_cmdletPassedIn.WriteDebug($"Found package -- '{foundPkgName}' version '{foundPkgVersion}'"); + + return addedToHash; + } + + private bool TryAddToKnownLatestPkgVersion(PSResourceInfo foundPkg) + { + // This handles prerelease versions as well. + bool addedToHash = false; + string foundPkgName = foundPkg.Name; + string foundPkgVersion = FormatPkgVersionString(foundPkg); + + if (_knownLatestPkgVersion.ContainsKey(foundPkgName)) + { + _knownLatestPkgVersion.TryGetValue(foundPkgName, out PSResourceInfo oldPkgVersion); + + _knownLatestPkgVersion.TryUpdate(foundPkgName, foundPkg, oldPkgVersion); + + addedToHash = true; + + } + else + { + _knownLatestPkgVersion.TryAdd(foundPkg.Name, foundPkg); + addedToHash = true; + } + + //_cmdletPassedIn.WriteDebug($"Found package -- '{foundPkgName}' version '{foundPkgVersion}'"); return addedToHash; } @@ -1140,7 +1167,7 @@ private string FormatPkgVersionString(PSResourceInfo pkg) { fullPkgVersion += $"-{pkg.Prerelease}"; } - _cmdletPassedIn.WriteDebug($"Formatted full package version is: '{fullPkgVersion}'"); + // _cmdletPassedIn.WriteDebug($"Formatted full package version is: '{fullPkgVersion}'"); return fullPkgVersion; } @@ -1151,13 +1178,13 @@ private string FormatPkgVersionString(PSResourceInfo pkg) internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { - depPkgsFound = new ConcurrentDictionary(); - _cmdletPassedIn.WriteDebug($"In FindDependencyPackages {currentPkg.Name}"); + depPkgsFound = new ConcurrentDictionary(); + //_cmdletPassedIn.WriteDebug($"In FindDependencyPackages {currentPkg.Name}"); FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); - _cmdletPassedIn.WriteDebug("In FindDependencyPackages 2"); + // _cmdletPassedIn.WriteDebug("In FindDependencyPackages 2"); - return depPkgsFound.Keys.ToList(); + return depPkgsFound.Values.ToList(); } // Method 2 @@ -1167,7 +1194,7 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response if (currentPkg.Dependencies.Length > 0) { // If finding more than 3 packages, do so concurrently - const int PARALLEL_THRESHOLD = 3; + const int PARALLEL_THRESHOLD = 5; // Trottle limit from user, defaults to 5; other tools? int processorCount = Environment.ProcessorCount; // 8 int maxDegreeOfParallelism = processorCount * 4; if (currentPkg.Dependencies.Length > PARALLEL_THRESHOLD) @@ -1178,7 +1205,7 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); }); - + // todo: what is perf if parallel.ForEach is always run? // todo: write any errors here } else @@ -1204,15 +1231,16 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS // Case 1: No upper bound, eg: "*" or "(1.0.0, )" if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedDepPkg)) { - // 1) Check if the latest version is cached - // if a package is already cached, its dependencies will have been cached as well. - // But if for some reason it wasn't added to depPkgs, add it now - depPkg = cachedDepPkg; - if (!depPkgsFound.ContainsKey(depPkg)) - { - depPkgsFound.TryAdd(depPkg, true); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - } + + //// 1) Check if the latest version is cached + //// if a package is already cached, its dependencies will have been cached as well. + //// But if for some reason it wasn't added to depPkgs, add it now + //depPkg = cachedDepPkg; + //if (!depPkgsFound.ContainsKey(depPkg)) + //{ + // depPkgsFound.TryAdd(depPkg, true); + // FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + //} } else { @@ -1232,17 +1260,20 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS // write error } - if (dep.VersionRange.Satisfies(cachedPkgVersion)) + // The cached package does not satisfy the version range, look for one that does. + if (!dep.VersionRange.Satisfies(cachedPkgVersion)) { - // if a package is already cached, its dependencies will have been cached as well. - // But if for some reason it wasn't added to depPkgs, add it now - depPkg = cachedDepPkg; - if (!depPkgsFound.ContainsKey(depPkg)) - { - depPkgsFound.TryAdd(depPkg, true); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - } + //// if a package is already cached, its dependencies will have been cached as well. + //// But if for some reason it wasn't added to depPkgs, add it now + //depPkg = cachedDepPkg; + //if (!depPkgsFound.ContainsKey(depPkg)) + //{ + // depPkgsFound.TryAdd(depPkg, true); + // FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + //} + depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); } + } else { @@ -1261,16 +1292,18 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS // write error } - if (dep.VersionRange.Satisfies(cachedPkgVersion)) + if (!dep.VersionRange.Satisfies(cachedPkgVersion)) { - // if a package is already cached, its dependencies will have been cached as well. - // But if for some reason it wasn't added to depPkgs, add it now - depPkg = cachedRangePkg; - if (!depPkgsFound.ContainsKey(depPkg)) - { - depPkgsFound.TryAdd(depPkg, true); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - } + //// if a package is already cached, its dependencies will have been cached as well. + //// But if for some reason it wasn't added to depPkgs, add it now + //depPkg = cachedRangePkg; + //if (!depPkgsFound.ContainsKey(depPkg)) + //{ + // depPkgsFound.TryAdd(depPkg, true); + // FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + //} + depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } } else @@ -1323,12 +1356,17 @@ private PSResourceInfo FindDependencyWithSpecificVersion(Dependency dep, ServerA else { depPkg = currentResult.returnedObject; - if (!depPkgsFound.ContainsKey(depPkg)) + TryAddToKnownLatestPkgVersion(depPkg); + + + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) { // Add pkg to collection of packages found then find dependencies // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. // This will eventually return the PSResourceInfo object to the main cmdlet class. - depPkgsFound.TryAdd(depPkg, true); + depPkgsFound.TryAdd(key, depPkg); FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } } @@ -1378,12 +1416,16 @@ private PSResourceInfo FindDependencyWithLowerBound(Dependency dep, ServerApiCal else { depPkg = currentResult.returnedObject; - if (!depPkgsFound.ContainsKey(depPkg)) + TryAddToKnownLatestPkgVersion(depPkg); + + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) { // Add pkg to collection of packages found then find dependencies // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. // This will eventually return the PSResourceInfo object to the main cmdlet class. - depPkgsFound.TryAdd(depPkg, true); + depPkgsFound.TryAdd(key, depPkg); FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } } @@ -1440,12 +1482,17 @@ private PSResourceInfo FindDependencyWithUpperBound(Dependency dep, ServerApiCal else { depPkg = currentResult.returnedObject; - if (!depPkgsFound.ContainsKey(depPkg)) + + TryAddToKnownLatestPkgVersion(depPkg); + + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) { // Add pkg to collection of packages found then find dependencies // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. // This will eventually return the PSResourceInfo object to the main cmdlet class. - depPkgsFound.TryAdd(depPkg, true); + depPkgsFound.TryAdd(key, depPkg); FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } } diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index 1f408cc0e..7f11c602a 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -6,6 +6,7 @@ using NuGet.Versioning; using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; @@ -531,7 +532,7 @@ private List InstallPackages( // and value as a Hashtable of specific package info: // packageName, { version = "", isScript = "", isModule = "", pkg = "", etc. } // Install parent package to the temp directory. - Hashtable packagesHash = BeginPackageInstall( + ConcurrentDictionary packagesHash = BeginPackageInstall( searchVersionType: _versionType, specificVersion: _nugetVersion, versionRange: _versionRange, @@ -541,7 +542,7 @@ private List InstallPackages( currentResponseUtil: currentResponseUtil, tempInstallPath: tempInstallPath, skipDependencyCheck: skipDependencyCheck, - packagesHash: new Hashtable(StringComparer.InvariantCultureIgnoreCase), + packagesHash: new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase), errRecord: out ErrorRecord errRecord); // At this point all packages are installed to temp path. @@ -617,7 +618,7 @@ private List InstallPackages( /// /// Installs a single package to the temporary path. /// - private Hashtable BeginPackageInstall( + private ConcurrentDictionary BeginPackageInstall( VersionType searchVersionType, NuGetVersion specificVersion, VersionRange versionRange, @@ -627,7 +628,7 @@ private Hashtable BeginPackageInstall( ResponseUtil currentResponseUtil, string tempInstallPath, bool skipDependencyCheck, - Hashtable packagesHash, + ConcurrentDictionary packagesHash, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackage()"); @@ -759,14 +760,14 @@ private Hashtable BeginPackageInstall( } - Hashtable updatedPackagesHash = packagesHash; + ConcurrentDictionary updatedPackagesHash = packagesHash; // -WhatIf processing. if (_savePkg && !_cmdletPassedIn.ShouldProcess($"Package to save: '{pkgToInstall.Name}', version: '{pkgVersion}'")) { if (!updatedPackagesHash.ContainsKey(pkgToInstall.Name)) { - updatedPackagesHash.Add(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -782,7 +783,7 @@ private Hashtable BeginPackageInstall( { if (!updatedPackagesHash.ContainsKey(pkgToInstall.Name)) { - updatedPackagesHash.Add(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -804,7 +805,9 @@ private Hashtable BeginPackageInstall( List parentAndDeps = _findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); // List returned only includes dependencies, so we'll add the parent pkg to this list to pass on to installation method parentAndDeps.Add(pkgToInstall); - + + _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackage(), found all dependencies"); + return InstallParentAndDependencyPackages(parentAndDeps, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); } else { @@ -831,16 +834,16 @@ private Hashtable BeginPackageInstall( } // Concurrently Updates - private Hashtable InstallParentAndDependencyPackages(List parentAndDeps, ServerApiCall currentServer, string tempInstallPath, Hashtable packagesHash, Hashtable updatedPackagesHash, PSResourceInfo pkgToInstall) + private ConcurrentDictionary InstallParentAndDependencyPackages(List parentAndDeps, ServerApiCall currentServer, string tempInstallPath, ConcurrentDictionary packagesHash, ConcurrentDictionary updatedPackagesHash, PSResourceInfo pkgToInstall) { - List errors = new List(); + List errors = new List(); + int azaccounts = 0; // TODO: figure out a good threshold and parallel count int processorCount = Environment.ProcessorCount; if (parentAndDeps.Count > processorCount) { // Set the maximum degree of parallelism to 32? (Invoke-Command has default of 32, that's where we got this number from) - // If installing more than 3 packages, do so concurrently // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. int maxDegreeOfParallelism = processorCount * 4; @@ -967,8 +970,8 @@ private bool TryInstallToTempPath( string pkgName, string normalizedPkgVersion, PSResourceInfo pkgToInstall, - Hashtable packagesHash, - out Hashtable updatedPackagesHash, + ConcurrentDictionary packagesHash, + out ConcurrentDictionary updatedPackagesHash, out ErrorRecord error) { //_cmdletPassedIn.WriteDebug("In InstallHelper::TryInstallToTempPath()"); @@ -1108,7 +1111,7 @@ private bool TryInstallToTempPath( if (!updatedPackagesHash.ContainsKey(pkgName)) { // Add pkg info to hashtable. - updatedPackagesHash.Add(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", isModule }, { "isScript", isScript }, @@ -1145,8 +1148,8 @@ private bool TrySaveNupkgToTempPath( string pkgName, string normalizedPkgVersion, PSResourceInfo pkgToInstall, - Hashtable packagesHash, - out Hashtable updatedPackagesHash, + ConcurrentDictionary packagesHash, + out ConcurrentDictionary updatedPackagesHash, out ErrorRecord error) { // _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); @@ -1173,7 +1176,7 @@ private bool TrySaveNupkgToTempPath( if (!updatedPackagesHash.ContainsKey(pkgName)) { // Add pkg info to hashtable. - updatedPackagesHash.Add(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -1269,7 +1272,7 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error /// /// Moves package files/directories from the temp install path into the final install path location. /// - private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hashtable packagesHash) + private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, ConcurrentDictionary packagesHash) { //_cmdletPassedIn.WriteDebug("In InstallHelper::TryMoveInstallContent()"); foreach (string pkgName in packagesHash.Keys)