diff --git a/src/code/ContainerRegistryServerAPICalls.cs b/src/code/ContainerRegistryServerAPICalls.cs index 9c17c0db0..20f084f10 100644 --- a/src/code/ContainerRegistryServerAPICalls.cs +++ b/src/code/ContainerRegistryServerAPICalls.cs @@ -81,6 +81,16 @@ public ContainerRegistryServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmd #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return null; + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } + /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// @@ -146,6 +156,12 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[] { }, hashtableResponse: pkgResult.ToArray(), responseType: containerRegistryFindResponseType); } + + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index d8287c689..d2627d34c 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -1,16 +1,22 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using Microsoft.PowerShell.PSResourceGet.UtilClasses; -using NuGet.Versioning; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Management.Automation; using System.Net; using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; +using Azure; +using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -35,7 +41,19 @@ internal class FindHelper private bool _includeDependencies = false; private bool _repositoryNameContainsWildcard = true; private NetworkCredential _networkCredential; - private Dictionary> _packagesFound; + + // Gets intantiated each time a cmdlet is run. + // If running 'Install-PSResource Az, TestModule, NewTestModule', it will contain one parent and its dependencies. + private ConcurrentDictionary> _packagesFound; + + // Creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + private ConcurrentDictionary depPkgsFound; + + // Contains the latest found version of a particular package. + private ConcurrentDictionary _knownLatestPkgVersion; + + ConcurrentDictionary> _cachedNetworkCalls; #endregion @@ -48,7 +66,10 @@ public FindHelper(CancellationToken cancellationToken, PSCmdlet cmdletPassedIn, _cancellationToken = cancellationToken; _cmdletPassedIn = cmdletPassedIn; _networkCredential = networkCredential; - _packagesFound = new Dictionary>(StringComparer.OrdinalIgnoreCase); + _packagesFound = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); + _knownLatestPkgVersion = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + _type = ResourceType.None; + _cachedNetworkCalls = new ConcurrentDictionary>(); } #endregion @@ -740,7 +761,7 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R { parentPkgs.Add(foundPkg); TryAddToPackagesFound(foundPkg); - _cmdletPassedIn.WriteDebug($"Found package '{foundPkg.Name}' version '{foundPkg.Version}'"); + // _cmdletPassedIn.WriteDebug($"Found package '{foundPkg.Name}' version '{foundPkg.Version}'"); yield return foundPkg; } @@ -884,7 +905,20 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R FindResults responses = null; if (_tag.Length == 0) { - responses = currentServer.FindVersion(pkgName, _nugetVersion.ToNormalizedString(), _type, out errRecord); + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { + string key = $"{pkgName}|{_nugetVersion.ToNormalizedString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(pkgName, _nugetVersion.ToNormalizedString(), _type)); + + responses = response.GetAwaiter().GetResult(); + + } + else { + responses = currentServer.FindVersion(pkgName, _nugetVersion.ToNormalizedString(), _type, out errRecord); + } + + } else { @@ -956,7 +990,22 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R FindResults responses = null; if (_tag.Length == 0) { - responses = currentServer.FindVersionGlobbing(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, out errRecord); + + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + Task response = null; + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) { + string key = $"{pkgName}|{_versionRange.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false)); + + responses = response.GetAwaiter().GetResult(); + + } + else { + responses = currentServer.FindVersionGlobbing(pkgName, _versionRange, _prerelease, _type, getOnlyLatest: false, out errRecord); + + } + + } else { @@ -1023,7 +1072,7 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R { foreach (PSResourceInfo currentPkg in parentPkgs) { - _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{currentPkg.Name}'"); + _cmdletPassedIn.WriteDebug($"Finding dependency packages (SearchByNames) for '{currentPkg.Name}'"); foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) { yield return pkgDep; @@ -1050,31 +1099,62 @@ private HashSet GetPackageNamesPopulated(string[] pkgNames) return pkgsToDiscover; } - private bool TryAddToPackagesFound(PSResourceInfo foundPkg) - { + { + // This handles prerelease versions as well. bool addedToHash = false; string foundPkgName = foundPkg.Name; - string foundPkgVersion = Utils.GetNormalizedVersionString(foundPkg.Version.ToString(), foundPkg.Prerelease); + string foundPkgVersion = FormatPkgVersionString(foundPkg); if (_packagesFound.ContainsKey(foundPkgName)) { - List pkgVersions = _packagesFound[foundPkgName] as List; + _packagesFound.TryGetValue(foundPkgName, out List pkgVersions); if (!pkgVersions.Contains(foundPkgVersion)) { - pkgVersions.Add(foundPkgVersion); - _packagesFound[foundPkgName] = pkgVersions; + List newPkgVersions = new List(pkgVersions) + { + foundPkgVersion + }; + _packagesFound.TryUpdate(foundPkgName, newPkgVersions, pkgVersions); + addedToHash = true; } } else { - _packagesFound.Add(foundPkg.Name, new List { foundPkgVersion }); + _packagesFound.TryAdd(foundPkg.Name, new List { foundPkgVersion }); addedToHash = true; } - _cmdletPassedIn.WriteDebug($"Found package '{foundPkg.Name}' version '{foundPkg.Version}'"); + //_cmdletPassedIn.WriteDebug($"Found package -- '{foundPkgName}' version '{foundPkgVersion}'"); + + return addedToHash; + } + + private bool TryAddToKnownLatestPkgVersion(PSResourceInfo foundPkg) + { + // This handles prerelease versions as well. + bool addedToHash = false; + string foundPkgName = foundPkg.Name; + string foundPkgVersion = FormatPkgVersionString(foundPkg); + + if (_knownLatestPkgVersion.ContainsKey(foundPkgName)) + { + _knownLatestPkgVersion.TryGetValue(foundPkgName, out PSResourceInfo oldPkgVersion); + + _knownLatestPkgVersion.TryUpdate(foundPkgName, foundPkg, oldPkgVersion); + + addedToHash = true; + + } + else + { + _knownLatestPkgVersion.TryAdd(foundPkg.Name, foundPkg); + addedToHash = true; + } + + //_cmdletPassedIn.WriteDebug($"Found package -- '{foundPkgName}' version '{foundPkgVersion}'"); return addedToHash; } @@ -1087,7 +1167,7 @@ private string FormatPkgVersionString(PSResourceInfo pkg) { fullPkgVersion += $"-{pkg.Prerelease}"; } - _cmdletPassedIn.WriteDebug($"Formatted full package version is: '{fullPkgVersion}'"); + // _cmdletPassedIn.WriteDebug($"Formatted full package version is: '{fullPkgVersion}'"); return fullPkgVersion; } @@ -1096,186 +1176,376 @@ private string FormatPkgVersionString(PSResourceInfo pkg) #region Internal Client Search Methods - internal IEnumerable FindDependencyPackages( - ServerApiCall currentServer, - ResponseUtil currentResponseUtil, - PSResourceInfo currentPkg, - PSRepositoryInfo repository) + internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) + { + depPkgsFound = new ConcurrentDictionary(); + //_cmdletPassedIn.WriteDebug($"In FindDependencyPackages {currentPkg.Name}"); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); + + // _cmdletPassedIn.WriteDebug("In FindDependencyPackages 2"); + + return depPkgsFound.Values.ToList(); + } + + // Method 2 + internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { + List errors = new List(); if (currentPkg.Dependencies.Length > 0) { - foreach (var dep in currentPkg.Dependencies) + // If finding more than 3 packages, do so concurrently + const int PARALLEL_THRESHOLD = 5; // Trottle limit from user, defaults to 5; other tools? + int processorCount = Environment.ProcessorCount; // 8 + int maxDegreeOfParallelism = processorCount * 4; + if (currentPkg.Dependencies.Length > PARALLEL_THRESHOLD) { - PSResourceInfo depPkg = null; + Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, dep => + { + // Console.WriteLine($"FindDependencyPackages Processing number: {dep}, Thread ID: {Task.CurrentId}"); + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); - if (dep.VersionRange.Equals(VersionRange.All)) + }); + // todo: what is perf if parallel.ForEach is always run? + // todo: write any errors here + } + else + { + foreach (var dep in currentPkg.Dependencies) { - FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - yield return null; - continue; - } + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + // todo write out errors here + } + } + } - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + // Method 3 + private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + //ErrorRecord errRecord = null; - depPkg = currentResult.returnedObject; + if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) + { + // Case 1: No upper bound, eg: "*" or "(1.0.0, )" + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedDepPkg)) + { - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - } + //// 1) Check if the latest version is cached + //// if a package is already cached, its dependencies will have been cached as well. + //// But if for some reason it wasn't added to depPkgs, add it now + //depPkg = cachedDepPkg; + //if (!depPkgsFound.ContainsKey(depPkg)) + //{ + // depPkgsFound.TryAdd(depPkg, true); + // FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + //} + } + else + { + // 2) Find this version from the server + depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + + } + else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) + { + // Case 2: Exact package version, eg: "1.0.0" or "[1.0.0, 1.0.0]" + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedDepPkg)) + { + // 1) Check if the latest version is cached, and if this latest version is the version we're looking for + if (!NuGetVersion.TryParse(cachedDepPkg.Version.ToString(), out NuGetVersion cachedPkgVersion)) + { + // write error } - else + + // The cached package does not satisfy the version range, look for one that does. + if (!dep.VersionRange.Satisfies(cachedPkgVersion)) { - FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - yield return null; - continue; - } + //// if a package is already cached, its dependencies will have been cached as well. + //// But if for some reason it wasn't added to depPkgs, add it now + //depPkg = cachedDepPkg; + //if (!depPkgsFound.ContainsKey(depPkg)) + //{ + // depPkgsFound.TryAdd(depPkg, true); + // FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + //} + depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } - if (responses.IsFindResultsEmpty()) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), - "FindDepPackagesFindVersionGlobbingFailure", - ErrorCategory.InvalidResult, - this)); - yield return null; - continue; - } + } + else + { + // 2) Find this version from the server + depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + } + else + { + // Case 3: Version range with an upper bound, eg: "(1.0.0, 3.0.0)" + if (_knownLatestPkgVersion.TryGetValue(dep.Name, out PSResourceInfo cachedRangePkg)) + { + // 1) Check if the latest version is cached, and if this latest version is the version we're looking for + if (!NuGetVersion.TryParse(cachedRangePkg.Version.ToString(), out NuGetVersion cachedPkgVersion)) + { + // write error + } - foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) - { - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); + if (!dep.VersionRange.Satisfies(cachedPkgVersion)) + { + //// if a package is already cached, its dependencies will have been cached as well. + //// But if for some reason it wasn't added to depPkgs, add it now + //depPkg = cachedRangePkg; + //if (!depPkgsFound.ContainsKey(depPkg)) + //{ + // depPkgsFound.TryAdd(depPkg, true); + // FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + //} + depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); - yield return null; - continue; - } + } + } + else + { + depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + } + } + } - // Check to see if version falls within version range - PSResourceInfo foundDep = currentResult.returnedObject; - string depVersionStr = $"{foundDep.Version}"; - if (foundDep.IsPrerelease) - { - depVersionStr += $"-{foundDep.Prerelease}"; - } - if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) - && dep.VersionRange.Satisfies(depVersion)) - { - depPkg = foundDep; - } - } - if (depPkg == null) - { - continue; - } + // Method 4 + private PSResourceInfo FindDependencyWithSpecificVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + ErrorRecord errRecord = null; + FindResults responses = null; + Task response = null; - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) - { - yield return depRes; - } - } - } + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + { + // See if the network call we're making is already caced, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionAsync(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type)); + + responses = response.GetAwaiter().GetResult(); + } + else + { + responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out errRecord); + } + + + // Error handling and Convert to PSResource object + if (errRecord != null) + { + errors = ProcessErrorRecord(errRecord, errors); + } + else + { + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + TryAddToKnownLatestPkgVersion(depPkg); + + + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) + { + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(key, depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } } } - if (!_packagesFound.ContainsKey(currentPkg.Name)) + return depPkg; + } + + // Method 5 + private PSResourceInfo FindDependencyWithLowerBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + FindResults responses = null; + ErrorRecord errRecord = null; + Task response = null; + + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + { + // See if the network call we're making is already caced, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|*|{_type}"; + response = _cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindNameAsync(dep.Name, includePrerelease: true, _type)); + + responses = response.GetAwaiter().GetResult(); + } + else { - TryAddToPackagesFound(currentPkg); + responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out errRecord); + } - yield return currentPkg; + + // Error handling and Convert to PSResource object + if (errRecord != null) + { + errors = ProcessErrorRecord(errRecord, errors); } else { - List pkgVersions = _packagesFound[currentPkg.Name] as List; - // _packagesFound has currentPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(currentPkg))) + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) { - TryAddToPackagesFound(currentPkg); + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + TryAddToKnownLatestPkgVersion(depPkg); - yield return currentPkg; + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) + { + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(key, depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } } } + return depPkg; } - #endregion + + + + // Method 6 + private PSResourceInfo FindDependencyWithUpperBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + + PSResourceInfo depPkg = null; + ErrorRecord errRecord = null; + FindResults responses = null; + Task response = null; + + ConcurrentDictionary> cachedNetworkCalls = new ConcurrentDictionary>(); + + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V2) + { + // See if the network call we're making is already caced, if not, call FindNameAsync() and cache results + string key = $"{dep.Name}|{dep.VersionRange.MaxVersion.ToString()}|{_type}"; + response = cachedNetworkCalls.GetOrAdd(key, _ => currentServer.FindVersionGlobbingAsync(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true)); + + responses = response.GetAwaiter().GetResult(); + + } + else + { + responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out errRecord); + } + + + // Error handling and Convert to PSResource object + if (errRecord != null) + { + errors = ProcessErrorRecord(errRecord, errors); + } + else + { + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + + TryAddToKnownLatestPkgVersion(depPkg); + + string pkgVersion = FormatPkgVersionString(depPkg); + string key = $"{depPkg.Name}{pkgVersion}"; + if (!depPkgsFound.ContainsKey(key)) + { + // Add pkg to collection of packages found then find dependencies + // depPkgsFound creates a new instance of depPkgsFound each time FindDependencyPackages() is called. + // This will eventually return the PSResourceInfo object to the main cmdlet class. + depPkgsFound.TryAdd(key, depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + } + } + + return depPkg; + } + + private List ProcessErrorRecord(ErrorRecord errRecord, List errors) + { + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package could not be found: '{errRecord.Exception.Message}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package could not be found: '{errRecord.Exception.Message}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + + return errors; + } + + private List ProcessPSResourceResult(PSResourceResult currentResult, List errors) + { + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{currentResult.returnedObject.Name}' could not be found in repository '{currentResult.returnedObject.Repository}'."), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{currentResult.returnedObject.Name}' could not be found in repository '{currentResult.returnedObject.Repository}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + + return errors; + } + + #endregion } } diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index fef419a4f..7f11c602a 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -2,10 +2,13 @@ // Licensed under the MIT License. using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; using NuGet.Versioning; using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.IO.Compression; @@ -15,6 +18,7 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -53,6 +57,7 @@ internal class InstallHelper private string _tmpPath; private NetworkCredential _networkCredential; private HashSet _packagesOnMachine; + private FindHelper _findHelper; #endregion @@ -64,6 +69,8 @@ public InstallHelper(PSCmdlet cmdletPassedIn, NetworkCredential networkCredentia _cancellationToken = source.Token; _cmdletPassedIn = cmdletPassedIn; _networkCredential = networkCredential; + + _findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); } /// @@ -396,7 +403,7 @@ private void MoveFilesIntoInstallPath( string moduleManifestVersion, string scriptPath) { - _cmdletPassedIn.WriteDebug("In InstallHelper::MoveFilesIntoInstallPath()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::MoveFilesIntoInstallPath()"); // Creating the proper installation path depending on whether pkg is a module or script var newPathParent = isModule ? Path.Combine(installPath, pkgInfo.Name) : installPath; var finalModuleVersionDir = isModule ? Path.Combine(installPath, pkgInfo.Name, moduleManifestVersion) : installPath; @@ -405,32 +412,32 @@ private void MoveFilesIntoInstallPath( var tempModuleVersionDir = (!isModule || isLocalRepo) ? dirNameVersion : Path.Combine(tempInstallPath, pkgInfo.Name, newVersion); - _cmdletPassedIn.WriteVerbose($"Installation source path is: '{tempModuleVersionDir}'"); - _cmdletPassedIn.WriteVerbose($"Installation destination path is: '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Installation source path is: '{tempModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Installation destination path is: '{finalModuleVersionDir}'"); if (isModule) { // If new path does not exist if (!Directory.Exists(newPathParent)) { - _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); Directory.CreateDirectory(newPathParent); Utils.MoveDirectory(tempModuleVersionDir, finalModuleVersionDir); } else { - _cmdletPassedIn.WriteVerbose($"Temporary module version directory is: '{tempModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Temporary module version directory is: '{tempModuleVersionDir}'"); if (Directory.Exists(finalModuleVersionDir)) { // Delete the directory path before replacing it with the new module. // If deletion fails (usually due to binary file in use), then attempt restore so that the currently // installed module is not corrupted. - _cmdletPassedIn.WriteVerbose($"Attempting to delete with restore on failure. '{finalModuleVersionDir}'"); + //_cmdletPassedIn.WriteVerbose($"Attempting to delete with restore on failure. '{finalModuleVersionDir}'"); Utils.DeleteDirectoryWithRestore(finalModuleVersionDir); } - _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); Utils.MoveDirectory(tempModuleVersionDir, finalModuleVersionDir); } } @@ -456,27 +463,27 @@ private void MoveFilesIntoInstallPath( if (!Directory.Exists(scriptInfoFolderPath)) { - _cmdletPassedIn.WriteVerbose($"Created '{scriptInfoFolderPath}' path for scripts"); + // _cmdletPassedIn.WriteVerbose($"Created '{scriptInfoFolderPath}' path for scripts"); Directory.CreateDirectory(scriptInfoFolderPath); } // Need to delete old xml files because there can only be 1 per script - _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: '{1}'", scriptXmlFilePath, File.Exists(scriptXmlFilePath))); + // _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: '{1}'", scriptXmlFilePath, File.Exists(scriptXmlFilePath))); if (File.Exists(scriptXmlFilePath)) { - _cmdletPassedIn.WriteVerbose("Deleting script metadata XML"); + // _cmdletPassedIn.WriteVerbose("Deleting script metadata XML"); File.Delete(Path.Combine(scriptInfoFolderPath, scriptXML)); } - _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML))); + // _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML))); Utils.MoveFiles(Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML)); // Need to delete old script file, if that exists - _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: ", File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)))); + // _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: ", File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)))); if (File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))) { - _cmdletPassedIn.WriteVerbose("Deleting script file"); + // _cmdletPassedIn.WriteVerbose("Deleting script file"); File.Delete(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)); } } @@ -489,7 +496,7 @@ private void MoveFilesIntoInstallPath( } } - _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))); + // _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))); Utils.MoveFiles(scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)); } } @@ -508,6 +515,7 @@ private List InstallPackages( FindHelper findHelper) { _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackages()"); + List pkgsSuccessfullyInstalled = new(); // Install parent package to the temp directory, @@ -524,7 +532,7 @@ private List InstallPackages( // and value as a Hashtable of specific package info: // packageName, { version = "", isScript = "", isModule = "", pkg = "", etc. } // Install parent package to the temp directory. - Hashtable packagesHash = BeginPackageInstall( + ConcurrentDictionary packagesHash = BeginPackageInstall( searchVersionType: _versionType, specificVersion: _nugetVersion, versionRange: _versionRange, @@ -533,10 +541,11 @@ private List InstallPackages( currentServer: currentServer, currentResponseUtil: currentResponseUtil, tempInstallPath: tempInstallPath, - packagesHash: new Hashtable(StringComparer.InvariantCultureIgnoreCase), + skipDependencyCheck: skipDependencyCheck, + packagesHash: new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase), errRecord: out ErrorRecord errRecord); - // At this point parent package is installed to temp path. + // At this point all packages are installed to temp path. if (errRecord != null) { if (errRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) @@ -559,70 +568,6 @@ private List InstallPackages( Hashtable parentPkgInfo = packagesHash[parentPackage] as Hashtable; PSResourceInfo parentPkgObj = parentPkgInfo["psResourceInfoPkg"] as PSResourceInfo; - if (!skipDependencyCheck) - { - // Get the dependencies from the installed package. - if (parentPkgObj.Dependencies.Length > 0) - { - bool depFindFailed = false; - foreach (PSResourceInfo depPkg in findHelper.FindDependencyPackages(currentServer, currentResponseUtil, parentPkgObj, repository)) - { - if (depPkg == null) - { - depFindFailed = true; - continue; - } - - if (String.Equals(depPkg.Name, parentPkgObj.Name, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - NuGetVersion depVersion = null; - if (depPkg.AdditionalMetadata.ContainsKey("NormalizedVersion")) - { - if (!NuGetVersion.TryParse(depPkg.AdditionalMetadata["NormalizedVersion"] as string, out depVersion)) - { - NuGetVersion.TryParse(depPkg.Version.ToString(), out depVersion); - } - } - - string depPkgNameVersion = $"{depPkg.Name}{depPkg.Version.ToString()}"; - if (_packagesOnMachine.Contains(depPkgNameVersion) && !depPkg.IsPrerelease) - { - // if a dependency package is already installed, do not install it again. - // to determine if the package version is already installed, _packagesOnMachine is used but it only contains name, version info, not version with prerelease info - // if the dependency package is found to be prerelease, it is safer to install it (and worse case it reinstalls) - _cmdletPassedIn.WriteVerbose($"Dependency '{depPkg.Name}' with version '{depPkg.Version}' is already installed."); - continue; - } - - packagesHash = BeginPackageInstall( - searchVersionType: VersionType.SpecificVersion, - specificVersion: depVersion, - versionRange: null, - pkgNameToInstall: depPkg.Name, - repository: repository, - currentServer: currentServer, - currentResponseUtil: currentResponseUtil, - tempInstallPath: tempInstallPath, - packagesHash: packagesHash, - errRecord: out ErrorRecord installPkgErrRecord); - - if (installPkgErrRecord != null) - { - _cmdletPassedIn.WriteError(installPkgErrRecord); - continue; - } - } - - if (depFindFailed) - { - continue; - } - } - } - // If -WhatIf is passed in, early out. if (_cmdletPassedIn.MyInvocation.BoundParameters.ContainsKey("WhatIf")) { @@ -673,7 +618,7 @@ private List InstallPackages( /// /// Installs a single package to the temporary path. /// - private Hashtable BeginPackageInstall( + private ConcurrentDictionary BeginPackageInstall( VersionType searchVersionType, NuGetVersion specificVersion, VersionRange versionRange, @@ -682,13 +627,15 @@ private Hashtable BeginPackageInstall( ServerApiCall currentServer, ResponseUtil currentResponseUtil, string tempInstallPath, - Hashtable packagesHash, + bool skipDependencyCheck, + ConcurrentDictionary packagesHash, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackage()"); FindResults responses = null; errRecord = null; + // Find the parent package that needs to be installed switch (searchVersionType) { case VersionType.VersionRange: @@ -717,6 +664,7 @@ private Hashtable BeginPackageInstall( default: // VersionType.NoVersion responses = currentServer.FindName(pkgNameToInstall, _prerelease, ResourceType.None, out ErrorRecord findNameErrRecord); + if (findNameErrRecord != null) { errRecord = findNameErrRecord; @@ -726,6 +674,7 @@ private Hashtable BeginPackageInstall( break; } + // Convert parent package to PSResourceInfo PSResourceInfo pkgToInstall = null; foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) { @@ -789,9 +738,11 @@ private Hashtable BeginPackageInstall( } // Check to see if the pkg is already installed (ie the pkg is installed and the version satisfies the version range provided via param) + // TODO: can use cache for this if (!_reinstall) { string currPkgNameVersion = $"{pkgToInstall.Name}{pkgToInstall.Version}"; + // Use HashSet lookup instead of Contains for O(1) performance if (_packagesOnMachine.Contains(currPkgNameVersion)) { _cmdletPassedIn.WriteWarning($"Resource '{pkgToInstall.Name}' with version '{pkgVersion}' is already installed. If you would like to reinstall, please run the cmdlet again with the -Reinstall parameter"); @@ -809,14 +760,14 @@ private Hashtable BeginPackageInstall( } - Hashtable updatedPackagesHash = packagesHash; + ConcurrentDictionary updatedPackagesHash = packagesHash; // -WhatIf processing. if (_savePkg && !_cmdletPassedIn.ShouldProcess($"Package to save: '{pkgToInstall.Name}', version: '{pkgVersion}'")) { if (!updatedPackagesHash.ContainsKey(pkgToInstall.Name)) { - updatedPackagesHash.Add(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -832,7 +783,7 @@ private Hashtable BeginPackageInstall( { if (!updatedPackagesHash.ContainsKey(pkgToInstall.Name)) { - updatedPackagesHash.Add(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -846,25 +797,118 @@ private Hashtable BeginPackageInstall( } else { - // Download the package. - string pkgName = pkgToInstall.Name; - Stream responseStream = currentServer.InstallPackage(pkgName, pkgVersion, _prerelease, out ErrorRecord installNameErrRecord); - if (installNameErrRecord != null) + // Concurrently Updates + // Find all dependencies + if (!skipDependencyCheck) { - errRecord = installNameErrRecord; - return packagesHash; + // concurrency updates + List parentAndDeps = _findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + // List returned only includes dependencies, so we'll add the parent pkg to this list to pass on to installation method + parentAndDeps.Add(pkgToInstall); + + _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackage(), found all dependencies"); + + return InstallParentAndDependencyPackages(parentAndDeps, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); } + else { + // If we don't install dependencies, we're only installing the parent pkg so we can short circut and simply install the parent pkg. + // TODO: check this version and prerelease combo + Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgToInstall.Version.ToString(), true, out ErrorRecord installNameErrRecord); - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + if (installNameErrRecord != null) + { + errRecord = installNameErrRecord; + return packagesHash; + } - if (!installedToTempPathSuccessfully) + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + if (!installedToTempPathSuccessfully) + { + return packagesHash; + } + } + } + + return updatedPackagesHash; + } + + // Concurrently Updates + private ConcurrentDictionary InstallParentAndDependencyPackages(List parentAndDeps, ServerApiCall currentServer, string tempInstallPath, ConcurrentDictionary packagesHash, ConcurrentDictionary updatedPackagesHash, PSResourceInfo pkgToInstall) + { + List errors = new List(); + + int azaccounts = 0; + // TODO: figure out a good threshold and parallel count + int processorCount = Environment.ProcessorCount; + if (parentAndDeps.Count > processorCount) + { + // Set the maximum degree of parallelism to 32? (Invoke-Command has default of 32, that's where we got this number from) + // If installing more than 3 packages, do so concurrently + // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. + int maxDegreeOfParallelism = processorCount * 4; + Parallel.ForEach(parentAndDeps, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, depPkg => { + var depPkgName = depPkg.Name; + var depPkgVersion = depPkg.Version.ToString(); + // Console.WriteLine($"Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); + + Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + errors.Add(installNameErrRecord); + } + + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + + if (!installedToTempPathSuccessfully) + { + errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); + } + }); + + if (errors.Count > 0) + { + // Write out all errors collected from Parallel.ForEach + foreach (var err in errors) + { + _cmdletPassedIn.WriteError(err); + } + return packagesHash; } + + return updatedPackagesHash; } + else + { + // Install the good old fashioned way + foreach (var pkgToBeInstalled in parentAndDeps) + { + var pkgToInstallName = pkgToBeInstalled.Name; + var pkgToInstallVersion = pkgToBeInstalled.Version.ToString(); + Stream responseStream = currentServer.InstallPackage(pkgToInstallName, pkgToInstallVersion, true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + _cmdletPassedIn.WriteError(installNameErrRecord); + return packagesHash; + } - return updatedPackagesHash; + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + + if (!installedToTempPathSuccessfully) + { + _cmdletPassedIn.WriteError(tempSaveErrRecord ?? tempInstallErrRecord); + return packagesHash; + } + } + + return updatedPackagesHash; + } } /// @@ -926,11 +970,11 @@ private bool TryInstallToTempPath( string pkgName, string normalizedPkgVersion, PSResourceInfo pkgToInstall, - Hashtable packagesHash, - out Hashtable updatedPackagesHash, + ConcurrentDictionary packagesHash, + out ConcurrentDictionary updatedPackagesHash, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TryInstallToTempPath()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::TryInstallToTempPath()"); error = null; updatedPackagesHash = packagesHash; try @@ -1031,7 +1075,7 @@ private bool TryInstallToTempPath( { foreach (ErrorRecord parseError in parseScriptFileErrors) { - _cmdletPassedIn.WriteError(parseError); + // _cmdletPassedIn.WriteError(parseError); } error = new ErrorRecord( @@ -1048,7 +1092,7 @@ private bool TryInstallToTempPath( // This package is not a PowerShell package (eg a resource from the NuGet Gallery). installPath = _pathsToInstallPkg.Find(path => path.EndsWith("Modules", StringComparison.InvariantCultureIgnoreCase)); - _cmdletPassedIn.WriteVerbose($"This resource is not a PowerShell package and will be installed to the modules path: {installPath}."); + //_cmdletPassedIn.WriteVerbose($"This resource is not a PowerShell package and will be installed to the modules path: {installPath}."); isModule = true; } @@ -1067,7 +1111,7 @@ private bool TryInstallToTempPath( if (!updatedPackagesHash.ContainsKey(pkgName)) { // Add pkg info to hashtable. - updatedPackagesHash.Add(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", isModule }, { "isScript", isScript }, @@ -1104,11 +1148,11 @@ private bool TrySaveNupkgToTempPath( string pkgName, string normalizedPkgVersion, PSResourceInfo pkgToInstall, - Hashtable packagesHash, - out Hashtable updatedPackagesHash, + ConcurrentDictionary packagesHash, + out ConcurrentDictionary updatedPackagesHash, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); + // _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); error = null; updatedPackagesHash = packagesHash; @@ -1132,7 +1176,7 @@ private bool TrySaveNupkgToTempPath( if (!updatedPackagesHash.ContainsKey(pkgName)) { // Add pkg info to hashtable. - updatedPackagesHash.Add(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) + updatedPackagesHash.TryAdd(pkgName, new Hashtable(StringComparer.InvariantCultureIgnoreCase) { { "isModule", "" }, { "isScript", "" }, @@ -1228,9 +1272,9 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error /// /// Moves package files/directories from the temp install path into the final install path location. /// - private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hashtable packagesHash) + private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, ConcurrentDictionary packagesHash) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TryMoveInstallContent()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::TryMoveInstallContent()"); foreach (string pkgName in packagesHash.Keys) { Hashtable pkgInfo = packagesHash[pkgName] as Hashtable; @@ -1255,7 +1299,7 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash moduleManifestVersion: pkgVersion, scriptPath); - _cmdletPassedIn.WriteVerbose($"Successfully installed package '{pkgName}' to location '{installPath}'"); + //_cmdletPassedIn.WriteVerbose($"Successfully installed package '{pkgName}' to location '{installPath}'"); if (!_savePkg && isScript) { @@ -1274,20 +1318,20 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash if (!String.IsNullOrEmpty(envPATHVarValue) && !envPATHVarValue.Contains(installPath) && !envPATHVarValue.Contains(installPathwithBackSlash)) { - _cmdletPassedIn.WriteWarning(String.Format(ScriptPATHWarning, scope, installPath)); + //_cmdletPassedIn.WriteWarning(String.Format(ScriptPATHWarning, scope, installPath)); } } } - catch (Exception e) + catch (Exception) { - _cmdletPassedIn.WriteError(new ErrorRecord( + /*_cmdletPassedIn.WriteError(new ErrorRecord( new PSInvalidOperationException( message: $"Unable to successfully install package '{pkgName}': '{e.Message}'", innerException: e), "InstallPackageFailed", ErrorCategory.InvalidOperation, _cmdletPassedIn)); - + */ return false; } } @@ -1300,7 +1344,7 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash /// private bool CallAcceptLicense(PSResourceInfo p, string moduleManifest, string tempInstallPath, string newVersion, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::CallAcceptLicense()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::CallAcceptLicense()"); error = null; var requireLicenseAcceptance = false; @@ -1404,7 +1448,7 @@ private bool CallAcceptLicense(PSResourceInfo p, string moduleManifest, string t /// private bool DetectClobber(string pkgName, Hashtable parsedMetadataHashtable, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::DetectClobber()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::DetectClobber()"); error = null; bool foundClobber = false; @@ -1468,7 +1512,7 @@ private bool DetectClobber(string pkgName, Hashtable parsedMetadataHashtable, ou /// private bool CreateMetadataXMLFile(string dirNameVersion, string installPath, PSResourceInfo pkg, bool isModule, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::CreateMetadataXMLFile()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::CreateMetadataXMLFile()"); error = null; bool success = true; // Script will have a metadata file similar to: "TestScript_InstalledScriptInfo.xml" @@ -1498,7 +1542,7 @@ private bool CreateMetadataXMLFile(string dirNameVersion, string installPath, PS /// private void DeleteExtraneousFiles(string packageName, string dirNameVersion) { - _cmdletPassedIn.WriteDebug("In InstallHelper::DeleteExtraneousFiles()"); + // _cmdletPassedIn.WriteDebug("In InstallHelper::DeleteExtraneousFiles()"); // Deleting .nupkg SHA file, .nuspec, and .nupkg after unpacking the module // since we download as .zip for HTTP calls, we shouldn't have .nupkg* files // var nupkgSHAToDelete = Path.Combine(dirNameVersion, pkgIdString + ".nupkg.sha512"); @@ -1511,22 +1555,22 @@ private void DeleteExtraneousFiles(string packageName, string dirNameVersion) if (File.Exists(nuspecToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{nuspecToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{nuspecToDelete}'"); File.Delete(nuspecToDelete); } if (File.Exists(contentTypesToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{contentTypesToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{contentTypesToDelete}'"); File.Delete(contentTypesToDelete); } if (Directory.Exists(relsDirToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{relsDirToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{relsDirToDelete}'"); Utils.DeleteDirectory(relsDirToDelete); } if (Directory.Exists(packageDirToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{packageDirToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{packageDirToDelete}'"); Utils.DeleteDirectory(packageDirToDelete); } } diff --git a/src/code/LocalServerApiCalls.cs b/src/code/LocalServerApiCalls.cs index cc43c340d..1726dda17 100644 --- a/src/code/LocalServerApiCalls.cs +++ b/src/code/LocalServerApiCalls.cs @@ -13,6 +13,7 @@ using System.Net; using System.Management.Automation; using System.Runtime.ExceptionServices; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -39,6 +40,15 @@ public LocalServerAPICalls (PSRepositoryInfo repository, PSCmdlet cmdletPassedIn #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return null; + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Examples: Search -Repository PSGallery @@ -109,6 +119,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return FindNameHelper(packageName, Utils.EmptyStrArray, includePrerelease, type, out errRecord); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/NuGetServerAPICalls.cs b/src/code/NuGetServerAPICalls.cs index 1497c83da..523844fc8 100644 --- a/src/code/NuGetServerAPICalls.cs +++ b/src/code/NuGetServerAPICalls.cs @@ -48,6 +48,15 @@ public NuGetServerAPICalls (PSRepositoryInfo repository, PSCmdlet cmdletPassedIn #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return null; + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Examples: Search -Repository MyNuGetServer @@ -183,6 +192,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: FindResponseType); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support diff --git a/src/code/ServerApiCall.cs b/src/code/ServerApiCall.cs index 4580e362e..f88f14053 100644 --- a/src/code/ServerApiCall.cs +++ b/src/code/ServerApiCall.cs @@ -10,6 +10,7 @@ using System.Text; using System.Runtime.ExceptionServices; using System.Management.Automation; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -81,6 +82,13 @@ public ServerApiCall(PSRepositoryInfo repository, NetworkCredential networkCrede /// public abstract FindResults FindName(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord); + /// + /// Find method which allows for searching for package by single name and returns latest version. + /// Name: no wildcard support + /// Examples: Search "PowerShellGet" + /// + public abstract Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type); + /// /// Find method which allows for searching for package by single name and tag and returns latest version. /// Name: no wildcard support @@ -118,6 +126,24 @@ public ServerApiCall(PSRepositoryInfo repository, NetworkCredential networkCrede /// public abstract FindResults FindVersion(string packageName, string version, ResourceType type, out ErrorRecord errRecord); + /// + /// Find method which allows for searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// Examples: Search "PowerShellGet" "2.2.5" + /// + public abstract Task FindVersionAsync(string packageName, string version, ResourceType type); + + + /// + /// Find method which allows for searching for single name with specific version. + /// Name: no wildcard support + /// Version: no wildcard support + /// Examples: Search "PowerShellGet" "2.2.5" + /// + + public abstract Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest); + /// /// Find method which allows for searching for single name with specific version. /// Name: no wildcard support diff --git a/src/code/Utils.cs b/src/code/Utils.cs index 26d3ab25e..8dc6bec57 100644 --- a/src/code/Utils.cs +++ b/src/code/Utils.cs @@ -1342,37 +1342,85 @@ private static bool TryReadPSDataFile( out Hashtable dataFileInfo, out Exception error) { + dataFileInfo = null; + error = null; try { if (filePath is null) { throw new PSArgumentNullException(nameof(filePath)); } - string contents = System.IO.File.ReadAllText(filePath); - var scriptBlock = System.Management.Automation.ScriptBlock.Create(contents); - - // Ensure that the content script block is safe to convert into a PSDataFile Hashtable. - // This will throw for unsafe content. - scriptBlock.CheckRestrictedLanguage( - allowedCommands: allowedCommands, - allowedVariables: allowedVariables, - allowEnvironmentVariables: allowEnvironmentVariables); - - // Convert contents into PSDataFile Hashtable by executing content as script. - object result = scriptBlock.InvokeReturnAsIs(); - if (result is PSObject psObject) + + // Parallel.ForEach calls into this method. + // Each thread needs its own runspace created to provide a separate environment for operations to run independently. + Runspace runspace = RunspaceFactory.CreateRunspace(); + runspace.Open(); + runspace.SessionStateProxy.LanguageMode = PSLanguageMode.ConstrainedLanguage; + + // Set the created runspace as the default for the current thread + Runspace.DefaultRunspace = runspace; + + using (System.Management.Automation.PowerShell pwsh = System.Management.Automation.PowerShell.Create()) { - result = psObject.BaseObject; - } + pwsh.Runspace = runspace; - dataFileInfo = (Hashtable)result; - error = null; - return true; + var cmd = new Command( + command: contents, + isScript: true, + useLocalScope: true); + cmd.MergeMyResults( + myResult: PipelineResultTypes.Error | PipelineResultTypes.Warning | PipelineResultTypes.Verbose | PipelineResultTypes.Debug | PipelineResultTypes.Information, + toResult: PipelineResultTypes.Output); + pwsh.Commands.AddCommand(cmd); + + + try + { + // Invoke the pipeline and retrieve the results + var results = pwsh.Invoke(); + + if (results[0] is PSObject pwshObj) + { + switch (pwshObj.BaseObject) + { + case ErrorRecord err: + //_cmdletPassedIn.WriteError(error); + break; + + case WarningRecord warning: + //cmdlet.WriteWarning(warning.Message); + break; + + case VerboseRecord verbose: + //cmdlet.WriteVerbose(verbose.Message); + break; + + case DebugRecord debug: + //cmdlet.WriteDebug(debug.Message); + break; + + case InformationRecord info: + //cmdlet.WriteInformation(info); + break; + + case Hashtable result: + dataFileInfo = result; + return true; + } + } + } + catch (Exception ex) + { + error = ex; + } + } + runspace.Close(); + + return false; } catch (Exception ex) { - dataFileInfo = null; error = ex; return false; } diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index 94d0b3a0b..3aba02b64 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -336,7 +336,7 @@ public override FindResults FindCommandOrDscResource(string[] tags, bool include /// public override FindResults FindName(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindName()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindName()"); // Make sure to include quotations around the package name // This should return the latest stable version or the latest prerelease version (respectively) @@ -397,6 +397,72 @@ public override FindResults FindName(string packageName, bool includePrerelease, return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); } + /// + /// Find method which allows for searching for single name and returns latest version. + /// Name: no wildcard support + /// Examples: Search "PowerShellGet" + /// API call: + /// - No prerelease: http://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet' + /// - Include prerelease: http://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet' + /// Implementation Note: Need to filter further for latest version (prerelease or non-prerelease depending on user preference) + /// + public override async Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindNameAsync()"); + // Make sure to include quotations around the package name + + // This should return the latest stable version or the latest prerelease version (respectively) + // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet'&$filter=IsLatestVersion and substringof('PSModule', Tags) eq true + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary{ + { "$inlinecount", "allpages" }, + { "id", $"'{packageName}'" }, + }); + var filterBuilder = queryBuilder.FilterBuilder; + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + filterBuilder.AddCriterion(includePrerelease ? "IsAbsoluteLatestVersion eq true" : "IsLatestVersion eq true"); + if (type != ResourceType.None) { + filterBuilder.AddCriterion(GetTypeFilterForRequest(type)); + } + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + string response; + try + { + response = await HttpRequestCallAsync(requestUrlV2); + } + catch (Exception e) + { + // usually this is for errors in calling the V2 server, but for ADO V2 this error will include package not found errors which we want to deliver in a standard message + if (_isADORepo && e is ResourceNotFoundException) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'. For ADO feed, if the package is in an upstream feed make sure you are authenticated to the upstream feed.", e); + } + + throw; + } + + int count = GetCountFromResponse(response, out ErrorRecord errRecord); + if (errRecord != null) + { + throw errRecord.Exception; + } + + if (count == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'."); + } + + return new FindResults(stringResponse: new string[]{ response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + /// /// Find method which allows for searching for single name and tag and returns latest version. /// Name: no wildcard support @@ -633,7 +699,7 @@ public override FindResults FindVersionGlobbing(string packageName, VersionRange /// public override FindResults FindVersion(string packageName, string version, ResourceType type, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersion()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersion()"); // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion eq '1.1.0' and substringof('PSModule', Tags) eq true // Quotations around package name and version do not matter, same metadata gets returned. // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. @@ -674,7 +740,7 @@ public override FindResults FindVersion(string packageName, string version, Reso } int count = GetCountFromResponse(response, out errRecord); - _cmdletPassedIn.WriteDebug($"Count from response is '{count}'"); + //_cmdletPassedIn.WriteDebug($"Count from response is '{count}'"); if (errRecord != null) { @@ -694,6 +760,64 @@ public override FindResults FindVersion(string packageName, string version, Reso return new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); } + + public override async Task FindVersionAsync(string packageName, string version, ResourceType type) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionAsync()"); + // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion eq '1.1.0' and substringof('PSModule', Tags) eq true + // Quotations around package name and version do not matter, same metadata gets returned. + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary{ + { "$inlinecount", "allpages" }, + { "id", $"'{packageName}'" }, + }); + var filterBuilder = queryBuilder.FilterBuilder; + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + filterBuilder.AddCriterion($"NormalizedVersion eq '{version}'"); + if (type != ResourceType.None) { + filterBuilder.AddCriterion(GetTypeFilterForRequest(type)); + } + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + string response; + try + { + response = await HttpRequestCallAsync(requestUrlV2); + } + catch (Exception e) + { + // usually this is for errors in calling the V2 server, but for ADO V2 this error will include package not found errors which we want to deliver with a standard message + if (_isADORepo && e is ResourceNotFoundException) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' and version '{version}' could not be found in repository '{Repository.Name}'. For ADO feed, if the package is in an upstream feed make sure you are authenticated to the upstream feed.", e); + } + + throw; + } + + int count = GetCountFromResponse(response, out ErrorRecord errRecord); + //_cmdletPassedIn.WriteDebug($"Count from response is '{count}'"); + + if (errRecord != null) + { + throw errRecord.Exception; + } + + if (count == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}', version '{version}' could not be found in repository '{Repository.Name}'."); + } + + return new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + /// /// Find method which allows for searching for single name with specific version and tag. /// Name: no wildcard support @@ -767,6 +891,7 @@ public override FindResults FindVersionWithTag(string packageName, string versio /// public override Stream InstallPackage(string packageName, string packageVersion, bool includePrerelease, out ErrorRecord errRecord) { + errRecord = null; Stream results = new MemoryStream(); if (string.IsNullOrEmpty(packageVersion)) { @@ -779,7 +904,8 @@ public override Stream InstallPackage(string packageName, string packageVersion, return results; } - results = InstallVersion(packageName, packageVersion, out errRecord); + //results = InstallVersion(packageName, packageVersion, out errRecord); + results = InstallVersionAsync(packageName, packageVersion).GetAwaiter().GetResult(); return results; } @@ -788,13 +914,13 @@ public override Stream InstallPackage(string packageName, string packageVersion, /// private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCall()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCall()"); errRecord = null; string response = string.Empty; try { - _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + // _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); response = SendV2RequestAsync(request, _sessionClient).GetAwaiter().GetResult(); @@ -834,7 +960,63 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) if (string.IsNullOrEmpty(response)) { - _cmdletPassedIn.WriteDebug("Response is empty"); + // _cmdletPassedIn.WriteDebug("Response is empty"); + } + + return response; + } + + /// + /// Helper method that makes the HTTP request for the V2 server protocol url passed in for find APIs. + /// + private async Task HttpRequestCallAsync(string requestUrlV2) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCall()"); + string response = string.Empty; + + try + { + // _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); + + response = await SendV2RequestAsync(request, _sessionClient); + } + catch (ResourceNotFoundException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "ResourceNotFound", + // ErrorCategory.InvalidResult, + // this); + } + catch (UnauthorizedException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "UnauthorizedRequest", + // ErrorCategory.InvalidResult, + // this); + } + catch (HttpRequestException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestCallFailure", + // ErrorCategory.ConnectionError, + // this); + } + catch (Exception) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestCallFailure", + // ErrorCategory.ConnectionError, + // this); + } + + if (string.IsNullOrEmpty(response)) + { + // _cmdletPassedIn.WriteDebug("Response is empty"); } return response; @@ -843,15 +1025,60 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) /// /// Helper method that makes the HTTP request for the V2 server protocol url passed in for install APIs. /// + private async Task HttpRequestCallForContentAsync(string requestUrlV2) + { + // _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContentAsync()"); + HttpContent content = null; + + try + { + //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); + + content = await SendV2RequestForContentAsync(request, _sessionClient); + } + catch (HttpRequestException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestFailure", + // ErrorCategory.ConnectionError, + // this); + } + catch (ArgumentNullException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestFailure", + // ErrorCategory.InvalidData, + // this); + } + catch (InvalidOperationException) + { + // errRecord = new ErrorRecord( + // exception: e, + // "HttpRequestFailure", + // ErrorCategory.InvalidOperation, + // this); + } + + if (content == null || string.IsNullOrEmpty(content.ToString())) + { + //_cmdletPassedIn.WriteDebug("Response is empty"); + } + + return content; + } + private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); + // _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); errRecord = null; HttpContent content = null; try { - _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); content = SendV2RequestForContentAsync(request, _sessionClient).GetAwaiter().GetResult(); @@ -883,7 +1110,7 @@ private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorReco if (content == null || string.IsNullOrEmpty(content.ToString())) { - _cmdletPassedIn.WriteDebug("Response is empty"); + //_cmdletPassedIn.WriteDebug("Response is empty"); } return content; @@ -1276,7 +1503,7 @@ private string FindNameGlobbingWithTag(string packageName, string[] tags, Resour /// private string FindVersionGlobbing(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, int skip, bool getOnlyLatest, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion gt '1.0.0' and NormalizedVersion lt '2.2.5' and substringof('PSModule', Tags) eq true //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= NormalizedVersion gt '1.1.1' and NormalizedVersion lt '2.2.5' // NormalizedVersion doesn't include trailing zeroes @@ -1372,6 +1599,157 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange return HttpRequestCall(requestUrlV2, out errRecord); } + + public override async Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + List responses = new List(); + int skip = 0; + + var initialResponse = await FindVersionGlobbingAsync(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest); + // if (errRecord != null) + // { + // return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + // } + + int initialCount = GetCountFromResponse(initialResponse, out ErrorRecord errRecord); + if (errRecord != null) + { + return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + + if (initialCount == 0) + { + throw new ResourceNotFoundException($"Package with name '{packageName}' could not be found in repository '{Repository.Name}'."); + } + + responses.Add(initialResponse); + + if (!getOnlyLatest) + { + int count = (int)Math.Ceiling((double)(initialCount / 100)); + + while (count > 0) + { + //_cmdletPassedIn.WriteDebug($"Count is '{count}'"); + // skip 100 + skip += 100; + var tmpResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, skip, getOnlyLatest, out errRecord); + if (errRecord != null) + { + return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + responses.Add(tmpResponse); + count--; + } + } + + return new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); + } + + + /// + /// Helper method for string[] FindVersionGlobbing() + /// + private async Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, int skip, bool getOnlyLatest) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion gt '1.0.0' and NormalizedVersion lt '2.2.5' and substringof('PSModule', Tags) eq true + //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= NormalizedVersion gt '1.1.1' and NormalizedVersion lt '2.2.5' + // NormalizedVersion doesn't include trailing zeroes + // Notes: this could allow us to take a version range (i.e (2.0.0, 3.0.0.0]) and deconstruct it and add options to the Filter for Version to describe that range + // will need to filter additionally, if IncludePrerelease=false, by default we get stable + prerelease both back + // Current bug: Find PSGet -Version "2.0.*" -> https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= Version gt '2.0.*' and Version lt '2.1' + // Make sure to include quotations around the package name + + //and IsPrerelease eq false + // ex: + // (2.0.0, 3.0.0) + // $filter= NVersion gt '2.0.0' and NVersion lt '3.0.0' + + // [2.0.0, 3.0.0] + // $filter= NVersion ge '2.0.0' and NVersion le '3.0.0' + + // [2.0.0, 3.0.0) + // $filter= NVersion ge '2.0.0' and NVersion lt '3.0.0' + + // (2.0.0, 3.0.0] + // $filter= NVersion gt '2.0.0' and NVersion le '3.0.0' + + // [, 2.0.0] + // $filter= NVersion le '2.0.0' + + string format = "NormalizedVersion {0} {1}"; + string minPart = String.Empty; + string maxPart = String.Empty; + + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary { + {"$inlinecount", "allpages"}, + {"$skip", skip.ToString()}, + {"$orderby", "NormalizedVersion desc"}, + {"id", $"'{packageName}'"} + }); + + var filterBuilder = queryBuilder.FilterBuilder; + + if (versionRange.MinVersion != null) + { + string operation = versionRange.IsMinInclusive ? "ge" : "gt"; + minPart = String.Format(format, operation, $"'{versionRange.MinVersion.ToNormalizedString()}'"); + } + + if (versionRange.MaxVersion != null) + { + string operation = versionRange.IsMaxInclusive ? "le" : "lt"; + // Adding '9' as a digit to the end of the patch portion of the version + // because we want to retrieve all the prerelease versions for the upper end of the range + // and PSGallery views prerelease as higher than its stable. + // eg 3.0.0-prerelease > 3.0.0 + // If looking for versions within '[1.9.9,1.9.9]' including prerelease values, this will change it to search for '[1.9.9,1.9.99]' + // and find any pkg versions that are 1.9.9-prerelease. + string maxString = includePrerelease ? $"{versionRange.MaxVersion.Major}.{versionRange.MaxVersion.Minor}.{versionRange.MaxVersion.Patch.ToString() + "9"}" : + $"{versionRange.MaxVersion.ToNormalizedString()}"; + if (NuGetVersion.TryParse(maxString, out NuGetVersion maxVersion)) + { + maxPart = String.Format(format, operation, $"'{maxVersion.ToNormalizedString()}'"); + } + else { + maxPart = String.Format(format, operation, $"'{versionRange.MaxVersion.ToNormalizedString()}'"); + } + } + + string versionFilterParts = String.Empty; + if (!String.IsNullOrEmpty(minPart)) + { + filterBuilder.AddCriterion(minPart); + } + if (!String.IsNullOrEmpty(maxPart)) + { + filterBuilder.AddCriterion(maxPart); + } + if (!includePrerelease) { + filterBuilder.AddCriterion("IsPrerelease eq false"); + } + + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + + // If it's a JFrog repository do not include the Id filter portion since JFrog uses 'Title' instead of 'Id', + // however filtering on 'and Title eq '' returns "Response status code does not indicate success: 500". + if (!_isJFrogRepo) { + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + } + + if (type == ResourceType.Script) { + filterBuilder.AddCriterion($"substringof('PS{type.ToString()}', Tags) eq true"); + } + + + var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + + return await HttpRequestCallAsync(requestUrlV2); + } + + /// /// Installs package with specific name and version. /// Name: no wildcard support. @@ -1382,7 +1760,7 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange /// private Stream InstallVersion(string packageName, string version, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersion()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersion()"); string requestUrlV2; if (_isADORepo) @@ -1421,6 +1799,40 @@ private Stream InstallVersion(string packageName, string version, out ErrorRecor return response.ReadAsStreamAsync().Result; } + private async Task InstallVersionAsync(string packageName, string version) + { + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersionAsync()"); + string requestUrlV2; + + if (_isADORepo) + { + // eg: https://pkgs.dev.azure.com///_packaging//nuget/v2?id=test_module&version=5.0.0 + requestUrlV2 = $"{Repository.Uri}?id={packageName.ToLower()}&version={version}"; + } + else if (_isJFrogRepo) + { + // eg: https://.jfrog.io/artifactory/api/nuget//Download/test_module/5.0.0 + requestUrlV2 = $"{Repository.Uri}/Download/{packageName}/{version}"; + } + else + { + requestUrlV2 = $"{Repository.Uri}/package/{packageName}/{version}"; + } + + + var response = await HttpRequestCallForContentAsync(requestUrlV2); + + + if (response is null) + { + throw new Exception($"No content was returned by repository '{Repository.Name}'"); + } + + var retResponse = await response.ReadAsStreamAsync(); + + return retResponse; + } + private string GetTypeFilterForRequest(ResourceType type) { string typeFilterPart = string.Empty; if (type == ResourceType.Script) @@ -1483,7 +1895,7 @@ public int GetCountFromResponse(string httpResponse, out ErrorRecord errRecord) } else { - _cmdletPassedIn.WriteDebug($"Property 'count' and 'd:Id' could not be found in response. This may indicate that the package could not be found"); + //_cmdletPassedIn.WriteDebug($"Property 'count' and 'd:Id' could not be found in response. This may indicate that the package could not be found"); } } } diff --git a/src/code/V3ServerAPICalls.cs b/src/code/V3ServerAPICalls.cs index 903b1da55..2bd701559 100644 --- a/src/code/V3ServerAPICalls.cs +++ b/src/code/V3ServerAPICalls.cs @@ -88,6 +88,16 @@ public V3ServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmdletPassedIn, Ne #region Overridden Methods + public override Task FindVersionAsync(string packageName, string version, ResourceType type) + { + return Task.FromResult(null); + } + + public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest) + { + return null; + } + /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. /// Not supported for V3 repository. @@ -155,6 +165,11 @@ public override FindResults FindName(string packageName, bool includePrerelease, return FindNameHelper(packageName, tags: Utils.EmptyStrArray, includePrerelease, type, out errRecord); } + public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type) + { + return null; + } + /// /// Find method which allows for searching for single name and specified tag(s) and returns latest version. /// Name: no wildcard support