Fix resuming downloads and add download tests

Author: shana

.yamato/test.yml

@@ -115,17 +115,6 @@ test_production_trigger:
     {% endfor %}
     {% endfor %}
   triggers:
-    pull_requests:
-      - targets:
-          only:
-            - "master"
-            - "/v\\d+\\.\\d+.*/"
-            - "/release.*/v\\d+\\.\\d+.*/"
-    branches:
-      only:
-      - "master"
-      - "/v\\d+\\.\\d+.*/"
-      - "/release.*/v\\d+\\.\\d+.*/"
     tags:
       only:
-      - "/.*/"
+      - "/release.*/v\\d+\\.\\d+.*/"

Unity.Editor.Tasks.sln

@@ -15,6 +15,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestBase", "tests\Helpers\T
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Helper.CommandLine", "tests\Helpers\Helper.CommandLine\Helper.CommandLine.csproj", "{9A8F0974-A7AE-433B-8C49-5DBD30B0E479}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Helper.WebServer", "tests\Helpers\Helper.WebServer\Helper.WebServer.csproj", "{F946F77C-25DD-44CE-AB8A-5E23528AC70F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -37,6 +39,10 @@ Global
 		{9A8F0974-A7AE-433B-8C49-5DBD30B0E479}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9A8F0974-A7AE-433B-8C49-5DBD30B0E479}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9A8F0974-A7AE-433B-8C49-5DBD30B0E479}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F946F77C-25DD-44CE-AB8A-5E23528AC70F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F946F77C-25DD-44CE-AB8A-5E23528AC70F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F946F77C-25DD-44CE-AB8A-5E23528AC70F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F946F77C-25DD-44CE-AB8A-5E23528AC70F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -45,6 +51,7 @@ Global
 		{452FEC2C-09AD-4FB5-965C-29D1FB872A4E} = {13E9D517-C9FB-4DBE-B4B2-10748698E27E}
 		{93F4BC16-1A50-4DB1-8FA3-78FCDB54B365} = {93CD1CE7-1B0E-47A0-ADC3-F30702D65605}
 		{9A8F0974-A7AE-433B-8C49-5DBD30B0E479} = {93CD1CE7-1B0E-47A0-ADC3-F30702D65605}
+		{F946F77C-25DD-44CE-AB8A-5E23528AC70F} = {93CD1CE7-1B0E-47A0-ADC3-F30702D65605}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {11F9BFE4-27A0-4CCF-A3C1-3683D75126A6}

src/com.unity.editor.tasks/Editor/Extensions/PathExtensions.cs

@@ -9,14 +9,16 @@
 
 namespace Unity.Editor.Tasks.Extensions
 {
-	public static class PathExtensions
+	using Internal.IO;
+
+	internal static class PathExtensions
 	{
-		public static string ToMD5(this string path)
+		public static string ToMD5(this SPath path)
 		{
 			byte[] computeHash;
 			using (var hash = System.Security.Cryptography.MD5.Create())
 			{
-				using (var stream = File.OpenRead(path))
+				using (var stream = path.OpenRead())
 				{
 					computeHash = hash.ComputeHash(stream);
 				}
@@ -25,12 +27,12 @@ public static string ToMD5(this string path)
 			return BitConverter.ToString(computeHash).Replace("-", string.Empty).ToLower();
 		}
 
-		public static string ToSha256(this string path)
+		public static string ToSha256(this SPath path)
 		{
 			byte[] computeHash;
 			using (var hash = System.Security.Cryptography.SHA256.Create())
 			{
-				using (var stream = File.OpenRead(path))
+				using (var stream = path.OpenRead())
 				{
 					computeHash = hash.ComputeHash(stream);
 				}
@@ -39,5 +41,4 @@ public static string ToSha256(this string path)
 			return BitConverter.ToString(computeHash).Replace("-", string.Empty).ToLower();
 		}
 	}
-
 }

src/com.unity.editor.tasks/Editor/Helpers/Utils.cs

@@ -10,6 +10,8 @@
 
 namespace Unity.Editor.Tasks.Helpers
 {
+	using Internal.IO;
+
 	public static class Utils
 	{
 		public static bool Copy(Stream source,
@@ -93,9 +95,9 @@ public static bool VerifyFileIntegrity(string file, string hash)
 				return false;
 			string actual;
 			if (hash.Length == 32)
-				actual = file.ToMD5();
+				actual = file.ToSPath().ToMD5();
 			else
-				actual = file.ToSha256();
+				actual = file.ToSPath().ToSha256();
 			return hash.Equals(actual, StringComparison.InvariantCultureIgnoreCase);
 		}
 	}

src/com.unity.editor.tasks/Editor/IO/SimpleIO.cs

@@ -1693,7 +1693,7 @@ public Stream OpenWrite(string path, FileMode mode)
 		{
 			if (!Path.IsPathRooted(path))
 				throw new ArgumentException("OpenWrite requires a rooted path", nameof(path));
-			return new FileStream(path, mode);
+			return new FileStream(path, mode, FileAccess.Write);
 		}
 
 		public string CurrentDirectory

src/com.unity.editor.tasks/Editor/Properties/AssemblyInfo.cs

@@ -9,4 +9,5 @@
 using System.Runtime.CompilerServices;
 
 [assembly: InternalsVisibleTo("Tasks.Tests")]
+[assembly: InternalsVisibleTo("Helper.WebServer")]
 [assembly: InternalsVisibleTo("com.unity.editor.tasks.tests")]

src/com.unity.editor.tasks/Editor/Tasks/DownloadTask.cs

@@ -11,6 +11,7 @@
 
 namespace Unity.Editor.Tasks
 {
+	using Internal.IO;
 	using Logging;
 	using Unity.Editor.Tasks.Helpers;
 
@@ -31,7 +32,7 @@ public DownloadTask(
 			RetryCount = retryCount;
 			Url = url;
 			Filename = string.IsNullOrEmpty(filename) ? url.Filename : filename;
-			TargetDirectory = targetDirectory;
+			this.targetDirectory = targetDirectory.ToSPath();
 			Name = $"Download {Url}";
 			Message = Filename;
 		}
@@ -73,8 +74,8 @@ protected virtual string RunDownload(bool success)
 			Exception exception = null;
 			var attempts = 0;
 			bool result = false;
-			var partialFile = Path.Combine(TargetDirectory, Filename + ".partial");
-			Directory.CreateDirectory(TargetDirectory);
+			var partialFile = targetDirectory.Combine(Filename + ".partial");
+			targetDirectory.EnsureDirectoryExists();
 			do
 			{
 				exception = null;
@@ -86,7 +87,7 @@ protected virtual string RunDownload(bool success)
 				{
 					Logger.Trace($"Download of {Url} to {Destination} Attempt {attempts + 1} of {RetryCount + 1}");
 
-					using (var destinationStream = File.OpenWrite(partialFile))
+					using (var destinationStream = partialFile.OpenWrite(FileMode.Append))
 					{
 						result = Downloader.Download(Logger, Url, destinationStream,
 							 (value, total) => {
@@ -97,7 +98,7 @@ protected virtual string RunDownload(bool success)
 
 					if (result)
 					{
-						File.Move(partialFile, Destination);
+						partialFile.Move(Destination);
 					}
 				}
 				catch (Exception ex)
@@ -118,11 +119,12 @@ protected virtual string RunDownload(bool success)
 
 		public UriString Url { get; }
 
-		public string TargetDirectory { get; }
+		private SPath targetDirectory;
+		public string TargetDirectory => targetDirectory.ToString();
 
 		public string Filename { get; }
 
-		public string Destination => Path.Combine(TargetDirectory, Filename);
+		public string Destination => targetDirectory.Combine(Filename).ToString();
 
 		protected int RetryCount { get; }
 	}
@@ -170,10 +172,25 @@ public DownloadData(UriString url, string file)
 
 	public class Downloader : TaskQueue<string, DownloadData>
 	{
+		/// <summary>
+		/// Called for every queued download task when it finishes.
+		/// </summary>
 		public event Action<UriString, string> OnDownloadComplete;
+		/// <summary>
+		/// Called for every queued download task when it fails.
+		/// </summary>
 		public event Action<UriString, Exception> OnDownloadFailed;
+		/// <summary>
+		/// Called for every queued download task when it starts.
+		/// </summary>
 		public event Action<UriString> OnDownloadStart;
 
+		/// <summary>
+		/// TaskQueue of DownloaderTask objects that can download multiple
+		/// things in parallel.
+		/// </summary>
+		/// <param name="taskManager"></param>
+		/// <param name="token"></param>
 		public Downloader(ITaskManager taskManager, CancellationToken token = default)
 			 : base(taskManager, t => {
 				 var dt = t as DownloadTask;

src/com.unity.editor.tasks/Tests/Editor/BaseTest_Shared.cs

@@ -11,7 +11,9 @@
 
 namespace BaseTests
 {
+	using SpoiledCat.Tests.TestWebServer;
 	using System.Threading;
+	using Unity.Editor.Tasks.Helpers;
 	using Unity.Editor.Tasks.Internal.IO;
 
 	internal class TestData : IDisposable
@@ -24,12 +26,18 @@ internal class TestData : IDisposable
 		public readonly IEnvironment Environment;
 		public readonly IProcessManager ProcessManager;
 		private readonly CancellationTokenSource cts;
+		public readonly SPath SourceDirectory;
+#if NUNIT
+		public readonly HttpServer HttpServer;
+#endif
+
 
-		public TestData(string testName, ILogging logger)
+		public TestData(string testName, ILogging logger, bool withHttpServer = false)
 		{
 			TestName = testName;
 			Logger = logger;
 			Watch = new Stopwatch();
+			SourceDirectory = TestContext.CurrentContext.TestDirectory.ToSPath();
 			TestPath = SPath.CreateTempDirectory(testName);
 			TaskManager = new TaskManager();
 			cts = CancellationTokenSource.CreateLinkedTokenSource(TaskManager.Token);
@@ -49,6 +57,19 @@ public TestData(string testName, ILogging logger)
 			InitializeEnvironment();
 			ProcessManager = new ProcessManager(Environment);
 
+#if NUNIT
+			if (withHttpServer)
+			{
+				var filesToServePath = SourceDirectory.Combine("files");
+				HttpServer = new HttpServer(filesToServePath, 0);
+				var started = new ManualResetEventSlim();
+				var task = TaskManager.With(HttpServer.Start, TaskAffinity.None);
+				task.OnStart += _ => started.Set();
+				task.Start();
+				started.Wait();
+			}
+#endif
+
 			Logger.Trace($"START {testName}");
 			Watch.Start();
 		}
@@ -84,6 +105,16 @@ private void InitializeEnvironment()
 		public void Dispose()
 		{
 			Watch.Stop();
+#if NUNIT
+			try
+			{
+				if (HttpServer != null)
+				{
+					HttpServer.Stop();
+				}
+			}
+			catch { }
+#endif
 
 			ProcessManager.Dispose();
 			if (SynchronizationContext.Current is IMainThreadSynchronizationContext ourContext)
@@ -98,11 +129,9 @@ public void Dispose()
 
 	public partial class BaseTest
 	{
-		protected const int Timeout = 30000;
+		protected const int Timeout = 3000;
 		protected const int RandomSeed = 120938;
 
-
-
 		protected void StartTrackTime(Stopwatch watch, ILogging logger, string message = "")
 		{
 			if (!string.IsNullOrEmpty(message))
@@ -181,6 +210,12 @@ public static void MatchesUnsorted<T>(this IEnumerable<T> actual, IEnumerable<T>
 		public static void Matches(this string actual, string expected) => Assert.AreEqual(expected, actual);
 		public static void Matches(this int actual, int expected) => Assert.AreEqual(expected, actual);
 		public static void Matches(this SPath actual, SPath expected) => Assert.AreEqual(expected, actual);
+
+		public static UriString FixPort(this UriString url, int port)
+		{
+			var uri = url.ToUri();
+			return UriString.TryParse(new UriBuilder(uri.Scheme, uri.Host, port, uri.PathAndQuery).Uri.ToString());
+		}
 	}
 
 	static class KeyValuePair

tests/Helpers/Helper.WebServer/Helper.WebServer.csproj

@@ -9,7 +9,6 @@
     </None>
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\..\..\src\com.spoiledcat.logging\Editor\SpoiledCat.Unity.Logging.csproj" />
-    <ProjectReference Include="..\..\..\src\com.spoiledcat.simplejson\Editor\SpoiledCat.Unity.SimpleJson.csproj" />
+    <ProjectReference Include="..\..\..\src\com.unity.editor.tasks\Editor\Unity.Editor.Tasks.csproj" />
   </ItemGroup>
 </Project>

tests/Helpers/Helper.WebServer/HttpServer.cs

@@ -9,9 +9,7 @@
 
 namespace SpoiledCat.Tests.TestWebServer
 {
-	using System.Xml;
-	using Logging;
-	using Json;
+	using Unity.Editor.Tasks.Logging;
 
 	public class HttpServer
 	{
@@ -39,7 +37,7 @@ public class HttpServer
 		/// <param name="port">Port of the server.</param>
 		public HttpServer(string path = null, int port = 0)
 		{
-			if (String.IsNullOrEmpty(path) || !Directory.Exists(path))
+			if (string.IsNullOrEmpty(path) || !Directory.Exists(path))
 			{
 				path = Path.Combine(Path.GetDirectoryName(GetType().Assembly.Location), "files");
 			}
@@ -92,31 +90,6 @@ private void Process(HttpListenerContext context)
 		{
 			Logger.Info("Handling request {0}", context.Request.Url.AbsolutePath);
 
-			if (context.Request.Url.AbsolutePath == "/api/usage/unity")
-			{
-				var streamReader = new StreamReader(context.Request.InputStream);
-				string body = null;
-				using (streamReader)
-				{
-					body = streamReader.ReadToEnd();
-				}
-				
-				Logger.Info(body);
-
-				var json = new { result = "Cool unity usage" }.ToJson();
-				context.Response.StatusCode = (int)HttpStatusCode.OK;
-				context.Response.ContentLength64 = json.Length;
-
-				string mime;
-				context.Response.ContentType = mimeTypeMappings.TryGetValue(".json", out mime)
-					? mime
-					: "application/octet-stream";
-				Utils.Copy(new MemoryStream(Encoding.UTF8.GetBytes(json)), context.Response.OutputStream, json.Length);
-				context.Response.OutputStream.Flush();
-				context.Response.Close();
-				return;
-			}
-
 			var filename = context.Request.Url.AbsolutePath;
 			filename = filename.TrimStart('/');
 			filename = filename.Replace('/', Path.DirectorySeparatorChar);
@@ -139,6 +112,7 @@ private void Process(HttpListenerContext context)
 
 				context.Response.AddHeader("Date", DateTime.Now.ToString("r"));
 				context.Response.AddHeader("Last-Modified", File.GetLastWriteTime(filename).ToString("r"));
+				context.Response.AddHeader("Accept-Ranges", "bytes");
 
 				using (var input = new FileStream(filename, FileMode.Open))
 				{
@@ -168,7 +142,7 @@ private void Process(HttpListenerContext context)
 						if (input.CanSeek && (input.Length > start) && (end <= input.Length))
 						{
 							context.Response.StatusCode = (int)HttpStatusCode.PartialContent;
-							context.Response.Headers.Add("Content-Range", $"{start}-{end}/{input.Length}");
+							context.Response.Headers.Add("Content-Range", $"bytes {start}-{end}/{input.Length}");
 							input.Seek(start, SeekOrigin.Current);
 						}
 						else