fix(uploads): resolve hash calculation memory crash and add hashing progress

src/lang/en/home.json

@@ -112,6 +112,7 @@
     "no_files_drag": "No files were dragged in.",
     "upload_files": "Choose Files",
     "upload_folder": "Choose Folder",
+    "hashing": "Hashing",
     "pending": "Pending",
     "uploading": "Uploading",
     "backending": "Uploading in the backend",

src/pages/home/uploads/form.ts

@@ -24,11 +24,15 @@ export const FormUpload: Upload = async (
     Overwrite: overwrite.toString(),
   }
   if (rapid) {
-    const { md5, sha1, sha256 } = await calculateHash(file)
+    setUpload("status", "hashing")
+    const { md5, sha1, sha256 } = await calculateHash(file, (p) => {
+      setUpload("progress", p | 0)
+    })
     headers["X-File-Md5"] = md5
     headers["X-File-Sha1"] = sha1
     headers["X-File-Sha256"] = sha256
   }
+  setUpload("status", "uploading")
   const resp: EmptyResp = await r.put("/fs/form", form, {
     headers: headers,
     onUploadProgress: (progressEvent) => {

src/pages/home/uploads/hash-worker.ts

@@ -0,0 +1,67 @@
+import { createMD5, createSHA1, createSHA256 } from "hash-wasm"
+
+interface WorkerProgressMessage {
+  type: "progress"
+  progress: number
+}
+
+interface WorkerResultMessage {
+  type: "result"
+  hash: { md5: string; sha1: string; sha256: string }
+}
+
+interface WorkerErrorMessage {
+  type: "error"
+  error: string
+}
+
+export type WorkerMessage =
+  | WorkerProgressMessage
+  | WorkerResultMessage
+  | WorkerErrorMessage
+
+self.onmessage = async (e: MessageEvent<{ file: File }>) => {
+  const { file } = e.data
+  try {
+    const [md5Digest, sha1Digest, sha256Digest] = await Promise.all([
+      createMD5(),
+      createSHA1(),
+      createSHA256(),
+    ])
+
+    const reader = file.stream().getReader()
+    let loaded = 0
+
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+
+      loaded += value.length
+      md5Digest.update(value)
+      sha1Digest.update(value)
+      sha256Digest.update(value)
+
+      const progress: WorkerProgressMessage = {
+        type: "progress",
+        progress: (loaded / file.size) * 100,
+      }
+      self.postMessage(progress)
+    }
+
+    const result: WorkerResultMessage = {
+      type: "result",
+      hash: {
+        md5: md5Digest.digest("hex"),
+        sha1: sha1Digest.digest("hex"),
+        sha256: sha256Digest.digest("hex"),
+      },
+    }
+    self.postMessage(result)
+  } catch (error) {
+    const err: WorkerErrorMessage = {
+      type: "error",
+      error: error instanceof Error ? error.message : String(error),
+    }
+    self.postMessage(err)
+  }
+}

src/pages/home/uploads/stream.ts

@@ -22,11 +22,15 @@ export const StreamUpload: Upload = async (
     Overwrite: overwrite.toString(),
   }
   if (rapid) {
-    const { md5, sha1, sha256 } = await calculateHash(file)
+    setUpload("status", "hashing")
+    const { md5, sha1, sha256 } = await calculateHash(file, (p) => {
+      setUpload("progress", p | 0)
+    })
     headers["X-File-Md5"] = md5
     headers["X-File-Sha1"] = sha1
     headers["X-File-Sha256"] = sha256
   }
+  setUpload("status", "uploading")
   const resp: EmptyResp = await r.put("/fs/put", file, {
     headers: headers,
     onUploadProgress: (progressEvent) => {

src/pages/home/uploads/types.ts

@@ -1,4 +1,10 @@
-type Status = "pending" | "uploading" | "backending" | "success" | "error"
+type Status =
+  | "pending"
+  | "hashing"
+  | "uploading"
+  | "backending"
+  | "success"
+  | "error"
 export interface UploadFileProps {
   name: string
   path: string
@@ -10,6 +16,7 @@ export interface UploadFileProps {
 }
 export const StatusBadge = {
   pending: "neutral",
+  hashing: "warning",
   uploading: "info",
   backending: "info",
   success: "success",

src/pages/home/uploads/util.ts

@@ -1,10 +1,11 @@
 import { UploadFileProps } from "./types"
-import { createMD5, createSHA1, createSHA256 } from "hash-wasm"
+import type { WorkerMessage } from "./hash-worker"
 
 export const traverseFileTree = async (entry: FileSystemEntry) => {
-  let res: File[] = []
+  const res: File[] = []
+
   const internalProcess = async (entry: FileSystemEntry, path: string) => {
-    const promise = new Promise<{}>((resolve, reject) => {
+    await new Promise<void>((resolve, reject) => {
       const errorCallback: ErrorCallback = (e) => {
         console.error(e)
         reject(e)
@@ -16,7 +17,7 @@ export const traverseFileTree = async (entry: FileSystemEntry) => {
           })
           res.push(newFile)
           console.log(newFile)
-          resolve({})
+          resolve()
         }, errorCallback)
       } else if (entry.isDirectory) {
         const dirReader = (entry as FileSystemDirectoryEntry).createReader()
@@ -28,10 +29,9 @@ export const traverseFileTree = async (entry: FileSystemEntry) => {
             if (entries.length > 0) {
               readEntries()
             } else {
-              resolve({})
+              resolve()
             }
 
-            /*  resolve({})
             /**
             why? https://stackoverflow.com/questions/3590058/does-html5-allow-drag-drop-upload-of-folders-or-a-folder-tree/53058574#53058574
             Unfortunately none of the existing answers are completely correct because 
@@ -42,16 +42,12 @@ export const traverseFileTree = async (entry: FileSystemEntry) => {
             until it returns an empty array. If we don't, we will miss some files/sub-directories in a directory 
             e.g. in Chrome, readEntries will only return at most 100 entries at a time.
 
-            if (entries.length > 0) {
-              readEntries()
-            }
             */
           }, errorCallback)
         }
         readEntries()
       }
     })
-    await promise
   }
   await internalProcess(entry, "")
   return res
@@ -60,32 +56,47 @@ export const traverseFileTree = async (entry: FileSystemEntry) => {
 export const File2Upload = (file: File): UploadFileProps => {
   return {
     name: file.name,
-    path: file.webkitRelativePath ? file.webkitRelativePath : file.name,
+    path: file.webkitRelativePath || file.name,
     size: file.size,
     progress: 0,
     speed: 0,
     status: "pending",
   }
 }
 
-export const calculateHash = async (file: File) => {
-  const md5Digest = await createMD5()
-  const sha1Digest = await createSHA1()
-  const sha256Digest = await createSHA256()
-  const reader = file.stream().getReader()
-  const read = async () => {
-    const { done, value } = await reader.read()
-    if (done) {
-      return
-    }
-    md5Digest.update(value)
-    sha1Digest.update(value)
-    sha256Digest.update(value)
-    await read()
-  }
-  await read()
-  const md5 = md5Digest.digest("hex")
-  const sha1 = sha1Digest.digest("hex")
-  const sha256 = sha256Digest.digest("hex")
-  return { md5, sha1, sha256 }
+export const calculateHash = async (
+  file: File,
+  onProgress?: (progress: number) => void,
+) => {
+  return new Promise<{ md5: string; sha1: string; sha256: string }>(
+    (resolve, reject) => {
+      const worker = new Worker(new URL("./hash-worker.ts", import.meta.url), {
+        type: "module",
+      })
+
+      const terminate = (fn: () => void) => {
+        worker.terminate()
+        fn()
+      }
+
+      worker.postMessage({ file })
+
+      worker.onmessage = (e: MessageEvent<WorkerMessage>) => {
+        const data = e.data
+        switch (data.type) {
+          case "progress":
+            onProgress?.(data.progress)
+            break
+          case "result":
+            terminate(() => resolve(data.hash))
+            break
+          case "error":
+            terminate(() => reject(new Error(data.error)))
+            break
+        }
+      }
+
+      worker.onerror = (e) => terminate(() => reject(e))
+    },
+  )
 }