improvement(uploads): add multipart upload + batching + retries (#938)

Sg312 · web-flow · commit 70fa628a2a55 · 2025-08-13T15:18:14.000-07:00
* File upload retries + multipart uploads

* Lint

* FIle uploads

* File uploads 2

* Lint

* Fix file uploads

* Add auth to file upload routes

* Lint
diff --git a/apps/sim/app/api/files/multipart/route.ts b/apps/sim/app/api/files/multipart/route.ts
@@ -0,0 +1,164 @@
+import {
+  AbortMultipartUploadCommand,
+  CompleteMultipartUploadCommand,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+} from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import { type NextRequest, NextResponse } from 'next/server'
+import { v4 as uuidv4 } from 'uuid'
+import { getSession } from '@/lib/auth'
+import { createLogger } from '@/lib/logs/console/logger'
+import { getStorageProvider, isUsingCloudStorage } from '@/lib/uploads'
+import { S3_KB_CONFIG } from '@/lib/uploads/setup'
+
+const logger = createLogger('MultipartUploadAPI')
+
+interface InitiateMultipartRequest {
+  fileName: string
+  contentType: string
+  fileSize: number
+}
+
+interface GetPartUrlsRequest {
+  uploadId: string
+  key: string
+  partNumbers: number[]
+}
+
+interface CompleteMultipartRequest {
+  uploadId: string
+  key: string
+  parts: Array<{
+    ETag: string
+    PartNumber: number
+  }>
+}
+
+export async function POST(request: NextRequest) {
+  try {
+    const session = await getSession()
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const action = request.nextUrl.searchParams.get('action')
+
+    if (!isUsingCloudStorage() || getStorageProvider() !== 's3') {
+      return NextResponse.json(
+        { error: 'Multipart upload is only available with S3 storage' },
+        { status: 400 }
+      )
+    }
+
+    const { getS3Client } = await import('@/lib/uploads/s3/s3-client')
+    const s3Client = getS3Client()
+
+    switch (action) {
+      case 'initiate': {
+        const data: InitiateMultipartRequest = await request.json()
+        const { fileName, contentType } = data
+
+        const safeFileName = fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_')
+        const uniqueKey = `kb/${uuidv4()}-${safeFileName}`
+
+        const command = new CreateMultipartUploadCommand({
+          Bucket: S3_KB_CONFIG.bucket,
+          Key: uniqueKey,
+          ContentType: contentType,
+          Metadata: {
+            originalName: fileName,
+            uploadedAt: new Date().toISOString(),
+            purpose: 'knowledge-base',
+          },
+        })
+
+        const response = await s3Client.send(command)
+
+        logger.info(`Initiated multipart upload for ${fileName}: ${response.UploadId}`)
+
+        return NextResponse.json({
+          uploadId: response.UploadId,
+          key: uniqueKey,
+        })
+      }
+
+      case 'get-part-urls': {
+        const data: GetPartUrlsRequest = await request.json()
+        const { uploadId, key, partNumbers } = data
+
+        const presignedUrls = await Promise.all(
+          partNumbers.map(async (partNumber) => {
+            const command = new UploadPartCommand({
+              Bucket: S3_KB_CONFIG.bucket,
+              Key: key,
+              PartNumber: partNumber,
+              UploadId: uploadId,
+            })
+
+            const url = await getSignedUrl(s3Client, command, { expiresIn: 3600 })
+            return { partNumber, url }
+          })
+        )
+
+        return NextResponse.json({ presignedUrls })
+      }
+
+      case 'complete': {
+        const data: CompleteMultipartRequest = await request.json()
+        const { uploadId, key, parts } = data
+
+        const command = new CompleteMultipartUploadCommand({
+          Bucket: S3_KB_CONFIG.bucket,
+          Key: key,
+          UploadId: uploadId,
+          MultipartUpload: {
+            Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
+          },
+        })
+
+        const response = await s3Client.send(command)
+
+        logger.info(`Completed multipart upload for key ${key}`)
+
+        const finalPath = `/api/files/serve/s3/${encodeURIComponent(key)}`
+
+        return NextResponse.json({
+          success: true,
+          location: response.Location,
+          path: finalPath,
+          key,
+        })
+      }
+
+      case 'abort': {
+        const data = await request.json()
+        const { uploadId, key } = data
+
+        const command = new AbortMultipartUploadCommand({
+          Bucket: S3_KB_CONFIG.bucket,
+          Key: key,
+          UploadId: uploadId,
+        })
+
+        await s3Client.send(command)
+
+        logger.info(`Aborted multipart upload for key ${key}`)
+
+        return NextResponse.json({ success: true })
+      }
+
+      default:
+        return NextResponse.json(
+          { error: 'Invalid action. Use: initiate, get-part-urls, complete, or abort' },
+          { status: 400 }
+        )
+    }
+  } catch (error) {
+    logger.error('Multipart upload error:', error)
+    return NextResponse.json(
+      { error: error instanceof Error ? error.message : 'Multipart upload failed' },
+      { status: 500 }
+    )
+  }
+}
diff --git a/apps/sim/app/api/files/presigned/route.ts b/apps/sim/app/api/files/presigned/route.ts
@@ -2,6 +2,7 @@ import { PutObjectCommand } from '@aws-sdk/client-s3'
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
 import { type NextRequest, NextResponse } from 'next/server'
 import { v4 as uuidv4 } from 'uuid'
+import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getStorageProvider, isUsingCloudStorage } from '@/lib/uploads'
 // Dynamic imports for storage clients to avoid client-side bundling
@@ -54,14 +55,19 @@ class ValidationError extends PresignedUrlError {
 
 export async function POST(request: NextRequest) {
   try {
+    const session = await getSession()
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
     let data: PresignedUrlRequest
     try {
       data = await request.json()
     } catch {
       throw new ValidationError('Invalid JSON in request body')
     }
 
-    const { fileName, contentType, fileSize, userId, chatId } = data
+    const { fileName, contentType, fileSize } = data
 
     if (!fileName?.trim()) {
       throw new ValidationError('fileName is required and cannot be empty')
@@ -90,10 +96,13 @@ export async function POST(request: NextRequest) {
             ? 'copilot'
             : 'general'
 
-    // Validate copilot-specific requirements
+    // Evaluate user id from session for copilot uploads
+    const sessionUserId = session.user.id
+
+    // Validate copilot-specific requirements (use session user)
     if (uploadType === 'copilot') {
-      if (!userId?.trim()) {
-        throw new ValidationError('userId is required for copilot uploads')
+      if (!sessionUserId?.trim()) {
+        throw new ValidationError('Authenticated user session is required for copilot uploads')
       }
     }
 
@@ -108,9 +117,21 @@ export async function POST(request: NextRequest) {
 
     switch (storageProvider) {
       case 's3':
-        return await handleS3PresignedUrl(fileName, contentType, fileSize, uploadType, userId)
+        return await handleS3PresignedUrl(
+          fileName,
+          contentType,
+          fileSize,
+          uploadType,
+          sessionUserId
+        )
       case 'blob':
-        return await handleBlobPresignedUrl(fileName, contentType, fileSize, uploadType, userId)
+        return await handleBlobPresignedUrl(
+          fileName,
+          contentType,
+          fileSize,
+          uploadType,
+          sessionUserId
+        )
       default:
         throw new StorageConfigError(`Unknown storage provider: ${storageProvider}`)
     }
diff --git a/apps/sim/app/api/files/upload/route.ts b/apps/sim/app/api/files/upload/route.ts
@@ -2,6 +2,7 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getPresignedUrl, isUsingCloudStorage, uploadFile } from '@/lib/uploads'
 import '@/lib/uploads/setup.server'
+import { getSession } from '@/lib/auth'
 import {
   createErrorResponse,
   createOptionsResponse,
@@ -14,6 +15,11 @@ const logger = createLogger('FilesUploadAPI')
 
 export async function POST(request: NextRequest) {
   try {
+    const session = await getSession()
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
     const formData = await request.formData()
 
     // Check if multiple files are being uploaded or a single file
diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/upload-modal/upload-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/upload-modal/upload-modal.tsx
@@ -1,10 +1,11 @@
 'use client'
 
 import { useRef, useState } from 'react'
-import { X } from 'lucide-react'
+import { Check, Loader2, X } from 'lucide-react'
 import { Button } from '@/components/ui/button'
 import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'
 import { Label } from '@/components/ui/label'
+import { Progress } from '@/components/ui/progress'
 import { createLogger } from '@/lib/logs/console/logger'
 import { useKnowledgeUpload } from '@/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload'
 
@@ -151,9 +152,15 @@ export function UploadModal({
     }
   }
 
+  // Calculate progress percentage
+  const progressPercentage =
+    uploadProgress.totalFiles > 0
+      ? Math.round((uploadProgress.filesCompleted / uploadProgress.totalFiles) * 100)
+      : 0
+
   return (
     <Dialog open={open} onOpenChange={handleClose}>
-      <DialogContent className='flex max-h-[90vh] max-w-2xl flex-col overflow-hidden'>
+      <DialogContent className='flex max-h-[95vh] max-w-2xl flex-col overflow-hidden'>
         <DialogHeader>
           <DialogTitle>Upload Documents</DialogTitle>
         </DialogHeader>
@@ -218,30 +225,55 @@ export function UploadModal({
                   </p>
                 </div>
 
-                <div className='max-h-40 space-y-2 overflow-auto'>
-                  {files.map((file, index) => (
-                    <div
-                      key={index}
-                      className='flex items-center justify-between rounded-md border p-3'
-                    >
-                      <div className='min-w-0 flex-1'>
-                        <p className='truncate font-medium text-sm'>{file.name}</p>
-                        <p className='text-muted-foreground text-xs'>
-                          {(file.size / 1024 / 1024).toFixed(2)} MB
-                        </p>
+                <div className='max-h-60 space-y-1.5 overflow-auto'>
+                  {files.map((file, index) => {
+                    const fileStatus = uploadProgress.fileStatuses?.[index]
+                    const isCurrentlyUploading = fileStatus?.status === 'uploading'
+                    const isCompleted = fileStatus?.status === 'completed'
+                    const isFailed = fileStatus?.status === 'failed'
+
+                    return (
+                      <div key={index} className='space-y-1.5 rounded-md border p-2'>
+                        <div className='flex items-center justify-between'>
+                          <div className='min-w-0 flex-1'>
+                            <div className='flex items-center gap-2'>
+                              {isCurrentlyUploading && (
+                                <Loader2 className='h-4 w-4 animate-spin text-blue-500' />
+                              )}
+                              {isCompleted && <Check className='h-4 w-4 text-green-500' />}
+                              {isFailed && <X className='h-4 w-4 text-red-500' />}
+                              {!isCurrentlyUploading && !isCompleted && !isFailed && (
+                                <div className='h-4 w-4' />
+                              )}
+                              <p className='truncate text-sm'>
+                                <span className='font-medium'>{file.name}</span>
+                                <span className='text-muted-foreground'>
+                                  {' '}
+                                  • {(file.size / 1024 / 1024).toFixed(2)} MB
+                                </span>
+                              </p>
+                            </div>
+                          </div>
+                          <Button
+                            type='button'
+                            variant='ghost'
+                            size='sm'
+                            onClick={() => removeFile(index)}
+                            disabled={isUploading}
+                            className='h-8 w-8 p-0'
+                          >
+                            <X className='h-4 w-4' />
+                          </Button>
+                        </div>
+                        {isCurrentlyUploading && (
+                          <Progress value={fileStatus?.progress || 0} className='h-1' />
+                        )}
+                        {isFailed && fileStatus?.error && (
+                          <p className='text-red-500 text-xs'>{fileStatus.error}</p>
+                        )}
                       </div>
-                      <Button
-                        type='button'
-                        variant='ghost'
-                        size='sm'
-                        onClick={() => removeFile(index)}
-                        disabled={isUploading}
-                        className='h-8 w-8 p-0'
-                      >
-                        <X className='h-4 w-4' />
-                      </Button>
-                    </div>
-                  ))}
+                    )
+                  })}
                 </div>
               </div>
             )}
diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts
