wip: 切片上传、文件秒传、断点续传陆续适配中当前进度（本地存储已适配）

Author: vastsa

.github/workflows/docker-image.yml

@@ -41,7 +41,7 @@ jobs:
           context: .
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ secrets.DOCKER_USERNAME }}/filecodebox:latest
+          tags: ${{ secrets.DOCKER_USERNAME }}/filecodebox:beta
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
       - name: Move cache

.gitignore

@@ -154,3 +154,6 @@ for_test.py
 data/.env
 .backup/
 /cloc-1.64.exe
+
+# Ignore node_modules
+node_modules/

apps/base/migrations/migrations_002.py

@@ -0,0 +1,28 @@
+from tortoise import connections
+
+
+async def create_upload_chunk_and_update_file_codes_table():
+    conn = connections.get("default")
+    await conn.execute_script(
+        """
+        ALTER TABLE "filecodes" ADD "file_hash" VARCHAR(128);
+        ALTER TABLE "filecodes" ADD "is_chunked" BOOL NOT NULL DEFAULT False;
+        ALTER TABLE "filecodes" ADD "upload_id" VARCHAR(128);
+        CREATE TABLE "uploadchunk" (
+            id  INTEGER  not null   primary key autoincrement,
+            "upload_id" VARCHAR(36) NOT NULL,
+            "chunk_index" INT NOT NULL,
+            "chunk_hash" VARCHAR(128) NOT NULL,
+            "total_chunks" INT NOT NULL,
+            "file_size" BIGINT NOT NULL,
+            "chunk_size" INT NOT NULL,
+            "created_at" TIMESTAMPTZ NOT NULL,
+            "file_name" VARCHAR(255) NOT NULL,
+            "completed" BOOL NOT NULL
+        );
+    """
+    )
+
+
+async def migrate():
+    await create_upload_chunk_and_update_file_codes_table()

apps/base/models.py

@@ -2,119 +2,55 @@
 # @Author  : Lan
 # @File    : models.py
 # @Software: PyCharm
-from datetime import datetime
 from typing import Optional
 
-from tortoise import fields
 from tortoise.models import Model
 from tortoise.contrib.pydantic import pydantic_model_creator
 
+from tortoise import fields, models
+from datetime import datetime
 from core.utils import get_now
 
 
-class FileCodes(Model):
-    id: Optional[int] = fields.IntField(pk=True)
-    code: Optional[int] = fields.CharField(
-        description="分享码", max_length=255, index=True, unique=True
-    )
-    prefix: Optional[str] = fields.CharField(
-        max_length=255, description="前缀", default=""
-    )
-    suffix: Optional[str] = fields.CharField(
-        max_length=255, description="后缀", default=""
-    )
-    uuid_file_name: Optional[str] = fields.CharField(
-        max_length=255, description="uuid文件名", null=True
-    )
-    file_path: Optional[str] = fields.CharField(
-        max_length=255, description="文件路径", null=True
-    )
-    size: Optional[int] = fields.IntField(description="文件大小", default=0)
-    text: Optional[str] = fields.TextField(description="文本内容", null=True)
-    expired_at: Optional[datetime] = fields.DatetimeField(
-        null=True, description="过期时间"
-    )
-    expired_count: Optional[int] = fields.IntField(description="可用次数", default=0)
-    used_count: Optional[int] = fields.IntField(description="已用次数", default=0)
-    created_at: Optional[datetime] = fields.DatetimeField(
-        auto_now_add=True, description="创建时间"
-    )
-    #
-    # file_hash = fields.CharField(
-    #     max_length=128,  # SHA-256需要64字符，这里预留扩展空间
-    #     description="文件哈希值",
-    #     unique=True,
-    #     null=True  # 允许旧数据为空
-    # )
-    # hash_algorithm = fields.CharField(
-    #     max_length=20,
-    #     description="哈希算法类型",
-    #     null=True,
-    #     default="sha256"
-    # )
-
-    # # 新增分片字段
-    # chunk_size = fields.IntField(
-    #     description="分片大小(字节)",
-    #     default=0
-    # )
-    # total_chunks = fields.IntField(
-    #     description="总分片数",
-    #     default=0
-    # )
-    # uploaded_chunks = fields.IntField(
-    #     description="已上传分片数",
-    #     default=0
-    # )
-    # upload_status = fields.CharField(
-    #     max_length=20,
-    #     description="上传状态",
-    #     default="pending",  # pending/in_progress/completed
-    #     choices=["pending", "in_progress", "completed"]
-    # )
-    # is_chunked = fields.BooleanField(
-    #     description="是否分片上传",
-    #     default=False
-    # )
+class FileCodes(models.Model):
+    id = fields.IntField(pk=True)
+    code = fields.CharField(max_length=255, unique=True, index=True)
+    prefix = fields.CharField(max_length=255, default="")
+    suffix = fields.CharField(max_length=255, default="")
+    uuid_file_name = fields.CharField(max_length=255, null=True)
+    file_path = fields.CharField(max_length=255, null=True)
+    size = fields.IntField(default=0)
+    text = fields.TextField(null=True)
+    expired_at = fields.DatetimeField(null=True)
+    expired_count = fields.IntField(default=0)
+    used_count = fields.IntField(default=0)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    file_hash = fields.CharField(max_length=64, null=True)
+    is_chunked = fields.BooleanField(default=False)
+    upload_id = fields.CharField(max_length=36, null=True)
 
     async def is_expired(self):
-        # 按时间
         if self.expired_at is None:
             return False
         if self.expired_at and self.expired_count < 0:
             return self.expired_at < await get_now()
-        # 按次数
-        else:
-            return self.expired_count <= 0
+        return self.expired_count <= 0
 
     async def get_file_path(self):
         return f"{self.file_path}/{self.uuid_file_name}"
 
 
-#
-# class FileChunks(Model):
-#     id = fields.IntField(pk=True)
-#     file_code = fields.ForeignKeyField(
-#         "models.FileCodes",
-#         related_name="chunks",
-#         on_delete=fields.CASCADE
-#     )
-#     chunk_number = fields.IntField(description="分片序号")
-#     chunk_hash = fields.CharField(
-#         max_length=128,
-#         description="分片哈希校验值"
-#     )
-#     chunk_path = fields.CharField(
-#         max_length=255,
-#         description="分片存储路径"
-#     )
-#     created_at = fields.DatetimeField(
-#         auto_now_add=True,
-#         description="上传时间"
-#     )
-#
-#     class Meta:
-#         unique_together = [("file_code", "chunk_number")]
+class UploadChunk(models.Model):
+    id = fields.IntField(pk=True)
+    upload_id = fields.CharField(max_length=36, index=True)
+    chunk_index = fields.IntField()
+    chunk_hash = fields.CharField(max_length=64)
+    total_chunks = fields.IntField()
+    file_size = fields.BigIntField()
+    chunk_size = fields.IntField()
+    file_name = fields.CharField(max_length=255)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    completed = fields.BooleanField(default=False)
 
 
 class KeyValue(Model):
@@ -129,3 +65,5 @@ class KeyValue(Model):
 
 
 file_codes_pydantic = pydantic_model_creator(FileCodes, name="FileCodes")
+upload_chunk_pydantic = pydantic_model_creator(UploadChunk, name="UploadChunk")
+key_value_pydantic = pydantic_model_creator(KeyValue, name="KeyValue")

apps/base/schemas.py

@@ -3,3 +3,15 @@
 
 class SelectFileModel(BaseModel):
     code: str
+
+
+class InitChunkUploadModel(BaseModel):
+    file_name: str
+    chunk_size: int = 5 * 1024 * 1024
+    file_size: int
+    file_hash: str
+
+
+class CompleteUploadModel(BaseModel):
+    expire_value: int
+    expire_style: str

apps/base/utils.py

@@ -1,4 +1,5 @@
 import datetime
+import hashlib
 import os
 import uuid
 
@@ -29,8 +30,19 @@ async def get_file_path_name(file: UploadFile) -> Tuple[str, str, str, str, str]
     return path, suffix, prefix, file.filename, save_path
 
 
+async def get_chunk_file_path_name(file_name: str, upload_id: str) -> Tuple[str, str, str, str, str]:
+    """获取切片文件的路径和文件名"""
+    today = datetime.datetime.now()
+    storage_path = settings.storage_path.strip("/")  # 移除开头和结尾的斜杠
+    base_path = f"share/data/{today.strftime('%Y/%m/%d')}/{upload_id}"
+    path = f"{storage_path}/{base_path}" if storage_path else base_path
+    prefix, suffix = os.path.splitext(file_name)
+    save_path = f"{path}/{prefix}{suffix}"
+    return path, suffix, prefix, file_name, save_path
+
+
 async def get_expire_info(
-    expire_value: int, expire_style: str
+        expire_value: int, expire_style: str
 ) -> Tuple[Optional[datetime.datetime], int, int, str]:
     """获取过期信息"""
     expired_count, used_count = -1, 0
@@ -86,6 +98,18 @@ async def get_random_code(style="num") -> str:
             return code
 
 
+async def calculate_file_hash(file: UploadFile, chunk_size=1024 * 1024) -> str:
+    sha = hashlib.sha256()
+    await file.seek(0)
+    while True:
+        chunk = await file.read(chunk_size)
+        if not chunk:
+            break
+        sha.update(chunk)
+    await file.seek(0)
+    return sha.hexdigest()
+
+
 ip_limit = {
     "error": IPRateLimit(count=settings.uploadCount, minutes=settings.errorMinute),
     "upload": IPRateLimit(count=settings.errorCount, minutes=settings.errorMinute),