fix(drivers/teldrive): enhance file listing and upload functionality with pagination and random chunk naming (OpenListTeam#2034)

* fix(drivers/teldrive): enhance file listing and upload functionality with pagination and random chunk naming

* fix(drivers/teldrive): optimize file listing by removing unnecessary mutex and restructuring data handling

* Update drivers/teldrive/meta.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Chaloemchai <chaloemchai.yy@gmail.com>

---------

Signed-off-by: Chaloemchai <chaloemchai.yy@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: totza2010

drivers/teldrive/driver.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strconv"
 	"strings"
 
 	"github.com/OpenListTeam/OpenList/v4/drivers/base"
@@ -17,6 +18,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	"github.com/go-resty/resty/v2"
 	"github.com/google/uuid"
+	"golang.org/x/sync/errgroup"
 )
 
 type Teldrive struct {
@@ -53,18 +55,58 @@ func (d *Teldrive) Drop(ctx context.Context) error {
 }
 
 func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
-	var listResp ListResp
+	var firstResp ListResp
 	err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) {
 		req.SetQueryParams(map[string]string{
 			"path":  dir.GetPath(),
-			"limit": "1000", // overide default 500, TODO pagination
+			"limit": "500",
+			"page":  "1",
 		})
-	}, &listResp)
+	}, &firstResp)
+
 	if err != nil {
 		return nil, err
 	}
 
-	return utils.SliceConvert(listResp.Items, func(src Object) (model.Obj, error) {
+	pagesData := make([][]Object, firstResp.Meta.TotalPages)
+	pagesData[0] = firstResp.Items
+
+	if firstResp.Meta.TotalPages > 1 {
+		g, _ := errgroup.WithContext(ctx)
+		g.SetLimit(8)
+
+		for i := 2; i <= firstResp.Meta.TotalPages; i++ {
+			page := i
+			g.Go(func() error {
+				var resp ListResp
+				err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) {
+					req.SetQueryParams(map[string]string{
+						"path":  dir.GetPath(),
+						"limit": "500",
+						"page":  strconv.Itoa(page),
+					})
+				}, &resp)
+
+				if err != nil {
+					return err
+				}
+
+				pagesData[page-1] = resp.Items
+				return nil
+			})
+		}
+
+		if err := g.Wait(); err != nil {
+			return nil, err
+		}
+	}
+
+	var allItems []Object
+	for _, items := range pagesData {
+		allItems = append(allItems, items...)
+	}
+
+	return utils.SliceConvert(allItems, func(src Object) (model.Obj, error) {
 		return &model.Object{
 			Path: path.Join(dir.GetPath(), src.Name),
 			ID:   src.ID,
@@ -184,7 +226,7 @@ func (d *Teldrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStr
 	}
 
 	if totalParts <= 1 {
-		return d.doSingleUpload(ctx, dstDir, file, up, totalParts, chunkSize, fileId)
+		return d.doSingleUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId)
 	}
 
 	return d.doMultiUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId)

drivers/teldrive/meta.go

@@ -11,6 +11,7 @@ type Addition struct {
 	Cookie            string `json:"cookie" type:"string" required:"true" help:"access_token=xxx"`
 	UseShareLink      bool   `json:"use_share_link" type:"bool" default:"false" help:"Create share link when getting link to support 302. If disabled, you need to enable web proxy."`
 	ChunkSize         int64  `json:"chunk_size" type:"number" default:"10" help:"Chunk size in MiB"`
+	RandomChunkName   bool   `json:"random_chunk_name" type:"bool" default:"true" help:"Random chunk name"`
 	UploadConcurrency int64  `json:"upload_concurrency" type:"number" default:"4" help:"Concurrency upload requests"`
 }
 

drivers/teldrive/upload.go

@@ -1,6 +1,8 @@
 package teldrive
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"net/http"
@@ -16,6 +18,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	"github.com/avast/retry-go"
 	"github.com/go-resty/resty/v2"
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"golang.org/x/sync/errgroup"
@@ -38,6 +41,11 @@ func (d *Teldrive) touch(name, path string) error {
 	return nil
 }
 
+func getMD5Hash(text string) string {
+	hash := md5.Sum([]byte(text))
+	return hex.EncodeToString(hash[:])
+}
+
 func (d *Teldrive) createFileOnUploadSuccess(name, id, path string, uploadedFileParts []FilePart, totalSize int64) error {
 	remoteFileParts, err := d.getFilePart(id)
 	if err != nil {
@@ -101,12 +109,10 @@ func (d *Teldrive) getFilePart(fileId string) ([]FilePart, error) {
 	return uploadedParts, nil
 }
 
-func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, resp interface{}) error {
+func (d *Teldrive) singleUploadRequest(ctx context.Context, fileId string, callback base.ReqCallback, resp any) error {
 	url := d.Address + "/api/uploads/" + fileId
 	client := resty.New().SetTimeout(0)
 
-	ctx := context.Background()
-
 	req := client.R().
 		SetContext(ctx)
 	req.SetHeader("Cookie", d.Cookie)
@@ -135,16 +141,18 @@ func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback,
 }
 
 func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up model.UpdateProgress,
-	totalParts int, chunkSize int64, fileId string) error {
+	maxRetried, totalParts int, chunkSize int64, fileId string) error {
 
 	totalSize := file.GetSize()
 	var fileParts []FilePart
 	var uploaded int64 = 0
-	ss, err := stream.NewStreamSectionReader(file, int(totalSize), &up)
+	var partName string
+	chunkSize = min(totalSize, chunkSize)
+	ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up)
 	if err != nil {
 		return err
 	}
-
+	chunkCnt := 0
 	for uploaded < totalSize {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -154,20 +162,27 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo
 		if err != nil {
 			return err
 		}
+		chunkCnt += 1
 		filePart := &FilePart{}
 		if err := retry.Do(func() error {
 
 			if _, err := rd.Seek(0, io.SeekStart); err != nil {
 				return err
 			}
 
-			if err := d.singleUploadRequest(fileId, func(req *resty.Request) {
+			if d.RandomChunkName {
+				partName = getMD5Hash(uuid.New().String())
+			} else {
+				partName = file.GetName()
+				if totalParts > 1 {
+					partName = fmt.Sprintf("%s.part.%03d", file.GetName(), chunkCnt)
+				}
+			}
+
+			if err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) {
 				uploadParams := map[string]string{
-					"partName": func() string {
-						digits := len(strconv.Itoa(totalParts))
-						return file.GetName() + fmt.Sprintf(".%0*d", digits, 1)
-					}(),
-					"partNo":   strconv.Itoa(1),
+					"partName": partName,
+					"partNo":   strconv.Itoa(chunkCnt),
 					"fileName": file.GetName(),
 				}
 				req.SetQueryParams(uploadParams)
@@ -180,7 +195,7 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo
 			return nil
 		},
 			retry.Context(ctx),
-			retry.Attempts(3),
+			retry.Attempts(uint(maxRetried)),
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second)); err != nil {
 			return err
@@ -189,8 +204,11 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo
 		if filePart.Name != "" {
 			fileParts = append(fileParts, *filePart)
 			uploaded += curChunkSize
-			up(float64(uploaded) / float64(totalSize))
+			up(float64(uploaded) / float64(totalSize) * 100)
 			ss.FreeSectionReader(rd)
+		} else {
+			// For common situation this code won't reach
+			return fmt.Errorf("[Teldrive] upload chunk %d failed: filePart Somehow missing", chunkCnt)
 		}
 
 	}
@@ -318,6 +336,7 @@ func (d *Teldrive) doMultiUpload(ctx context.Context, dstDir model.Obj, file mod
 func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task chunkTask, totalParts, maxRetried int) (*FilePart, error) {
 	filePart := &FilePart{}
 	retryCount := 0
+	var partName string
 	defer task.ss.FreeSectionReader(task.reader)
 
 	for {
@@ -331,12 +350,22 @@ func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task ch
 			return &existingPart, nil
 		}
 
-		err := d.singleUploadRequest(fileId, func(req *resty.Request) {
+		if _, err := task.reader.Seek(0, io.SeekStart); err != nil {
+			return nil, err
+		}
+
+		if d.RandomChunkName {
+			partName = getMD5Hash(uuid.New().String())
+		} else {
+			partName = task.fileName
+			if totalParts > 1 {
+				partName = fmt.Sprintf("%s.part.%03d", task.fileName, task.chunkIdx)
+			}
+		}
+
+		err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) {
 			uploadParams := map[string]string{
-				"partName": func() string {
-					digits := len(strconv.Itoa(totalParts))
-					return task.fileName + fmt.Sprintf(".%0*d", digits, task.chunkIdx)
-				}(),
+				"partName": partName,
 				"partNo":   strconv.Itoa(task.chunkIdx),
 				"fileName": task.fileName,
 			}