From 0a22887145db7e1267bc408dec882681271e4918 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Tue, 26 Aug 2025 02:48:28 +0800 Subject: [PATCH 01/25] feat(chunk): add a driver that divides large files into multiple chunks --- drivers/all.go | 1 + drivers/chunk/driver.go | 397 ++++++++++++++++++++++++++++++++++++++++ drivers/chunk/meta.go | 27 +++ 3 files changed, 425 insertions(+) create mode 100644 drivers/chunk/driver.go create mode 100644 drivers/chunk/meta.go diff --git a/drivers/all.go b/drivers/all.go index 2932ee227..7d72b282a 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -20,6 +20,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" _ "github.com/OpenListTeam/OpenList/v4/drivers/chaoxing" + _ "github.com/OpenListTeam/OpenList/v4/drivers/chunk" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve_v4" _ "github.com/OpenListTeam/OpenList/v4/drivers/crypt" diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go new file mode 100644 index 000000000..46ae48a14 --- /dev/null +++ b/drivers/chunk/driver.go @@ -0,0 +1,397 @@ +package chunk + +import ( + "context" + "errors" + "fmt" + "io" + stdpath "path" + "regexp" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" +) + +type Chunk struct { + model.Storage + Addition + remoteStorage driver.Driver + partNameMatchRe *regexp.Regexp +} + +func (d *Chunk) Config() driver.Config { + return config +} + +func (d *Chunk) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Chunk) Init(ctx context.Context) error { + if d.PartSize <= 0 { + return errors.New("part size must be positive") + } + d.RemotePath = utils.FixAndCleanPath(d.RemotePath) + d.partNameMatchRe = regexp.MustCompile("^(.+)\\.openlist_chunk_(\\d+)" + regexp.QuoteMeta(d.CustomExt) + "$") + return nil +} + +func (d *Chunk) Drop(ctx context.Context) error { + return nil +} + +func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { + if utils.PathEqual(path, "/") { + return &model.Object{ + Name: "Root", + IsFolder: true, + Path: "/", + }, nil + } + dir, base := stdpath.Split(stdpath.Join(d.RemotePath, path)) + objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) + if err != nil { + return nil, err + } + partPrefix := base + ".openlist_chunk_" + var first model.Obj + var totalSize int64 = 0 + for _, obj := range objs { + if obj.GetName() == base { + first = obj + if obj.IsDir() { + totalSize = obj.GetSize() + break + } else { + totalSize += obj.GetSize() + } + } else if strings.HasPrefix(obj.GetName(), partPrefix) { + totalSize += obj.GetSize() + } + } + if first == nil { + return nil, errs.ObjectNotFound + } + return &model.Object{ + Path: path, + Name: base, + Size: totalSize, + Modified: first.ModTime(), + Ctime: first.CreateTime(), + IsFolder: first.IsDir(), + }, nil +} + +func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + objs, err := fs.List(ctx, stdpath.Join(d.RemotePath, dir.GetPath()), &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) + if err != nil { + return nil, err + } + ret := make([]model.Obj, 0) + sizeMap := make(map[string]int64) + for _, obj := range objs { + if obj.IsDir() { + ret = append(ret, &model.Object{ + Name: obj.GetName(), + Size: obj.GetSize(), + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: true, + }) + continue + } + var name string + matches := d.partNameMatchRe.FindStringSubmatch(obj.GetName()) + if len(matches) < 3 { + ret = append(ret, &model.Object{ + Name: obj.GetName(), + Size: 0, + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: false, + }) + name = obj.GetName() + } else { + name = matches[1] + } + _, ok := sizeMap[name] + if !ok { + sizeMap[name] = obj.GetSize() + } else { + sizeMap[name] += obj.GetSize() + } + } + for _, obj := range ret { + if !obj.IsDir() { + obj.(*model.Object).Size = sizeMap[obj.GetName()] + } + } + return ret, nil +} + +func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + path := stdpath.Join(d.RemotePath, file.GetPath()) + links := make([]*model.Link, 0) + rrfs := make([]model.RangeReaderIF, 0) + totalLength := int64(0) + for { + l, o, err := link(ctx, d.getPartName(path, len(links)), args) + if errors.Is(err, errs.ObjectNotFound) { + break + } + if err != nil { + for _, l1 := range links { + _ = l1.Close() + } + return nil, fmt.Errorf("failed get Part %d link: %+v", len(links), err) + } + if l.ContentLength <= 0 { + l.ContentLength = o.GetSize() + } + rrf, err := stream.GetRangeReaderFromLink(l.ContentLength, l) + if err != nil { + for _, l1 := range links { + _ = l1.Close() + } + _ = l.Close() + return nil, fmt.Errorf("failed get Part %d range reader: %+v", len(links), err) + } + links = append(links, l) + rrfs = append(rrfs, rrf) + totalLength += l.ContentLength + } + mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + if httpRange.Length == -1 { + httpRange.Length = totalLength - httpRange.Start + } + firstPartIdx := 0 + firstPartStart := httpRange.Start + for firstPartIdx < len(links) && firstPartStart > links[firstPartIdx].ContentLength { + firstPartStart -= links[firstPartIdx].ContentLength + firstPartIdx++ + } + if firstPartIdx == len(links) { + return nil, io.EOF + } + if firstPartStart+httpRange.Length <= links[firstPartIdx].ContentLength { + return rrfs[firstPartIdx].RangeRead(ctx, http_range.Range{ + Start: firstPartStart, + Length: httpRange.Length, + }) + } + + lastPartIdx := firstPartIdx + tailLength := firstPartStart + httpRange.Length + for lastPartIdx < len(links) && tailLength > links[lastPartIdx].ContentLength { + tailLength -= links[lastPartIdx].ContentLength + lastPartIdx++ + } + if lastPartIdx == len(links) || tailLength == 0 { + lastPartIdx-- + tailLength = links[lastPartIdx].ContentLength + } + + rs := make([]io.Reader, 0, lastPartIdx-firstPartIdx+1) + cs := make(utils.Closers, 0, lastPartIdx-firstPartIdx+1) + firstRc, err := rrfs[firstPartIdx].RangeRead(ctx, http_range.Range{ + Start: firstPartStart, + Length: links[firstPartIdx].ContentLength - firstPartStart, + }) + if err != nil { + return nil, err + } + rs = append(rs, firstRc) + cs = append(cs, firstRc) + partIdx := firstPartIdx + 1 + for partIdx < lastPartIdx { + rc, err := rrfs[partIdx].RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + return nil, err + } + rs = append(rs, rc) + cs = append(cs, rc) + partIdx++ + } + lastRc, err := rrfs[lastPartIdx].RangeRead(ctx, http_range.Range{ + Start: 0, + Length: tailLength, + }) + if err != nil { + return nil, err + } + rs = append(rs, lastRc) + cs = append(cs, lastRc) + return &struct { + io.Reader + utils.Closers + }{ + Reader: io.MultiReader(rs...), + Closers: cs, + }, nil + } + linkClosers := make([]io.Closer, 0, len(links)) + for _, l := range links { + linkClosers = append(linkClosers, l) + } + return &model.Link{ + RangeReader: stream.RangeReaderFunc(mergedRrf), + SyncClosers: utils.NewSyncClosers(linkClosers...), + }, nil +} + +func link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + return nil, nil, err + } + if !args.Redirect { + return op.Link(ctx, storage, reqActualPath, args) + } + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, nil, err + } + if common.ShouldProxy(storage, stdpath.Base(reqPath)) { + return nil, obj, nil + } + return op.Link(ctx, storage, reqActualPath, args) +} + +func (d *Chunk) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + path := stdpath.Join(d.RemotePath, parentDir.GetPath(), dirName) + return fs.MakeDir(ctx, path) +} + +func (d *Chunk) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + path := stdpath.Join(d.RemotePath, srcObj.GetPath()) + dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) + if srcObj.IsDir() { + _, err := fs.Move(ctx, path, dst) + return err + } + dir, base := stdpath.Split(path) + objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) + if err != nil { + return err + } + for _, obj := range objs { + suffix := strings.TrimPrefix(obj.GetName(), base) + if suffix != obj.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { + _, e := fs.Move(ctx, path+suffix, dst, true) + err = errors.Join(err, e) + } + } + _, e := fs.Move(ctx, path, dst) + return errors.Join(err, e) +} + +func (d *Chunk) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + path := stdpath.Join(d.RemotePath, srcObj.GetPath()) + if srcObj.IsDir() { + return fs.Rename(ctx, path, newName) + } + dir, base := stdpath.Split(path) + objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) + if err != nil { + return err + } + for _, obj := range objs { + suffix := strings.TrimPrefix(obj.GetName(), base) + if suffix != obj.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { + err = errors.Join(err, fs.Rename(ctx, path+suffix, newName+suffix, true)) + } + } + return errors.Join(err, fs.Rename(ctx, path, newName)) +} + +func (d *Chunk) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + path := stdpath.Join(d.RemotePath, srcObj.GetPath()) + dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) + if srcObj.IsDir() { + _, err := fs.Copy(ctx, path, dst) + return err + } + dir, base := stdpath.Split(path) + objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) + if err != nil { + return err + } + for _, obj := range objs { + suffix := strings.TrimPrefix(obj.GetName(), base) + if suffix != obj.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { + _, e := fs.Copy(ctx, path+suffix, dst, true) + err = errors.Join(err, e) + } + } + _, e := fs.Copy(ctx, path, dst) + return errors.Join(err, e) +} + +func (d *Chunk) Remove(ctx context.Context, obj model.Obj) error { + path := stdpath.Join(d.RemotePath, obj.GetPath()) + if obj.IsDir() { + return fs.Remove(ctx, path) + } + dir, base := stdpath.Split(path) + objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) + if err != nil { + return err + } + for _, o := range objs { + suffix := strings.TrimPrefix(o.GetName(), base) + if suffix != obj.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { + err = errors.Join(err, fs.Remove(ctx, path+suffix)) + } + } + return errors.Join(err, fs.Remove(ctx, path)) +} + +func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + upReader := &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + } + dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) + fullPartCount := int(file.GetSize() / d.PartSize) + tailSize := file.GetSize() % d.PartSize + partIndex := 0 + var err error + for partIndex < fullPartCount { + err = errors.Join(err, fs.PutDirectly(ctx, dst, &stream.FileStream{ + Obj: &model.Object{ + Name: d.getPartName(file.GetName(), partIndex), + Size: d.PartSize, + Modified: file.ModTime(), + }, + Mimetype: file.GetMimetype(), + Reader: io.LimitReader(upReader, d.PartSize), + }, true)) + partIndex++ + } + return errors.Join(err, fs.PutDirectly(ctx, dst, &stream.FileStream{ + Obj: &model.Object{ + Name: d.getPartName(file.GetName(), fullPartCount), + Size: tailSize, + Modified: file.ModTime(), + }, + Mimetype: file.GetMimetype(), + Reader: upReader, + })) +} + +func (d *Chunk) getPartName(name string, part int) string { + if part == 0 { + return name + } + return fmt.Sprintf("%s.openlist_chunk_%d%s", name, part, d.CustomExt) +} + +var _ driver.Driver = (*Chunk)(nil) diff --git a/drivers/chunk/meta.go b/drivers/chunk/meta.go new file mode 100644 index 000000000..3adb70586 --- /dev/null +++ b/drivers/chunk/meta.go @@ -0,0 +1,27 @@ +package chunk + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + RemotePath string `json:"remote_path" required:"true"` + PartSize int64 `json:"part_size" required:"true" type:"number" help:"bytes"` + CustomExt string `json:"custom_ext" type:"string"` +} + +var config = driver.Config{ + Name: "Chunk", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + DefaultRoot: "/", + NoLinkURL: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Chunk{} + }) +} From 9defe892cd27bfff9949ee39785065ed91edee7f Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Tue, 26 Aug 2025 20:14:34 +0800 Subject: [PATCH 02/25] fix --- drivers/chunk/driver.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 46ae48a14..22fc714e5 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -347,7 +347,7 @@ func (d *Chunk) Remove(ctx context.Context, obj model.Obj) error { } for _, o := range objs { suffix := strings.TrimPrefix(o.GetName(), base) - if suffix != obj.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { + if suffix != o.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { err = errors.Join(err, fs.Remove(ctx, path+suffix)) } } @@ -362,6 +362,10 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) fullPartCount := int(file.GetSize() / d.PartSize) tailSize := file.GetSize() % d.PartSize + if tailSize == 0 && fullPartCount > 0 { + fullPartCount-- + tailSize = d.PartSize + } partIndex := 0 var err error for partIndex < fullPartCount { From 6d538f065bb692e9cb97f59930bf2e6557d315b9 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Thu, 28 Aug 2025 01:22:13 +0800 Subject: [PATCH 03/25] refactor(chunk): refactor Link and Put methods --- drivers/chunk/driver.go | 54 +++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 22fc714e5..cf5989dd4 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -17,13 +17,11 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" ) type Chunk struct { model.Storage Addition - remoteStorage driver.Driver partNameMatchRe *regexp.Regexp } @@ -138,12 +136,16 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - path := stdpath.Join(d.RemotePath, file.GetPath()) + storage, reqActualPath, err := op.GetStorageAndActualPath(d.RemotePath) + if err != nil { + return nil, err + } + path := stdpath.Join(reqActualPath, file.GetPath()) links := make([]*model.Link, 0) rrfs := make([]model.RangeReaderIF, 0) totalLength := int64(0) for { - l, o, err := link(ctx, d.getPartName(path, len(links)), args) + l, o, err := op.Link(ctx, storage, d.getPartName(path, len(links)), args) if errors.Is(err, errs.ObjectNotFound) { break } @@ -169,7 +171,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( totalLength += l.ContentLength } mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { - if httpRange.Length == -1 { + if httpRange.Length < 0 || httpRange.Start+httpRange.Length > totalLength { httpRange.Length = totalLength - httpRange.Start } firstPartIdx := 0 @@ -229,12 +231,9 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } rs = append(rs, lastRc) cs = append(cs, lastRc) - return &struct { - io.Reader - utils.Closers - }{ - Reader: io.MultiReader(rs...), - Closers: cs, + return utils.ReadCloser{ + Reader: io.MultiReader(rs...), + Closer: &cs, }, nil } linkClosers := make([]io.Closer, 0, len(links)) @@ -247,24 +246,6 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( }, nil } -func link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { - storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) - if err != nil { - return nil, nil, err - } - if !args.Redirect { - return op.Link(ctx, storage, reqActualPath, args) - } - obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) - if err != nil { - return nil, nil, err - } - if common.ShouldProxy(storage, stdpath.Base(reqPath)) { - return nil, obj, nil - } - return op.Link(ctx, storage, reqActualPath, args) -} - func (d *Chunk) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { path := stdpath.Join(d.RemotePath, parentDir.GetPath(), dirName) return fs.MakeDir(ctx, path) @@ -355,11 +336,15 @@ func (d *Chunk) Remove(ctx context.Context, obj model.Obj) error { } func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + storage, reqActualPath, err := op.GetStorageAndActualPath(d.RemotePath) + if err != nil { + return err + } upReader := &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, } - dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) + dst := stdpath.Join(reqActualPath, dstDir.GetPath()) fullPartCount := int(file.GetSize() / d.PartSize) tailSize := file.GetSize() % d.PartSize if tailSize == 0 && fullPartCount > 0 { @@ -367,9 +352,8 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream tailSize = d.PartSize } partIndex := 0 - var err error for partIndex < fullPartCount { - err = errors.Join(err, fs.PutDirectly(ctx, dst, &stream.FileStream{ + err = errors.Join(err, op.Put(ctx, storage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(file.GetName(), partIndex), Size: d.PartSize, @@ -377,10 +361,10 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, Mimetype: file.GetMimetype(), Reader: io.LimitReader(upReader, d.PartSize), - }, true)) + }, nil, true)) partIndex++ } - return errors.Join(err, fs.PutDirectly(ctx, dst, &stream.FileStream{ + return errors.Join(err, op.Put(ctx, storage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(file.GetName(), fullPartCount), Size: tailSize, @@ -388,7 +372,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, Mimetype: file.GetMimetype(), Reader: upReader, - })) + }, nil)) } func (d *Chunk) getPartName(name string, part int) string { From b4b49c153ffd0ecb11f8c2fd7fa0be48d1a564fa Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Thu, 28 Aug 2025 11:41:24 +0800 Subject: [PATCH 04/25] refactor(chunk): optimize Link method for improved readability and performance --- drivers/chunk/driver.go | 111 ++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 61 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index cf5989dd4..c03090ecb 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net/http" stdpath "path" "regexp" "strings" @@ -13,6 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" @@ -140,6 +142,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if err != nil { return nil, err } + args.Redirect = false path := stdpath.Join(reqActualPath, file.GetPath()) links := make([]*model.Link, 0) rrfs := make([]model.RangeReaderIF, 0) @@ -171,70 +174,56 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( totalLength += l.ContentLength } mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { - if httpRange.Length < 0 || httpRange.Start+httpRange.Length > totalLength { - httpRange.Length = totalLength - httpRange.Start + start := httpRange.Start + length := httpRange.Length + if length < 0 || start+length > totalLength { + length = totalLength - start } - firstPartIdx := 0 - firstPartStart := httpRange.Start - for firstPartIdx < len(links) && firstPartStart > links[firstPartIdx].ContentLength { - firstPartStart -= links[firstPartIdx].ContentLength - firstPartIdx++ - } - if firstPartIdx == len(links) { - return nil, io.EOF - } - if firstPartStart+httpRange.Length <= links[firstPartIdx].ContentLength { - return rrfs[firstPartIdx].RangeRead(ctx, http_range.Range{ - Start: firstPartStart, - Length: httpRange.Length, - }) - } - - lastPartIdx := firstPartIdx - tailLength := firstPartStart + httpRange.Length - for lastPartIdx < len(links) && tailLength > links[lastPartIdx].ContentLength { - tailLength -= links[lastPartIdx].ContentLength - lastPartIdx++ - } - if lastPartIdx == len(links) || tailLength == 0 { - lastPartIdx-- - tailLength = links[lastPartIdx].ContentLength - } - - rs := make([]io.Reader, 0, lastPartIdx-firstPartIdx+1) - cs := make(utils.Closers, 0, lastPartIdx-firstPartIdx+1) - firstRc, err := rrfs[firstPartIdx].RangeRead(ctx, http_range.Range{ - Start: firstPartStart, - Length: links[firstPartIdx].ContentLength - firstPartStart, - }) - if err != nil { - return nil, err - } - rs = append(rs, firstRc) - cs = append(cs, firstRc) - partIdx := firstPartIdx + 1 - for partIdx < lastPartIdx { - rc, err := rrfs[partIdx].RangeRead(ctx, http_range.Range{Length: -1}) - if err != nil { - return nil, err + rs := make([]io.Reader, 0) + cs := make(utils.Closers, 0) + var ( + rc io.ReadCloser + err error + readFrom bool + ) + for idx, l := range links { + if readFrom { + newLength := length - l.ContentLength + if newLength >= 0 { + length = newLength + rc, err = rrfs[idx].RangeRead(ctx, http_range.Range{Length: -1}) + } else { + rc, err = rrfs[idx].RangeRead(ctx, http_range.Range{Length: length}) + } + if err != nil { + _ = cs.Close() + return nil, err + } + rs = append(rs, rc) + cs = append(cs, rc) + if newLength <= 0 { + return utils.ReadCloser{ + Reader: io.MultiReader(rs...), + Closer: &cs, + }, nil + } + } else if newStart := start - l.ContentLength; newStart >= 0 { + start = newStart + } else { + rc, err = rrfs[idx].RangeRead(ctx, http_range.Range{Start: start, Length: -1}) + if err != nil { + return nil, err + } + length -= l.ContentLength - start + if length <= 0 { + return rc, nil + } + rs = append(rs, rc) + cs = append(cs, rc) + readFrom = true } - rs = append(rs, rc) - cs = append(cs, rc) - partIdx++ } - lastRc, err := rrfs[lastPartIdx].RangeRead(ctx, http_range.Range{ - Start: 0, - Length: tailLength, - }) - if err != nil { - return nil, err - } - rs = append(rs, lastRc) - cs = append(cs, lastRc) - return utils.ReadCloser{ - Reader: io.MultiReader(rs...), - Closer: &cs, - }, nil + return nil, fmt.Errorf("invalid range: start=%d,length=%d,totalLength=%d, status: %w", httpRange.Start, httpRange.Length, totalLength, net.ErrorHttpStatusCode(http.StatusRequestedRangeNotSatisfiable)) } linkClosers := make([]io.Closer, 0, len(links)) for _, l := range links { From 9472752f5ab57f2d7c1e167ef598b3bf47e296e1 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Thu, 28 Aug 2025 11:44:58 +0800 Subject: [PATCH 05/25] refactor(net): rename ErrorHttpStatusCode to HttpStatusCodeError --- drivers/chunk/driver.go | 2 +- internal/net/request.go | 4 ++-- internal/net/serve.go | 12 ++++++------ internal/stream/util.go | 2 +- server/handles/down.go | 2 +- server/webdav/webdav.go | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index c03090ecb..4eaedaedb 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -223,7 +223,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( readFrom = true } } - return nil, fmt.Errorf("invalid range: start=%d,length=%d,totalLength=%d, status: %w", httpRange.Start, httpRange.Length, totalLength, net.ErrorHttpStatusCode(http.StatusRequestedRangeNotSatisfiable)) + return nil, fmt.Errorf("invalid range: start=%d,length=%d,totalLength=%d, status: %w", httpRange.Start, httpRange.Length, totalLength, net.HttpStatusCodeError(http.StatusRequestedRangeNotSatisfiable)) } linkClosers := make([]io.Closer, 0, len(links)) for _, l := range links { diff --git a/internal/net/request.go b/internal/net/request.go index 399e01f37..1306bc549 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -125,7 +125,7 @@ type ConcurrencyLimit struct { Limit int // 需要大于0 } -var ErrExceedMaxConcurrency = ErrorHttpStatusCode(http.StatusTooManyRequests) +var ErrExceedMaxConcurrency = HttpStatusCodeError(http.StatusTooManyRequests) func (l *ConcurrencyLimit) sub() error { l._m.Lock() @@ -403,7 +403,7 @@ var errInfiniteRetry = errors.New("infinite retry") func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { resp, err := d.cfg.HttpClient(d.ctx, params) if err != nil { - statusCode, ok := errors.Unwrap(err).(ErrorHttpStatusCode) + statusCode, ok := errors.Unwrap(err).(HttpStatusCodeError) if !ok { return 0, err } diff --git a/internal/net/serve.go b/internal/net/serve.go index 1fd40b1c1..6ffe41204 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -114,7 +114,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time reader, err := RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) if err != nil { code = http.StatusRequestedRangeNotSatisfiable - if statusCode, ok := errors.Unwrap(err).(ErrorHttpStatusCode); ok { + if statusCode, ok := errors.Unwrap(err).(HttpStatusCodeError); ok { code = int(statusCode) } http.Error(w, err.Error(), code) @@ -137,7 +137,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time sendContent, err = RangeReadCloser.RangeRead(ctx, ra) if err != nil { code = http.StatusRequestedRangeNotSatisfiable - if statusCode, ok := errors.Unwrap(err).(ErrorHttpStatusCode); ok { + if statusCode, ok := errors.Unwrap(err).(HttpStatusCodeError); ok { code = int(statusCode) } http.Error(w, err.Error(), code) @@ -199,7 +199,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time log.Warnf("Maybe size incorrect or reader not giving correct/full data, or connection closed before finish. written bytes: %d ,sendSize:%d, ", written, sendSize) } code = http.StatusInternalServerError - if statusCode, ok := errors.Unwrap(err).(ErrorHttpStatusCode); ok { + if statusCode, ok := errors.Unwrap(err).(HttpStatusCodeError); ok { code = int(statusCode) } w.WriteHeader(code) @@ -253,14 +253,14 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea _ = res.Body.Close() msg := string(all) log.Debugln(msg) - return nil, fmt.Errorf("http request [%s] failure,status: %w response:%s", URL, ErrorHttpStatusCode(res.StatusCode), msg) + return nil, fmt.Errorf("http request [%s] failure,status: %w response:%s", URL, HttpStatusCodeError(res.StatusCode), msg) } return res, nil } -type ErrorHttpStatusCode int +type HttpStatusCodeError int -func (e ErrorHttpStatusCode) Error() string { +func (e HttpStatusCodeError) Error() string { return fmt.Sprintf("%d|%s", e, http.StatusText(int(e))) } diff --git a/internal/stream/util.go b/internal/stream/util.go index 20cb4be07..4f51a46d2 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -77,7 +77,7 @@ func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, response, err := net.RequestHttp(ctx, "GET", header, link.URL) if err != nil { - if _, ok := errors.Unwrap(err).(net.ErrorHttpStatusCode); ok { + if _, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { return nil, err } return nil, fmt.Errorf("http request failure, err:%w", err) diff --git a/server/handles/down.go b/server/handles/down.go index 6f9d48c29..d43dd3829 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -148,7 +148,7 @@ func proxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { if Writer.IsWritten() { log.Errorf("%s %s local proxy error: %+v", c.Request.Method, c.Request.URL.Path, err) } else { - if statusCode, ok := errors.Unwrap(err).(net.ErrorHttpStatusCode); ok { + if statusCode, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { common.ErrorPage(c, err, int(statusCode), true) } else { common.ErrorPage(c, err, 500, true) diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index b6f7cdac8..671f9bd4f 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -271,7 +271,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta } err = common.Proxy(w, r, link, fi) if err != nil { - if statusCode, ok := errors.Unwrap(err).(net.ErrorHttpStatusCode); ok { + if statusCode, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { return int(statusCode), err } return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) From 439c3c119adcbd6d0f3314892c101f69db295d2f Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Thu, 28 Aug 2025 12:01:53 +0800 Subject: [PATCH 06/25] fix(chunk): improve error handling in Link method --- drivers/chunk/driver.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 4eaedaedb..b7f33d90b 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -149,14 +149,16 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( totalLength := int64(0) for { l, o, err := op.Link(ctx, storage, d.getPartName(path, len(links)), args) - if errors.Is(err, errs.ObjectNotFound) { - break - } if err != nil { - for _, l1 := range links { - _ = l1.Close() + if len(links) > 0 { + if errors.Is(err, errs.ObjectNotFound) { + break + } + for _, l1 := range links { + _ = l1.Close() + } } - return nil, fmt.Errorf("failed get Part %d link: %+v", len(links), err) + return nil, fmt.Errorf("failed get Part %d link: %w", len(links), err) } if l.ContentLength <= 0 { l.ContentLength = o.GetSize() @@ -167,7 +169,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( _ = l1.Close() } _ = l.Close() - return nil, fmt.Errorf("failed get Part %d range reader: %+v", len(links), err) + return nil, fmt.Errorf("failed get Part %d range reader: %w", len(links), err) } links = append(links, l) rrfs = append(rrfs, rrf) From 7f61346e92cbe0d3c20ce4a6356c16253c9cf059 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 1 Sep 2025 16:14:30 +0800 Subject: [PATCH 07/25] =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index b7f33d90b..54d9a3925 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "net/http" stdpath "path" "regexp" "strings" @@ -14,7 +13,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" @@ -225,7 +223,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( readFrom = true } } - return nil, fmt.Errorf("invalid range: start=%d,length=%d,totalLength=%d, status: %w", httpRange.Start, httpRange.Length, totalLength, net.HttpStatusCodeError(http.StatusRequestedRangeNotSatisfiable)) + return nil, fmt.Errorf("invalid range: start=%d,length=%d,totalLength=%d", httpRange.Start, httpRange.Length, totalLength) } linkClosers := make([]io.Closer, 0, len(links)) for _, l := range links { From d9fd154255769c8eca728b1e0c94a96418f485b1 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 1 Sep 2025 23:18:34 +0800 Subject: [PATCH 08/25] =?UTF-8?q?perf(chunk):=20=E6=94=B9=E8=BF=9BLink?= =?UTF-8?q?=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 202 ++++++++++++++++++++++++++-------------- drivers/chunk/obj.go | 8 ++ 2 files changed, 140 insertions(+), 70 deletions(-) create mode 100644 drivers/chunk/obj.go diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 54d9a3925..41e791f25 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -7,6 +7,7 @@ import ( "io" stdpath "path" "regexp" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -62,6 +63,8 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { partPrefix := base + ".openlist_chunk_" var first model.Obj var totalSize int64 = 0 + var idxMask int64 = 0 + chunkSizes := make([]int64, 1) for _, obj := range objs { if obj.GetName() == base { first = obj @@ -70,21 +73,44 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { break } else { totalSize += obj.GetSize() + chunkSizes[0] = obj.GetSize() + idxMask |= 1 + } + } else if suffix, ok := strings.CutPrefix(obj.GetName(), partPrefix); ok { + idx, err := strconv.Atoi(strings.TrimSuffix(suffix, d.CustomExt)) + if err != nil { + return nil, fmt.Errorf("invalid chunk part name: %s", obj.GetName()) } - } else if strings.HasPrefix(obj.GetName(), partPrefix) { totalSize += obj.GetSize() + if len(chunkSizes) > idx { + chunkSizes[idx] = obj.GetSize() + } else if len(chunkSizes) == idx { + chunkSizes = append(chunkSizes, obj.GetSize()) + } else { + newChunkSizes := make([]int64, idx+1) + copy(newChunkSizes, chunkSizes) + chunkSizes = newChunkSizes + chunkSizes[idx] = obj.GetSize() + } + idxMask |= 1 << idx } } if first == nil { return nil, errs.ObjectNotFound } - return &model.Object{ - Path: path, - Name: base, - Size: totalSize, - Modified: first.ModTime(), - Ctime: first.CreateTime(), - IsFolder: first.IsDir(), + if (idxMask+1)&idxMask != 0 { + return nil, fmt.Errorf("some chunk parts are missing for file: %s", path) + } + return &chunkObject{ + Object: model.Object{ + Path: path, + Name: base, + Size: totalSize, + Modified: first.ModTime(), + Ctime: first.CreateTime(), + IsFolder: first.IsDir(), + }, + chunkSizes: chunkSizes, }, nil } @@ -95,6 +121,8 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } ret := make([]model.Obj, 0) sizeMap := make(map[string]int64) + chunkSizeMap := make(map[string][]int64) + chunkIdxMask := make(map[string]int64) for _, obj := range objs { if obj.IsDir() { ret = append(ret, &model.Object{ @@ -108,28 +136,49 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } var name string matches := d.partNameMatchRe.FindStringSubmatch(obj.GetName()) + idx := 0 if len(matches) < 3 { - ret = append(ret, &model.Object{ - Name: obj.GetName(), - Size: 0, - Modified: obj.ModTime(), - Ctime: obj.CreateTime(), - IsFolder: false, + ret = append(ret, &chunkObject{ + Object: model.Object{ + Name: obj.GetName(), + Size: 0, + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: false, + }, }) name = obj.GetName() } else { name = matches[1] + idx, _ = strconv.Atoi(matches[2]) } - _, ok := sizeMap[name] - if !ok { - sizeMap[name] = obj.GetSize() + sizeMap[name] += obj.GetSize() + // Mark this chunk index as present + chunkIdxMask[name] |= 1 << idx + // Collect chunk sizes + chunkSizes := chunkSizeMap[name] + if len(chunkSizes) > idx { + chunkSizes[idx] = obj.GetSize() + } else if len(chunkSizes) == idx { + chunkSizes = append(chunkSizes, obj.GetSize()) } else { - sizeMap[name] += obj.GetSize() + newChunkSizes := make([]int64, idx+1) + copy(newChunkSizes, chunkSizes) + chunkSizes = newChunkSizes + chunkSizes[idx] = obj.GetSize() } + chunkSizeMap[name] = chunkSizes } for _, obj := range ret { if !obj.IsDir() { - obj.(*model.Object).Size = sizeMap[obj.GetName()] + // Check if all chunk parts are present + idxMask := chunkIdxMask[obj.GetName()] + if idxMask > 0 && (idxMask+1)&idxMask != 0 { + return nil, fmt.Errorf("some chunk parts are missing for file: %s", obj.GetName()) + } + cObj := obj.(*chunkObject) + cObj.Size = sizeMap[cObj.Name] + cObj.chunkSizes = chunkSizeMap[cObj.Name] } } return ret, nil @@ -140,98 +189,111 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if err != nil { return nil, err } + chunkFile := file.(*chunkObject) args.Redirect = false - path := stdpath.Join(reqActualPath, file.GetPath()) - links := make([]*model.Link, 0) - rrfs := make([]model.RangeReaderIF, 0) - totalLength := int64(0) - for { - l, o, err := op.Link(ctx, storage, d.getPartName(path, len(links)), args) - if err != nil { - if len(links) > 0 { - if errors.Is(err, errs.ObjectNotFound) { - break - } - for _, l1 := range links { - _ = l1.Close() - } - } - return nil, fmt.Errorf("failed get Part %d link: %w", len(links), err) - } - if l.ContentLength <= 0 { - l.ContentLength = o.GetSize() - } - rrf, err := stream.GetRangeReaderFromLink(l.ContentLength, l) - if err != nil { - for _, l1 := range links { - _ = l1.Close() - } - _ = l.Close() - return nil, fmt.Errorf("failed get Part %d range reader: %w", len(links), err) - } - links = append(links, l) - rrfs = append(rrfs, rrf) - totalLength += l.ContentLength - } + path := stdpath.Join(reqActualPath, chunkFile.GetPath()) + fileSize := chunkFile.GetSize() mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { start := httpRange.Start length := httpRange.Length - if length < 0 || start+length > totalLength { - length = totalLength - start + if length < 0 || start+length > fileSize { + length = fileSize - start + } + if length == 0 { + return io.NopCloser(strings.NewReader("")), nil } rs := make([]io.Reader, 0) cs := make(utils.Closers, 0) var ( rc io.ReadCloser - err error readFrom bool ) - for idx, l := range links { + for idx, chunkSize := range chunkFile.chunkSizes { if readFrom { - newLength := length - l.ContentLength + l, o, err := op.Link(ctx, storage, d.getPartName(path, idx), args) + if err != nil { + _ = cs.Close() + return nil, err + } + chunkSize2 := l.ContentLength + if chunkSize2 <= 0 { + chunkSize2 = o.GetSize() + } + if chunkSize2 != chunkSize { + _ = cs.Close() + _ = l.Close() + return nil, fmt.Errorf("chunk part size changed, please retry: %s", o.GetPath()) + } + rrf, err := stream.GetRangeReaderFromLink(chunkSize2, l) + if err != nil { + _ = cs.Close() + _ = l.Close() + return nil, err + } + newLength := length - chunkSize2 if newLength >= 0 { length = newLength - rc, err = rrfs[idx].RangeRead(ctx, http_range.Range{Length: -1}) + rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) } else { - rc, err = rrfs[idx].RangeRead(ctx, http_range.Range{Length: length}) + rc, err = rrf.RangeRead(ctx, http_range.Range{Length: length}) } if err != nil { _ = cs.Close() return nil, err } rs = append(rs, rc) - cs = append(cs, rc) + cs = append(cs, rc, l) if newLength <= 0 { return utils.ReadCloser{ Reader: io.MultiReader(rs...), Closer: &cs, }, nil } - } else if newStart := start - l.ContentLength; newStart >= 0 { + } else if newStart := start - chunkSize; newStart >= 0 { start = newStart } else { - rc, err = rrfs[idx].RangeRead(ctx, http_range.Range{Start: start, Length: -1}) + l, o, err := op.Link(ctx, storage, d.getPartName(path, idx), args) + if err != nil { + _ = cs.Close() + _ = l.Close() + return nil, err + } + chunkSize2 := l.ContentLength + if chunkSize2 <= 0 { + chunkSize2 = o.GetSize() + } + if chunkSize2 != chunkSize { + _ = cs.Close() + _ = l.Close() + return nil, fmt.Errorf("chunk part size not match, need refresh cache: %s", o.GetPath()) + } + rrf, err := stream.GetRangeReaderFromLink(chunkSize2, l) if err != nil { + _ = cs.Close() + _ = l.Close() return nil, err } - length -= l.ContentLength - start + rc, err = rrf.RangeRead(ctx, http_range.Range{Start: start, Length: -1}) + if err != nil { + return nil, err + } + length -= chunkSize2 - start + cs = append(cs, rc, l) if length <= 0 { - return rc, nil + return utils.ReadCloser{ + Reader: rc, + Closer: &cs, + }, nil } rs = append(rs, rc) - cs = append(cs, rc) + cs = append(cs, rc, l) readFrom = true } } - return nil, fmt.Errorf("invalid range: start=%d,length=%d,totalLength=%d", httpRange.Start, httpRange.Length, totalLength) - } - linkClosers := make([]io.Closer, 0, len(links)) - for _, l := range links { - linkClosers = append(linkClosers, l) + return nil, fmt.Errorf("invalid range: start=%d,length=%d,fileSize=%d", httpRange.Start, httpRange.Length, fileSize) } return &model.Link{ RangeReader: stream.RangeReaderFunc(mergedRrf), - SyncClosers: utils.NewSyncClosers(linkClosers...), }, nil } diff --git a/drivers/chunk/obj.go b/drivers/chunk/obj.go new file mode 100644 index 000000000..1885a9257 --- /dev/null +++ b/drivers/chunk/obj.go @@ -0,0 +1,8 @@ +package chunk + +import "github.com/OpenListTeam/OpenList/v4/internal/model" + +type chunkObject struct { + model.Object + chunkSizes []int64 +} From 10a6677a850fcde0b28585f6cb375e27e5fd99e7 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 1 Sep 2025 23:28:24 +0800 Subject: [PATCH 09/25] =?UTF-8?q?fix(chunk):=20=E4=BC=98=E5=8C=96Link?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E4=BB=A5=E5=A4=84=E7=90=86=E5=8D=95=E4=B8=AA?= =?UTF-8?q?chunk=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 41e791f25..dfc80436e 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -192,6 +192,10 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( chunkFile := file.(*chunkObject) args.Redirect = false path := stdpath.Join(reqActualPath, chunkFile.GetPath()) + if len(chunkFile.chunkSizes) <= 1 { + l, _, err := op.Link(ctx, storage, path, args) + return l, err + } fileSize := chunkFile.GetSize() mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { start := httpRange.Start From 95ec9fd5baed9062ee6ca3e64c431e4e04266480 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Tue, 2 Sep 2025 00:54:10 +0800 Subject: [PATCH 10/25] =?UTF-8?q?=E6=94=B9=E8=BF=9BGet=E5=92=8CList?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E4=B8=AD=E7=9A=84chunk=E9=83=A8=E5=88=86?= =?UTF-8?q?=E7=BC=BA=E5=A4=B1=E6=A3=80=E6=9F=A5=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index dfc80436e..67c6fca5a 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -63,8 +63,7 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { partPrefix := base + ".openlist_chunk_" var first model.Obj var totalSize int64 = 0 - var idxMask int64 = 0 - chunkSizes := make([]int64, 1) + chunkSizes := []int64{-1} for _, obj := range objs { if obj.GetName() == base { first = obj @@ -74,7 +73,6 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { } else { totalSize += obj.GetSize() chunkSizes[0] = obj.GetSize() - idxMask |= 1 } } else if suffix, ok := strings.CutPrefix(obj.GetName(), partPrefix); ok { idx, err := strconv.Atoi(strings.TrimSuffix(suffix, d.CustomExt)) @@ -92,14 +90,15 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { chunkSizes = newChunkSizes chunkSizes[idx] = obj.GetSize() } - idxMask |= 1 << idx } } if first == nil { return nil, errs.ObjectNotFound } - if (idxMask+1)&idxMask != 0 { - return nil, fmt.Errorf("some chunk parts are missing for file: %s", path) + for i, l := 0, len(chunkSizes)-1; i <= l; i++ { + if (i == 0 && chunkSizes[i] == -1) || chunkSizes[i] == 0 { + return nil, fmt.Errorf("some chunk parts are missing for file: %s", base) + } } return &chunkObject{ Object: model.Object{ @@ -122,7 +121,6 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ ret := make([]model.Obj, 0) sizeMap := make(map[string]int64) chunkSizeMap := make(map[string][]int64) - chunkIdxMask := make(map[string]int64) for _, obj := range objs { if obj.IsDir() { ret = append(ret, &model.Object{ @@ -153,10 +151,11 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ idx, _ = strconv.Atoi(matches[2]) } sizeMap[name] += obj.GetSize() - // Mark this chunk index as present - chunkIdxMask[name] |= 1 << idx // Collect chunk sizes - chunkSizes := chunkSizeMap[name] + chunkSizes, ok := chunkSizeMap[name] + if !ok { + chunkSizes = []int64{-1} + } if len(chunkSizes) > idx { chunkSizes[idx] = obj.GetSize() } else if len(chunkSizes) == idx { @@ -171,14 +170,15 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } for _, obj := range ret { if !obj.IsDir() { - // Check if all chunk parts are present - idxMask := chunkIdxMask[obj.GetName()] - if idxMask > 0 && (idxMask+1)&idxMask != 0 { - return nil, fmt.Errorf("some chunk parts are missing for file: %s", obj.GetName()) + chunkSizes := chunkSizeMap[obj.GetName()] + for i, l := 0, len(chunkSizes)-1; i < l; i++ { + if (i == 0 && chunkSizes[i] == -1) || chunkSizes[i] == 0 { + return nil, fmt.Errorf("some chunk parts are missing for file: %s", obj.GetName()) + } } cObj := obj.(*chunkObject) - cObj.Size = sizeMap[cObj.Name] - cObj.chunkSizes = chunkSizeMap[cObj.Name] + cObj.Size = sizeMap[cObj.GetName()] + cObj.chunkSizes = chunkSizes } } return ret, nil From 06e762c1e9d3e3c158351e094edaec769ed100ab Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 7 Sep 2025 14:25:20 +0800 Subject: [PATCH 11/25] =?UTF-8?q?refactor(List):=20=E7=A7=BB=E9=99=A4?= =?UTF-8?q?=E4=B8=8D=E5=BF=85=E8=A6=81=E7=9A=84chunk=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E6=98=A0=E5=B0=84=E9=80=BB=E8=BE=91=EF=BC=8C=E7=AE=80=E5=8C=96?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 45 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 36 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 67c6fca5a..6016f9ae2 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -120,7 +120,6 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } ret := make([]model.Obj, 0) sizeMap := make(map[string]int64) - chunkSizeMap := make(map[string][]int64) for _, obj := range objs { if obj.IsDir() { ret = append(ret, &model.Object{ @@ -134,51 +133,25 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } var name string matches := d.partNameMatchRe.FindStringSubmatch(obj.GetName()) - idx := 0 if len(matches) < 3 { - ret = append(ret, &chunkObject{ - Object: model.Object{ - Name: obj.GetName(), - Size: 0, - Modified: obj.ModTime(), - Ctime: obj.CreateTime(), - IsFolder: false, - }, - }) + ret = append(ret, &model.Object{ + Name: obj.GetName(), + Size: 0, + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: false, + }, + ) name = obj.GetName() } else { name = matches[1] - idx, _ = strconv.Atoi(matches[2]) } sizeMap[name] += obj.GetSize() - // Collect chunk sizes - chunkSizes, ok := chunkSizeMap[name] - if !ok { - chunkSizes = []int64{-1} - } - if len(chunkSizes) > idx { - chunkSizes[idx] = obj.GetSize() - } else if len(chunkSizes) == idx { - chunkSizes = append(chunkSizes, obj.GetSize()) - } else { - newChunkSizes := make([]int64, idx+1) - copy(newChunkSizes, chunkSizes) - chunkSizes = newChunkSizes - chunkSizes[idx] = obj.GetSize() - } - chunkSizeMap[name] = chunkSizes } for _, obj := range ret { if !obj.IsDir() { - chunkSizes := chunkSizeMap[obj.GetName()] - for i, l := 0, len(chunkSizes)-1; i < l; i++ { - if (i == 0 && chunkSizes[i] == -1) || chunkSizes[i] == 0 { - return nil, fmt.Errorf("some chunk parts are missing for file: %s", obj.GetName()) - } - } - cObj := obj.(*chunkObject) + cObj := obj.(*model.Object) cObj.Size = sizeMap[cObj.GetName()] - cObj.chunkSizes = chunkSizes } } return ret, nil From a495e1c1503814cf0edba37e3386b7669ce10f6a Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 7 Sep 2025 16:21:05 +0800 Subject: [PATCH 12/25] =?UTF-8?q?fix(chunk):=20=E4=BC=98=E5=8C=96Link?= =?UTF-8?q?=E6=96=B9=E6=B3=95=EF=BC=8C=E7=AE=80=E5=8C=96=E4=BB=A3=E7=A0=81?= =?UTF-8?q?;=20refactor(utils):=20=E6=94=B9=E8=BF=9BSyncClosers=E7=9A=84?= =?UTF-8?q?=E5=BC=95=E7=94=A8=E7=AE=A1=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 12 ++++-------- internal/op/archive.go | 5 +---- internal/op/fs.go | 5 +---- pkg/utils/io.go | 41 ++++++++++++++++++++++++++--------------- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 6016f9ae2..ccff52d42 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -192,19 +192,18 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( _ = cs.Close() return nil, err } + cs = append(cs, l) chunkSize2 := l.ContentLength if chunkSize2 <= 0 { chunkSize2 = o.GetSize() } if chunkSize2 != chunkSize { _ = cs.Close() - _ = l.Close() return nil, fmt.Errorf("chunk part size changed, please retry: %s", o.GetPath()) } rrf, err := stream.GetRangeReaderFromLink(chunkSize2, l) if err != nil { _ = cs.Close() - _ = l.Close() return nil, err } newLength := length - chunkSize2 @@ -219,7 +218,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, err } rs = append(rs, rc) - cs = append(cs, rc, l) + cs = append(cs, rc) if newLength <= 0 { return utils.ReadCloser{ Reader: io.MultiReader(rs...), @@ -232,22 +231,20 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( l, o, err := op.Link(ctx, storage, d.getPartName(path, idx), args) if err != nil { _ = cs.Close() - _ = l.Close() return nil, err } + cs = append(cs, l) chunkSize2 := l.ContentLength if chunkSize2 <= 0 { chunkSize2 = o.GetSize() } if chunkSize2 != chunkSize { _ = cs.Close() - _ = l.Close() return nil, fmt.Errorf("chunk part size not match, need refresh cache: %s", o.GetPath()) } rrf, err := stream.GetRangeReaderFromLink(chunkSize2, l) if err != nil { _ = cs.Close() - _ = l.Close() return nil, err } rc, err = rrf.RangeRead(ctx, http_range.Range{Start: start, Length: -1}) @@ -255,7 +252,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, err } length -= chunkSize2 - start - cs = append(cs, rc, l) + cs = append(cs, rc) if length <= 0 { return utils.ReadCloser{ Reader: rc, @@ -263,7 +260,6 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( }, nil } rs = append(rs, rc) - cs = append(cs, rc, l) readFrom = true } } diff --git a/internal/op/archive.go b/internal/op/archive.go index 964e93970..4d85d2064 100644 --- a/internal/op/archive.go +++ b/internal/op/archive.go @@ -405,11 +405,8 @@ func DriverExtract(ctx context.Context, storage driver.Driver, path string, args return nil }) link, err, _ := extractG.Do(key, fn) - if err == nil && !link.AcquireReference() { + for err == nil && !link.AcquireReference() { link, err, _ = extractG.Do(key, fn) - if err == nil { - link.AcquireReference() - } } if err == errLinkMFileCache { if linkM != nil { diff --git a/internal/op/fs.go b/internal/op/fs.go index bdf0567be..c668503cc 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -327,11 +327,8 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li return nil }) link, err, _ := linkG.Do(key, fn) - if err == nil && !link.AcquireReference() { + for err == nil && !link.AcquireReference() { link, err, _ = linkG.Do(key, fn) - if err == nil { - link.AcquireReference() - } } if err == errLinkMFileCache { diff --git a/pkg/utils/io.go b/pkg/utils/io.go index 172dc41c0..7ce6a9125 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -200,26 +200,37 @@ type SyncClosers struct { var _ SyncClosersIF = (*SyncClosers)(nil) func (c *SyncClosers) AcquireReference() bool { - ref := atomic.AddInt32(&c.ref, 1) - if ref > 0 { - // log.Debugf("SyncClosers.AcquireReference %p,ref=%d\n", c, ref) - return true + for { + ref := atomic.LoadInt32(&c.ref) + if ref < 0 { + return false + } + newRef := ref + 1 + if atomic.CompareAndSwapInt32(&c.ref, ref, newRef) { + log.Debugf("AcquireReference %p: %d", c, newRef) + return true + } } - atomic.StoreInt32(&c.ref, math.MinInt16) - return false } func (c *SyncClosers) Close() error { - ref := atomic.AddInt32(&c.ref, -1) - if ref < -1 { - atomic.StoreInt32(&c.ref, math.MinInt16) - return nil - } - // log.Debugf("SyncClosers.Close %p,ref=%d\n", c, ref+1) - if ref > 0 { - return nil + for { + ref := atomic.LoadInt32(&c.ref) + if ref < 0 { + return nil + } + newRef := ref - 1 + if newRef <= 0 { + newRef = math.MinInt16 + } + if atomic.CompareAndSwapInt32(&c.ref, ref, newRef) { + log.Debugf("Close %p: %d", c, ref) + if newRef > 0 { + return nil + } + break + } } - atomic.StoreInt32(&c.ref, math.MinInt16) var errs []error for _, closer := range c.closers { From c48fba13781e1c5479f2cc2e929aaa5b640264af Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 7 Sep 2025 19:19:27 +0800 Subject: [PATCH 13/25] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=A4=B9=E4=BF=9D=E5=AD=98=E5=88=86=E7=89=87=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 282 +++++++++++++++++----------------------- internal/op/fs.go | 3 + 2 files changed, 124 insertions(+), 161 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index ccff52d42..d9dd6a0df 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -6,12 +6,10 @@ import ( "fmt" "io" stdpath "path" - "regexp" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -23,7 +21,6 @@ import ( type Chunk struct { model.Storage Addition - partNameMatchRe *regexp.Regexp } func (d *Chunk) Config() driver.Config { @@ -39,7 +36,6 @@ func (d *Chunk) Init(ctx context.Context) error { return errors.New("part size must be positive") } d.RemotePath = utils.FixAndCleanPath(d.RemotePath) - d.partNameMatchRe = regexp.MustCompile("^(.+)\\.openlist_chunk_(\\d+)" + regexp.QuoteMeta(d.CustomExt) + "$") return nil } @@ -55,106 +51,126 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { Path: "/", }, nil } - dir, base := stdpath.Split(stdpath.Join(d.RemotePath, path)) - objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) + storage, reqActualPath, err := op.GetStorageAndActualPath(d.RemotePath) + if err != nil { + return nil, err + } + reqPath := stdpath.Join(reqActualPath, path) + obj, err := op.Get(ctx, storage, reqPath) + if err == nil { + return &model.Object{ + Path: path, + Name: obj.GetName(), + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + }, nil + } + + dir, base := stdpath.Split(reqPath) + reqPath = stdpath.Join(dir, "[openlist_chunk]"+base) + obj, err = op.Get(ctx, storage, reqPath) + if err != nil { + return nil, err + } + chunkObjs, err := op.List(ctx, storage, reqPath, model.ListArgs{}) if err != nil { return nil, err } - partPrefix := base + ".openlist_chunk_" - var first model.Obj var totalSize int64 = 0 chunkSizes := []int64{-1} - for _, obj := range objs { - if obj.GetName() == base { - first = obj - if obj.IsDir() { - totalSize = obj.GetSize() - break - } else { - totalSize += obj.GetSize() - chunkSizes[0] = obj.GetSize() - } - } else if suffix, ok := strings.CutPrefix(obj.GetName(), partPrefix); ok { - idx, err := strconv.Atoi(strings.TrimSuffix(suffix, d.CustomExt)) - if err != nil { - return nil, fmt.Errorf("invalid chunk part name: %s", obj.GetName()) - } - totalSize += obj.GetSize() - if len(chunkSizes) > idx { - chunkSizes[idx] = obj.GetSize() - } else if len(chunkSizes) == idx { - chunkSizes = append(chunkSizes, obj.GetSize()) - } else { - newChunkSizes := make([]int64, idx+1) - copy(newChunkSizes, chunkSizes) - chunkSizes = newChunkSizes - chunkSizes[idx] = obj.GetSize() - } + for _, o := range chunkObjs { + if o.IsDir() { + continue + } + idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) + if err != nil { + continue + } + totalSize += o.GetSize() + if len(chunkSizes) > idx { + chunkSizes[idx] = o.GetSize() + } else if len(chunkSizes) == idx { + chunkSizes = append(chunkSizes, o.GetSize()) + } else { + newChunkSizes := make([]int64, idx+1) + copy(newChunkSizes, chunkSizes) + chunkSizes = newChunkSizes + chunkSizes[idx] = o.GetSize() } } - if first == nil { - return nil, errs.ObjectNotFound - } - for i, l := 0, len(chunkSizes)-1; i <= l; i++ { + for i, l := 0, len(chunkSizes)-2; i <= l; i++ { if (i == 0 && chunkSizes[i] == -1) || chunkSizes[i] == 0 { - return nil, fmt.Errorf("some chunk parts are missing for file: %s", base) + return nil, fmt.Errorf("chunk part[%d] are missing", i) } } + reqDir, _ := stdpath.Split(path) return &chunkObject{ Object: model.Object{ - Path: path, + Path: stdpath.Join(reqDir, "[openlist_chunk]"+base), Name: base, Size: totalSize, - Modified: first.ModTime(), - Ctime: first.CreateTime(), - IsFolder: first.IsDir(), + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), }, chunkSizes: chunkSizes, }, nil } func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - objs, err := fs.List(ctx, stdpath.Join(d.RemotePath, dir.GetPath()), &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) + storage, reqActualPath, err := op.GetStorageAndActualPath(d.RemotePath) + if err != nil { + return nil, err + } + dirPath := stdpath.Join(reqActualPath, dir.GetPath()) + objs, err := op.List(ctx, storage, dirPath, model.ListArgs{Refresh: args.Refresh}) if err != nil { return nil, err } - ret := make([]model.Obj, 0) - sizeMap := make(map[string]int64) - for _, obj := range objs { - if obj.IsDir() { - ret = append(ret, &model.Object{ + return utils.SliceConvert(objs, func(obj model.Obj) (model.Obj, error) { + if !obj.IsDir() || !strings.HasPrefix(obj.GetName(), "[openlist_chunk]") { + thumb, ok := model.GetThumb(obj) + objRes := model.Object{ Name: obj.GetName(), Size: obj.GetSize(), Modified: obj.ModTime(), - Ctime: obj.CreateTime(), - IsFolder: true, - }) - continue + IsFolder: obj.IsDir(), + } + if !ok { + return &objRes, nil + } + return &model.ObjThumb{ + Object: objRes, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + }, nil } - var name string - matches := d.partNameMatchRe.FindStringSubmatch(obj.GetName()) - if len(matches) < 3 { - ret = append(ret, &model.Object{ - Name: obj.GetName(), - Size: 0, - Modified: obj.ModTime(), - Ctime: obj.CreateTime(), - IsFolder: false, - }, - ) - name = obj.GetName() - } else { - name = matches[1] + reqPath := stdpath.Join(dirPath, obj.GetName()) + chunkObjs, err := op.List(ctx, storage, reqPath, model.ListArgs{Refresh: args.Refresh}) + if err != nil { + return nil, err } - sizeMap[name] += obj.GetSize() - } - for _, obj := range ret { - if !obj.IsDir() { - cObj := obj.(*model.Object) - cObj.Size = sizeMap[cObj.GetName()] + totalSize := int64(0) + for _, o := range chunkObjs { + if o.IsDir() { + continue + } + _, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) + if err != nil { + continue + } + totalSize += o.GetSize() } - } - return ret, nil + return &model.Object{ + Name: strings.TrimPrefix(obj.GetName(), "[openlist_chunk]"), + Path: reqPath, + Size: totalSize, + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + }, nil + }) } func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -162,12 +178,17 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if err != nil { return nil, err } - chunkFile := file.(*chunkObject) args.Redirect = false - path := stdpath.Join(reqActualPath, chunkFile.GetPath()) - if len(chunkFile.chunkSizes) <= 1 { - l, _, err := op.Link(ctx, storage, path, args) - return l, err + chunkFile, ok := file.(*chunkObject) + reqPath := stdpath.Join(reqActualPath, file.GetPath()) + if !ok { + l, _, err := op.Link(ctx, storage, reqPath, args) + if err != nil { + return nil, err + } + resultLink := *l + resultLink.SyncClosers = utils.NewSyncClosers(l) + return &resultLink, nil } fileSize := chunkFile.GetSize() mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { @@ -187,7 +208,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( ) for idx, chunkSize := range chunkFile.chunkSizes { if readFrom { - l, o, err := op.Link(ctx, storage, d.getPartName(path, idx), args) + l, o, err := op.Link(ctx, storage, stdpath.Join(reqPath, d.getPartName(idx)), args) if err != nil { _ = cs.Close() return nil, err @@ -199,7 +220,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } if chunkSize2 != chunkSize { _ = cs.Close() - return nil, fmt.Errorf("chunk part size changed, please retry: %s", o.GetPath()) + return nil, fmt.Errorf("chunk part[%d] size not match", idx) } rrf, err := stream.GetRangeReaderFromLink(chunkSize2, l) if err != nil { @@ -228,7 +249,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } else if newStart := start - chunkSize; newStart >= 0 { start = newStart } else { - l, o, err := op.Link(ctx, storage, d.getPartName(path, idx), args) + l, o, err := op.Link(ctx, storage, stdpath.Join(reqPath, d.getPartName(idx)), args) if err != nil { _ = cs.Close() return nil, err @@ -240,7 +261,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } if chunkSize2 != chunkSize { _ = cs.Close() - return nil, fmt.Errorf("chunk part size not match, need refresh cache: %s", o.GetPath()) + return nil, fmt.Errorf("chunk part[%d] size not match", idx) } rrf, err := stream.GetRangeReaderFromLink(chunkSize2, l) if err != nil { @@ -276,87 +297,29 @@ func (d *Chunk) MakeDir(ctx context.Context, parentDir model.Obj, dirName string } func (d *Chunk) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - path := stdpath.Join(d.RemotePath, srcObj.GetPath()) + src := stdpath.Join(d.RemotePath, srcObj.GetPath()) dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) - if srcObj.IsDir() { - _, err := fs.Move(ctx, path, dst) - return err - } - dir, base := stdpath.Split(path) - objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) - if err != nil { - return err - } - for _, obj := range objs { - suffix := strings.TrimPrefix(obj.GetName(), base) - if suffix != obj.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { - _, e := fs.Move(ctx, path+suffix, dst, true) - err = errors.Join(err, e) - } - } - _, e := fs.Move(ctx, path, dst) - return errors.Join(err, e) + _, err := fs.Move(ctx, src, dst) + return err } func (d *Chunk) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - path := stdpath.Join(d.RemotePath, srcObj.GetPath()) - if srcObj.IsDir() { - return fs.Rename(ctx, path, newName) + chunkObj, ok := srcObj.(*chunkObject) + if !ok { + return fs.Rename(ctx, stdpath.Join(d.RemotePath, srcObj.GetPath()), newName) } - dir, base := stdpath.Split(path) - objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) - if err != nil { - return err - } - for _, obj := range objs { - suffix := strings.TrimPrefix(obj.GetName(), base) - if suffix != obj.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { - err = errors.Join(err, fs.Rename(ctx, path+suffix, newName+suffix, true)) - } - } - return errors.Join(err, fs.Rename(ctx, path, newName)) + return fs.Rename(ctx, stdpath.Join(d.RemotePath, chunkObj.GetPath()), "[openlist_chunk]"+newName) } func (d *Chunk) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - path := stdpath.Join(d.RemotePath, srcObj.GetPath()) dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) - if srcObj.IsDir() { - _, err := fs.Copy(ctx, path, dst) - return err - } - dir, base := stdpath.Split(path) - objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) - if err != nil { - return err - } - for _, obj := range objs { - suffix := strings.TrimPrefix(obj.GetName(), base) - if suffix != obj.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { - _, e := fs.Copy(ctx, path+suffix, dst, true) - err = errors.Join(err, e) - } - } - _, e := fs.Copy(ctx, path, dst) - return errors.Join(err, e) + src := stdpath.Join(d.RemotePath, srcObj.GetPath()) + _, err := fs.Copy(ctx, src, dst) + return err } func (d *Chunk) Remove(ctx context.Context, obj model.Obj) error { - path := stdpath.Join(d.RemotePath, obj.GetPath()) - if obj.IsDir() { - return fs.Remove(ctx, path) - } - dir, base := stdpath.Split(path) - objs, err := fs.List(ctx, dir, &fs.ListArgs{NoLog: true}) - if err != nil { - return err - } - for _, o := range objs { - suffix := strings.TrimPrefix(o.GetName(), base) - if suffix != o.GetName() && strings.HasPrefix(suffix, ".openlist_chunk_") { - err = errors.Join(err, fs.Remove(ctx, path+suffix)) - } - } - return errors.Join(err, fs.Remove(ctx, path)) + return fs.Remove(ctx, stdpath.Join(d.RemotePath, obj.GetPath())) } func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { @@ -368,7 +331,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream Reader: file, UpdateProgress: up, } - dst := stdpath.Join(reqActualPath, dstDir.GetPath()) + dst := stdpath.Join(reqActualPath, dstDir.GetPath(), "[openlist_chunk]"+file.GetName()) fullPartCount := int(file.GetSize() / d.PartSize) tailSize := file.GetSize() % d.PartSize if tailSize == 0 && fullPartCount > 0 { @@ -379,7 +342,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream for partIndex < fullPartCount { err = errors.Join(err, op.Put(ctx, storage, dst, &stream.FileStream{ Obj: &model.Object{ - Name: d.getPartName(file.GetName(), partIndex), + Name: d.getPartName(partIndex), Size: d.PartSize, Modified: file.ModTime(), }, @@ -390,7 +353,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream } return errors.Join(err, op.Put(ctx, storage, dst, &stream.FileStream{ Obj: &model.Object{ - Name: d.getPartName(file.GetName(), fullPartCount), + Name: d.getPartName(fullPartCount), Size: tailSize, Modified: file.ModTime(), }, @@ -399,11 +362,8 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, nil)) } -func (d *Chunk) getPartName(name string, part int) string { - if part == 0 { - return name - } - return fmt.Sprintf("%s.openlist_chunk_%d%s", name, part, d.CustomExt) +func (d *Chunk) getPartName(part int) string { + return fmt.Sprintf("%d%s", part, d.CustomExt) } var _ driver.Driver = (*Chunk)(nil) diff --git a/internal/op/fs.go b/internal/op/fs.go index c668503cc..9330ceb6a 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -184,6 +184,9 @@ func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, er if err == nil { return model.WrapObjName(obj), nil } + if !errs.IsNotImplement(err) { + return nil, errors.WithMessage(err, "failed to get obj") + } } // is root folder From 87a48e2ed5f742250fd5fabd9c4411385f019c86 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 7 Sep 2025 20:19:10 +0800 Subject: [PATCH 14/25] =?UTF-8?q?fix(chunk):=20=E4=BC=98=E5=8C=96Get?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E4=B8=AD=E7=9A=84=E6=A3=80=E6=9F=A5chunk?= =?UTF-8?q?=E9=83=A8=E5=88=86=E7=9A=84=E7=BC=BA=E5=A4=B1=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index d9dd6a0df..2ec7a107d 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -100,10 +100,13 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { chunkSizes[idx] = o.GetSize() } } - for i, l := 0, len(chunkSizes)-2; i <= l; i++ { + for i, l := 0, len(chunkSizes)-2; ; i++ { if (i == 0 && chunkSizes[i] == -1) || chunkSizes[i] == 0 { return nil, fmt.Errorf("chunk part[%d] are missing", i) } + if i >= l { + break + } } reqDir, _ := stdpath.Split(path) return &chunkObject{ From fda06d1e60d25a2f5d517e0315916ae3e72f7b42 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sun, 7 Sep 2025 20:43:00 +0800 Subject: [PATCH 15/25] feat(chunk): support hash --- drivers/chunk/driver.go | 40 ++++++++++++++++++++++++++++++++++++++++ drivers/chunk/meta.go | 1 + pkg/utils/hash.go | 5 +++++ 3 files changed, 46 insertions(+) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index d9dd6a0df..8b4b8299d 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -1,6 +1,7 @@ package chunk import ( + "bytes" "context" "errors" "fmt" @@ -80,10 +81,22 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { } var totalSize int64 = 0 chunkSizes := []int64{-1} + h := make(map[*utils.HashType]string) for _, o := range chunkObjs { if o.IsDir() { continue } + if strings.HasPrefix(o.GetName(), "hash_") { + typeValue := strings.TrimSuffix(strings.TrimPrefix(o.GetName(), "hash_"), d.CustomExt) + hn, value, ok := strings.Cut(typeValue, "_") + if ok { + ht, ok := utils.GetHashByName(hn) + if ok { + h[ht] = value + } + continue + } + } idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) if err != nil { continue @@ -113,6 +126,7 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { Size: totalSize, Modified: obj.ModTime(), Ctime: obj.CreateTime(), + HashInfo: utils.NewHashInfoByMap(h), }, chunkSizes: chunkSizes, }, nil @@ -153,10 +167,22 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ return nil, err } totalSize := int64(0) + h := make(map[*utils.HashType]string) for _, o := range chunkObjs { if o.IsDir() { continue } + if strings.HasPrefix(o.GetName(), "hash_") { + typeValue := strings.TrimSuffix(strings.TrimPrefix(o.GetName(), "hash_"), d.CustomExt) + hn, value, ok := strings.Cut(typeValue, "_") + if ok { + ht, ok := utils.GetHashByName(hn) + if ok { + h[ht] = value + } + continue + } + } _, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) if err != nil { continue @@ -169,6 +195,7 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ Size: totalSize, Modified: obj.ModTime(), Ctime: obj.CreateTime(), + HashInfo: utils.NewHashInfoByMap(h), }, nil }) } @@ -332,6 +359,19 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream UpdateProgress: up, } dst := stdpath.Join(reqActualPath, dstDir.GetPath(), "[openlist_chunk]"+file.GetName()) + if d.StoreHash { + for ht, value := range file.GetHash().All() { + _ = op.Put(ctx, storage, dst, &stream.FileStream{ + Obj: &model.Object{ + Name: fmt.Sprintf("hash_%s_%s%s", ht.Name, value, d.CustomExt), + Size: 1, + Modified: file.ModTime(), + }, + Mimetype: "application/octet-stream", + Reader: bytes.NewReader([]byte{0}), // 兼容不支持空文件的驱动 + }, nil, true) + } + } fullPartCount := int(file.GetSize() / d.PartSize) tailSize := file.GetSize() % d.PartSize if tailSize == 0 && fullPartCount > 0 { diff --git a/drivers/chunk/meta.go b/drivers/chunk/meta.go index 3adb70586..9f3c5800f 100644 --- a/drivers/chunk/meta.go +++ b/drivers/chunk/meta.go @@ -9,6 +9,7 @@ type Addition struct { RemotePath string `json:"remote_path" required:"true"` PartSize int64 `json:"part_size" required:"true" type:"number" help:"bytes"` CustomExt string `json:"custom_ext" type:"string"` + StoreHash bool `json:"store_hash" type:"bool" default:"true"` } var config = driver.Config{ diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index 0b70e4e1f..596e61e54 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -57,6 +57,11 @@ var ( Supported []*HashType ) +func GetHashByName(name string) (ht *HashType, ok bool) { + ht, ok = name2hash[name] + return +} + // RegisterHash adds a new Hash to the list and returns its Type func RegisterHash(name, alias string, width int, newFunc func() hash.Hash) *HashType { return RegisterHashWithParam(name, alias, width, func(a ...any) hash.Hash { return newFunc() }) From af1f3d1d6bec444db792a285f6e34a4d47379573 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 8 Sep 2025 13:28:33 +0800 Subject: [PATCH 16/25] =?UTF-8?q?fix(chunk):=20=E4=BC=98=E5=8C=96Get?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E4=B8=AD=E8=8E=B7=E5=8F=96=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=97=B6=E9=97=B4=E5=92=8C=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 44c902f0f..5c79de9d9 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -71,10 +71,6 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { dir, base := stdpath.Split(reqPath) reqPath = stdpath.Join(dir, "[openlist_chunk]"+base) - obj, err = op.Get(ctx, storage, reqPath) - if err != nil { - return nil, err - } chunkObjs, err := op.List(ctx, storage, reqPath, model.ListArgs{}) if err != nil { return nil, err @@ -82,6 +78,7 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { var totalSize int64 = 0 chunkSizes := []int64{-1} h := make(map[*utils.HashType]string) + var first model.Obj for _, o := range chunkObjs { if o.IsDir() { continue @@ -103,6 +100,9 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { } totalSize += o.GetSize() if len(chunkSizes) > idx { + if idx == 0 { + first = o + } chunkSizes[idx] = o.GetSize() } else if len(chunkSizes) == idx { chunkSizes = append(chunkSizes, o.GetSize()) @@ -127,8 +127,8 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { Path: stdpath.Join(reqDir, "[openlist_chunk]"+base), Name: base, Size: totalSize, - Modified: obj.ModTime(), - Ctime: obj.CreateTime(), + Modified: first.ModTime(), + Ctime: first.CreateTime(), HashInfo: utils.NewHashInfoByMap(h), }, chunkSizes: chunkSizes, @@ -171,6 +171,7 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } totalSize := int64(0) h := make(map[*utils.HashType]string) + first := obj for _, o := range chunkObjs { if o.IsDir() { continue @@ -186,18 +187,20 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ continue } } - _, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) + idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) if err != nil { continue } + if idx == 0 { + first = o + } totalSize += o.GetSize() } return &model.Object{ Name: strings.TrimPrefix(obj.GetName(), "[openlist_chunk]"), - Path: reqPath, Size: totalSize, - Modified: obj.ModTime(), - Ctime: obj.CreateTime(), + Modified: first.ModTime(), + Ctime: first.CreateTime(), HashInfo: utils.NewHashInfoByMap(h), }, nil }) From c9d640f608f606ca44539dfbb391d290fc79fcae Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 8 Sep 2025 15:19:42 +0800 Subject: [PATCH 17/25] =?UTF-8?q?feat(chunk):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E7=BC=A9=E7=95=A5=E5=9B=BE=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 45 +++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 5c79de9d9..bc602230f 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -14,9 +14,11 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" ) type Chunk struct { @@ -83,8 +85,8 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { if o.IsDir() { continue } - if strings.HasPrefix(o.GetName(), "hash_") { - typeValue := strings.TrimSuffix(strings.TrimPrefix(o.GetName(), "hash_"), d.CustomExt) + if after, ok := strings.CutPrefix(o.GetName(), "hash_"); ok { + typeValue := strings.TrimSuffix(after, d.CustomExt) hn, value, ok := strings.Cut(typeValue, "_") if ok { ht, ok := utils.GetHashByName(hn) @@ -122,17 +124,20 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { } } reqDir, _ := stdpath.Split(path) - return &chunkObject{ + objRes := chunkObject{ Object: model.Object{ Path: stdpath.Join(reqDir, "[openlist_chunk]"+base), Name: base, Size: totalSize, Modified: first.ModTime(), Ctime: first.CreateTime(), - HashInfo: utils.NewHashInfoByMap(h), }, chunkSizes: chunkSizes, - }, nil + } + if len(h) > 0 { + objRes.HashInfo = utils.NewHashInfoByMap(h) + } + return &objRes, nil } func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { @@ -153,6 +158,7 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ Size: obj.GetSize(), Modified: obj.ModTime(), IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), } if !ok { return &objRes, nil @@ -172,13 +178,20 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ totalSize := int64(0) h := make(map[*utils.HashType]string) first := obj + thumb := "" for _, o := range chunkObjs { if o.IsDir() { continue } - if strings.HasPrefix(o.GetName(), "hash_") { - typeValue := strings.TrimSuffix(strings.TrimPrefix(o.GetName(), "hash_"), d.CustomExt) - hn, value, ok := strings.Cut(typeValue, "_") + if o.GetName() == "thumbnail.webp" { + thumbPath := stdpath.Join(args.ReqPath, obj.GetName(), o.GetName()) + thumb = fmt.Sprintf("%s/d%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(thumbPath, true), + sign.Sign(thumbPath)) + } + if after, ok := strings.CutPrefix(strings.TrimSuffix(o.GetName(), d.CustomExt), "hash_"); ok { + hn, value, ok := strings.Cut(after, "_") if ok { ht, ok := utils.GetHashByName(hn) if ok { @@ -196,12 +209,23 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } totalSize += o.GetSize() } - return &model.Object{ + objRes := model.Object{ Name: strings.TrimPrefix(obj.GetName(), "[openlist_chunk]"), Size: totalSize, Modified: first.ModTime(), Ctime: first.CreateTime(), - HashInfo: utils.NewHashInfoByMap(h), + } + if len(h) > 0 { + objRes.HashInfo = utils.NewHashInfoByMap(h) + } + if thumb == "" { + return &objRes, nil + } + return &model.ObjThumb{ + Object: objRes, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, }, nil }) } @@ -211,7 +235,6 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if err != nil { return nil, err } - args.Redirect = false chunkFile, ok := file.(*chunkObject) reqPath := stdpath.Join(reqActualPath, file.GetPath()) if !ok { From 93ac17beb43f4dc68f57c1ec9257066a13299d43 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Tue, 9 Sep 2025 09:23:51 +0800 Subject: [PATCH 18/25] =?UTF-8?q?fix(chunk):=20=E4=BC=98=E5=8C=96Get?= =?UTF-8?q?=E5=92=8CList=E6=96=B9=E6=B3=95=E4=B8=AD=E7=9A=84=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 193 +++++++++++++++++++++------------------- 1 file changed, 100 insertions(+), 93 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index bc602230f..abd9a15c4 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -54,26 +54,25 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { Path: "/", }, nil } - storage, reqActualPath, err := op.GetStorageAndActualPath(d.RemotePath) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, err } - reqPath := stdpath.Join(reqActualPath, path) - obj, err := op.Get(ctx, storage, reqPath) - if err == nil { + remoteActualPath = stdpath.Join(remoteActualPath, path) + if remoteObj, err := op.Get(ctx, remoteStorage, remoteActualPath); err == nil { return &model.Object{ Path: path, - Name: obj.GetName(), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - HashInfo: obj.GetHash(), + Name: remoteObj.GetName(), + Size: remoteObj.GetSize(), + Modified: remoteObj.ModTime(), + IsFolder: remoteObj.IsDir(), + HashInfo: remoteObj.GetHash(), }, nil } - dir, base := stdpath.Split(reqPath) - reqPath = stdpath.Join(dir, "[openlist_chunk]"+base) - chunkObjs, err := op.List(ctx, storage, reqPath, model.ListArgs{}) + remoteActualDir, name := stdpath.Split(remoteActualPath) + chunkName := "[openlist_chunk]" + name + chunkObjs, err := op.List(ctx, remoteStorage, stdpath.Join(remoteActualDir, chunkName), model.ListArgs{}) if err != nil { return nil, err } @@ -86,15 +85,14 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { continue } if after, ok := strings.CutPrefix(o.GetName(), "hash_"); ok { - typeValue := strings.TrimSuffix(after, d.CustomExt) - hn, value, ok := strings.Cut(typeValue, "_") + hn, value, ok := strings.Cut(strings.TrimSuffix(after, d.CustomExt), "_") if ok { ht, ok := utils.GetHashByName(hn) if ok { h[ht] = value } - continue } + continue } idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) if err != nil { @@ -126,8 +124,8 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { reqDir, _ := stdpath.Split(path) objRes := chunkObject{ Object: model.Object{ - Path: stdpath.Join(reqDir, "[openlist_chunk]"+base), - Name: base, + Path: stdpath.Join(reqDir, chunkName), + Name: name, Size: totalSize, Modified: first.ModTime(), Ctime: first.CreateTime(), @@ -141,84 +139,93 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { } func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - storage, reqActualPath, err := op.GetStorageAndActualPath(d.RemotePath) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, err } - dirPath := stdpath.Join(reqActualPath, dir.GetPath()) - objs, err := op.List(ctx, storage, dirPath, model.ListArgs{Refresh: args.Refresh}) + remoteActualDir := stdpath.Join(remoteActualPath, dir.GetPath()) + remoteObjs, err := op.List(ctx, remoteStorage, remoteActualDir, model.ListArgs{ + ReqPath: args.ReqPath, + Refresh: args.Refresh, + }) if err != nil { return nil, err } - return utils.SliceConvert(objs, func(obj model.Obj) (model.Obj, error) { - if !obj.IsDir() || !strings.HasPrefix(obj.GetName(), "[openlist_chunk]") { - thumb, ok := model.GetThumb(obj) - objRes := model.Object{ - Name: obj.GetName(), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - HashInfo: obj.GetHash(), - } - if !ok { - return &objRes, nil - } - return &model.ObjThumb{ - Object: objRes, - Thumbnail: model.Thumbnail{ - Thumbnail: thumb, - }, - }, nil - } - reqPath := stdpath.Join(dirPath, obj.GetName()) - chunkObjs, err := op.List(ctx, storage, reqPath, model.ListArgs{Refresh: args.Refresh}) - if err != nil { - return nil, err - } - totalSize := int64(0) - h := make(map[*utils.HashType]string) - first := obj - thumb := "" - for _, o := range chunkObjs { - if o.IsDir() { - continue - } - if o.GetName() == "thumbnail.webp" { - thumbPath := stdpath.Join(args.ReqPath, obj.GetName(), o.GetName()) - thumb = fmt.Sprintf("%s/d%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(thumbPath, true), - sign.Sign(thumbPath)) - } - if after, ok := strings.CutPrefix(strings.TrimSuffix(o.GetName(), d.CustomExt), "hash_"); ok { - hn, value, ok := strings.Cut(after, "_") - if ok { - ht, ok := utils.GetHashByName(hn) - if ok { - h[ht] = value + return utils.SliceConvert(remoteObjs, func(obj model.Obj) (model.Obj, error) { + rawName := obj.GetName() + if obj.IsDir() { + if name, ok := strings.CutPrefix(rawName, "[openlist_chunk]"); ok { + chunkObjs, err := op.List(ctx, remoteStorage, stdpath.Join(remoteActualDir, rawName), model.ListArgs{ + ReqPath: stdpath.Join(args.ReqPath, rawName), + Refresh: args.Refresh, + }) + if err != nil { + return nil, err + } + totalSize := int64(0) + h := make(map[*utils.HashType]string) + first := obj + thumb := "" + for _, o := range chunkObjs { + if o.IsDir() { + continue + } + if o.GetName() == "thumbnail.webp" { + thumbPath := stdpath.Join(args.ReqPath, rawName, o.GetName()) + thumb = fmt.Sprintf("%s/d%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(thumbPath, true), + sign.Sign(thumbPath)) } - continue + if after, ok := strings.CutPrefix(strings.TrimSuffix(o.GetName(), d.CustomExt), "hash_"); ok { + hn, value, ok := strings.Cut(after, "_") + if ok { + ht, ok := utils.GetHashByName(hn) + if ok { + h[ht] = value + } + continue + } + } + idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) + if err != nil { + continue + } + if idx == 0 { + first = o + } + totalSize += o.GetSize() } + objRes := model.Object{ + Name: name, + Size: totalSize, + Modified: first.ModTime(), + Ctime: first.CreateTime(), + } + if len(h) > 0 { + objRes.HashInfo = utils.NewHashInfoByMap(h) + } + if thumb == "" { + return &objRes, nil + } + return &model.ObjThumb{ + Object: objRes, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + }, nil } - idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) - if err != nil { - continue - } - if idx == 0 { - first = o - } - totalSize += o.GetSize() } + + thumb, ok := model.GetThumb(obj) objRes := model.Object{ - Name: strings.TrimPrefix(obj.GetName(), "[openlist_chunk]"), - Size: totalSize, - Modified: first.ModTime(), - Ctime: first.CreateTime(), - } - if len(h) > 0 { - objRes.HashInfo = utils.NewHashInfoByMap(h) + Name: rawName, + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), } - if thumb == "" { + if !ok { return &objRes, nil } return &model.ObjThumb{ @@ -231,14 +238,14 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - storage, reqActualPath, err := op.GetStorageAndActualPath(d.RemotePath) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, err } chunkFile, ok := file.(*chunkObject) - reqPath := stdpath.Join(reqActualPath, file.GetPath()) + remoteActualPath = stdpath.Join(remoteActualPath, file.GetPath()) if !ok { - l, _, err := op.Link(ctx, storage, reqPath, args) + l, _, err := op.Link(ctx, remoteStorage, remoteActualPath, args) if err != nil { return nil, err } @@ -264,7 +271,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( ) for idx, chunkSize := range chunkFile.chunkSizes { if readFrom { - l, o, err := op.Link(ctx, storage, stdpath.Join(reqPath, d.getPartName(idx)), args) + l, o, err := op.Link(ctx, remoteStorage, stdpath.Join(remoteActualPath, d.getPartName(idx)), args) if err != nil { _ = cs.Close() return nil, err @@ -305,7 +312,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } else if newStart := start - chunkSize; newStart >= 0 { start = newStart } else { - l, o, err := op.Link(ctx, storage, stdpath.Join(reqPath, d.getPartName(idx)), args) + l, o, err := op.Link(ctx, remoteStorage, stdpath.Join(remoteActualPath, d.getPartName(idx)), args) if err != nil { _ = cs.Close() return nil, err @@ -379,7 +386,7 @@ func (d *Chunk) Remove(ctx context.Context, obj model.Obj) error { } func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { - storage, reqActualPath, err := op.GetStorageAndActualPath(d.RemotePath) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return err } @@ -387,10 +394,10 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream Reader: file, UpdateProgress: up, } - dst := stdpath.Join(reqActualPath, dstDir.GetPath(), "[openlist_chunk]"+file.GetName()) + dst := stdpath.Join(remoteActualPath, dstDir.GetPath(), "[openlist_chunk]"+file.GetName()) if d.StoreHash { for ht, value := range file.GetHash().All() { - _ = op.Put(ctx, storage, dst, &stream.FileStream{ + _ = op.Put(ctx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: fmt.Sprintf("hash_%s_%s%s", ht.Name, value, d.CustomExt), Size: 1, @@ -409,7 +416,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream } partIndex := 0 for partIndex < fullPartCount { - err = errors.Join(err, op.Put(ctx, storage, dst, &stream.FileStream{ + err = errors.Join(err, op.Put(ctx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(partIndex), Size: d.PartSize, @@ -420,7 +427,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, nil, true)) partIndex++ } - return errors.Join(err, op.Put(ctx, storage, dst, &stream.FileStream{ + return errors.Join(err, op.Put(ctx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(fullPartCount), Size: tailSize, From ef9e0285a8b1663a6b2c926b18dd6ec74e4045c6 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 13 Sep 2025 15:13:36 +0800 Subject: [PATCH 19/25] =?UTF-8?q?fix(chunk):=20=E4=BF=AE=E5=A4=8DLink?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E4=B8=AD=E6=9C=AA=E5=85=B3=E9=97=AD=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E7=9A=84=E6=BD=9C=E5=9C=A8=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Suyunmeng --- drivers/chunk/driver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index abd9a15c4..c42425554 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -333,6 +333,7 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } rc, err = rrf.RangeRead(ctx, http_range.Range{Start: start, Length: -1}) if err != nil { + _ = cs.Close() return nil, err } length -= chunkSize2 - start From b10eb75d56e36b3d8d15a43a8e134cd899870e4c Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 13 Sep 2025 15:45:16 +0800 Subject: [PATCH 20/25] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Thumbnail=E5=92=8CShowH?= =?UTF-8?q?idden=E5=AD=97=E6=AE=B5=E4=BB=A5=E6=94=AF=E6=8C=81=E7=BC=A9?= =?UTF-8?q?=E7=95=A5=E5=9B=BE=E5=92=8C=E9=9A=90=E8=97=8F=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=EF=BC=8C=E9=80=BB=E8=BE=91=E4=B8=8Ecrypt?= =?UTF-8?q?=E9=A9=B1=E5=8A=A8=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 55 ++++++++++++++++++++++------------------- drivers/chunk/meta.go | 3 +++ 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index c42425554..a511d953e 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -151,7 +151,8 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if err != nil { return nil, err } - return utils.SliceConvert(remoteObjs, func(obj model.Obj) (model.Obj, error) { + result := make([]model.Obj, len(remoteObjs)) + for _, obj := range remoteObjs { rawName := obj.GetName() if obj.IsDir() { if name, ok := strings.CutPrefix(rawName, "[openlist_chunk]"); ok { @@ -165,18 +166,10 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ totalSize := int64(0) h := make(map[*utils.HashType]string) first := obj - thumb := "" for _, o := range chunkObjs { if o.IsDir() { continue } - if o.GetName() == "thumbnail.webp" { - thumbPath := stdpath.Join(args.ReqPath, rawName, o.GetName()) - thumb = fmt.Sprintf("%s/d%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(thumbPath, true), - sign.Sign(thumbPath)) - } if after, ok := strings.CutPrefix(strings.TrimSuffix(o.GetName(), d.CustomExt), "hash_"); ok { hn, value, ok := strings.Cut(after, "_") if ok { @@ -205,18 +198,28 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if len(h) > 0 { objRes.HashInfo = utils.NewHashInfoByMap(h) } - if thumb == "" { - return &objRes, nil + if !d.Thumbnail { + result = append(result, &objRes) + } else { + thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") + thumb := fmt.Sprintf("%s/d%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(thumbPath, true), + sign.Sign(thumbPath)) + result = append(result, &model.ObjThumb{ + Object: objRes, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + }) } - return &model.ObjThumb{ - Object: objRes, - Thumbnail: model.Thumbnail{ - Thumbnail: thumb, - }, - }, nil + continue } } + if !d.ShowHidden && strings.HasPrefix(rawName, ".") { + continue + } thumb, ok := model.GetThumb(obj) objRes := model.Object{ Name: rawName, @@ -226,15 +229,17 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ HashInfo: obj.GetHash(), } if !ok { - return &objRes, nil + result = append(result, &objRes) + } else { + result = append(result, &model.ObjThumb{ + Object: objRes, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + }) } - return &model.ObjThumb{ - Object: objRes, - Thumbnail: model.Thumbnail{ - Thumbnail: thumb, - }, - }, nil - }) + } + return result, nil } func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { diff --git a/drivers/chunk/meta.go b/drivers/chunk/meta.go index 9f3c5800f..45429231f 100644 --- a/drivers/chunk/meta.go +++ b/drivers/chunk/meta.go @@ -10,6 +10,9 @@ type Addition struct { PartSize int64 `json:"part_size" required:"true" type:"number" help:"bytes"` CustomExt string `json:"custom_ext" type:"string"` StoreHash bool `json:"store_hash" type:"bool" default:"true"` + + Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ From 275c844ef5d23607f1f9c0b37c6a87c759263691 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 13 Sep 2025 15:49:52 +0800 Subject: [PATCH 21/25] =?UTF-8?q?=E5=9C=A8Put=E6=96=B9=E6=B3=95=E4=B8=AD?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9=E7=BC=A9=E7=95=A5=E5=9B=BE=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index a511d953e..03f6fd99a 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -396,6 +396,9 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream if err != nil { return err } + if d.Thumbnail && dstDir.GetName() == ".thumbnails" { + return op.Put(ctx, remoteStorage, stdpath.Join(remoteActualPath, dstDir.GetPath()), file, up) + } upReader := &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, From 99cd410c7093f4864c4bdc91699a62befceb1928 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 13 Sep 2025 15:56:20 +0800 Subject: [PATCH 22/25] =?UTF-8?q?fix(chunk):=20=E4=BC=98=E5=8C=96List?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E4=B8=AD=E7=BB=93=E6=9E=9C=E5=88=87=E7=89=87?= =?UTF-8?q?=E7=9A=84=E5=88=9D=E5=A7=8B=E5=8C=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 03f6fd99a..bb0271cf9 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -151,7 +151,7 @@ func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if err != nil { return nil, err } - result := make([]model.Obj, len(remoteObjs)) + result := make([]model.Obj, 0, len(remoteObjs)) for _, obj := range remoteObjs { rawName := obj.GetName() if obj.IsDir() { From 9c6dd2b302bf3a9853fda6bd72f1f1142b332ac2 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 13 Sep 2025 20:31:46 +0800 Subject: [PATCH 23/25] =?UTF-8?q?fix(chunk):=20=E6=9B=B4=E6=96=B0Get?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E4=B8=AD=E7=9A=84=E6=B3=A8=E9=87=8A=E4=BB=A5?= =?UTF-8?q?=E5=A2=9E=E5=BC=BA=E4=BB=A3=E7=A0=81=E5=8F=AF=E8=AF=BB=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index bb0271cf9..5a636da7c 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -77,6 +77,7 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { return nil, err } var totalSize int64 = 0 + // 0号块必须存在 chunkSizes := []int64{-1} h := make(map[*utils.HashType]string) var first model.Obj @@ -113,8 +114,15 @@ func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { chunkSizes[idx] = o.GetSize() } } + // 检查0号块不等于-1 以支持空文件 + // 如果块数量大于1 最后一块不可能为0 + // 只检查中间块是否有0 for i, l := 0, len(chunkSizes)-2; ; i++ { - if (i == 0 && chunkSizes[i] == -1) || chunkSizes[i] == 0 { + if i == 0 { + if chunkSizes[i] == -1 { + return nil, fmt.Errorf("chunk part[%d] are missing", i) + } + } else if chunkSizes[i] == 0 { return nil, fmt.Errorf("chunk part[%d] are missing", i) } if i >= l { From 957d2cf7bacbfbdc80035673ed275389782d63b2 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 15 Sep 2025 16:56:36 +0800 Subject: [PATCH 24/25] =?UTF-8?q?fix(chunk):=20=E4=BC=98=E5=8C=96Rename?= =?UTF-8?q?=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/chunk/driver.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 5a636da7c..b0a486379 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -381,11 +381,10 @@ func (d *Chunk) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Chunk) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - chunkObj, ok := srcObj.(*chunkObject) - if !ok { - return fs.Rename(ctx, stdpath.Join(d.RemotePath, srcObj.GetPath()), newName) + if _, ok := srcObj.(*chunkObject); ok { + newName = "[openlist_chunk]" + newName } - return fs.Rename(ctx, stdpath.Join(d.RemotePath, chunkObj.GetPath()), "[openlist_chunk]"+newName) + return fs.Rename(ctx, stdpath.Join(d.RemotePath, srcObj.GetPath()), newName) } func (d *Chunk) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { From 05c5bd4d87e142bc78e9d5c71c326141bc02ae5e Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Fri, 19 Sep 2025 14:33:14 +0800 Subject: [PATCH 25/25] feat: autoremove unsuccessfully uploaded file-parts & add disk usage --- drivers/chunk/driver.go | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index b0a486379..763469740 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -432,7 +433,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream } partIndex := 0 for partIndex < fullPartCount { - err = errors.Join(err, op.Put(ctx, remoteStorage, dst, &stream.FileStream{ + err = op.Put(ctx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(partIndex), Size: d.PartSize, @@ -440,10 +441,14 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, Mimetype: file.GetMimetype(), Reader: io.LimitReader(upReader, d.PartSize), - }, nil, true)) + }, nil, true) + if err != nil { + _ = op.Remove(ctx, remoteStorage, dst) + return err + } partIndex++ } - return errors.Join(err, op.Put(ctx, remoteStorage, dst, &stream.FileStream{ + err = op.Put(ctx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(fullPartCount), Size: tailSize, @@ -451,11 +456,33 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, Mimetype: file.GetMimetype(), Reader: upReader, - }, nil)) + }, nil) + if err != nil { + _ = op.Remove(ctx, remoteStorage, dst) + } + return err } func (d *Chunk) getPartName(part int) string { return fmt.Sprintf("%d%s", part, d.CustomExt) } +func (d *Chunk) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + remoteStorage, err := fs.GetStorage(d.RemotePath, &fs.GetStoragesArgs{}) + if err != nil { + return nil, errs.NotImplement + } + wd, ok := remoteStorage.(driver.WithDetails) + if !ok { + return nil, errs.NotImplement + } + remoteDetails, err := wd.GetDetails(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: remoteDetails.DiskUsage, + }, nil +} + var _ driver.Driver = (*Chunk)(nil)