From d9de036001cb2db6901acd114a05d7796bc2a1d1 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sun, 7 Dec 2025 18:34:13 +0800 Subject: [PATCH 01/30] feat(alias): support load balance --- drivers/alias/driver.go | 380 +++++++++++++++------------------------- drivers/alias/meta.go | 12 +- drivers/alias/types.go | 43 +++++ drivers/alias/util.go | 237 ++++++++++++++++++++++--- 4 files changed, 403 insertions(+), 269 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 3b079a15a..958084ec1 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math/rand" "net/url" stdpath "path" "strings" @@ -83,45 +84,55 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { if !ok { return nil, errs.ObjectNotFound } - var ret *model.Object - provider := "" + var objs []model.Obj for _, dst := range dsts { - rawPath := stdpath.Join(dst, sub) - obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) + p := stdpath.Join(dst, sub) + obj, err := fs.Get(ctx, p, &fs.GetArgs{NoLog: true}) if err != nil { continue } - storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}) - if ret == nil { - ret = &model.Object{ - Path: path, - Name: obj.GetName(), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - HashInfo: obj.GetHash(), + object := model.Object{ + Path: path, + Name: obj.GetName(), + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + } + if !obj.IsDir() { + if d.ProviderPassThrough { + storage, e := fs.GetStorage(p, &fs.GetStoragesArgs{}) + provider := "" + if e == nil { + provider = storage.Config().Name + } + obj = &model.ObjectProvider{ + Object: object, + Provider: model.Provider{ + Provider: provider, + }, + } + } else { + obj = &object } - if !d.ProviderPassThrough || err != nil { - break + obj = &BalancedObj{ + Obj: obj, + ExactReqPath: p, } - provider = storage.Config().Name - } else if err != nil || provider != storage.GetStorage().Driver { - provider = "" - break + } + if d.ReadConflictPolicy == FirstRWP { + return obj, nil + } else { + objs = append(objs, obj) } } - if ret == nil { + if len(objs) == 0 { return nil, errs.ObjectNotFound } - if provider != "" { - return &model.ObjectProvider{ - Object: *ret, - Provider: model.Provider{ - Provider: provider, - }, - }, nil + if d.ReadConflictPolicy == RandomBalancedRP { + return objs[rand.Intn(len(objs))], nil } - return ret, nil + return nil, ErrInvalidConflictPolicy } func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { @@ -175,195 +186,126 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - root, sub := d.getRootAndPath(file.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := GetExactReqPath(file) + if reqPath == "" { + return nil, errs.NotFile } // proxy || ftp,s3 if common.GetApiUrl(ctx) == "" { args.Redirect = false } - for _, dst := range dsts { - reqPath := stdpath.Join(dst, sub) - link, fi, err := d.link(ctx, reqPath, args) - if err != nil { - continue - } - if link == nil { - // 重定向且需要通过代理 - return &model.Link{ - URL: fmt.Sprintf("%s/p%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)), - }, nil - } - - resultLink := *link - resultLink.SyncClosers = utils.NewSyncClosers(link) - if args.Redirect { - return &resultLink, nil - } + link, fi, err := d.link(ctx, reqPath, args) + if err != nil { + return nil, err + } + if link == nil { + // 重定向且需要通过代理 + return &model.Link{ + URL: fmt.Sprintf("%s/p%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(reqPath, true), + sign.Sign(reqPath)), + }, nil + } - if resultLink.ContentLength == 0 { - resultLink.ContentLength = fi.GetSize() - } - if d.DownloadConcurrency > 0 { - resultLink.Concurrency = d.DownloadConcurrency - } - if d.DownloadPartSize > 0 { - resultLink.PartSize = d.DownloadPartSize * utils.KB - } + resultLink := *link + resultLink.SyncClosers = utils.NewSyncClosers(link) + if args.Redirect { return &resultLink, nil } - return nil, errs.ObjectNotFound + + if resultLink.ContentLength == 0 { + resultLink.ContentLength = fi.GetSize() + } + if d.DownloadConcurrency > 0 { + resultLink.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + resultLink.PartSize = d.DownloadPartSize * utils.KB + } + return &resultLink, nil } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { - root, sub := d.getRootAndPath(args.Obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := GetExactReqPath(args.Obj) + if reqPath == "" { + return nil, errs.NotImplement } - for _, dst := range dsts { - rawPath := stdpath.Join(dst, sub) - storage, actualPath, err := op.GetStorageAndActualPath(rawPath) - if err != nil { - continue - } - return op.Other(ctx, storage, model.FsOtherArgs{ - Path: actualPath, - Method: args.Method, - Data: args.Data, - }) + storage, actualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + return nil, err } - return nil, errs.NotImplement + return op.Other(ctx, storage, model.FsOtherArgs{ + Path: actualPath, + Method: args.Method, + Data: args.Data, + }) } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, parentDir, true) + reqPath, err := d.getWritePath(ctx, parentDir, true) if err == nil { for _, path := range reqPath { - err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(*path, dirName))) + err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(path, dirName))) } return err } - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot make sub-dir") - } return err } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be moved") - } + srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) if err != nil { return err } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be moved to") - } - if err != nil { - return err - } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.Move(ctx, *srcPath[i], *dstPath[i]) - err = errors.Join(err, e) - } - return err - } else { - return errors.New("parallel paths mismatch") + for i, src := range srcs { + dst := dsts[i] + _, e := fs.Move(ctx, src, dst) + err = errors.Join(err, e) } + return err } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, srcObj, false) + reqPath, err := d.getWritePath(ctx, srcObj, false) if err == nil { for _, path := range reqPath { - err = errors.Join(err, fs.Rename(ctx, *path, newName)) + err = errors.Join(err, fs.Rename(ctx, path, newName)) } return err } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be Rename") - } return err } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be copied") - } + srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) if err != nil { return err } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be copied to") - } - if err != nil { - return err - } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.Copy(ctx, *srcPath[i], *dstPath[i]) - err = errors.Join(err, e) - } - return err - } else if len(srcPath) == 1 || !d.ProtectSameName { - for _, path := range dstPath { - _, e := fs.Copy(ctx, *srcPath[0], *path) - err = errors.Join(err, e) - } - return err - } else { - return errors.New("parallel paths mismatch") + for i, src := range srcs { + dst := dsts[i] + _, e := fs.Copy(ctx, src, dst) + err = errors.Join(err, e) } + return err } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, obj, false) + reqPath, err := d.getWritePath(ctx, obj, false) if err == nil { for _, path := range reqPath { - err = errors.Join(err, fs.Remove(ctx, *path)) + err = errors.Join(err, fs.Remove(ctx, path)) } return err } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be Delete") - } return err } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, dstDir, true) + reqPath, err := d.getPutPath(ctx, dstDir) if err == nil { if len(reqPath) == 1 { - storage, reqActualPath, err := op.GetStorageAndActualPath(*reqPath[0]) + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath[0]) if err != nil { return err } @@ -380,7 +322,7 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, count := float64(len(reqPath) + 1) up(100 / count) for i, path := range reqPath { - err = errors.Join(err, fs.PutDirectly(ctx, *path, &stream.FileStream{ + err = errors.Join(err, fs.PutDirectly(ctx, path, &stream.FileStream{ Obj: s, Mimetype: s.GetMimetype(), Reader: file, @@ -394,55 +336,40 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, return err } } - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be Put") - } return err } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, dstDir, true) + reqPath, err := d.getPutPath(ctx, dstDir) if err == nil { for _, path := range reqPath { - err = errors.Join(err, fs.PutURL(ctx, *path, name, url)) + err = errors.Join(err, fs.PutURL(ctx, path, name, url)) } return err } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot offline download") - } return err } func (d *Alias) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := GetExactReqPath(obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - meta, err := d.getArchiveMeta(ctx, dst, sub, args) - if err == nil { - return meta, nil - } + meta, err := d.getArchiveMeta(ctx, reqPath, args) + if err == nil { + return meta, nil } return nil, errs.NotImplement } func (d *Alias) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := GetExactReqPath(obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - l, err := d.listArchive(ctx, dst, sub, args) - if err == nil { - return l, nil - } + l, err := d.listArchive(ctx, reqPath, args) + if err == nil { + return l, nil } return nil, errs.NotImplement } @@ -451,67 +378,40 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn // alias的两个驱动，一个支持驱动提取，一个不支持，如何兼容？ // 如果访问的是不支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回errs.NotImplement，提取URL前缀就会是/ae，Extract就不会被调用 // 如果访问的是支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回有效值，提取URL前缀就会是/ad，Extract就会被调用 - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := GetExactReqPath(obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - reqPath := stdpath.Join(dst, sub) - link, err := d.extract(ctx, reqPath, args) - if err != nil { - continue - } - if link == nil { - return &model.Link{ - URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - utils.EncodePath(args.InnerPath, true), - url.QueryEscape(args.Password), - sign.SignArchive(reqPath)), - }, nil - } - resultLink := *link - resultLink.SyncClosers = utils.NewSyncClosers(link) - return &resultLink, nil + link, err := d.extract(ctx, reqPath, args) + if err != nil { + return nil, errs.NotImplement + } + if link == nil { + return &model.Link{ + URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(reqPath, true), + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + sign.SignArchive(reqPath)), + }, nil } - return nil, errs.NotImplement + resultLink := *link + resultLink.SyncClosers = utils.NewSyncClosers(link) + return &resultLink, nil } func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { - if !d.Writable { - return errs.PermissionDenied - } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be decompressed") - } + srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) if err != nil { return err } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be decompressed to") - } - if err != nil { - return err - } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.ArchiveDecompress(ctx, *srcPath[i], *dstPath[i], args) - err = errors.Join(err, e) - } - return err - } else if len(srcPath) == 1 || !d.ProtectSameName { - for _, path := range dstPath { - _, e := fs.ArchiveDecompress(ctx, *srcPath[0], *path, args) - err = errors.Join(err, e) - } - return err - } else { - return errors.New("parallel paths mismatch") + for i, src := range srcs { + dst := dsts[i] + _, e := fs.ArchiveDecompress(ctx, src, dst, args) + err = errors.Join(err, e) } + return err } func (d *Alias) ResolveLinkCacheMode(path string) driver.LinkCacheMode { diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 763e66472..8c6f4cf34 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -10,11 +10,11 @@ type Addition struct { // driver.RootPath // define other Paths string `json:"paths" required:"true" type:"text"` - ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"` - ParallelWrite bool `json:"parallel_write" type:"bool" default:"false"` + ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random" default:"random"` + WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled"` + PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled"` DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` - Writable bool `json:"writable" type:"bool" default:"false"` ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` } @@ -31,10 +31,6 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { - return &Alias{ - Addition: Addition{ - ProtectSameName: true, - }, - } + return &Alias{} }) } diff --git a/drivers/alias/types.go b/drivers/alias/types.go index e560393da..1b588bfbd 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -1 +1,44 @@ package alias + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/pkg/errors" +) + +const ( + DisabledWP = "disabled" + FirstRWP = "first" + DeterministicWP = "deterministic" + DeterministicOrAllWP = "deterministic_or_all" + AllWP = "all" + AllStrictWP = "all_strict" + RandomBalancedRP = "random" + BalancedByQuotaP = "quota" + BalancedByQuotaStrictP = "quota_strict" +) + +var ( + ErrPathConflict = errors.New("path conflict") + ErrSamePathLeak = errors.New("leak some of same-name dirs") + ErrInvalidConflictPolicy = errors.New("invalid conflict policy") + ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") +) + +type BalancedObj struct { + model.Obj + ExactReqPath string +} + +func (b *BalancedObj) Unwrap() model.Obj { + return b.Obj +} + +func GetExactReqPath(obj model.Obj) string { + if b, ok := obj.(*BalancedObj); ok { + return b.ExactReqPath + } + if unwrap, ok := obj.(model.ObjUnwrap); ok { + return GetExactReqPath(unwrap.Unwrap()) + } + return "" +} diff --git a/drivers/alias/util.go b/drivers/alias/util.go index e47695fa8..87ef36230 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -2,7 +2,7 @@ package alias import ( "context" - "errors" + "math/rand" stdpath "path" "strings" "time" @@ -12,7 +12,9 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -118,46 +120,240 @@ func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) ( return op.Link(ctx, storage, reqActualPath, args) } -func (d *Alias) getReqPath(ctx context.Context, obj model.Obj, isParent bool) ([]*string, error) { - root, sub := d.getRootAndPath(obj.GetPath()) +func (d *Alias) getAllReqPath(ctx context.Context, objPath string, isParent bool, ifContinue func(err error) (bool, error)) ([]string, error) { + root, sub := d.getRootAndPath(objPath) if sub == "" && !isParent { return nil, errs.NotSupport } dsts, ok := d.pathMap[root] - all := true if !ok { return nil, errs.ObjectNotFound } - var reqPath []*string + var reqPath []string for _, dst := range dsts { path := stdpath.Join(dst, sub) _, err := fs.Get(ctx, path, &fs.GetArgs{NoLog: true}) + cont, err := ifContinue(err) + if !cont { + if err == nil { + return []string{path}, nil + } else { + return nil, err + } + } + reqPath = append(reqPath, path) + } + if len(reqPath) == 0 { + return nil, errs.ObjectNotFound + } + return reqPath, nil +} + +func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { + if policy == AllWP { + return func(_ error) (bool, error) { + return true, nil + } + } + all := true + l := 0 + return func(err error) (bool, error) { if err != nil { + switch policy { + case AllStrictWP: + return false, ErrSamePathLeak + case DeterministicOrAllWP: + if l >= 2 { + return false, ErrSamePathLeak + } + } all = false - if d.ProtectSameName && d.ParallelWrite && len(reqPath) >= 2 { - return nil, errs.NotImplement + } else { + switch policy { + case FirstRWP: + return false, nil + case DeterministicWP: + if l > 0 { + return false, ErrPathConflict + } + case DeterministicOrAllWP: + if l > 0 && !all { + return false, ErrSamePathLeak + } } + l += 1 + } + return true, nil + } +} + +func (d *Alias) getWritePath(ctx context.Context, obj model.Obj, isParent bool) ([]string, error) { + if d.WriteConflictPolicy == DisabledWP { + return nil, errs.PermissionDenied + } + reqPath, err := d.getAllReqPath(ctx, obj.GetPath(), isParent, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) + if err != nil { + return nil, err + } + return reqPath, nil +} + +func (d *Alias) getPutPath(ctx context.Context, obj model.Obj) ([]string, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, errs.PermissionDenied + } + reqPath, err := d.getAllReqPath(ctx, obj.GetPath(), true, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, err + } + if d.PutConflictPolicy == RandomBalancedRP { + ri := rand.Intn(len(reqPath)) + return []string{reqPath[ri]}, nil + } else if d.PutConflictPolicy == BalancedByQuotaP || d.PutConflictPolicy == BalancedByQuotaStrictP { + r, ok := getRandomPathByQuotaBalanced(ctx, reqPath, d.PutConflictPolicy == BalancedByQuotaStrictP, uint64(obj.GetSize())) + if !ok { + return nil, ErrNoEnoughSpace + } + return []string{r}, nil + } else { + return reqPath, nil + } +} + +func getRandomPathByQuotaBalanced(ctx context.Context, reqPath []string, strict bool, objSize uint64) (string, bool) { + // Get all space + details := make([]*model.StorageDetails, len(reqPath)) + detailsChan := make(chan detailWithIndex, len(reqPath)) + workerCount := 0 + for i, p := range reqPath { + s, err := fs.GetStorage(p, &fs.GetStoragesArgs{}) + if err != nil { continue } - if !d.ProtectSameName && !d.ParallelWrite { - return []*string{&path}, nil + if _, ok := s.(driver.WithDetails); !ok { + continue + } + workerCount++ + go func(dri driver.Driver, i int) { + d, e := op.GetStorageDetails(ctx, dri) + if e != nil { + if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { + log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e) + } + } + detailsChan <- detailWithIndex{idx: i, val: d} + }(s, i) + } + for workerCount > 0 { + select { + case r := <-detailsChan: + details[r.idx] = r.val + workerCount-- + case <-time.After(time.Second): + workerCount = 0 + } + } + + // Try select one that has space info + selected, ok := selectRandom(details, func(d *model.StorageDetails) uint64 { + if d == nil || d.FreeSpace < objSize { + return 0 + } + return d.FreeSpace + }) + if !ok { + if strict { + return "", false + } else { + // No strict mode, return any of non-details ones + noDetails := make([]int, 0, len(details)) + for i, d := range details { + if d == nil { + noDetails = append(noDetails, i) + } + } + if len(noDetails) == 0 { + return "", false + } + return reqPath[noDetails[rand.Intn(len(noDetails))]], true + } + } + return reqPath[selected], true +} + +func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) { + var totalWeight uint64 = 0 + for _, i := range arr { + totalWeight += getWeight(i) + } + if totalWeight == 0 { + return 0, false + } + r := rand.Uint64() % totalWeight + for i, item := range arr { + w := getWeight(item) + if r < w { + return i, true + } + r -= w + } + return 0, false +} + +func (d *Alias) getCopyMovePath(ctx context.Context, srcObj, dstDir model.Obj) ([]string, []string, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, nil, errs.PermissionDenied + } + if utils.SliceContains([]string{RandomBalancedRP, BalancedByQuotaP, BalancedByQuotaStrictP}, d.PutConflictPolicy) { + return nil, nil, errs.NotImplement + } + dstPath, err := d.getAllReqPath(ctx, dstDir.GetPath(), true, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, nil, err + } + dstStorageMap := make(map[string][]string) + allocatingDst := make(map[string]struct{}) + for _, dp := range dstPath { + storage, e := fs.GetStorage(dp, &fs.GetStoragesArgs{}) + if e != nil { + return nil, nil, errors.WithMessagef(e, "cannot copy or move to virtual path [%s]", dp) } - reqPath = append(reqPath, &path) - if d.ProtectSameName && !d.ParallelWrite && len(reqPath) >= 2 { - return nil, errs.NotImplement + mp := storage.GetStorage().MountPath + dstStorageMap[mp] = append(dstStorageMap[mp], dp) + allocatingDst[dp] = struct{}{} + } + srcPath, err := d.getAllReqPath(ctx, srcObj.GetPath(), false, getWriteAndPutFilterFunc(AllWP)) + if err != nil { + return nil, nil, err + } + srcs := make([]string, 0) + dsts := make([]string, 0) + for _, sp := range srcPath { + storage, e := fs.GetStorage(sp, &fs.GetStoragesArgs{}) + if e != nil { + continue } - if d.ProtectSameName && d.ParallelWrite && len(reqPath) >= 2 && !all { - return nil, errs.NotImplement + if dstPaths, ok := dstStorageMap[storage.GetStorage().MountPath]; ok { + for _, dp := range dstPaths { + srcs = append(srcs, sp) + dsts = append(dsts, dp) + delete(allocatingDst, dp) + } + delete(dstStorageMap, storage.GetStorage().MountPath) } } - if len(reqPath) == 0 { - return nil, errs.ObjectNotFound + for dp := range allocatingDst { + sp := srcs[0] + if d.ReadConflictPolicy == RandomBalancedRP { + sp = srcs[rand.Intn(len(srcs))] + } + srcs = append(srcs, sp) + dsts = append(dsts, dp) } - return reqPath, nil + return srcs, dsts, nil } -func (d *Alias) getArchiveMeta(ctx context.Context, dst, sub string, args model.ArchiveArgs) (model.ArchiveMeta, error) { - reqPath := stdpath.Join(dst, sub) +func (d *Alias) getArchiveMeta(ctx context.Context, reqPath string, args model.ArchiveArgs) (model.ArchiveMeta, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err @@ -171,8 +367,7 @@ func (d *Alias) getArchiveMeta(ctx context.Context, dst, sub string, args model. return nil, errs.NotImplement } -func (d *Alias) listArchive(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) ([]model.Obj, error) { - reqPath := stdpath.Join(dst, sub) +func (d *Alias) listArchive(ctx context.Context, reqPath string, args model.ArchiveInnerArgs) ([]model.Obj, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err From 08ef98ae9ebd1e13227d8c4db2ced05e8c5a4549 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sun, 7 Dec 2025 19:53:56 +0800 Subject: [PATCH 02/30] feat(alias): support storage match for load balance --- drivers/alias/util.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 87ef36230..33dbbc942 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -12,7 +12,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -304,9 +303,6 @@ func (d *Alias) getCopyMovePath(ctx context.Context, srcObj, dstDir model.Obj) ( if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } - if utils.SliceContains([]string{RandomBalancedRP, BalancedByQuotaP, BalancedByQuotaStrictP}, d.PutConflictPolicy) { - return nil, nil, errs.NotImplement - } dstPath, err := d.getAllReqPath(ctx, dstDir.GetPath(), true, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, nil, err From 5cefee58d38d16dbd8a8d43d46219b308d10766f Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 8 Dec 2025 15:25:42 +0800 Subject: [PATCH 03/30] feat(patch): add alias addition upgrade patch --- internal/bootstrap/patch/all.go | 7 ++ internal/bootstrap/patch/v4_1_8/alias.go | 81 ++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 internal/bootstrap/patch/v4_1_8/alias.go diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index 44ba17376..0d0098597 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -5,6 +5,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_32_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_41_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_all" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v4_1_8" ) type VersionPatches struct { @@ -39,4 +40,10 @@ var UpgradePatches = []VersionPatches{ v3_all.RenameAlistV3Driver, }, }, + { + Version: "v4.1.8", + Patches: []func(){ + v4_1_8.FixAliasConfig, + }, + }, } diff --git a/internal/bootstrap/patch/v4_1_8/alias.go b/internal/bootstrap/patch/v4_1_8/alias.go new file mode 100644 index 000000000..7fedd1ef4 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_8/alias.go @@ -0,0 +1,81 @@ +package v4_1_8 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// FixAliasConfig upgrade the old version of the Addition of the Alias driver +func FixAliasConfig() { + storages, _, err := db.GetStorages(1, -1) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to get storages: %s", err.Error()) + return + } + for _, s := range storages { + if s.Driver != "Alias" { + continue + } + addition := make(map[string]any) + err = utils.Json.UnmarshalFromString(s.Addition, addition) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to unmarshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) + continue + } + if _, ok := addition["read_conflict_policy"]; ok { + utils.Log.Infof("[FixAliasConfig] skip fixing [%d]%s because the addition already has \"read_conflict_policy\" key", s.ID, s.MountPath) + continue + } + var protectSameName, parallelWrite, writable bool + protectSameNameAny, ok := addition["protect_same_name"] + if ok { + delete(addition, "protect_same_name") + protectSameName, ok = protectSameNameAny.(bool) + } + if !ok { + protectSameName = false + } + parallelWriteAny, ok := addition["parallel_write"] + if ok { + delete(addition, "parallel_write") + parallelWrite, ok = parallelWriteAny.(bool) + } + if !ok { + parallelWrite = false + } + writableAny, ok := addition["writable"] + if ok { + delete(addition, "writable") + writable, ok = writableAny.(bool) + } + if !ok { + writable = false + } + if !writable { + addition["write_conflict_policy"] = "disabled" + addition["put_conflict_policy"] = "disabled" + } else if !protectSameName && !parallelWrite { + addition["write_conflict_policy"] = "first" + addition["put_conflict_policy"] = "first" + } else if protectSameName && !parallelWrite { + addition["write_conflict_policy"] = "deterministic" + addition["put_conflict_policy"] = "deterministic" + } else if !protectSameName && parallelWrite { + addition["write_conflict_policy"] = "all" + addition["put_conflict_policy"] = "all" + } else { + addition["write_conflict_policy"] = "deterministic_or_all" + addition["put_conflict_policy"] = "deterministic_or_all" + } + addition["read_conflict_policy"] = "first" + s.Addition, err = utils.Json.MarshalToString(addition) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to marshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) + continue + } + err = db.UpdateStorage(&s) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) + } + } +} From a08c1d65c7fcd5f97474d931281e6269adbfbce1 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 8 Dec 2025 16:43:43 +0800 Subject: [PATCH 04/30] fix bugs --- drivers/alias/driver.go | 56 ++++++++++++++++++------ drivers/alias/types.go | 15 +++++-- drivers/alias/util.go | 12 ++--- internal/bootstrap/patch.go | 1 - internal/bootstrap/patch/v4_1_8/alias.go | 2 +- 5 files changed, 61 insertions(+), 25 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 958084ec1..eb632484a 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -42,6 +42,15 @@ func (d *Alias) Init(ctx context.Context) error { if d.Paths == "" { return errors.New("paths is required") } + if !utils.SliceContains(ValidReadConflictPolicy, d.ReadConflictPolicy) { + d.ReadConflictPolicy = FirstRWP + } + if !utils.SliceContains(ValidWriteConflictPolicy, d.WriteConflictPolicy) { + d.WriteConflictPolicy = DisabledWP + } + if !utils.SliceContains(ValidPutConflictPolicy, d.PutConflictPolicy) { + d.PutConflictPolicy = DisabledWP + } paths := strings.Split(d.Paths, "\n") d.rootOrder = make([]string, 0, len(paths)) d.pathMap = make(map[string][]string) @@ -119,6 +128,8 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { Obj: obj, ExactReqPath: p, } + } else { + obj = &object } if d.ReadConflictPolicy == FirstRWP { return obj, nil @@ -129,10 +140,7 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { if len(objs) == 0 { return nil, errs.ObjectNotFound } - if d.ReadConflictPolicy == RandomBalancedRP { - return objs[rand.Intn(len(objs))], nil - } - return nil, ErrInvalidConflictPolicy + return objs[rand.Intn(len(objs))], nil } func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { @@ -145,9 +153,10 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if !ok { return nil, errs.ObjectNotFound } - var objs []model.Obj + objs := make(map[string][]model.Obj) for _, dst := range dsts { - tmp, err := fs.List(ctx, stdpath.Join(dst, sub), &fs.ListArgs{ + exactPath := stdpath.Join(dst, sub) + tmp, err := fs.List(ctx, exactPath, &fs.ListArgs{ NoLog: true, Refresh: args.Refresh, WithStorageDetails: args.WithStorageDetails && d.DetailsPassThrough, @@ -161,28 +170,47 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ Modified: obj.ModTime(), IsFolder: obj.IsDir(), } + var objRet model.Obj if thumb, ok := model.GetThumb(obj); ok { - return &model.ObjThumb{ + objRet = &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, - }, nil + } + } else { + objRet = &objRes } if details, ok := model.GetStorageDetails(obj); ok { - return &model.ObjStorageDetails{ - Obj: &objRes, + objRet = &model.ObjStorageDetails{ + Obj: objRet, StorageDetailsWithName: *details, - }, nil + } } - return &objRes, nil + if !objRet.IsDir() { + objRet = &BalancedObj{ + Obj: objRet, + ExactReqPath: stdpath.Join(exactPath, objRet.GetName()), + } + } + return objRet, nil }) } if err == nil { - objs = append(objs, tmp...) + for _, o := range tmp { + objs[o.GetName()] = append(objs[o.GetName()], o) + } + } + } + ret := make([]model.Obj, 0, len(objs)) + for _, snObjs := range objs { + if d.ReadConflictPolicy == RandomBalancedRP { + ret = append(ret, snObjs[rand.Intn(len(snObjs))]) + } else { + ret = append(ret, snObjs[0]) } } - return objs, nil + return ret, nil } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { diff --git a/drivers/alias/types.go b/drivers/alias/types.go index 1b588bfbd..fdca777ea 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -18,10 +18,17 @@ const ( ) var ( - ErrPathConflict = errors.New("path conflict") - ErrSamePathLeak = errors.New("leak some of same-name dirs") - ErrInvalidConflictPolicy = errors.New("invalid conflict policy") - ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") + ValidReadConflictPolicy = []string{FirstRWP, RandomBalancedRP} + ValidWriteConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllWP, + AllStrictWP} + ValidPutConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllWP, + AllStrictWP, RandomBalancedRP, BalancedByQuotaP, BalancedByQuotaStrictP} +) + +var ( + ErrPathConflict = errors.New("path conflict") + ErrSamePathLeak = errors.New("leak some of same-name dirs") + ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") ) type BalancedObj struct { diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 33dbbc942..20744a283 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -140,7 +140,9 @@ func (d *Alias) getAllReqPath(ctx context.Context, objPath string, isParent bool return nil, err } } - reqPath = append(reqPath, path) + if err == nil { + reqPath = append(reqPath, path) + } } if len(reqPath) == 0 { return nil, errs.ObjectNotFound @@ -150,8 +152,8 @@ func (d *Alias) getAllReqPath(ctx context.Context, objPath string, isParent bool func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { if policy == AllWP { - return func(_ error) (bool, error) { - return true, nil + return func(err error) (bool, error) { + return true, err } } all := true @@ -170,7 +172,7 @@ func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { } else { switch policy { case FirstRWP: - return false, nil + return true, nil case DeterministicWP: if l > 0 { return false, ErrPathConflict @@ -182,7 +184,7 @@ func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { } l += 1 } - return true, nil + return true, err } } diff --git a/internal/bootstrap/patch.go b/internal/bootstrap/patch.go index 1e76190c8..e8baeeac9 100644 --- a/internal/bootstrap/patch.go +++ b/internal/bootstrap/patch.go @@ -2,7 +2,6 @@ package bootstrap import ( "fmt" - "strings" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch" diff --git a/internal/bootstrap/patch/v4_1_8/alias.go b/internal/bootstrap/patch/v4_1_8/alias.go index 7fedd1ef4..e41742654 100644 --- a/internal/bootstrap/patch/v4_1_8/alias.go +++ b/internal/bootstrap/patch/v4_1_8/alias.go @@ -17,7 +17,7 @@ func FixAliasConfig() { continue } addition := make(map[string]any) - err = utils.Json.UnmarshalFromString(s.Addition, addition) + err = utils.Json.UnmarshalFromString(s.Addition, &addition) if err != nil { utils.Log.Errorf("[FixAliasConfig] failed to unmarshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) continue From 9d453ed2767c47d3886dfef1fa8e8fb2c091222d Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 8 Dec 2025 17:33:06 +0800 Subject: [PATCH 05/30] fix(op/balance): optimize compatibility --- drivers/alias/util.go | 8 +++++--- drivers/strm/hook.go | 2 +- internal/op/cache.go | 9 +++++---- internal/op/fs.go | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 20744a283..f6fd4a3ca 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -12,6 +12,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -316,7 +317,7 @@ func (d *Alias) getCopyMovePath(ctx context.Context, srcObj, dstDir model.Obj) ( if e != nil { return nil, nil, errors.WithMessagef(e, "cannot copy or move to virtual path [%s]", dp) } - mp := storage.GetStorage().MountPath + mp := utils.GetActualMountPath(storage.GetStorage().MountPath) dstStorageMap[mp] = append(dstStorageMap[mp], dp) allocatingDst[dp] = struct{}{} } @@ -331,13 +332,14 @@ func (d *Alias) getCopyMovePath(ctx context.Context, srcObj, dstDir model.Obj) ( if e != nil { continue } - if dstPaths, ok := dstStorageMap[storage.GetStorage().MountPath]; ok { + mp := utils.GetActualMountPath(storage.GetStorage().MountPath) + if dstPaths, ok := dstStorageMap[mp]; ok { for _, dp := range dstPaths { srcs = append(srcs, sp) dsts = append(dsts, dp) delete(allocatingDst, dp) } - delete(dstStorageMap, storage.GetStorage().MountPath) + delete(dstStorageMap, mp) } } for dp := range allocatingDst { diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go index 392bb014d..e07b337e3 100644 --- a/drivers/strm/hook.go +++ b/drivers/strm/hook.go @@ -21,7 +21,7 @@ var strmTrie = patricia.NewTrie() func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) { path = utils.FixAndCleanPath(path) updateLocal := func(driver *Strm, basePath string, objs []model.Obj) { - relParent := strings.TrimPrefix(basePath, driver.MountPath) + relParent := strings.TrimPrefix(basePath, utils.GetActualMountPath(driver.MountPath)) localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent) for _, obj := range objs { localPath := stdpath.Join(localParentPath, obj.GetName()) diff --git a/internal/op/cache.go b/internal/op/cache.go index 52e430b1e..54c285965 100644 --- a/internal/op/cache.go +++ b/internal/op/cache.go @@ -8,6 +8,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/cache" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type CacheManager struct { @@ -32,7 +33,7 @@ func NewCacheManager() *CacheManager { var Cache = NewCacheManager() func Key(storage driver.Driver, path string) string { - return stdpath.Join(storage.GetStorage().MountPath, path) + return stdpath.Join(utils.GetActualMountPath(storage.GetStorage().MountPath), path) } // update object in dirCache. @@ -162,15 +163,15 @@ func (cm *CacheManager) SetStorageDetails(storage driver.Driver, details *model. return } expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) - cm.detailCache.SetWithTTL(storage.GetStorage().MountPath, details, expiration) + cm.detailCache.SetWithTTL(utils.GetActualMountPath(storage.GetStorage().MountPath), details, expiration) } func (cm *CacheManager) GetStorageDetails(storage driver.Driver) (*model.StorageDetails, bool) { - return cm.detailCache.Get(storage.GetStorage().MountPath) + return cm.detailCache.Get(utils.GetActualMountPath(storage.GetStorage().MountPath)) } func (cm *CacheManager) InvalidateStorageDetails(storage driver.Driver) { - cm.detailCache.Delete(storage.GetStorage().MountPath) + cm.detailCache.Delete(utils.GetActualMountPath(storage.GetStorage().MountPath)) } // clears all caches diff --git a/internal/op/fs.go b/internal/op/fs.go index 08e25e311..03d4fb00a 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -62,7 +62,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li // call hooks go func(reqPath string, files []model.Obj) { HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files) - }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) + }(utils.GetFullPath(utils.GetActualMountPath(storage.GetStorage().MountPath), path), files) // sort objs if storage.Config().LocalSort { From 686b56378639a9db5404bd63062685e17cfc97c4 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 8 Dec 2025 21:27:49 +0800 Subject: [PATCH 06/30] chore: change default read conflict policy --- drivers/alias/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 8c6f4cf34..486b1025d 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -10,7 +10,7 @@ type Addition struct { // driver.RootPath // define other Paths string `json:"paths" required:"true" type:"text"` - ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random" default:"random"` + ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random" default:"first"` WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled"` PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled"` DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` From 379b17d3ec1040273e274ba3b192aebcf64162d8 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Fri, 12 Dec 2025 00:42:28 +0800 Subject: [PATCH 07/30] feat(alias): refactor Alias initialization and enhance path handling --- drivers/alias/driver.go | 280 +++++++++++++++++++--------------------- drivers/alias/types.go | 50 ++++--- drivers/alias/util.go | 143 +++++++++++--------- 3 files changed, 245 insertions(+), 228 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index dd8be1d0c..5bf61e159 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "math/rand" "net/url" stdpath "path" "strings" @@ -24,10 +23,9 @@ import ( type Alias struct { model.Storage Addition - rootOrder []string - pathMap map[string][]string - autoFlatten bool - oneKey string + rootOrder []string + pathMap map[string][]string + root model.Obj } func (d *Alias) Config() driver.Config { @@ -39,18 +37,6 @@ func (d *Alias) GetAddition() driver.Additional { } func (d *Alias) Init(ctx context.Context) error { - if d.Paths == "" { - return errors.New("paths is required") - } - if !utils.SliceContains(ValidReadConflictPolicy, d.ReadConflictPolicy) { - d.ReadConflictPolicy = FirstRWP - } - if !utils.SliceContains(ValidWriteConflictPolicy, d.WriteConflictPolicy) { - d.WriteConflictPolicy = DisabledWP - } - if !utils.SliceContains(ValidPutConflictPolicy, d.PutConflictPolicy) { - d.PutConflictPolicy = DisabledWP - } paths := strings.Split(d.Paths, "\n") d.rootOrder = make([]string, 0, len(paths)) d.pathMap = make(map[string][]string) @@ -65,14 +51,42 @@ func (d *Alias) Init(ctx context.Context) error { } d.pathMap[k] = append(d.pathMap[k], v) } - if len(d.pathMap) == 1 { - for k := range d.pathMap { - d.oneKey = k + + switch len(d.rootOrder) { + case 0: + return errors.New("paths is required") + case 1: + paths := d.pathMap[d.rootOrder[0]] + roots := make(BalancedObjs, 0, len(paths)) + roots = append(roots, &model.Object{ + Name: "root", + Path: paths[0], + IsFolder: true, + Modified: d.Modified, + }) + for _, path := range paths[1:] { + roots = append(roots, &model.Object{ + Path: path, + }) + } + d.root = roots + default: + d.root = &model.Object{ + Name: "root", + Path: "/", + IsFolder: true, + Modified: d.Modified, } - d.autoFlatten = true - } else { - d.oneKey = "" - d.autoFlatten = false + } + + if !utils.SliceContains(ValidReadConflictPolicy, d.ReadConflictPolicy) { + d.ReadConflictPolicy = FirstRWP + } + if !utils.SliceContains(ValidWriteConflictPolicy, d.WriteConflictPolicy) { + d.WriteConflictPolicy = DisabledWP + } + if !utils.SliceContains(ValidPutConflictPolicy, d.PutConflictPolicy) { + d.PutConflictPolicy = DisabledWP } return nil } @@ -80,28 +94,25 @@ func (d *Alias) Init(ctx context.Context) error { func (d *Alias) Drop(ctx context.Context) error { d.rootOrder = nil d.pathMap = nil + d.root = nil return nil } -func (Addition) GetRootPath() string { - return "/" -} - func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { - root, sub := d.getRootAndPath(path) - dsts, ok := d.pathMap[root] - if !ok { + dsts, sub := d.getRootsAndPath(path) + if len(dsts) == 0 { return nil, errs.ObjectNotFound } - var objs []model.Obj - for _, dst := range dsts { + for idx, dst := range dsts { rawPath := stdpath.Join(dst, sub) obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) if err != nil { continue } + mask := model.GetObjMask(obj) + mask &^= model.Temp ret := model.Object{ - Path: path, + Path: rawPath, Name: obj.GetName(), Size: obj.GetSize(), Modified: obj.ModTime(), @@ -109,9 +120,8 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { HashInfo: obj.GetHash(), } obj = &ret - if !obj.IsDir() && d.ProviderPassThrough { - storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}) - if err == nil { + if d.ProviderPassThrough && !obj.IsDir() { + if storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}); err == nil { obj = &model.ObjectProvider{ Object: ret, Provider: model.Provider{ @@ -120,108 +130,85 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } } } - mask := model.GetObjMask(obj) - mask &^= model.Temp obj = model.ObjAddMask(obj, mask) - if !obj.IsDir() { - obj = &BalancedObj{ - Obj: obj, - ExactReqPath: rawPath, - } - } - if d.ReadConflictPolicy == FirstRWP { - return obj, nil - } else { - objs = append(objs, obj) + + dsts = dsts[idx+1:] + objs := make(BalancedObjs, 0, len(dsts)+1) + objs = append(objs, obj) + for _, d := range dsts { + objs = append(objs, &tempObj{model.Object{ + Path: stdpath.Join(d, sub), + }}) } + return objs, nil } - if len(objs) == 0 { - return nil, errs.ObjectNotFound + return nil, errs.ObjectNotFound +} + +func (d *Alias) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root == nil { + return nil, errs.StorageNotInit } - return objs[rand.Intn(len(objs))], nil + return d.root, nil } func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - if utils.PathEqual(path, "/") && !d.autoFlatten { - return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil - } - root, sub := d.getRootAndPath(path) - dsts, ok := d.pathMap[root] + dirs, ok := dir.(BalancedObjs) if !ok { - return nil, errs.ObjectNotFound + return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil } - objs := make(map[string][]model.Obj) - for _, dst := range dsts { - exactPath := stdpath.Join(dst, sub) - tmp, err := fs.List(ctx, exactPath, &fs.ListArgs{ + objMap := make(map[string]BalancedObjs) + for _, dir := range dirs { + dirPath := dir.GetPath() + tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ NoLog: true, Refresh: args.Refresh, WithStorageDetails: args.WithStorageDetails && d.DetailsPassThrough, }) - if err == nil { - tmp, err = utils.SliceConvert(tmp, func(obj model.Obj) (model.Obj, error) { - mask := model.GetObjMask(obj) - mask &^= model.Temp - objRes := model.Object{ - Name: obj.GetName(), - Path: stdpath.Join(path, obj.GetName()), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - } - var objRet model.Obj - if thumb, ok := model.GetThumb(obj); ok { - objRet = &model.ObjThumb{ - Object: objRes, - Thumbnail: model.Thumbnail{ - Thumbnail: thumb, - }, - } - } else { - objRet = &objRes - } - if details, ok := model.GetStorageDetails(obj); ok { - objRet = &model.ObjStorageDetails{ - Obj: objRet, - StorageDetailsWithName: *details, - } + if err != nil { + continue + } + for _, obj := range tmp { + name := obj.GetName() + mask := model.GetObjMask(obj) + mask &^= model.Temp + objRes := model.Object{ + Name: name, + Path: stdpath.Join(dirPath, name), + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + } + var objRet model.Obj + if thumb, ok := model.GetThumb(obj); ok { + objRet = &model.ObjThumb{ + Object: objRes, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, } - if !objRet.IsDir() { - objRet = &BalancedObj{ - Obj: objRet, - ExactReqPath: stdpath.Join(exactPath, objRet.GetName()), - } + } else { + objRet = &objRes + } + if details, ok := model.GetStorageDetails(obj); ok { + objRet = &model.ObjStorageDetails{ + Obj: objRet, + StorageDetailsWithName: *details, } - return model.ObjAddMask(objRet, mask), nil - }) - } - if err == nil { - for _, o := range tmp { - objs[o.GetName()] = append(objs[o.GetName()], o) } + obj = model.ObjAddMask(objRet, mask) + objMap[name] = append(objMap[name], obj) } } - ret := make([]model.Obj, 0, len(objs)) - for _, snObjs := range objs { - if d.ReadConflictPolicy == RandomBalancedRP { - ret = append(ret, snObjs[rand.Intn(len(snObjs))]) - } else { - ret = append(ret, snObjs[0]) - } + ret := make([]model.Obj, 0, len(objMap)) + for _, snObjs := range objMap { + ret = append(ret, snObjs) } return ret, nil } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - reqPath := GetExactReqPath(file) - if reqPath == "" { - return nil, errs.NotFile - } - // proxy || ftp,s3 - if common.GetApiUrl(ctx) == "" { - args.Redirect = false - } + reqPath := d.getBalancedPath(ctx, file) link, fi, err := d.link(ctx, reqPath, args) if err != nil { return nil, err @@ -255,11 +242,10 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { - reqPath := GetExactReqPath(args.Obj) - if reqPath == "" { - return nil, errs.NotImplement + if d.ReadConflictPolicy != FirstRWP { + return nil, errs.NotSupport } - storage, actualPath, err := op.GetStorageAndActualPath(reqPath) + storage, actualPath, err := op.GetStorageAndActualPath(args.Obj.GetPath()) if err != nil { return nil, err } @@ -271,12 +257,11 @@ func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, e } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - reqPath, err := d.getWritePath(ctx, parentDir, true) + objs, err := d.getWritePath(ctx, parentDir) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(path, dirName))) + for _, obj := range objs { + err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(obj.GetPath(), dirName))) } - return err } return err } @@ -295,12 +280,11 @@ func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - reqPath, err := d.getWritePath(ctx, srcObj, false) + objs, err := d.getWritePath(ctx, srcObj) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.Rename(ctx, path, newName)) + for _, obj := range objs { + err = errors.Join(err, fs.Rename(ctx, obj.GetPath(), newName)) } - return err } return err } @@ -319,21 +303,20 @@ func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { - reqPath, err := d.getWritePath(ctx, obj, false) + objs, err := d.getWritePath(ctx, obj) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.Remove(ctx, path)) + for _, obj := range objs { + err = errors.Join(err, fs.Remove(ctx, obj.GetPath())) } - return err } return err } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - reqPath, err := d.getPutPath(ctx, dstDir) + objs, err := d.getPutPath(ctx, dstDir) if err == nil { - if len(reqPath) == 1 { - storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath[0]) + if len(objs) == 1 { + storage, reqActualPath, err := op.GetStorageAndActualPath(objs.GetPath()) if err != nil { return err } @@ -347,10 +330,10 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, if err != nil { return err } - count := float64(len(reqPath) + 1) + count := float64(len(objs) + 1) up(100 / count) - for i, path := range reqPath { - err = errors.Join(err, fs.PutDirectly(ctx, path, &stream.FileStream{ + for i, obj := range objs { + err = errors.Join(err, fs.PutDirectly(ctx, obj.GetPath(), &stream.FileStream{ Obj: s, Mimetype: s.GetMimetype(), Reader: file, @@ -368,10 +351,10 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { - reqPath, err := d.getPutPath(ctx, dstDir) + objs, err := d.getPutPath(ctx, dstDir) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.PutURL(ctx, path, name, url)) + for _, obj := range objs { + err = errors.Join(err, fs.PutURL(ctx, obj.GetPath(), name, url)) } return err } @@ -379,7 +362,7 @@ func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) } func (d *Alias) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { - reqPath := GetExactReqPath(obj) + reqPath := d.getBalancedPath(ctx, obj) if reqPath == "" { return nil, errs.NotFile } @@ -391,7 +374,7 @@ func (d *Alias) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.Ar } func (d *Alias) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { - reqPath := GetExactReqPath(obj) + reqPath := d.getBalancedPath(ctx, obj) if reqPath == "" { return nil, errs.NotFile } @@ -406,7 +389,7 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn // alias的两个驱动，一个支持驱动提取，一个不支持，如何兼容？ // 如果访问的是不支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回errs.NotImplement，提取URL前缀就会是/ae，Extract就不会被调用 // 如果访问的是支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回有效值，提取URL前缀就会是/ad，Extract就会被调用 - reqPath := GetExactReqPath(obj) + reqPath := d.getBalancedPath(ctx, obj) if reqPath == "" { return nil, errs.NotFile } @@ -443,13 +426,12 @@ func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, } func (d *Alias) ResolveLinkCacheMode(path string) driver.LinkCacheMode { - root, sub := d.getRootAndPath(path) - dsts, ok := d.pathMap[root] - if !ok { + roots, sub := d.getRootsAndPath(path) + if len(roots) == 0 { return 0 } - for _, dst := range dsts { - storage, actualPath, err := op.GetStorageAndActualPath(stdpath.Join(dst, sub)) + for _, root := range roots { + storage, actualPath, err := op.GetStorageAndActualPath(stdpath.Join(root, sub)) if err != nil { continue } diff --git a/drivers/alias/types.go b/drivers/alias/types.go index 09edab877..201a2d36f 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -1,7 +1,10 @@ package alias import ( + "time" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) @@ -31,24 +34,37 @@ var ( ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") ) -type BalancedObj struct { - model.Obj - ExactReqPath string -} +type BalancedObjs []model.Obj -func (b *BalancedObj) Unwrap() model.Obj { - return b.Obj +func (b BalancedObjs) GetSize() int64 { + return b[0].GetSize() +} +func (b BalancedObjs) ModTime() time.Time { + return b[0].ModTime() +} +func (b BalancedObjs) CreateTime() time.Time { + return b[0].CreateTime() +} +func (b BalancedObjs) IsDir() bool { + return b[0].IsDir() +} +func (b BalancedObjs) GetHash() utils.HashInfo { + return b[0].GetHash() +} +func (b BalancedObjs) GetName() string { + return b[0].GetName() +} +func (b BalancedObjs) GetPath() string { + return b[0].GetPath() +} +func (b BalancedObjs) GetID() string { + return b[0].GetID() } -func GetExactReqPath(obj model.Obj) string { - for { - switch o := obj.(type) { - case *BalancedObj: - return o.ExactReqPath - case model.ObjUnwrap: - obj = o.Unwrap() - default: - return "" - } - } +func (b BalancedObjs) Unwrap() model.Obj { + return b[0] } + +var _ model.Obj = (BalancedObjs)(nil) + +type tempObj struct{ model.Object } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index c816d19b3..e5716be3b 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -90,16 +90,16 @@ func getPair(path string) (string, string) { return stdpath.Base(path), path } -func (d *Alias) getRootAndPath(path string) (string, string) { - if d.autoFlatten { - return d.oneKey, path +func (d *Alias) getRootsAndPath(path string) (roots []string, sub string) { + if len(d.rootOrder) == 1 { + return d.pathMap[d.rootOrder[0]], path } path = strings.TrimPrefix(path, "/") parts := strings.SplitN(path, "/", 2) if len(parts) == 1 { - return parts[0], "" + return d.pathMap[parts[0]], "" } - return parts[0], parts[1] + return d.pathMap[parts[0]], parts[1] } func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { @@ -107,48 +107,64 @@ func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) ( if err != nil { return nil, nil, err } - if !args.Redirect { - return op.Link(ctx, storage, reqActualPath, args) - } - obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) - if err != nil { - return nil, nil, err - } - if common.ShouldProxy(storage, stdpath.Base(reqPath)) { - return nil, obj, nil + if args.Redirect && common.ShouldProxy(storage, stdpath.Base(reqPath)) { + return nil, nil, nil } return op.Link(ctx, storage, reqActualPath, args) } -func (d *Alias) getAllReqPath(ctx context.Context, objPath string, isParent bool, ifContinue func(err error) (bool, error)) ([]string, error) { - root, sub := d.getRootAndPath(objPath) - if sub == "" && !isParent { - return nil, errs.NotSupport - } - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound - } - var reqPath []string - for _, dst := range dsts { - path := stdpath.Join(dst, sub) - _, err := fs.Get(ctx, path, &fs.GetArgs{NoLog: true}) +func getAllObjs(ctx context.Context, obj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { + objs := obj.(BalancedObjs) + isDir := obj.IsDir() + length := 0 + for _, o := range objs { + var err error + temp, isTemp := o.(*tempObj) + if isTemp { + obj, err = fs.Get(ctx, o.GetPath(), &fs.GetArgs{NoLog: true}) + if err == nil && isDir != obj.IsDir() { + err = errs.ObjectNotFound + } + } + cont, err := ifContinue(err) - if !cont { - if err == nil { - return []string{path}, nil - } else { - return nil, err + if err != nil { + if cont { + continue } + return nil, err } - if err == nil { - reqPath = append(reqPath, path) + if isTemp { + objRes := temp.Object + // objRes.Name = obj.GetName() + // objRes.Size = obj.GetSize() + // objRes.Modified = obj.ModTime() + // objRes.HashInfo = obj.GetHash() + objs[length] = &objRes + } else { + objs[length] = o + } + length++ + if !cont { + break } } - if len(reqPath) == 0 { + if length == 0 { return nil, errs.ObjectNotFound } - return reqPath, nil + return objs[:length], nil +} + +func (d *Alias) getBalancedPath(ctx context.Context, file model.Obj) string { + if d.ReadConflictPolicy == FirstRWP { + return file.GetPath() + } + files := file.(BalancedObjs) + if rand.Intn(len(files)) == 0 { + return file.GetPath() + } + files, _ = getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllWP)) + return files[rand.Intn(len(files))].GetPath() } func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { @@ -189,46 +205,47 @@ func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { } } -func (d *Alias) getWritePath(ctx context.Context, obj model.Obj, isParent bool) ([]string, error) { +func (d *Alias) getWritePath(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.WriteConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } - reqPath, err := d.getAllReqPath(ctx, obj.GetPath(), isParent, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) - if err != nil { - return nil, err - } - return reqPath, nil + return getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) } -func (d *Alias) getPutPath(ctx context.Context, obj model.Obj) ([]string, error) { +func (d *Alias) getPutPath(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } - reqPath, err := d.getAllReqPath(ctx, obj.GetPath(), true, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + objs, err := getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, err } - if d.PutConflictPolicy == RandomBalancedRP { - ri := rand.Intn(len(reqPath)) - return []string{reqPath[ri]}, nil - } else if d.PutConflictPolicy == BalancedByQuotaP || d.PutConflictPolicy == BalancedByQuotaStrictP { - r, ok := getRandomPathByQuotaBalanced(ctx, reqPath, d.PutConflictPolicy == BalancedByQuotaStrictP, uint64(obj.GetSize())) + strict := false + switch d.PutConflictPolicy { + case RandomBalancedRP: + ri := rand.Intn(len(objs)) + return objs[ri : ri+1], nil + case BalancedByQuotaStrictP: + strict = true + fallthrough + case BalancedByQuotaP: + objs, ok := getRandomPathByQuotaBalanced(ctx, objs, strict, uint64(obj.GetSize())) if !ok { return nil, ErrNoEnoughSpace } - return []string{r}, nil - } else { - return reqPath, nil + return objs, nil + default: + return objs, nil } } -func getRandomPathByQuotaBalanced(ctx context.Context, reqPath []string, strict bool, objSize uint64) (string, bool) { +func getRandomPathByQuotaBalanced(ctx context.Context, reqPath BalancedObjs, strict bool, objSize uint64) (BalancedObjs, bool) { // Get all space details := make([]*model.StorageDetails, len(reqPath)) detailsChan := make(chan detailWithIndex, len(reqPath)) workerCount := 0 for i, p := range reqPath { - s, err := fs.GetStorage(p, &fs.GetStoragesArgs{}) + s, err := fs.GetStorage(p.GetPath(), &fs.GetStoragesArgs{}) if err != nil { continue } @@ -265,7 +282,7 @@ func getRandomPathByQuotaBalanced(ctx context.Context, reqPath []string, strict }) if !ok { if strict { - return "", false + return nil, false } else { // No strict mode, return any of non-details ones noDetails := make([]int, 0, len(details)) @@ -275,12 +292,12 @@ func getRandomPathByQuotaBalanced(ctx context.Context, reqPath []string, strict } } if len(noDetails) == 0 { - return "", false + return nil, false } - return reqPath[noDetails[rand.Intn(len(noDetails))]], true + selected = noDetails[rand.Intn(len(noDetails))] } } - return reqPath[selected], true + return reqPath[selected : selected+1], true } func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) { @@ -306,13 +323,14 @@ func (d *Alias) getCopyMovePath(ctx context.Context, srcObj, dstDir model.Obj) ( if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } - dstPath, err := d.getAllReqPath(ctx, dstDir.GetPath(), true, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + dstObjs, err := getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, nil, err } dstStorageMap := make(map[string][]string) allocatingDst := make(map[string]struct{}) - for _, dp := range dstPath { + for _, o := range dstObjs { + dp := o.GetPath() storage, e := fs.GetStorage(dp, &fs.GetStoragesArgs{}) if e != nil { return nil, nil, errors.WithMessagef(e, "cannot copy or move to virtual path [%s]", dp) @@ -321,13 +339,14 @@ func (d *Alias) getCopyMovePath(ctx context.Context, srcObj, dstDir model.Obj) ( dstStorageMap[mp] = append(dstStorageMap[mp], dp) allocatingDst[dp] = struct{}{} } - srcPath, err := d.getAllReqPath(ctx, srcObj.GetPath(), false, getWriteAndPutFilterFunc(AllWP)) + srcObjs, err := getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) if err != nil { return nil, nil, err } srcs := make([]string, 0) dsts := make([]string, 0) - for _, sp := range srcPath { + for _, o := range srcObjs { + sp := o.GetPath() storage, e := fs.GetStorage(sp, &fs.GetStoragesArgs{}) if e != nil { continue From 05191befebfd8b9c87dfb3a049411a39d537f0e8 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Fri, 12 Dec 2025 20:03:09 +0800 Subject: [PATCH 08/30] feat(alias): enhance object masking and add support for operation restrictions --- drivers/alias/driver.go | 49 ++++++++++++++++++++++++++++++++--------- internal/model/obj.go | 4 ++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 5bf61e159..972c5bbe9 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -58,12 +58,12 @@ func (d *Alias) Init(ctx context.Context) error { case 1: paths := d.pathMap[d.rootOrder[0]] roots := make(BalancedObjs, 0, len(paths)) - roots = append(roots, &model.Object{ + roots = append(roots, model.ObjAddMask(&model.Object{ Name: "root", Path: paths[0], IsFolder: true, Modified: d.Modified, - }) + }, model.NoRename|model.NoDelete|model.NoMove)) for _, path := range paths[1:] { roots = append(roots, &model.Object{ Path: path, @@ -71,12 +71,12 @@ func (d *Alias) Init(ctx context.Context) error { } d.root = roots default: - d.root = &model.Object{ + d.root = model.ObjAddMask(&model.Object{ Name: "root", Path: "/", IsFolder: true, Modified: d.Modified, - } + }, model.NoRename|model.NoDelete|model.NoMove|model.NoWrite) } if !utils.SliceContains(ValidReadConflictPolicy, d.ReadConflictPolicy) { @@ -98,6 +98,14 @@ func (d *Alias) Drop(ctx context.Context) error { return nil } +func (d *Alias) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root == nil { + return nil, errs.StorageNotInit + } + return d.root, nil +} + +// 通过op.Get调用的话，path一定不是根目录"/" func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { dsts, sub := d.getRootsAndPath(path) if len(dsts) == 0 { @@ -130,6 +138,9 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } } } + if sub == "" { + mask |= model.Virtual | model.NoRename | model.NoDelete | model.NoMove + } obj = model.ObjAddMask(obj, mask) dsts = dsts[idx+1:] @@ -145,13 +156,6 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { return nil, errs.ObjectNotFound } -func (d *Alias) GetRoot(ctx context.Context) (model.Obj, error) { - if d.root == nil { - return nil, errs.StorageNotInit - } - return d.root, nil -} - func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { dirs, ok := dir.(BalancedObjs) if !ok { @@ -196,6 +200,8 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ StorageDetailsWithName: *details, } } + // alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 + // op.Get不会从List里寻找对象，所以这里不需要添加额外的mask obj = model.ObjAddMask(objRet, mask) objMap[name] = append(objMap[name], obj) } @@ -257,6 +263,9 @@ func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, e } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if model.ObjHasMask(parentDir, model.NoWrite) { + return errs.PermissionDenied + } objs, err := d.getWritePath(ctx, parentDir) if err == nil { for _, obj := range objs { @@ -267,6 +276,9 @@ func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if model.ObjHasMask(srcObj, model.NoMove) || model.ObjHasMask(dstDir, model.NoWrite) { + return errs.PermissionDenied + } srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) if err != nil { return err @@ -280,6 +292,9 @@ func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if model.ObjHasMask(srcObj, model.NoRename) { + return errs.PermissionDenied + } objs, err := d.getWritePath(ctx, srcObj) if err == nil { for _, obj := range objs { @@ -290,6 +305,9 @@ func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) er } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if model.ObjHasMask(dstDir, model.NoWrite) { + return errs.PermissionDenied + } srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) if err != nil { return err @@ -303,6 +321,9 @@ func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { + if model.ObjHasMask(obj, model.NoDelete) { + return errs.PermissionDenied + } objs, err := d.getWritePath(ctx, obj) if err == nil { for _, obj := range objs { @@ -313,6 +334,9 @@ func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + if model.ObjHasMask(dstDir, model.NoWrite) { + return errs.PermissionDenied + } objs, err := d.getPutPath(ctx, dstDir) if err == nil { if len(objs) == 1 { @@ -351,6 +375,9 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { + if model.ObjHasMask(dstDir, model.NoWrite) { + return errs.PermissionDenied + } objs, err := d.getPutPath(ctx, dstDir) if err == nil { for _, obj := range objs { diff --git a/internal/model/obj.go b/internal/model/obj.go index 74c140d9f..6ef14ccdd 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -241,6 +241,10 @@ type ObjMask uint8 const ( Virtual ObjMask = 1 << iota Temp + NoRename + NoDelete + NoMove + NoWrite ) type maskObj struct { From 14087e5f0403512787b8f30ec7c7b16ddb24a27d Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 13 Dec 2025 20:12:07 +0800 Subject: [PATCH 09/30] feat(alias): enhance object masking --- drivers/alias/driver.go | 32 ++++++++++++++++---------------- drivers/alias/util.go | 6 ++++-- drivers/crypt/driver.go | 21 ++++++++++----------- internal/model/obj.go | 30 ++++++++++++++++++------------ internal/model/object.go | 5 +++++ internal/op/fs.go | 23 ++++++++++++++++------- internal/op/storage.go | 31 +++++++++++++++++-------------- 7 files changed, 86 insertions(+), 62 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 972c5bbe9..6f16cd13e 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -58,12 +58,13 @@ func (d *Alias) Init(ctx context.Context) error { case 1: paths := d.pathMap[d.rootOrder[0]] roots := make(BalancedObjs, 0, len(paths)) - roots = append(roots, model.ObjAddMask(&model.Object{ + roots = append(roots, &model.Object{ Name: "root", Path: paths[0], IsFolder: true, Modified: d.Modified, - }, model.NoRename|model.NoDelete|model.NoMove)) + Mask: model.Static, + }) for _, path := range paths[1:] { roots = append(roots, &model.Object{ Path: path, @@ -71,12 +72,13 @@ func (d *Alias) Init(ctx context.Context) error { } d.root = roots default: - d.root = model.ObjAddMask(&model.Object{ + d.root = &model.Object{ Name: "root", Path: "/", IsFolder: true, Modified: d.Modified, - }, model.NoRename|model.NoDelete|model.NoMove|model.NoWrite) + Mask: model.Virtual, + } } if !utils.SliceContains(ValidReadConflictPolicy, d.ReadConflictPolicy) { @@ -117,8 +119,10 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { if err != nil { continue } - mask := model.GetObjMask(obj) - mask &^= model.Temp + mask := model.GetObjMask(obj) &^ model.Temp + if sub == "" { + mask |= model.Static + } ret := model.Object{ Path: rawPath, Name: obj.GetName(), @@ -126,6 +130,7 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { Modified: obj.ModTime(), IsFolder: obj.IsDir(), HashInfo: obj.GetHash(), + Mask: mask, } obj = &ret if d.ProviderPassThrough && !obj.IsDir() { @@ -138,10 +143,6 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } } } - if sub == "" { - mask |= model.Virtual | model.NoRename | model.NoDelete | model.NoMove - } - obj = model.ObjAddMask(obj, mask) dsts = dsts[idx+1:] objs := make(BalancedObjs, 0, len(dsts)+1) @@ -174,14 +175,16 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } for _, obj := range tmp { name := obj.GetName() - mask := model.GetObjMask(obj) - mask &^= model.Temp + // alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 + // 所以op.Get不会从List里寻找对象，所以这里不需要添加额外的mask + mask := model.GetObjMask(obj) &^ model.Temp objRes := model.Object{ Name: name, Path: stdpath.Join(dirPath, name), Size: obj.GetSize(), Modified: obj.ModTime(), IsFolder: obj.IsDir(), + Mask: mask, } var objRet model.Obj if thumb, ok := model.GetThumb(obj); ok { @@ -200,10 +203,7 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ StorageDetailsWithName: *details, } } - // alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 - // op.Get不会从List里寻找对象，所以这里不需要添加额外的mask - obj = model.ObjAddMask(objRet, mask) - objMap[name] = append(objMap[name], obj) + objMap[name] = append(objMap[name], objRet) } } ret := make([]model.Obj, 0, len(objMap)) diff --git a/drivers/alias/util.go b/drivers/alias/util.go index e5716be3b..0801cb2fc 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -28,14 +28,15 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model detailsChan := make(chan detailWithIndex, len(d.pathMap)) workerCount := 0 for _, k := range d.rootOrder { - obj := model.Object{ + obj := &model.Object{ Name: k, Path: "/" + k, IsFolder: true, Modified: d.Modified, + Mask: model.Static, } idx := len(objs) - objs = append(objs, model.ObjAddMask(&obj, model.Virtual)) + objs = append(objs, obj) v := d.pathMap[k] if !withDetails || len(v) != 1 { continue @@ -44,6 +45,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model if err != nil { continue } + obj.Modified = remoteDriver.GetStorage().Modified _, ok := remoteDriver.(driver.WithDetails) if !ok { continue diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index 47ac0c672..8ccde9ded 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -113,7 +113,7 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ size := obj.GetSize() mask := model.GetObjMask(obj) name := obj.GetName() - if mask&model.Virtual == 0 { + if mask&model.Static == 0 { if obj.IsDir() { name, err = d.cipher.DecryptDirName(model.UnwrapObjName(obj).GetName()) if err != nil { @@ -136,7 +136,6 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if !d.ShowHidden && strings.HasPrefix(name, ".") { continue } - mask &^= model.Temp objRes := &model.Object{ Path: stdpath.Join(remoteFullPath, obj.GetName()), Name: name, @@ -144,10 +143,11 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ Modified: obj.ModTime(), IsFolder: obj.IsDir(), Ctime: obj.CreateTime(), + Mask: mask &^ model.Temp, // discarding hash as it's encrypted } if !d.Thumbnail || !strings.HasPrefix(args.ReqPath, "/") { - result = append(result, model.ObjAddMask(objRes, mask)) + result = append(result, objRes) continue } thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") @@ -155,12 +155,12 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ common.GetApiUrl(ctx), utils.EncodePath(thumbPath, true), sign.Sign(thumbPath)) - result = append(result, model.ObjAddMask(&model.ObjThumb{ + result = append(result, &model.ObjThumb{ Object: *objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, - }, mask)) + }) } return result, nil @@ -196,9 +196,8 @@ func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { size := remoteObj.GetSize() name := remoteObj.GetName() - mask := model.GetObjMask(remoteObj) - mask &^= model.Temp - if mask&model.Virtual == 0 { + mask := model.GetObjMask(remoteObj) &^ model.Temp + if mask&model.Static == 0 { if !remoteObj.IsDir() { decryptedSize, err := d.cipher.DecryptedSize(size) if err != nil { @@ -221,15 +220,15 @@ func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { } } } - obj := &model.Object{ + return &model.Object{ Path: remoteFullPath, Name: name, Size: size, Modified: remoteObj.ModTime(), IsFolder: remoteObj.IsDir(), Ctime: remoteObj.CreateTime(), - } - return model.ObjAddMask(obj, mask), nil + Mask: mask, + }, nil } // https://github.com/rclone/rclone/blob/v1.67.0/backend/crypt/cipher.go#L37 diff --git a/internal/model/obj.go b/internal/model/obj.go index 6ef14ccdd..2340a0bad 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -239,13 +239,16 @@ func (om *ObjMerge) Reset() { type ObjMask uint8 const ( - Virtual ObjMask = 1 << iota - Temp + Temp ObjMask = 1 << iota NoRename NoDelete NoMove NoWrite ) +const ( + Static = NoRename | NoDelete | NoMove + Virtual = Static | NoWrite // NoRename | NoDelete | NoMove | NoWrite +) type maskObj struct { Obj @@ -255,12 +258,15 @@ type maskObj struct { func (m *maskObj) Unwrap() Obj { return m.Obj } +func (m *maskObj) ObjMask() *ObjMask { + return &m.mask +} -func getMaskObj(obj Obj) *maskObj { +func getObjMask(obj Obj) *ObjMask { for { switch o := obj.(type) { - case *maskObj: - return o + case interface{ ObjMask() *ObjMask }: + return o.ObjMask() case ObjUnwrap: obj = o.Unwrap() default: @@ -269,8 +275,8 @@ func getMaskObj(obj Obj) *maskObj { } } func ObjHasMask(obj Obj, mask ObjMask) bool { - if m := getMaskObj(obj); m != nil { - return m.mask&mask == mask + if m := getObjMask(obj); m != nil { + return *m&mask == mask } return false } @@ -278,15 +284,15 @@ func ObjAddMask(obj Obj, mask ObjMask) Obj { if mask == 0 { return obj } - if m := getMaskObj(obj); m != nil { - m.mask |= mask + if m := getObjMask(obj); m != nil { + *m |= mask return obj } - return &maskObj{Obj: obj, mask: mask} + return &maskObj{obj, mask} } func GetObjMask(obj Obj) ObjMask { - if m := getMaskObj(obj); m != nil { - return m.mask + if m := getObjMask(obj); m != nil { + return *m } return 0 } diff --git a/internal/model/object.go b/internal/model/object.go index 8e5cdf047..c1653697a 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -28,6 +28,7 @@ type Object struct { Ctime time.Time // file create time IsFolder bool HashInfo utils.HashInfo + Mask ObjMask } func (o *Object) GetName() string { @@ -68,6 +69,10 @@ func (o *Object) GetHash() utils.HashInfo { return o.HashInfo } +func (o *Object) ObjMask() *ObjMask { + return &o.Mask +} + type Thumbnail struct { Thumbnail string } diff --git a/internal/op/fs.go b/internal/op/fs.go index ddfedccb8..b7e15990f 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -149,12 +149,14 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Name: RootName, Modified: storage.GetStorage().Modified, IsFolder: true, + Mask: model.Static, }, nil case driver.IRootPath: return &model.Object{ Path: r.GetRootPath(), Name: RootName, Modified: storage.GetStorage().Modified, + Mask: model.Static, IsFolder: true, }, nil } @@ -347,12 +349,13 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { if dirCache, exist := Cache.dirCache.Get(Key(storage, parentPath)); exist { if newObj == nil { t := time.Now() - newObj = model.ObjAddMask(&model.Object{ + newObj = &model.Object{ Name: dirName, IsFolder: true, Modified: t, Ctime: t, - }, model.Temp) + Mask: model.Temp, + } } dirCache.UpdateObject("", wrapObjName(storage, newObj)) } @@ -456,7 +459,11 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) return errors.WithStack(err) } if newObj == nil { - newObj = model.ObjAddMask(&model.ObjWrapName{Name: dstName, Obj: srcObj}, model.Temp) + newObj = &struct { + model.ObjWrapName + }{ + model.ObjWrapName{Name: dstName, Obj: model.ObjAddMask(srcObj, model.Temp)}, + } } Cache.updateDirectoryObject(storage, stdpath.Dir(srcPath), srcRawObj, wrapObjName(storage, newObj)) @@ -631,12 +638,13 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { if newObj == nil { - newObj = model.ObjAddMask(&model.Object{ + newObj = &model.Object{ Name: file.GetName(), Size: file.GetSize(), Modified: file.ModTime(), Ctime: file.CreateTime(), - }, model.Temp) + Mask: model.Temp, + } } newObj = wrapObjName(storage, newObj) cache.UpdateObject(newObj.GetName(), newObj) @@ -696,11 +704,12 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { if newObj == nil { t := time.Now() - newObj = model.ObjAddMask(&model.Object{ + newObj = &model.Object{ Name: dstName, Modified: t, Ctime: t, - }, model.Temp) + Mask: model.Temp, + } } newObj = wrapObjName(storage, newObj) cache.UpdateObject(newObj.GetName(), newObj) diff --git a/internal/op/storage.go b/internal/op/storage.go index 453fa11f6..4c28d4fc8 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -397,32 +397,35 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver continue } names := strings.SplitN(strings.TrimPrefix(mountPath[len(prefix):], "/"), "/", 2) - idx, ok := set[names[0]] - if !ok { - set[names[0]] = len(files) - obj := model.ObjAddMask(&model.Object{ - Name: names[0], - Size: 0, - Modified: v.GetStorage().Modified, - IsFolder: true, - }, model.Virtual) + if idx, ok := set[names[0]]; ok { if len(names) == 1 { - idx = len(files) - files = append(files, obj) wg.Add(1) go func() { defer wg.Done() files[idx] = rootCallback(v, files[idx]) }() - } else { - files = append(files, obj) } - } else if len(names) == 1 { + continue + } + set[names[0]] = len(files) + obj := &model.Object{ + Name: names[0], + Size: 0, + Modified: v.GetStorage().Modified, + IsFolder: true, + } + if len(names) == 1 { + idx := len(files) + obj.Mask = model.Static + files = append(files, obj) wg.Add(1) go func() { defer wg.Done() files[idx] = rootCallback(v, files[idx]) }() + } else { + obj.Mask = model.Virtual + files = append(files, obj) } } wg.Wait() From c78f99f10b768089652dd4663990d1bd9bf98bc7 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 13 Dec 2025 20:58:08 +0800 Subject: [PATCH 10/30] feat(fs): add permission checks --- drivers/alias/driver.go | 21 ---------------- internal/model/obj.go | 5 ++-- internal/op/cache.go | 21 ---------------- internal/op/fs.go | 53 ++++++++++++++++++++++++++++++++++++----- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 6f16cd13e..ff87a6e2c 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -263,9 +263,6 @@ func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, e } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - if model.ObjHasMask(parentDir, model.NoWrite) { - return errs.PermissionDenied - } objs, err := d.getWritePath(ctx, parentDir) if err == nil { for _, obj := range objs { @@ -276,9 +273,6 @@ func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - if model.ObjHasMask(srcObj, model.NoMove) || model.ObjHasMask(dstDir, model.NoWrite) { - return errs.PermissionDenied - } srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) if err != nil { return err @@ -292,9 +286,6 @@ func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - if model.ObjHasMask(srcObj, model.NoRename) { - return errs.PermissionDenied - } objs, err := d.getWritePath(ctx, srcObj) if err == nil { for _, obj := range objs { @@ -305,9 +296,6 @@ func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) er } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if model.ObjHasMask(dstDir, model.NoWrite) { - return errs.PermissionDenied - } srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) if err != nil { return err @@ -321,9 +309,6 @@ func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { - if model.ObjHasMask(obj, model.NoDelete) { - return errs.PermissionDenied - } objs, err := d.getWritePath(ctx, obj) if err == nil { for _, obj := range objs { @@ -334,9 +319,6 @@ func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - if model.ObjHasMask(dstDir, model.NoWrite) { - return errs.PermissionDenied - } objs, err := d.getPutPath(ctx, dstDir) if err == nil { if len(objs) == 1 { @@ -375,9 +357,6 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { - if model.ObjHasMask(dstDir, model.NoWrite) { - return errs.PermissionDenied - } objs, err := d.getPutPath(ctx, dstDir) if err == nil { for _, obj := range objs { diff --git a/internal/model/obj.go b/internal/model/obj.go index 2340a0bad..4c23dff0e 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -241,12 +241,13 @@ type ObjMask uint8 const ( Temp ObjMask = 1 << iota NoRename - NoDelete + NoRemove NoMove + NoCopy NoWrite ) const ( - Static = NoRename | NoDelete | NoMove + Static = NoRename | NoRemove | NoMove Virtual = Static | NoWrite // NoRename | NoDelete | NoMove | NoWrite ) diff --git a/internal/op/cache.go b/internal/op/cache.go index 2ffc2bcc8..d8d32a74b 100644 --- a/internal/op/cache.go +++ b/internal/op/cache.go @@ -36,27 +36,6 @@ func Key(storage driver.Driver, path string) string { return utils.GetFullPath(storage.GetStorage().MountPath, path) } -// update object in dirCache. -// if it's a directory, remove all its children from dirCache too. -// if it's a file, remove its link from linkCache. -func (cm *CacheManager) updateDirectoryObject(storage driver.Driver, dirPath string, oldObj model.Obj, newObj model.Obj) { - key := Key(storage, dirPath) - if !oldObj.IsDir() { - cm.linkCache.DeleteKey(stdpath.Join(key, oldObj.GetName())) - cm.linkCache.DeleteKey(stdpath.Join(key, newObj.GetName())) - } - if storage.Config().NoCache { - return - } - - if cache, exist := cm.dirCache.Get(key); exist { - if oldObj.IsDir() { - cm.deleteDirectoryTree(stdpath.Join(key, oldObj.GetName())) - } - cache.UpdateObject(oldObj.GetName(), newObj) - } -} - // recursively delete directory and its children from dirCache func (cm *CacheManager) DeleteDirectoryTree(storage driver.Driver, dirPath string) { if storage.Config().NoCache { diff --git a/internal/op/fs.go b/internal/op/fs.go index b7e15990f..52710f5ef 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -330,6 +330,9 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { if err != nil { return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) } + if model.ObjHasMask(parentDir, model.NoWrite) { + return nil, errors.WithStack(errs.PermissionDenied) + } var newObj model.Obj switch s := storage.(type) { @@ -381,11 +384,17 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string if err != nil { return errors.WithMessage(err, "failed to get src object") } + if model.ObjHasMask(srcRawObj, model.NoMove) { + return errors.WithStack(errs.PermissionDenied) + } srcObj := model.UnwrapObjName(srcRawObj) dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } var newObj model.Obj switch s := storage.(type) { @@ -444,6 +453,9 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) if err != nil { return errors.WithMessage(err, "failed to get src object") } + if model.ObjHasMask(srcRawObj, model.NoRename) { + return errors.WithStack(errs.PermissionDenied) + } srcObj := model.UnwrapObjName(srcRawObj) var newObj model.Obj @@ -458,14 +470,28 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) if err != nil { return errors.WithStack(err) } - if newObj == nil { - newObj = &struct { - model.ObjWrapName - }{ - model.ObjWrapName{Name: dstName, Obj: model.ObjAddMask(srcObj, model.Temp)}, + + dirKey := Key(storage, stdpath.Dir(srcPath)) + if !srcRawObj.IsDir() { + Cache.linkCache.DeleteKey(stdpath.Join(dirKey, srcRawObj.GetName())) + Cache.linkCache.DeleteKey(stdpath.Join(dirKey, dstName)) + } + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(dirKey); exist { + if srcRawObj.IsDir() { + Cache.deleteDirectoryTree(stdpath.Join(dirKey, srcRawObj.GetName())) + } + if newObj == nil { + newObj = &struct { + model.ObjWrapName + }{ + model.ObjWrapName{Name: dstName, Obj: model.ObjAddMask(srcObj, model.Temp)}, + } + } + newObj = wrapObjName(storage, newObj) + cache.UpdateObject(srcRawObj.GetName(), newObj) } } - Cache.updateDirectoryObject(storage, stdpath.Dir(srcPath), srcRawObj, wrapObjName(storage, newObj)) if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { return nil @@ -493,11 +519,17 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string if err != nil { return errors.WithMessage(err, "failed to get src object") } + // if model.ObjHasMask(srcRawObj, model.NoCopy) { + // return errors.WithStack(errs.PermissionDenied) + // } srcObj := model.UnwrapObjName(srcRawObj) dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } var newObj model.Obj switch s := storage.(type) { @@ -553,6 +585,9 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { } return errors.WithMessage(err, "failed to get object") } + if model.ObjHasMask(rawObj, model.NoRemove) { + return errors.WithStack(errs.PermissionDenied) + } dirPath := stdpath.Dir(path) switch s := storage.(type) { @@ -613,6 +648,9 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod if err != nil { return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) } + if model.ObjHasMask(parentDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } // if up is nil, set a default to prevent panic if up == nil { up = func(p float64) {} @@ -689,6 +727,9 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url if err != nil { return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) } + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } var newObj model.Obj switch s := storage.(type) { case driver.PutURLResult: From d82f9e7af2ad958052bcc64b68649cfc5e8c34a3 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 14 Dec 2025 08:23:55 +0800 Subject: [PATCH 11/30] improve parsing --- drivers/alias/driver.go | 42 +++++++++++++++++++++++------------------ drivers/alias/util.go | 16 ++++++---------- drivers/chunk/driver.go | 14 +++++--------- internal/op/fs.go | 10 +++++----- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index ff87a6e2c..d1cc9717d 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -46,10 +46,11 @@ func (d *Alias) Init(ctx context.Context) error { continue } k, v := getPair(path) - if _, ok := d.pathMap[k]; !ok { + temp, ok := d.pathMap[k] + if !ok { d.rootOrder = append(d.rootOrder, k) } - d.pathMap[k] = append(d.pathMap[k], v) + d.pathMap[k] = append(temp, v) } switch len(d.rootOrder) { @@ -107,20 +108,21 @@ func (d *Alias) GetRoot(ctx context.Context) (model.Obj, error) { return d.root, nil } -// 通过op.Get调用的话，path一定不是根目录"/" +// 通过op.Get调用的话，path一定是子路径(/开头) func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { - dsts, sub := d.getRootsAndPath(path) - if len(dsts) == 0 { + roots, sub := d.getRootsAndPath(path) + if len(roots) == 0 { return nil, errs.ObjectNotFound } - for idx, dst := range dsts { - rawPath := stdpath.Join(dst, sub) + for idx, root := range roots { + rawPath := stdpath.Join(root, sub) obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) if err != nil { continue } mask := model.GetObjMask(obj) &^ model.Temp if sub == "" { + // 根目录 mask |= model.Static } ret := model.Object{ @@ -144,10 +146,10 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } } - dsts = dsts[idx+1:] - objs := make(BalancedObjs, 0, len(dsts)+1) + roots = roots[idx+1:] + objs := make(BalancedObjs, 0, len(roots)+1) objs = append(objs, obj) - for _, d := range dsts { + for _, d := range roots { objs = append(objs, &tempObj{model.Object{ Path: stdpath.Join(d, sub), }}) @@ -162,7 +164,10 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if !ok { return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil } - objMap := make(map[string]BalancedObjs) + + // 因为alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 + // 所以这里对象不会传回到alias，也就不需要返回BalancedObjs了 + objMap := make(map[string]model.Obj) for _, dir := range dirs { dirPath := dir.GetPath() tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ @@ -175,8 +180,9 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } for _, obj := range tmp { name := obj.GetName() - // alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 - // 所以op.Get不会从List里寻找对象，所以这里不需要添加额外的mask + if _, exists := objMap[name]; exists { + continue + } mask := model.GetObjMask(obj) &^ model.Temp objRes := model.Object{ Name: name, @@ -203,14 +209,14 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ StorageDetailsWithName: *details, } } - objMap[name] = append(objMap[name], objRet) + objMap[name] = objRet } } - ret := make([]model.Obj, 0, len(objMap)) - for _, snObjs := range objMap { - ret = append(ret, snObjs) + objs := make([]model.Obj, 0, len(objMap)) + for _, obj := range objMap { + objs = append(objs, obj) } - return ret, nil + return objs, nil } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 0801cb2fc..f4acae3b1 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -82,12 +82,8 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model // do others that not defined in Driver interface func getPair(path string) (string, string) { - // path = strings.TrimSpace(path) - if strings.Contains(path, ":") { - pair := strings.SplitN(path, ":", 2) - if !strings.Contains(pair[0], "/") { - return pair[0], pair[1] - } + if name, path, ok := strings.Cut(path, ":"); ok && !strings.Contains(name, "/") { + return name, path } return stdpath.Base(path), path } @@ -97,11 +93,11 @@ func (d *Alias) getRootsAndPath(path string) (roots []string, sub string) { return d.pathMap[d.rootOrder[0]], path } path = strings.TrimPrefix(path, "/") - parts := strings.SplitN(path, "/", 2) - if len(parts) == 1 { - return d.pathMap[parts[0]], "" + before, after, ok := strings.Cut(path, "/") + if !ok { + return d.pathMap[path], "" } - return d.pathMap[parts[0]], parts[1] + return d.pathMap[before], after } func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 60c5790fe..b544391dd 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -270,17 +270,13 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( // 检查0号块不等于-1 以支持空文件 // 如果块数量大于1 最后一块不可能为0 // 只检查中间块是否有0 - for i, l := 0, len(chunkFile.chunkSizes)-2; ; i++ { - if i == 0 { - if chunkFile.chunkSizes[i] == -1 { - return nil, fmt.Errorf("chunk part[%d] are missing", i) - } - } else if chunkFile.chunkSizes[i] == 0 { + if chunkFile.chunkSizes[0] == -1 { + return nil, fmt.Errorf("chunk part[%d] are missing", 0) + } + for i, l := 1, len(chunkFile.chunkSizes)-1; i < l; i++ { + if chunkFile.chunkSizes[i] == 0 { return nil, fmt.Errorf("chunk part[%d] are missing", i) } - if i >= l { - break - } } fileSize := chunkFile.GetSize() mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { diff --git a/internal/op/fs.go b/internal/op/fs.go index 52710f5ef..d30f40870 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -86,19 +86,19 @@ func list(ctx context.Context, storage driver.Driver, path string, args model.Li if len(customCachePolicies) > 0 { configPolicies := strings.Split(customCachePolicies, "\n") for _, configPolicy := range configPolicies { - policy := strings.Split(strings.TrimSpace(configPolicy), ":") - if len(policy) != 2 { + pattern, ttlstr, ok := strings.Cut(strings.TrimSpace(configPolicy), ":") + if !ok { log.Warnf("Malformed custom cache policy entry: %s in storage %s for path %s. Expected format: pattern:ttl", configPolicy, storage.GetStorage().MountPath, path) continue } - if match, err1 := doublestar.Match(policy[0], path); err1 != nil { - log.Warnf("Invalid glob pattern in custom cache policy: %s, error: %v", policy[0], err1) + if match, err1 := doublestar.Match(pattern, path); err1 != nil { + log.Warnf("Invalid glob pattern in custom cache policy: %s, error: %v", pattern, err1) continue } else if !match { continue } - if configTtl, err1 := strconv.ParseInt(policy[1], 10, 64); err1 == nil { + if configTtl, err1 := strconv.ParseInt(ttlstr, 10, 64); err1 == nil { ttl = int(configTtl) break } From e56f704f8a605986edb23ab9370e36e9703b1c22 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 14 Dec 2025 10:21:58 +0800 Subject: [PATCH 12/30] update object masks --- drivers/alias/driver.go | 6 ++--- drivers/alias/util.go | 2 +- drivers/crypt/driver.go | 4 ++-- internal/model/obj.go | 49 +++++++++++++++------------------------- internal/model/object.go | 4 ++-- internal/op/fs.go | 28 +++++++++++------------ internal/op/storage.go | 4 ++-- 7 files changed, 42 insertions(+), 55 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index d1cc9717d..bfdabb089 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -64,7 +64,7 @@ func (d *Alias) Init(ctx context.Context) error { Path: paths[0], IsFolder: true, Modified: d.Modified, - Mask: model.Static, + Mask: model.Locked, }) for _, path := range paths[1:] { roots = append(roots, &model.Object{ @@ -78,7 +78,7 @@ func (d *Alias) Init(ctx context.Context) error { Path: "/", IsFolder: true, Modified: d.Modified, - Mask: model.Virtual, + Mask: model.ReadOnly, } } @@ -123,7 +123,7 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { mask := model.GetObjMask(obj) &^ model.Temp if sub == "" { // 根目录 - mask |= model.Static + mask |= model.Locked | model.Virtual } ret := model.Object{ Path: rawPath, diff --git a/drivers/alias/util.go b/drivers/alias/util.go index f4acae3b1..53e165a39 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -33,7 +33,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model Path: "/" + k, IsFolder: true, Modified: d.Modified, - Mask: model.Static, + Mask: model.Locked | model.Virtual, } idx := len(objs) objs = append(objs, obj) diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index 8ccde9ded..9e8b5c5c7 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -113,7 +113,7 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ size := obj.GetSize() mask := model.GetObjMask(obj) name := obj.GetName() - if mask&model.Static == 0 { + if mask&model.Virtual == 0 { if obj.IsDir() { name, err = d.cipher.DecryptDirName(model.UnwrapObjName(obj).GetName()) if err != nil { @@ -197,7 +197,7 @@ func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { size := remoteObj.GetSize() name := remoteObj.GetName() mask := model.GetObjMask(remoteObj) &^ model.Temp - if mask&model.Static == 0 { + if mask&model.Virtual == 0 { if !remoteObj.IsDir() { decryptedSize, err := d.cipher.DecryptedSize(size) if err != nil { diff --git a/internal/model/obj.go b/internal/model/obj.go index 4c23dff0e..f0dc9c4a2 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -238,8 +238,13 @@ func (om *ObjMerge) Reset() { type ObjMask uint8 +func (m ObjMask) GetObjMask() ObjMask { + return m +} + const ( Temp ObjMask = 1 << iota + Virtual NoRename NoRemove NoMove @@ -247,53 +252,35 @@ const ( NoWrite ) const ( - Static = NoRename | NoRemove | NoMove - Virtual = Static | NoWrite // NoRename | NoDelete | NoMove | NoWrite + Locked = NoRename | NoRemove | NoMove + ReadOnly = Locked | NoWrite // NoRename | NoDelete | NoMove | NoWrite ) -type maskObj struct { +type ObjWrapMask struct { Obj - mask ObjMask + Mask ObjMask } -func (m *maskObj) Unwrap() Obj { +func (m *ObjWrapMask) Unwrap() Obj { return m.Obj } -func (m *maskObj) ObjMask() *ObjMask { - return &m.mask +func (m *ObjWrapMask) GetObjMask() ObjMask { + return m.Mask } -func getObjMask(obj Obj) *ObjMask { +func GetObjMask(obj Obj) ObjMask { for { switch o := obj.(type) { - case interface{ ObjMask() *ObjMask }: - return o.ObjMask() + case interface{ GetObjMask() ObjMask }: + return o.GetObjMask() case ObjUnwrap: obj = o.Unwrap() default: - return nil + return 0 } } } + func ObjHasMask(obj Obj, mask ObjMask) bool { - if m := getObjMask(obj); m != nil { - return *m&mask == mask - } - return false -} -func ObjAddMask(obj Obj, mask ObjMask) Obj { - if mask == 0 { - return obj - } - if m := getObjMask(obj); m != nil { - *m |= mask - return obj - } - return &maskObj{obj, mask} -} -func GetObjMask(obj Obj) ObjMask { - if m := getObjMask(obj); m != nil { - return *m - } - return 0 + return GetObjMask(obj)&mask != 0 } diff --git a/internal/model/object.go b/internal/model/object.go index c1653697a..b6cb0d7f7 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -69,8 +69,8 @@ func (o *Object) GetHash() utils.HashInfo { return o.HashInfo } -func (o *Object) ObjMask() *ObjMask { - return &o.Mask +func (o *Object) GetObjMask() ObjMask { + return o.Mask } type Thumbnail struct { diff --git a/internal/op/fs.go b/internal/op/fs.go index d30f40870..5116bbef5 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -142,21 +142,21 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj } return rootObj, nil } - switch r := storage.GetAddition().(type) { + switch r := storage.(type) { case driver.IRootId: return &model.Object{ ID: r.GetRootId(), Name: RootName, Modified: storage.GetStorage().Modified, IsFolder: true, - Mask: model.Static, + Mask: model.Locked, }, nil case driver.IRootPath: return &model.Object{ Path: r.GetRootPath(), Name: RootName, Modified: storage.GetStorage().Modified, - Mask: model.Static, + Mask: model.Locked, IsFolder: true, }, nil } @@ -235,10 +235,10 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path) } typeKey := args.Type - if mode&driver.LinkCacheIP == driver.LinkCacheIP { + if mode&driver.LinkCacheIP != 0 { typeKey += "/" + args.IP } - if mode&driver.LinkCacheUA == driver.LinkCacheUA { + if mode&driver.LinkCacheUA != 0 { typeKey += "/" + args.Header.Get("User-Agent") } key := Key(storage, path) @@ -424,9 +424,11 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string } if cache, exist := Cache.dirCache.Get(dstKey); exist { if newObj == nil { - newObj = model.ObjAddMask(srcObj, model.Temp) + newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp} + } else { + newObj = wrapObjName(storage, newObj) } - cache.UpdateObject(srcRawObj.GetName(), wrapObjName(storage, newObj)) + cache.UpdateObject(srcRawObj.GetName(), newObj) } } @@ -482,11 +484,7 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) Cache.deleteDirectoryTree(stdpath.Join(dirKey, srcRawObj.GetName())) } if newObj == nil { - newObj = &struct { - model.ObjWrapName - }{ - model.ObjWrapName{Name: dstName, Obj: model.ObjAddMask(srcObj, model.Temp)}, - } + newObj = &model.ObjWrapMask{Obj: &model.ObjWrapName{Name: dstName, Obj: srcObj}, Mask: model.Temp} } newObj = wrapObjName(storage, newObj) cache.UpdateObject(srcRawObj.GetName(), newObj) @@ -551,9 +549,11 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(dstKey); exist { if newObj == nil { - newObj = model.ObjAddMask(srcObj, model.Temp) + newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp} + } else { + newObj = wrapObjName(storage, newObj) } - cache.UpdateObject(srcRawObj.GetName(), wrapObjName(storage, newObj)) + cache.UpdateObject(srcRawObj.GetName(), newObj) } } diff --git a/internal/op/storage.go b/internal/op/storage.go index 4c28d4fc8..9f79dd078 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -416,7 +416,7 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver } if len(names) == 1 { idx := len(files) - obj.Mask = model.Static + obj.Mask = model.Locked | model.Virtual files = append(files, obj) wg.Add(1) go func() { @@ -424,7 +424,7 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver files[idx] = rootCallback(v, files[idx]) }() } else { - obj.Mask = model.Virtual + obj.Mask = model.ReadOnly | model.Virtual files = append(files, obj) } } From eacef0032f99ec6be6f414f68860e55d2026f26d Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 14 Dec 2025 10:37:59 +0800 Subject: [PATCH 13/30] feat(fs): enhance virtual file handling --- internal/fs/get.go | 9 +++--- internal/fs/list.go | 2 +- internal/model/obj.go | 4 +-- internal/op/storage.go | 71 ++++++++++++++++++++++++++---------------- 4 files changed, 51 insertions(+), 35 deletions(-) diff --git a/internal/fs/get.go b/internal/fs/get.go index 459282ce3..8a920065e 100644 --- a/internal/fs/get.go +++ b/internal/fs/get.go @@ -3,7 +3,6 @@ package fs import ( "context" stdpath "path" - "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -15,9 +14,10 @@ func get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) { path = utils.FixAndCleanPath(path) // maybe a virtual file if path != "/" { - virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, stdpath.Dir(path), !args.WithStorageDetails, false) + dir, name := stdpath.Split(path) + virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, dir, !args.WithStorageDetails, false, name) for _, f := range virtualFiles { - if f.GetName() == stdpath.Base(path) { + if f.GetName() == name { return f, nil } } @@ -28,9 +28,8 @@ func get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) { if path == "/" { return &model.Object{ Name: "root", - Size: 0, - Modified: time.Time{}, IsFolder: true, + Mask: model.ReadOnly | model.Virtual, }, nil } return nil, errors.WithMessage(err, "failed get storage") diff --git a/internal/fs/list.go b/internal/fs/list.go index fc3b25ab5..1f92c7d46 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -15,7 +15,7 @@ import ( func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) { meta, _ := ctx.Value(conf.MetaKey).(*model.Meta) user, _ := ctx.Value(conf.UserKey).(*model.User) - virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails, args.Refresh) + virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails, args.Refresh, "") storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil && len(virtualFiles) == 0 { return nil, errors.WithMessage(err, "failed get storage") diff --git a/internal/model/obj.go b/internal/model/obj.go index f0dc9c4a2..ba7467f74 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -243,13 +243,13 @@ func (m ObjMask) GetObjMask() ObjMask { } const ( - Temp ObjMask = 1 << iota - Virtual + Virtual ObjMask = 1 << iota NoRename NoRemove NoMove NoCopy NoWrite + Temp ) const ( Locked = NoRename | NoRemove | NoMove diff --git a/internal/op/storage.go b/internal/op/storage.go index 9f79dd078..90e231b4d 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -341,14 +341,12 @@ func getStoragesByPath(path string) []driver.Driver { // for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av // GetStorageVirtualFilesByPath(/a) => b,c,d func GetStorageVirtualFilesByPath(prefix string) []model.Obj { - return getStorageVirtualFilesByPath(prefix, func(_ driver.Driver, obj model.Obj) model.Obj { - return obj - }) + return getStorageVirtualFilesByPath(prefix, nil, "") } -func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, hideDetails, refresh bool) []model.Obj { +func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, hideDetails, refresh bool, filterByName string) []model.Obj { if hideDetails { - return GetStorageVirtualFilesByPath(prefix) + return getStorageVirtualFilesByPath(prefix, nil, filterByName) } return getStorageVirtualFilesByPath(prefix, func(d driver.Driver, obj model.Obj) model.Obj { ret := &model.ObjStorageDetails{ @@ -374,10 +372,10 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, case <-time.After(time.Second): } return ret - }) + }, filterByName) } -func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver, model.Obj) model.Obj) []model.Obj { +func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver, model.Obj) model.Obj, filterByName string) []model.Obj { files := make([]model.Obj, 0) storages := storagesMap.Values() sort.Slice(storages, func(i, j int) bool { @@ -391,44 +389,63 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver set := make(map[string]int) var wg sync.WaitGroup for _, v := range storages { - mountPath := utils.GetActualMountPath(v.GetStorage().MountPath) // Exclude prefix itself and non prefix - if len(prefix) >= len(mountPath) || !utils.IsSubPath(prefix, mountPath) { + p, found := strings.CutPrefix(utils.GetActualMountPath(v.GetStorage().MountPath), prefix) + if !found || p == "" { continue } - names := strings.SplitN(strings.TrimPrefix(mountPath[len(prefix):], "/"), "/", 2) - if idx, ok := set[names[0]]; ok { - if len(names) == 1 { - wg.Add(1) - go func() { - defer wg.Done() - files[idx] = rootCallback(v, files[idx]) - }() + slashIdx := strings.Index(p, "/") + var name string + switch slashIdx { + case 0: + name, _, found = strings.Cut(p[1:], "/") + case -1: + name, found = p, false + default: + continue + } + if filterByName != "" && name != filterByName { + continue + } + + if idx, ok := set[name]; ok { + if !found { + files[idx].(*model.Object).Mask = model.Locked | model.Virtual + if rootCallback != nil { + wg.Add(1) + go func() { + defer wg.Done() + files[idx] = rootCallback(v, files[idx]) + }() + } } continue } - set[names[0]] = len(files) + set[name] = len(files) obj := &model.Object{ - Name: names[0], - Size: 0, + Name: name, Modified: v.GetStorage().Modified, IsFolder: true, } - if len(names) == 1 { + if !found { idx := len(files) obj.Mask = model.Locked | model.Virtual files = append(files, obj) - wg.Add(1) - go func() { - defer wg.Done() - files[idx] = rootCallback(v, files[idx]) - }() + if rootCallback != nil { + wg.Add(1) + go func() { + defer wg.Done() + files[idx] = rootCallback(v, files[idx]) + }() + } } else { obj.Mask = model.ReadOnly | model.Virtual files = append(files, obj) } } - wg.Wait() + if rootCallback != nil { + wg.Wait() + } return files } From 5f18e458a7a61fe39d1b29758d982e39aa6e1d96 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 14 Dec 2025 19:55:34 +0800 Subject: [PATCH 14/30] feat(storage): enhance virtual file retrieval and path handling --- internal/op/storage.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/internal/op/storage.go b/internal/op/storage.go index 90e231b4d..20abfa49d 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -349,6 +349,9 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, return getStorageVirtualFilesByPath(prefix, nil, filterByName) } return getStorageVirtualFilesByPath(prefix, func(d driver.Driver, obj model.Obj) model.Obj { + if _, ok := obj.(*model.ObjStorageDetails); ok { + return obj + } ret := &model.ObjStorageDetails{ Obj: obj, StorageDetailsWithName: model.StorageDetailsWithName{ @@ -385,7 +388,9 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver return storages[i].GetStorage().Order < storages[j].GetStorage().Order }) - prefix = utils.FixAndCleanPath(prefix) + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } set := make(map[string]int) var wg sync.WaitGroup for _, v := range storages { @@ -394,16 +399,7 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver if !found || p == "" { continue } - slashIdx := strings.Index(p, "/") - var name string - switch slashIdx { - case 0: - name, _, found = strings.Cut(p[1:], "/") - case -1: - name, found = p, false - default: - continue - } + name, _, found := strings.Cut(p, "/") if filterByName != "" && name != filterByName { continue } From 652b57cbdfa9804dfde3763b8b63798849609596 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 14 Dec 2025 19:56:16 +0800 Subject: [PATCH 15/30] refactor(alias): rename path handling functions for clarity and consistency --- drivers/alias/driver.go | 55 ++++++++++++++++++-------------------- drivers/alias/util.go | 59 +++++++++++++++++++++-------------------- 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index bfdabb089..455cadeb8 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -269,7 +269,7 @@ func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, e } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - objs, err := d.getWritePath(ctx, parentDir) + objs, err := d.getWriteObjs(ctx, parentDir) if err == nil { for _, obj := range objs { err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(obj.GetPath(), dirName))) @@ -279,20 +279,19 @@ func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) - if err != nil { - return err - } - for i, src := range srcs { - dst := dsts[i] - _, e := fs.Move(ctx, src, dst) - err = errors.Join(err, e) + srcs, dsts, err := d.getCopyMoveObjs(ctx, srcObj, dstDir) + if err == nil { + for i, src := range srcs { + dst := dsts[i] + _, e := fs.Move(ctx, src.GetPath(), dst.GetPath()) + err = errors.Join(err, e) + } } return err } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - objs, err := d.getWritePath(ctx, srcObj) + objs, err := d.getWriteObjs(ctx, srcObj) if err == nil { for _, obj := range objs { err = errors.Join(err, fs.Rename(ctx, obj.GetPath(), newName)) @@ -302,20 +301,19 @@ func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) er } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) - if err != nil { - return err - } - for i, src := range srcs { - dst := dsts[i] - _, e := fs.Copy(ctx, src, dst) - err = errors.Join(err, e) + srcs, dsts, err := d.getCopyMoveObjs(ctx, srcObj, dstDir) + if err == nil { + for i, src := range srcs { + dst := dsts[i] + _, e := fs.Copy(ctx, src.GetPath(), dst.GetPath()) + err = errors.Join(err, e) + } } return err } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { - objs, err := d.getWritePath(ctx, obj) + objs, err := d.getWriteObjs(ctx, obj) if err == nil { for _, obj := range objs { err = errors.Join(err, fs.Remove(ctx, obj.GetPath())) @@ -325,7 +323,7 @@ func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - objs, err := d.getPutPath(ctx, dstDir) + objs, err := d.getPutObjs(ctx, dstDir) if err == nil { if len(objs) == 1 { storage, reqActualPath, err := op.GetStorageAndActualPath(objs.GetPath()) @@ -363,7 +361,7 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { - objs, err := d.getPutPath(ctx, dstDir) + objs, err := d.getPutObjs(ctx, dstDir) if err == nil { for _, obj := range objs { err = errors.Join(err, fs.PutURL(ctx, obj.GetPath(), name, url)) @@ -425,14 +423,13 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn } func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { - srcs, dsts, err := d.getCopyMovePath(ctx, srcObj, dstDir) - if err != nil { - return err - } - for i, src := range srcs { - dst := dsts[i] - _, e := fs.ArchiveDecompress(ctx, src, dst, args) - err = errors.Join(err, e) + srcs, dsts, err := d.getCopyMoveObjs(ctx, srcObj, dstDir) + if err == nil { + for i, src := range srcs { + dst := dsts[i] + _, e := fs.ArchiveDecompress(ctx, src.GetPath(), dst.GetPath(), args) + err = errors.Join(err, e) + } } return err } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 53e165a39..ec80c0895 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -187,7 +187,7 @@ func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { } else { switch policy { case FirstRWP: - return true, nil + return false, nil case DeterministicWP: if l > 0 { return false, ErrPathConflict @@ -203,14 +203,14 @@ func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { } } -func (d *Alias) getWritePath(ctx context.Context, obj model.Obj) (BalancedObjs, error) { +func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.WriteConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } return getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) } -func (d *Alias) getPutPath(ctx context.Context, obj model.Obj) (BalancedObjs, error) { +func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } @@ -227,7 +227,7 @@ func (d *Alias) getPutPath(ctx context.Context, obj model.Obj) (BalancedObjs, er strict = true fallthrough case BalancedByQuotaP: - objs, ok := getRandomPathByQuotaBalanced(ctx, objs, strict, uint64(obj.GetSize())) + objs, ok := getRandomObjByQuotaBalanced(ctx, objs, strict, uint64(obj.GetSize())) if !ok { return nil, ErrNoEnoughSpace } @@ -237,7 +237,7 @@ func (d *Alias) getPutPath(ctx context.Context, obj model.Obj) (BalancedObjs, er } } -func getRandomPathByQuotaBalanced(ctx context.Context, reqPath BalancedObjs, strict bool, objSize uint64) (BalancedObjs, bool) { +func getRandomObjByQuotaBalanced(ctx context.Context, reqPath BalancedObjs, strict bool, objSize uint64) (BalancedObjs, bool) { // Get all space details := make([]*model.StorageDetails, len(reqPath)) detailsChan := make(chan detailWithIndex, len(reqPath)) @@ -317,7 +317,7 @@ func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) return 0, false } -func (d *Alias) getCopyMovePath(ctx context.Context, srcObj, dstDir model.Obj) ([]string, []string, error) { +func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } @@ -325,49 +325,50 @@ func (d *Alias) getCopyMovePath(ctx context.Context, srcObj, dstDir model.Obj) ( if err != nil { return nil, nil, err } - dstStorageMap := make(map[string][]string) - allocatingDst := make(map[string]struct{}) + dstStorageMap := make(map[string][]model.Obj) + allocatingDst := make(map[model.Obj]struct{}) for _, o := range dstObjs { - dp := o.GetPath() - storage, e := fs.GetStorage(dp, &fs.GetStoragesArgs{}) + storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) if e != nil { - return nil, nil, errors.WithMessagef(e, "cannot copy or move to virtual path [%s]", dp) + return nil, nil, errors.WithMessagef(e, "cannot copy or move to virtual path [%s]", o.GetPath()) } mp := utils.GetActualMountPath(storage.GetStorage().MountPath) - dstStorageMap[mp] = append(dstStorageMap[mp], dp) - allocatingDst[dp] = struct{}{} + dstStorageMap[mp] = append(dstStorageMap[mp], o) + allocatingDst[o] = struct{}{} } srcObjs, err := getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) if err != nil { return nil, nil, err } - srcs := make([]string, 0) - dsts := make([]string, 0) - for _, o := range srcObjs { - sp := o.GetPath() - storage, e := fs.GetStorage(sp, &fs.GetStoragesArgs{}) + srcIdx, dstIdx := 0, 0 + for _, src := range srcObjs { + storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) if e != nil { continue } mp := utils.GetActualMountPath(storage.GetStorage().MountPath) - if dstPaths, ok := dstStorageMap[mp]; ok { - for _, dp := range dstPaths { - srcs = append(srcs, sp) - dsts = append(dsts, dp) - delete(allocatingDst, dp) + if tmp, ok := dstStorageMap[mp]; ok { + for _, dst := range tmp { + srcObjs[srcIdx] = src + srcIdx++ + dstObjs[dstIdx] = dst + dstIdx++ + delete(allocatingDst, dst) } delete(dstStorageMap, mp) } } - for dp := range allocatingDst { - sp := srcs[0] + srcObjs = srcObjs[:srcIdx] + dstObjs = dstObjs[:dstIdx] + for dst := range allocatingDst { + src := srcObjs[0] if d.ReadConflictPolicy == RandomBalancedRP { - sp = srcs[rand.Intn(len(srcs))] + src = srcObjs[rand.Intn(len(srcObjs[:srcIdx]))] } - srcs = append(srcs, sp) - dsts = append(dsts, dp) + srcObjs = append(srcObjs, src) + dstObjs = append(dstObjs, dst) } - return srcs, dsts, nil + return srcObjs, dstObjs, nil } func (d *Alias) getArchiveMeta(ctx context.Context, reqPath string, args model.ArchiveArgs) (model.ArchiveMeta, error) { From a24ff2b50b921e13cce9bf1ac2bf30973435c01d Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 14 Dec 2025 19:59:17 +0800 Subject: [PATCH 16/30] fix(alias): update path handling in Other method to use balanced path --- drivers/alias/driver.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 455cadeb8..2f3314ad2 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -254,10 +254,8 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { - if d.ReadConflictPolicy != FirstRWP { - return nil, errs.NotSupport - } - storage, actualPath, err := op.GetStorageAndActualPath(args.Obj.GetPath()) + reqPath := d.getBalancedPath(ctx, args.Obj) + storage, actualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err } From cb83366d99f7b50092a2a523d245760e8fa4a6fc Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 14 Dec 2025 20:27:13 +0800 Subject: [PATCH 17/30] fix bug --- drivers/alias/util.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/drivers/alias/util.go b/drivers/alias/util.go index ec80c0895..f9902ae75 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -336,12 +336,12 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( dstStorageMap[mp] = append(dstStorageMap[mp], o) allocatingDst[o] = struct{}{} } - srcObjs, err := getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) + tmpSrcObjs, err := getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) if err != nil { return nil, nil, err } - srcIdx, dstIdx := 0, 0 - for _, src := range srcObjs { + srcObjs := make(BalancedObjs, 0, len(dstObjs)) + for _, src := range tmpSrcObjs { storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) if e != nil { continue @@ -349,21 +349,18 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( mp := utils.GetActualMountPath(storage.GetStorage().MountPath) if tmp, ok := dstStorageMap[mp]; ok { for _, dst := range tmp { - srcObjs[srcIdx] = src - srcIdx++ - dstObjs[dstIdx] = dst - dstIdx++ + dstObjs[len(srcObjs)] = dst + srcObjs = append(srcObjs, src) delete(allocatingDst, dst) } delete(dstStorageMap, mp) } } - srcObjs = srcObjs[:srcIdx] - dstObjs = dstObjs[:dstIdx] + dstObjs = dstObjs[:len(srcObjs)] for dst := range allocatingDst { - src := srcObjs[0] + src := tmpSrcObjs[0] if d.ReadConflictPolicy == RandomBalancedRP { - src = srcObjs[rand.Intn(len(srcObjs[:srcIdx]))] + src = tmpSrcObjs[rand.Intn(len(tmpSrcObjs))] } srcObjs = append(srcObjs, src) dstObjs = append(dstObjs, dst) From 4d9e68b10d679ce1c2991d8b5ddae0a9a6881907 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sun, 14 Dec 2025 21:12:38 +0800 Subject: [PATCH 18/30] feat(alias): add file size validation --- drivers/alias/meta.go | 4 +--- drivers/alias/util.go | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 486b1025d..219f4f5c4 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -6,11 +6,9 @@ import ( ) type Addition struct { - // Usually one of two - // driver.RootPath - // define other Paths string `json:"paths" required:"true" type:"text"` ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random" default:"first"` + FileSizeStrict bool `json:"file_size_strict" type:"bool" default:"false"` WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled"` PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled"` DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` diff --git a/drivers/alias/util.go b/drivers/alias/util.go index f9902ae75..ed02b4093 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -111,17 +111,26 @@ func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) ( return op.Link(ctx, storage, reqActualPath, args) } -func getAllObjs(ctx context.Context, obj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { +func (d *Alias) getAllObjs(ctx context.Context, obj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { objs := obj.(BalancedObjs) isDir := obj.IsDir() + size := obj.GetSize() length := 0 for _, o := range objs { var err error temp, isTemp := o.(*tempObj) if isTemp { obj, err = fs.Get(ctx, o.GetPath(), &fs.GetArgs{NoLog: true}) - if err == nil && isDir != obj.IsDir() { - err = errs.ObjectNotFound + if err == nil { + if !isDir { + if obj.IsDir() { + err = errs.NotFile + } else if d.FileSizeStrict && size != obj.GetSize() { + err = errs.ObjectNotFound + } + } else if !obj.IsDir() { + err = errs.NotFolder + } } } @@ -161,7 +170,7 @@ func (d *Alias) getBalancedPath(ctx context.Context, file model.Obj) string { if rand.Intn(len(files)) == 0 { return file.GetPath() } - files, _ = getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllWP)) + files, _ = d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllWP)) return files[rand.Intn(len(files))].GetPath() } @@ -207,14 +216,14 @@ func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) (BalancedObjs, if d.WriteConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } - return getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) + return d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) } func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } - objs, err := getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + objs, err := d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, err } @@ -321,7 +330,7 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } - dstObjs, err := getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, nil, err } @@ -336,7 +345,7 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( dstStorageMap[mp] = append(dstStorageMap[mp], o) allocatingDst[o] = struct{}{} } - tmpSrcObjs, err := getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) + tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) if err != nil { return nil, nil, err } From 90360a4390b8f9a5a1cc7aee8455bbf572e0e6d4 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 15 Dec 2025 20:05:15 +0800 Subject: [PATCH 19/30] feat(alias): add hash consistency check --- drivers/alias/meta.go | 18 +++++++++--------- drivers/alias/types.go | 15 ++++++++++++++- drivers/alias/util.go | 26 +++++++++++++++++++------- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 219f4f5c4..3271ca625 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -6,15 +6,15 @@ import ( ) type Addition struct { - Paths string `json:"paths" required:"true" type:"text"` - ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random" default:"first"` - FileSizeStrict bool `json:"file_size_strict" type:"bool" default:"false"` - WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled"` - PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled"` - DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` - DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` - ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` - DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` + Paths string `json:"paths" required:"true" type:"text"` + ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random" default:"first"` + WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled"` + PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled"` + FileConsistencyCheck bool `json:"file_consistency_check" type:"bool" default:"false"` + DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` + DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` + ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` + DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` } var config = driver.Config{ diff --git a/drivers/alias/types.go b/drivers/alias/types.go index 201a2d36f..cab1260d1 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -39,24 +39,37 @@ type BalancedObjs []model.Obj func (b BalancedObjs) GetSize() int64 { return b[0].GetSize() } + func (b BalancedObjs) ModTime() time.Time { return b[0].ModTime() } + func (b BalancedObjs) CreateTime() time.Time { return b[0].CreateTime() } + func (b BalancedObjs) IsDir() bool { return b[0].IsDir() } + func (b BalancedObjs) GetHash() utils.HashInfo { - return b[0].GetHash() + ret := make(map[*utils.HashType]string) + for _, o := range b { + for ht, v := range o.GetHash().All() { + ret[ht] = v + } + } + return utils.NewHashInfoByMap(ret) } + func (b BalancedObjs) GetName() string { return b[0].GetName() } + func (b BalancedObjs) GetPath() string { return b[0].GetPath() } + func (b BalancedObjs) GetID() string { return b[0].GetID() } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index ed02b4093..8744a4749 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -111,25 +111,37 @@ func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) ( return op.Link(ctx, storage, reqActualPath, args) } -func (d *Alias) getAllObjs(ctx context.Context, obj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { - objs := obj.(BalancedObjs) - isDir := obj.IsDir() - size := obj.GetSize() +func isConsistent(a, b model.Obj) bool { + if a.GetSize() != b.GetSize() { + return false + } + for ht, v := range b.GetHash().All() { + ah := a.GetHash().GetHash(ht) + if ah != "" && ah != v { + return false + } + } + return true +} + +func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { + objs := bObj.(BalancedObjs) length := 0 for _, o := range objs { var err error + var obj model.Obj temp, isTemp := o.(*tempObj) if isTemp { obj, err = fs.Get(ctx, o.GetPath(), &fs.GetArgs{NoLog: true}) if err == nil { - if !isDir { + if !bObj.IsDir() { if obj.IsDir() { err = errs.NotFile - } else if d.FileSizeStrict && size != obj.GetSize() { + } else if d.FileConsistencyCheck && !isConsistent(bObj, obj) { err = errs.ObjectNotFound } } else if !obj.IsDir() { - err = errs.NotFolder + err = errs.NotFile } } } From bf3fb7e7f7ae38ca7cc078f5df294506e02464c9 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 15 Dec 2025 22:22:11 +0800 Subject: [PATCH 20/30] =?UTF-8?q?=E7=A7=BB=E9=99=A4=E5=93=88=E5=B8=8C?= =?UTF-8?q?=E5=90=88=E5=B9=B6=EF=BC=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/alias/types.go | 8 +------- drivers/alias/util.go | 6 +++--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/drivers/alias/types.go b/drivers/alias/types.go index cab1260d1..10d80f200 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -53,13 +53,7 @@ func (b BalancedObjs) IsDir() bool { } func (b BalancedObjs) GetHash() utils.HashInfo { - ret := make(map[*utils.HashType]string) - for _, o := range b { - for ht, v := range o.GetHash().All() { - ret[ht] = v - } - } - return utils.NewHashInfoByMap(ret) + return b[0].GetHash() } func (b BalancedObjs) GetName() string { diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 8744a4749..9b8e06ca6 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -115,8 +115,8 @@ func isConsistent(a, b model.Obj) bool { if a.GetSize() != b.GetSize() { return false } - for ht, v := range b.GetHash().All() { - ah := a.GetHash().GetHash(ht) + for ht, v := range a.GetHash().All() { + ah := b.GetHash().GetHash(ht) if ah != "" && ah != v { return false } @@ -141,7 +141,7 @@ func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func( err = errs.ObjectNotFound } } else if !obj.IsDir() { - err = errs.NotFile + err = errs.NotFolder } } } From f001f2dcd76d5711d2b1f44d04edf646f32c75a7 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 20 Dec 2025 00:47:46 +0800 Subject: [PATCH 21/30] fix(alias): wrong behavior for all_strict/deterministic_or_all --- drivers/alias/driver.go | 28 ++++++++++++++------- drivers/alias/types.go | 44 ++++++++++++++++++--------------- drivers/alias/util.go | 55 ++++++++++++++++++++++++----------------- 3 files changed, 75 insertions(+), 52 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 2f3314ad2..f3da5f8f9 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -58,8 +58,12 @@ func (d *Alias) Init(ctx context.Context) error { return errors.New("paths is required") case 1: paths := d.pathMap[d.rootOrder[0]] - roots := make(BalancedObjs, 0, len(paths)) - roots = append(roots, &model.Object{ + roots := &BalancedObjs{ + hasFailed: false, + unmappedPath: "/", + objs: make([]model.Obj, 0, len(paths)), + } + roots.objs = append(roots.objs, &model.Object{ Name: "root", Path: paths[0], IsFolder: true, @@ -67,7 +71,7 @@ func (d *Alias) Init(ctx context.Context) error { Mask: model.Locked, }) for _, path := range paths[1:] { - roots = append(roots, &model.Object{ + roots.objs = append(roots.objs, &model.Object{ Path: path, }) } @@ -114,10 +118,12 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { if len(roots) == 0 { return nil, errs.ObjectNotFound } + hasFailed := false for idx, root := range roots { rawPath := stdpath.Join(root, sub) obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) if err != nil { + hasFailed = true continue } mask := model.GetObjMask(obj) &^ model.Temp @@ -147,10 +153,14 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } roots = roots[idx+1:] - objs := make(BalancedObjs, 0, len(roots)+1) - objs = append(objs, obj) + objs := &BalancedObjs{ + objs: make([]model.Obj, 0, len(roots)+1), + hasFailed: hasFailed, + unmappedPath: path, + } + objs.objs = append(objs.objs, obj) for _, d := range roots { - objs = append(objs, &tempObj{model.Object{ + objs.objs = append(objs.objs, &tempObj{model.Object{ Path: stdpath.Join(d, sub), }}) } @@ -160,7 +170,7 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - dirs, ok := dir.(BalancedObjs) + dirs, ok := dir.(*BalancedObjs) if !ok { return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil } @@ -168,7 +178,7 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ // 因为alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 // 所以这里对象不会传回到alias，也就不需要返回BalancedObjs了 objMap := make(map[string]model.Obj) - for _, dir := range dirs { + for _, dir := range dirs.objs { dirPath := dir.GetPath() tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ NoLog: true, @@ -324,7 +334,7 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, objs, err := d.getPutObjs(ctx, dstDir) if err == nil { if len(objs) == 1 { - storage, reqActualPath, err := op.GetStorageAndActualPath(objs.GetPath()) + storage, reqActualPath, err := op.GetStorageAndActualPath(objs[0].GetPath()) if err != nil { return err } diff --git a/drivers/alias/types.go b/drivers/alias/types.go index 10d80f200..2291c8927 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -34,44 +34,48 @@ var ( ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") ) -type BalancedObjs []model.Obj +type BalancedObjs struct { + objs []model.Obj + hasFailed bool + unmappedPath string +} -func (b BalancedObjs) GetSize() int64 { - return b[0].GetSize() +func (b *BalancedObjs) GetSize() int64 { + return b.objs[0].GetSize() } -func (b BalancedObjs) ModTime() time.Time { - return b[0].ModTime() +func (b *BalancedObjs) ModTime() time.Time { + return b.objs[0].ModTime() } -func (b BalancedObjs) CreateTime() time.Time { - return b[0].CreateTime() +func (b *BalancedObjs) CreateTime() time.Time { + return b.objs[0].CreateTime() } -func (b BalancedObjs) IsDir() bool { - return b[0].IsDir() +func (b *BalancedObjs) IsDir() bool { + return b.objs[0].IsDir() } -func (b BalancedObjs) GetHash() utils.HashInfo { - return b[0].GetHash() +func (b *BalancedObjs) GetHash() utils.HashInfo { + return b.objs[0].GetHash() } -func (b BalancedObjs) GetName() string { - return b[0].GetName() +func (b *BalancedObjs) GetName() string { + return b.objs[0].GetName() } -func (b BalancedObjs) GetPath() string { - return b[0].GetPath() +func (b *BalancedObjs) GetPath() string { + return b.objs[0].GetPath() } -func (b BalancedObjs) GetID() string { - return b[0].GetID() +func (b *BalancedObjs) GetID() string { + return b.objs[0].GetID() } -func (b BalancedObjs) Unwrap() model.Obj { - return b[0] +func (b *BalancedObjs) Unwrap() model.Obj { + return b.objs[0] } -var _ model.Obj = (BalancedObjs)(nil) +var _ model.Obj = (*BalancedObjs)(nil) type tempObj struct{ model.Object } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 9b8e06ca6..65369e4a3 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -124,20 +124,19 @@ func isConsistent(a, b model.Obj) bool { return true } -func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { - objs := bObj.(BalancedObjs) +func (d *Alias) getAllObjs(ctx context.Context, objs *BalancedObjs, ifContinue func(err error) (bool, error)) ([]model.Obj, error) { length := 0 - for _, o := range objs { + for _, o := range objs.objs { var err error var obj model.Obj temp, isTemp := o.(*tempObj) if isTemp { obj, err = fs.Get(ctx, o.GetPath(), &fs.GetArgs{NoLog: true}) if err == nil { - if !bObj.IsDir() { + if !objs.IsDir() { if obj.IsDir() { err = errs.NotFile - } else if d.FileConsistencyCheck && !isConsistent(bObj, obj) { + } else if d.FileConsistencyCheck && !isConsistent(objs, obj) { err = errs.ObjectNotFound } } else if !obj.IsDir() { @@ -159,9 +158,9 @@ func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func( // objRes.Size = obj.GetSize() // objRes.Modified = obj.ModTime() // objRes.HashInfo = obj.GetHash() - objs[length] = &objRes + objs.objs[length] = &objRes } else { - objs[length] = o + objs.objs[length] = o } length++ if !cont { @@ -171,28 +170,28 @@ func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func( if length == 0 { return nil, errs.ObjectNotFound } - return objs[:length], nil + return objs.objs[:length], nil } func (d *Alias) getBalancedPath(ctx context.Context, file model.Obj) string { if d.ReadConflictPolicy == FirstRWP { return file.GetPath() } - files := file.(BalancedObjs) - if rand.Intn(len(files)) == 0 { + files := file.(*BalancedObjs) + if rand.Intn(len(files.objs)) == 0 { return file.GetPath() } - files, _ = d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllWP)) - return files[rand.Intn(len(files))].GetPath() + filesObjs, _ := d.getAllObjs(ctx, files, getWriteAndPutFilterFunc(AllWP, files.hasFailed)) + return filesObjs[rand.Intn(len(files.objs))].GetPath() } -func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { +func getWriteAndPutFilterFunc(policy string, hasFailed bool) func(error) (bool, error) { if policy == AllWP { return func(err error) (bool, error) { return true, err } } - all := true + all := !hasFailed l := 0 return func(err error) (bool, error) { if err != nil { @@ -224,18 +223,26 @@ func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { } } -func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { +func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) ([]model.Obj, error) { if d.WriteConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } - return d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) + bObj := obj.(*BalancedObjs) + if bObj.hasFailed && d.WriteConflictPolicy == AllStrictWP { + return nil, ErrSamePathLeak + } + return d.getAllObjs(ctx, bObj, getWriteAndPutFilterFunc(d.WriteConflictPolicy, bObj.hasFailed)) } -func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { +func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) ([]model.Obj, error) { if d.PutConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } - objs, err := d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + bObj := obj.(*BalancedObjs) + if bObj.hasFailed && d.PutConflictPolicy == AllStrictWP { + return nil, ErrSamePathLeak + } + objs, err := d.getAllObjs(ctx, bObj, getWriteAndPutFilterFunc(d.PutConflictPolicy, bObj.hasFailed)) if err != nil { return nil, err } @@ -258,7 +265,7 @@ func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) (BalancedObjs, er } } -func getRandomObjByQuotaBalanced(ctx context.Context, reqPath BalancedObjs, strict bool, objSize uint64) (BalancedObjs, bool) { +func getRandomObjByQuotaBalanced(ctx context.Context, reqPath []model.Obj, strict bool, objSize uint64) ([]model.Obj, bool) { // Get all space details := make([]*model.StorageDetails, len(reqPath)) detailsChan := make(chan detailWithIndex, len(reqPath)) @@ -338,11 +345,12 @@ func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) return 0, false } -func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { +func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ([]model.Obj, []model.Obj, error) { if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } - dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + bDstDir := dstDir.(*BalancedObjs) + dstObjs, err := d.getAllObjs(ctx, bDstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy, bDstDir.hasFailed)) if err != nil { return nil, nil, err } @@ -357,11 +365,12 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( dstStorageMap[mp] = append(dstStorageMap[mp], o) allocatingDst[o] = struct{}{} } - tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) + bSrcObj := srcObj.(*BalancedObjs) + tmpSrcObjs, err := d.getAllObjs(ctx, bSrcObj, getWriteAndPutFilterFunc(AllWP, bSrcObj.hasFailed)) if err != nil { return nil, nil, err } - srcObjs := make(BalancedObjs, 0, len(dstObjs)) + srcObjs := make([]model.Obj, 0, len(dstObjs)) for _, src := range tmpSrcObjs { storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) if e != nil { From 4c05e21c5ad04090846b3c3d0d241c46ff768287 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 20 Dec 2025 07:27:13 +0800 Subject: [PATCH 22/30] Revert "fix(alias): wrong behavior for all_strict/deterministic_or_all" This reverts commit f001f2dcd76d5711d2b1f44d04edf646f32c75a7. --- drivers/alias/driver.go | 28 +++++++-------------- drivers/alias/types.go | 44 +++++++++++++++------------------ drivers/alias/util.go | 55 +++++++++++++++++------------------------ 3 files changed, 52 insertions(+), 75 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index f3da5f8f9..2f3314ad2 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -58,12 +58,8 @@ func (d *Alias) Init(ctx context.Context) error { return errors.New("paths is required") case 1: paths := d.pathMap[d.rootOrder[0]] - roots := &BalancedObjs{ - hasFailed: false, - unmappedPath: "/", - objs: make([]model.Obj, 0, len(paths)), - } - roots.objs = append(roots.objs, &model.Object{ + roots := make(BalancedObjs, 0, len(paths)) + roots = append(roots, &model.Object{ Name: "root", Path: paths[0], IsFolder: true, @@ -71,7 +67,7 @@ func (d *Alias) Init(ctx context.Context) error { Mask: model.Locked, }) for _, path := range paths[1:] { - roots.objs = append(roots.objs, &model.Object{ + roots = append(roots, &model.Object{ Path: path, }) } @@ -118,12 +114,10 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { if len(roots) == 0 { return nil, errs.ObjectNotFound } - hasFailed := false for idx, root := range roots { rawPath := stdpath.Join(root, sub) obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) if err != nil { - hasFailed = true continue } mask := model.GetObjMask(obj) &^ model.Temp @@ -153,14 +147,10 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } roots = roots[idx+1:] - objs := &BalancedObjs{ - objs: make([]model.Obj, 0, len(roots)+1), - hasFailed: hasFailed, - unmappedPath: path, - } - objs.objs = append(objs.objs, obj) + objs := make(BalancedObjs, 0, len(roots)+1) + objs = append(objs, obj) for _, d := range roots { - objs.objs = append(objs.objs, &tempObj{model.Object{ + objs = append(objs, &tempObj{model.Object{ Path: stdpath.Join(d, sub), }}) } @@ -170,7 +160,7 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - dirs, ok := dir.(*BalancedObjs) + dirs, ok := dir.(BalancedObjs) if !ok { return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil } @@ -178,7 +168,7 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ // 因为alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 // 所以这里对象不会传回到alias，也就不需要返回BalancedObjs了 objMap := make(map[string]model.Obj) - for _, dir := range dirs.objs { + for _, dir := range dirs { dirPath := dir.GetPath() tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ NoLog: true, @@ -334,7 +324,7 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, objs, err := d.getPutObjs(ctx, dstDir) if err == nil { if len(objs) == 1 { - storage, reqActualPath, err := op.GetStorageAndActualPath(objs[0].GetPath()) + storage, reqActualPath, err := op.GetStorageAndActualPath(objs.GetPath()) if err != nil { return err } diff --git a/drivers/alias/types.go b/drivers/alias/types.go index 2291c8927..10d80f200 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -34,48 +34,44 @@ var ( ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") ) -type BalancedObjs struct { - objs []model.Obj - hasFailed bool - unmappedPath string -} +type BalancedObjs []model.Obj -func (b *BalancedObjs) GetSize() int64 { - return b.objs[0].GetSize() +func (b BalancedObjs) GetSize() int64 { + return b[0].GetSize() } -func (b *BalancedObjs) ModTime() time.Time { - return b.objs[0].ModTime() +func (b BalancedObjs) ModTime() time.Time { + return b[0].ModTime() } -func (b *BalancedObjs) CreateTime() time.Time { - return b.objs[0].CreateTime() +func (b BalancedObjs) CreateTime() time.Time { + return b[0].CreateTime() } -func (b *BalancedObjs) IsDir() bool { - return b.objs[0].IsDir() +func (b BalancedObjs) IsDir() bool { + return b[0].IsDir() } -func (b *BalancedObjs) GetHash() utils.HashInfo { - return b.objs[0].GetHash() +func (b BalancedObjs) GetHash() utils.HashInfo { + return b[0].GetHash() } -func (b *BalancedObjs) GetName() string { - return b.objs[0].GetName() +func (b BalancedObjs) GetName() string { + return b[0].GetName() } -func (b *BalancedObjs) GetPath() string { - return b.objs[0].GetPath() +func (b BalancedObjs) GetPath() string { + return b[0].GetPath() } -func (b *BalancedObjs) GetID() string { - return b.objs[0].GetID() +func (b BalancedObjs) GetID() string { + return b[0].GetID() } -func (b *BalancedObjs) Unwrap() model.Obj { - return b.objs[0] +func (b BalancedObjs) Unwrap() model.Obj { + return b[0] } -var _ model.Obj = (*BalancedObjs)(nil) +var _ model.Obj = (BalancedObjs)(nil) type tempObj struct{ model.Object } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 65369e4a3..9b8e06ca6 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -124,19 +124,20 @@ func isConsistent(a, b model.Obj) bool { return true } -func (d *Alias) getAllObjs(ctx context.Context, objs *BalancedObjs, ifContinue func(err error) (bool, error)) ([]model.Obj, error) { +func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { + objs := bObj.(BalancedObjs) length := 0 - for _, o := range objs.objs { + for _, o := range objs { var err error var obj model.Obj temp, isTemp := o.(*tempObj) if isTemp { obj, err = fs.Get(ctx, o.GetPath(), &fs.GetArgs{NoLog: true}) if err == nil { - if !objs.IsDir() { + if !bObj.IsDir() { if obj.IsDir() { err = errs.NotFile - } else if d.FileConsistencyCheck && !isConsistent(objs, obj) { + } else if d.FileConsistencyCheck && !isConsistent(bObj, obj) { err = errs.ObjectNotFound } } else if !obj.IsDir() { @@ -158,9 +159,9 @@ func (d *Alias) getAllObjs(ctx context.Context, objs *BalancedObjs, ifContinue f // objRes.Size = obj.GetSize() // objRes.Modified = obj.ModTime() // objRes.HashInfo = obj.GetHash() - objs.objs[length] = &objRes + objs[length] = &objRes } else { - objs.objs[length] = o + objs[length] = o } length++ if !cont { @@ -170,28 +171,28 @@ func (d *Alias) getAllObjs(ctx context.Context, objs *BalancedObjs, ifContinue f if length == 0 { return nil, errs.ObjectNotFound } - return objs.objs[:length], nil + return objs[:length], nil } func (d *Alias) getBalancedPath(ctx context.Context, file model.Obj) string { if d.ReadConflictPolicy == FirstRWP { return file.GetPath() } - files := file.(*BalancedObjs) - if rand.Intn(len(files.objs)) == 0 { + files := file.(BalancedObjs) + if rand.Intn(len(files)) == 0 { return file.GetPath() } - filesObjs, _ := d.getAllObjs(ctx, files, getWriteAndPutFilterFunc(AllWP, files.hasFailed)) - return filesObjs[rand.Intn(len(files.objs))].GetPath() + files, _ = d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllWP)) + return files[rand.Intn(len(files))].GetPath() } -func getWriteAndPutFilterFunc(policy string, hasFailed bool) func(error) (bool, error) { +func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { if policy == AllWP { return func(err error) (bool, error) { return true, err } } - all := !hasFailed + all := true l := 0 return func(err error) (bool, error) { if err != nil { @@ -223,26 +224,18 @@ func getWriteAndPutFilterFunc(policy string, hasFailed bool) func(error) (bool, } } -func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) ([]model.Obj, error) { +func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.WriteConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } - bObj := obj.(*BalancedObjs) - if bObj.hasFailed && d.WriteConflictPolicy == AllStrictWP { - return nil, ErrSamePathLeak - } - return d.getAllObjs(ctx, bObj, getWriteAndPutFilterFunc(d.WriteConflictPolicy, bObj.hasFailed)) + return d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) } -func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) ([]model.Obj, error) { +func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } - bObj := obj.(*BalancedObjs) - if bObj.hasFailed && d.PutConflictPolicy == AllStrictWP { - return nil, ErrSamePathLeak - } - objs, err := d.getAllObjs(ctx, bObj, getWriteAndPutFilterFunc(d.PutConflictPolicy, bObj.hasFailed)) + objs, err := d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, err } @@ -265,7 +258,7 @@ func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) ([]model.Obj, err } } -func getRandomObjByQuotaBalanced(ctx context.Context, reqPath []model.Obj, strict bool, objSize uint64) ([]model.Obj, bool) { +func getRandomObjByQuotaBalanced(ctx context.Context, reqPath BalancedObjs, strict bool, objSize uint64) (BalancedObjs, bool) { // Get all space details := make([]*model.StorageDetails, len(reqPath)) detailsChan := make(chan detailWithIndex, len(reqPath)) @@ -345,12 +338,11 @@ func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) return 0, false } -func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ([]model.Obj, []model.Obj, error) { +func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } - bDstDir := dstDir.(*BalancedObjs) - dstObjs, err := d.getAllObjs(ctx, bDstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy, bDstDir.hasFailed)) + dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, nil, err } @@ -365,12 +357,11 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( dstStorageMap[mp] = append(dstStorageMap[mp], o) allocatingDst[o] = struct{}{} } - bSrcObj := srcObj.(*BalancedObjs) - tmpSrcObjs, err := d.getAllObjs(ctx, bSrcObj, getWriteAndPutFilterFunc(AllWP, bSrcObj.hasFailed)) + tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) if err != nil { return nil, nil, err } - srcObjs := make([]model.Obj, 0, len(dstObjs)) + srcObjs := make(BalancedObjs, 0, len(dstObjs)) for _, src := range tmpSrcObjs { storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) if e != nil { From 6a828d3122db12947dcaa787702229ea84321be3 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 20 Dec 2025 08:02:29 +0800 Subject: [PATCH 23/30] fix(alias): wrong behavior for all_strict/deterministic_or_all --- drivers/alias/driver.go | 10 +++++++++- drivers/alias/util.go | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 2f3314ad2..a598a00dd 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -147,8 +147,16 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { } roots = roots[idx+1:] - objs := make(BalancedObjs, 0, len(roots)+1) + var objs BalancedObjs + if idx > 0 { + objs = make(BalancedObjs, 0, len(roots)+2) + } else { + objs = make(BalancedObjs, 0, len(roots)+1) + } objs = append(objs, obj) + if idx > 0 { + objs = append(objs, nil) + } for _, d := range roots { objs = append(objs, &tempObj{model.Object{ Path: stdpath.Join(d, sub), diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 9b8e06ca6..1c7857339 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -144,6 +144,8 @@ func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func( err = errs.NotFolder } } + } else if o == nil { + err = errs.ObjectNotFound } cont, err := ifContinue(err) From 2508cebba2304ed98041d416b0740fc9bfc4341f Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 20 Dec 2025 14:48:32 +0800 Subject: [PATCH 24/30] feat(alias): support part-based read load balance --- drivers/alias/driver.go | 82 +++++++++++++++++++++++++++++------------ drivers/alias/meta.go | 2 +- drivers/alias/types.go | 8 ++-- drivers/alias/util.go | 8 ++-- 4 files changed, 67 insertions(+), 33 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index a598a00dd..c568aff46 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math/rand" "net/url" stdpath "path" "strings" @@ -16,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ) @@ -228,29 +230,61 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - reqPath := d.getBalancedPath(ctx, file) - link, fi, err := d.link(ctx, reqPath, args) - if err != nil { - return nil, err - } - if link == nil { - // 重定向且需要通过代理 - return &model.Link{ - URL: fmt.Sprintf("%s/p%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)), - }, nil - } - - resultLink := *link - resultLink.SyncClosers = utils.NewSyncClosers(link) - if args.Redirect { - return &resultLink, nil - } - - if resultLink.ContentLength == 0 { - resultLink.ContentLength = fi.GetSize() + var resultLink *model.Link + if d.ReadConflictPolicy == AllRWP && !args.Redirect { + files, err := d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) + if err != nil { + return nil, err + } + linkClosers := make([]io.Closer, 0, len(files)) + rrf := make([]model.RangeReaderIF, 0, len(files)) + for _, f := range files { + link, fi, err := d.link(ctx, f.GetPath(), args) + if err != nil { + continue + } + if fi.GetSize() != files.GetSize() { + _ = link.Close() + continue + } + rr, err := stream.GetRangeReaderFromLink(fi.GetSize(), link) + if err != nil { + _ = link.Close() + continue + } + linkClosers = append(linkClosers, link) + rrf = append(rrf, rr) + } + rr := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return rrf[rand.Intn(len(rrf))].RangeRead(ctx, httpRange) + } + resultLink = &model.Link{ + RangeReader: stream.RangeReaderFunc(rr), + SyncClosers: utils.NewSyncClosers(linkClosers...), + } + } else { + reqPath := d.getBalancedPath(ctx, file) + link, fi, err := d.link(ctx, reqPath, args) + if err != nil { + return nil, err + } + if link == nil { + // 重定向且需要通过代理 + return &model.Link{ + URL: fmt.Sprintf("%s/p%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(reqPath, true), + sign.Sign(reqPath)), + }, nil + } + resultLink = link + resultLink.SyncClosers = utils.NewSyncClosers(link) + if args.Redirect { + return resultLink, nil + } + if resultLink.ContentLength == 0 { + resultLink.ContentLength = fi.GetSize() + } } if d.DownloadConcurrency > 0 { resultLink.Concurrency = d.DownloadConcurrency @@ -258,7 +292,7 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if d.DownloadPartSize > 0 { resultLink.PartSize = d.DownloadPartSize * utils.KB } - return &resultLink, nil + return resultLink, nil } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 3271ca625..c226bddc5 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -7,7 +7,7 @@ import ( type Addition struct { Paths string `json:"paths" required:"true" type:"text"` - ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random" default:"first"` + ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random,all" default:"first"` WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled"` PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled"` FileConsistencyCheck bool `json:"file_consistency_check" type:"bool" default:"false"` diff --git a/drivers/alias/types.go b/drivers/alias/types.go index 10d80f200..c32365d18 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -13,7 +13,7 @@ const ( FirstRWP = "first" DeterministicWP = "deterministic" DeterministicOrAllWP = "deterministic_or_all" - AllWP = "all" + AllRWP = "all" AllStrictWP = "all_strict" RandomBalancedRP = "random" BalancedByQuotaP = "quota" @@ -21,10 +21,10 @@ const ( ) var ( - ValidReadConflictPolicy = []string{FirstRWP, RandomBalancedRP} - ValidWriteConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllWP, + ValidReadConflictPolicy = []string{FirstRWP, RandomBalancedRP, AllRWP} + ValidWriteConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllRWP, AllStrictWP} - ValidPutConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllWP, + ValidPutConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllRWP, AllStrictWP, RandomBalancedRP, BalancedByQuotaP, BalancedByQuotaStrictP} ) diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 1c7857339..17ed1568b 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -184,12 +184,12 @@ func (d *Alias) getBalancedPath(ctx context.Context, file model.Obj) string { if rand.Intn(len(files)) == 0 { return file.GetPath() } - files, _ = d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllWP)) + files, _ = d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) return files[rand.Intn(len(files))].GetPath() } func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { - if policy == AllWP { + if policy == AllRWP { return func(err error) (bool, error) { return true, err } @@ -359,7 +359,7 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( dstStorageMap[mp] = append(dstStorageMap[mp], o) allocatingDst[o] = struct{}{} } - tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllWP)) + tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllRWP)) if err != nil { return nil, nil, err } @@ -382,7 +382,7 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( dstObjs = dstObjs[:len(srcObjs)] for dst := range allocatingDst { src := tmpSrcObjs[0] - if d.ReadConflictPolicy == RandomBalancedRP { + if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { src = tmpSrcObjs[rand.Intn(len(tmpSrcObjs))] } srcObjs = append(srcObjs, src) From 7274bf5c66f62d779d746b537a6a3246e6d1fa27 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 20 Dec 2025 15:05:36 +0800 Subject: [PATCH 25/30] fix(alias): list panic when leak conflict path --- drivers/alias/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index c568aff46..7cc472098 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -179,6 +179,9 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ // 所以这里对象不会传回到alias，也就不需要返回BalancedObjs了 objMap := make(map[string]model.Obj) for _, dir := range dirs { + if dir == nil { + continue + } dirPath := dir.GetPath() tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ NoLog: true, From e54d810aa0ff46a7c5f579e0b02668ffee492d41 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 20 Dec 2025 15:39:46 +0800 Subject: [PATCH 26/30] fix(alias): remove Other load balance --- drivers/alias/driver.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 7cc472098..166feaf27 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -299,8 +299,9 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { - reqPath := d.getBalancedPath(ctx, args.Obj) - storage, actualPath, err := op.GetStorageAndActualPath(reqPath) + // Other 不应负载均衡，这是因为前端是否调用 /fs/other 的判断条件是返回的 provider 的值 + // 而 ProviderPassThrough 开启时，返回的 provider 固定为第一个 obj 的后端驱动 + storage, actualPath, err := op.GetStorageAndActualPath(args.Obj.GetPath()) if err != nil { return nil, err } From d4f514b6c160e29475252fca60f6ac6bd4a9b127 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 22 Dec 2025 21:56:59 +0800 Subject: [PATCH 27/30] =?UTF-8?q?fix(alias):=20=E4=BF=AE=E5=A4=8D=20Link?= =?UTF-8?q?=20=E6=96=B9=E6=B3=95=E4=B8=AD=20resultLink=20=E7=9A=84?= =?UTF-8?q?=E8=BF=94=E5=9B=9E=E7=B1=BB=E5=9E=8B=E5=92=8C=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E5=A4=8D=E5=88=B6=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/alias/driver.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 166feaf27..dd6c16b71 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -233,7 +233,7 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - var resultLink *model.Link + var resultLink model.Link if d.ReadConflictPolicy == AllRWP && !args.Redirect { files, err := d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) if err != nil { @@ -250,7 +250,11 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( _ = link.Close() continue } - rr, err := stream.GetRangeReaderFromLink(fi.GetSize(), link) + size := link.ContentLength + if size == 0 { + size = fi.GetSize() + } + rr, err := stream.GetRangeReaderFromLink(size, link) if err != nil { _ = link.Close() continue @@ -261,7 +265,7 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( rr := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return rrf[rand.Intn(len(rrf))].RangeRead(ctx, httpRange) } - resultLink = &model.Link{ + resultLink = model.Link{ RangeReader: stream.RangeReaderFunc(rr), SyncClosers: utils.NewSyncClosers(linkClosers...), } @@ -280,10 +284,10 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( sign.Sign(reqPath)), }, nil } - resultLink = link + resultLink = *link // 复制一份，避免修改到原始link resultLink.SyncClosers = utils.NewSyncClosers(link) if args.Redirect { - return resultLink, nil + return &resultLink, nil } if resultLink.ContentLength == 0 { resultLink.ContentLength = fi.GetSize() @@ -295,7 +299,7 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if d.DownloadPartSize > 0 { resultLink.PartSize = d.DownloadPartSize * utils.KB } - return resultLink, nil + return &resultLink, nil } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { From 50bbd2023cf52799a0ff09b376961bdb98b44112 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Mon, 22 Dec 2025 22:22:02 +0800 Subject: [PATCH 28/30] =?UTF-8?q?fix(alias):=20=E6=9B=B4=E5=A5=BD=E7=9A=84?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=B9=B6=E5=8F=91=EF=BC=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/alias/driver.go | 65 ++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index dd6c16b71..78a94e8d4 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -233,7 +233,6 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - var resultLink model.Link if d.ReadConflictPolicy == AllRWP && !args.Redirect { files, err := d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) if err != nil { @@ -250,11 +249,17 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( _ = link.Close() continue } - size := link.ContentLength - if size == 0 { - size = fi.GetSize() + l := *link // 复制一份，避免修改到原始link + if l.ContentLength == 0 { + l.ContentLength = fi.GetSize() } - rr, err := stream.GetRangeReaderFromLink(size, link) + if d.DownloadConcurrency > 0 { + l.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + l.PartSize = d.DownloadPartSize * utils.KB + } + rr, err := stream.GetRangeReaderFromLink(l.ContentLength, &l) if err != nil { _ = link.Close() continue @@ -265,33 +270,33 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( rr := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return rrf[rand.Intn(len(rrf))].RangeRead(ctx, httpRange) } - resultLink = model.Link{ + return &model.Link{ RangeReader: stream.RangeReaderFunc(rr), SyncClosers: utils.NewSyncClosers(linkClosers...), - } - } else { - reqPath := d.getBalancedPath(ctx, file) - link, fi, err := d.link(ctx, reqPath, args) - if err != nil { - return nil, err - } - if link == nil { - // 重定向且需要通过代理 - return &model.Link{ - URL: fmt.Sprintf("%s/p%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)), - }, nil - } - resultLink = *link // 复制一份，避免修改到原始link - resultLink.SyncClosers = utils.NewSyncClosers(link) - if args.Redirect { - return &resultLink, nil - } - if resultLink.ContentLength == 0 { - resultLink.ContentLength = fi.GetSize() - } + }, nil + } + + reqPath := d.getBalancedPath(ctx, file) + link, fi, err := d.link(ctx, reqPath, args) + if err != nil { + return nil, err + } + if link == nil { + // 重定向且需要通过代理 + return &model.Link{ + URL: fmt.Sprintf("%s/p%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(reqPath, true), + sign.Sign(reqPath)), + }, nil + } + resultLink := *link // 复制一份，避免修改到原始link + resultLink.SyncClosers = utils.NewSyncClosers(link) + if args.Redirect { + return &resultLink, nil + } + if resultLink.ContentLength == 0 { + resultLink.ContentLength = fi.GetSize() } if d.DownloadConcurrency > 0 { resultLink.Concurrency = d.DownloadConcurrency From e72d67ab670ef46f46525bb70cfc07feb7057653 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Wed, 24 Dec 2025 15:52:59 +0800 Subject: [PATCH 29/30] chore(alias): all tips --- drivers/alias/meta.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index c226bddc5..72eb3c877 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -8,8 +8,8 @@ import ( type Addition struct { Paths string `json:"paths" required:"true" type:"text"` ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random,all" default:"first"` - WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled"` - PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled"` + WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled" help:"How the driver handles identical backend paths when renaming, removing, or making directories."` + PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled" help:"How the driver handles identical backend paths when uploading, copying, moving, or decompressing."` FileConsistencyCheck bool `json:"file_consistency_check" type:"bool" default:"false"` DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` From 88d720c7686deb558bec78c63ae5c19165d54266 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Thu, 25 Dec 2025 20:19:13 +0800 Subject: [PATCH 30/30] fix(alias): moving paths mismatch --- drivers/alias/driver.go | 15 ++++++---- drivers/alias/types.go | 7 +++-- drivers/alias/util.go | 66 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 78a94e8d4..672bd00b1 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -332,13 +332,18 @@ func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - srcs, dsts, err := d.getCopyMoveObjs(ctx, srcObj, dstDir) + srcs, dsts, err := d.getMoveObjs(ctx, srcObj, dstDir) if err == nil { - for i, src := range srcs { - dst := dsts[i] + for i, dst := range dsts { + src := srcs[i] _, e := fs.Move(ctx, src.GetPath(), dst.GetPath()) err = errors.Join(err, e) } + srcs = srcs[len(dsts):] + for _, src := range srcs { + e := fs.Remove(ctx, src.GetPath()) + err = errors.Join(err, e) + } } return err } @@ -354,7 +359,7 @@ func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) er } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - srcs, dsts, err := d.getCopyMoveObjs(ctx, srcObj, dstDir) + srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) if err == nil { for i, src := range srcs { dst := dsts[i] @@ -476,7 +481,7 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn } func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { - srcs, dsts, err := d.getCopyMoveObjs(ctx, srcObj, dstDir) + srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) if err == nil { for i, src := range srcs { dst := dsts[i] diff --git a/drivers/alias/types.go b/drivers/alias/types.go index c32365d18..9fade0c5b 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -29,9 +29,10 @@ var ( ) var ( - ErrPathConflict = errors.New("path conflict") - ErrSamePathLeak = errors.New("leak some of same-name dirs") - ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") + ErrPathConflict = errors.New("path conflict") + ErrSamePathLeak = errors.New("leak some of same-name dirs") + ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") + ErrNotEnoughSrcObjs = errors.New("cannot move fewer objs to more paths, please try copying") ) type BalancedObjs []model.Obj diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 17ed1568b..23b10e994 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -12,7 +12,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -340,7 +339,7 @@ func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) return 0, false } -func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { +func (d *Alias) getCopyObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } @@ -353,9 +352,9 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( for _, o := range dstObjs { storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) if e != nil { - return nil, nil, errors.WithMessagef(e, "cannot copy or move to virtual path [%s]", o.GetPath()) + return nil, nil, errors.WithMessagef(e, "cannot copy to virtual path [%s]", o.GetPath()) } - mp := utils.GetActualMountPath(storage.GetStorage().MountPath) + mp := storage.GetStorage().MountPath dstStorageMap[mp] = append(dstStorageMap[mp], o) allocatingDst[o] = struct{}{} } @@ -369,7 +368,7 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( if e != nil { continue } - mp := utils.GetActualMountPath(storage.GetStorage().MountPath) + mp := storage.GetStorage().MountPath if tmp, ok := dstStorageMap[mp]; ok { for _, dst := range tmp { dstObjs[len(srcObjs)] = dst @@ -391,6 +390,63 @@ func (d *Alias) getCopyMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) ( return srcObjs, dstObjs, nil } +func (d *Alias) getMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, nil, errs.PermissionDenied + } + dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, nil, err + } + tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllRWP)) + if err != nil { + return nil, nil, err + } + if len(tmpSrcObjs) < len(dstObjs) { + return nil, nil, ErrNotEnoughSrcObjs + } + dstStorageMap := make(map[string][]model.Obj) + allocatingDst := make(map[model.Obj]struct{}) + for _, o := range dstObjs { + storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + return nil, nil, errors.WithMessagef(e, "cannot move to virtual path [%s]", o.GetPath()) + } + mp := storage.GetStorage().MountPath + dstStorageMap[mp] = append(dstStorageMap[mp], o) + allocatingDst[o] = struct{}{} + } + srcObjs := make(BalancedObjs, 0, len(tmpSrcObjs)) + restSrcObjs := make(BalancedObjs, 0, len(tmpSrcObjs)-len(dstObjs)) + for _, src := range tmpSrcObjs { + storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + continue + } + mp := storage.GetStorage().MountPath + if tmp, ok := dstStorageMap[mp]; ok { + dst := tmp[0] + if len(tmp) == 1 { + delete(dstStorageMap, mp) + } else { + dstStorageMap[mp] = tmp[1:] + } + dstObjs[len(srcObjs)] = dst + srcObjs = append(srcObjs, src) + delete(allocatingDst, dst) + } else { + restSrcObjs = append(restSrcObjs, src) + } + } + dstObjs = dstObjs[:len(srcObjs)] + // len(restSrcObjs) >= len(allocatingDst) + srcObjs = append(srcObjs, restSrcObjs...) + for dst := range allocatingDst { + dstObjs = append(dstObjs, dst) + } + return srcObjs, dstObjs, nil +} + func (d *Alias) getArchiveMeta(ctx context.Context, reqPath string, args model.ArchiveArgs) (model.ArchiveMeta, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil {