Skip to content

Commit 5ac70cf

Browse files
Brownjyhunjixin
Brownjy
and
hunjixin
authored
feat: delete repo data (#76)
* feat: delete repo data --------- Co-authored-by: brown <zhangjy43@gmail.com> Co-authored-by: hunjixin <1084400399@qq.com>
1 parent 177d20b commit 5ac70cf

File tree

15 files changed

+241
-12
lines changed

15 files changed

+241
-12
lines changed

‎block/adapter.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ type Adapter interface {
139139
GetRange(ctx context.Context, obj ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)
140140
GetProperties(ctx context.Context, obj ObjectPointer) (Properties, error)
141141
Remove(ctx context.Context, obj ObjectPointer) error
142+
RemoveNameSpace(ctx context.Context, storageNamespace string) error
142143
Copy(ctx context.Context, sourceObj, destinationObj ObjectPointer) error
143144
CreateMultiPartUpload(ctx context.Context, obj ObjectPointer, r *http.Request, opts CreateMultiPartUploadOpts) (*CreateMultiPartUploadResponse, error)
144145
UploadPart(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int) (*UploadPartResponse, error)

‎block/azure/adapter.go‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ import (
99
"strings"
1010
"time"
1111

12-
"github.com/jiaozifs/jiaozifs/utils/hash"
13-
1412
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
1513
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
1614
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
@@ -19,6 +17,8 @@ import (
1917
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
2018
"github.com/jiaozifs/jiaozifs/block"
2119
"github.com/jiaozifs/jiaozifs/block/params"
20+
"github.com/jiaozifs/jiaozifs/utils"
21+
"github.com/jiaozifs/jiaozifs/utils/hash"
2222
)
2323

2424
const (
@@ -152,6 +152,7 @@ func (a *Adapter) Put(ctx context.Context, obj block.ObjectPointer, sizeBytes in
152152
if err != nil {
153153
return err
154154
}
155+
155156
_, err = containerClient.NewBlockBlobClient(qualifiedKey.BlobURL).UploadStream(ctx, reader, &azblob.UploadStreamOptions{})
156157
return err
157158
}
@@ -350,6 +351,40 @@ func (a *Adapter) Remove(ctx context.Context, obj block.ObjectPointer) error {
350351
return err
351352
}
352353

354+
func (a *Adapter) RemoveNameSpace(ctx context.Context, namespace string) error {
355+
var err error
356+
parsedNamespace, err := url.ParseRequestURI(namespace)
357+
if err != nil {
358+
return err
359+
}
360+
qp, err := ResolveBlobURLInfoFromURL(parsedNamespace)
361+
if err != nil {
362+
return err
363+
}
364+
containerClient, err := a.clientCache.NewContainerClient(qp.StorageAccountName, qp.ContainerName)
365+
if err != nil {
366+
return err
367+
}
368+
objs := containerClient.NewListBlobsFlatPager(&azblob.ListBlobsFlatOptions{
369+
Prefix: &qp.BlobURL,
370+
})
371+
for objs.More() {
372+
page, err := objs.NextPage(ctx)
373+
if err != nil {
374+
return err
375+
}
376+
for _, blob := range page.ListBlobsFlatSegmentResponse.Segment.BlobItems {
377+
blobClient := containerClient.NewBlobClient(utils.StringValue(blob.Name))
378+
_, err = blobClient.Delete(ctx, &azblob.DeleteBlobOptions{})
379+
if err != nil {
380+
return err
381+
}
382+
}
383+
}
384+
385+
return err
386+
}
387+
353388
func (a *Adapter) Copy(ctx context.Context, sourceObj, destinationObj block.ObjectPointer) error {
354389
var err error
355390
defer reportMetrics("Copy", time.Now(), nil, &err)

‎block/azure/adapter_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import (
1515
func TestAzureAdapter(t *testing.T) {
1616
basePath, err := url.JoinPath(blockURL, containerName)
1717
require.NoError(t, err)
18-
localPath, err := url.JoinPath(basePath, "lakefs")
18+
localPath, err := url.JoinPath(basePath, "jiaozfs")
1919
require.NoError(t, err)
2020
externalPath, err := url.JoinPath(basePath, "external")
2121
require.NoError(t, err)

‎block/blocktest/adapter.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,69 @@ func AdapterTest(t *testing.T, adapter block.Adapter, storageNamespace, external
2626
t.Run("Adapter_Exists", func(t *testing.T) { testAdapterExists(t, adapter, storageNamespace) })
2727
t.Run("Adapter_GetRange", func(t *testing.T) { testAdapterGetRange(t, adapter, storageNamespace) })
2828
t.Run("Adapter_Walker", func(t *testing.T) { testAdapterWalker(t, adapter, storageNamespace) })
29+
t.Run("Adapter_Clean", func(t *testing.T) { testAdapterClean(t, adapter, storageNamespace) })
30+
}
31+
32+
func testAdapterClean(t *testing.T, adapter block.Adapter, storageNamespace string) { //nolint
33+
ctx := context.Background()
34+
const content = "content used for testing"
35+
36+
tests := []struct {
37+
name string
38+
additionalObjects []string
39+
path string
40+
wantErr bool
41+
wantTree []string
42+
}{
43+
{
44+
name: "test_single",
45+
path: "README",
46+
wantErr: false,
47+
wantTree: []string{},
48+
},
49+
50+
{
51+
name: "test_under_folder",
52+
path: "src/tools.go",
53+
wantErr: false,
54+
wantTree: []string{},
55+
},
56+
{
57+
name: "test_under_multiple_folders",
58+
path: "a/b/c/d.txt",
59+
wantErr: false,
60+
wantTree: []string{},
61+
},
62+
{
63+
name: "file_in_the_way",
64+
path: "a/b/c/d.txt",
65+
additionalObjects: []string{"a/b/blocker.txt"},
66+
wantErr: false,
67+
wantTree: []string{"/a/b/blocker.txt"},
68+
},
69+
}
70+
71+
// setup env
72+
for _, tt := range tests {
73+
t.Run(tt.name, func(t *testing.T) {
74+
envObjects := tt.additionalObjects
75+
envObjects = append(envObjects, tt.path)
76+
for _, p := range envObjects {
77+
obj := block.ObjectPointer{
78+
StorageNamespace: storageNamespace,
79+
Identifier: tt.name + "/" + p,
80+
IdentifierType: block.IdentifierTypeRelative,
81+
}
82+
require.NoError(t, adapter.Put(ctx, obj, int64(len(content)), strings.NewReader(content), block.PutOpts{}))
83+
}
84+
})
85+
}
86+
87+
// clean
88+
t.Run("clean repo", func(t *testing.T) {
89+
err := adapter.RemoveNameSpace(ctx, storageNamespace)
90+
require.NoError(t, err)
91+
})
2992
}
3093

3194
func testAdapterPutGet(t *testing.T, adapter block.Adapter, storageNamespace, externalPath string) {

‎block/gs/adapter.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,38 @@ func (a *Adapter) Remove(ctx context.Context, obj block.ObjectPointer) error {
225225
return nil
226226
}
227227

228+
func (a *Adapter) RemoveNameSpace(ctx context.Context, namespace string) error {
229+
var err error
230+
defer reportMetrics("RemoveNameSpace", time.Now(), nil, &err)
231+
bucket, key, err := a.extractParamsFromObj(block.ObjectPointer{
232+
StorageNamespace: namespace,
233+
Identifier: "'",
234+
IdentifierType: block.IdentifierTypeRelative,
235+
})
236+
if err != nil {
237+
return err
238+
}
239+
iter := a.client.Bucket(bucket).Objects(ctx, &storage.Query{
240+
Delimiter: key,
241+
})
242+
243+
for {
244+
obj, err := iter.Next()
245+
if err != nil {
246+
return err
247+
}
248+
if iter.PageInfo().Remaining() == 0 {
249+
break
250+
}
251+
err = a.client.Bucket(bucket).Object(obj.Name).Delete(ctx)
252+
if err != nil {
253+
return fmt.Errorf("Object(%q).Delete: %w", key, err)
254+
}
255+
}
256+
257+
return nil
258+
}
259+
228260
func (a *Adapter) Copy(ctx context.Context, sourceObj, destinationObj block.ObjectPointer) error {
229261
var err error
230262
defer reportMetrics("Copy", time.Now(), nil, &err)

‎block/gs/adapter_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ func newAdapter() *gs.Adapter {
1717
func TestAdapter(t *testing.T) {
1818
basePath, err := url.JoinPath("gs://", bucketName)
1919
require.NoError(t, err)
20-
localPath, err := url.JoinPath(basePath, "lakefs")
20+
localPath, err := url.JoinPath(basePath, "jiaozfs")
2121
require.NoError(t, err)
2222
externalPath, err := url.JoinPath(basePath, "external")
2323
require.NoError(t, err)

‎block/local/adapter.go‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@ type Adapter struct {
3535
}
3636

3737
var (
38-
ErrPathNotWritable = errors.New("path provided is not writable")
39-
ErrInvalidUploadIDFormat = errors.New("invalid upload id format")
40-
ErrBadPath = errors.New("bad path traversal blocked")
38+
ErrPathNotWritable = errors.New("path provided is not writable")
39+
ErrInvalidUploadIDFormat = errors.New("invalid upload id format")
40+
ErrBadPath = errors.New("bad path traversal blocked")
41+
ErrInvalidStorageNamespace = errors.New("invalid storageNamespace")
4142
)
4243

4344
type QualifiedKey struct {
@@ -156,6 +157,7 @@ func (l *Adapter) Path() string {
156157

157158
func (l *Adapter) Put(_ context.Context, obj block.ObjectPointer, _ int64, reader io.Reader, _ block.PutOpts) error {
158159
p, err := l.extractParamsFromObj(obj)
160+
fmt.Println(p)
159161
if err != nil {
160162
return err
161163
}
@@ -189,6 +191,19 @@ func (l *Adapter) Remove(_ context.Context, obj block.ObjectPointer) error {
189191
return nil
190192
}
191193

194+
func (l *Adapter) RemoveNameSpace(_ context.Context, storageNamespace string) error {
195+
p, err := l.extractParamsFromObj(block.ObjectPointer{
196+
StorageNamespace: storageNamespace,
197+
Identifier: "",
198+
IdentifierType: block.IdentifierTypeRelative,
199+
})
200+
if err != nil {
201+
return err
202+
}
203+
p = filepath.Clean(p)
204+
return os.RemoveAll(p)
205+
}
206+
192207
func removeEmptyDirUntil(dir string, stopAt string) {
193208
if stopAt == "" {
194209
return

‎block/local/adapter_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ const testStorageNamespace = "local://test"
1515

1616
func TestLocalAdapter(t *testing.T) {
1717
tmpDir := t.TempDir()
18-
localPath := path.Join(tmpDir, "lakefs")
19-
externalPath := block.BlockstoreTypeLocal + "://" + path.Join(tmpDir, "lakefs", "external")
18+
localPath := path.Join(tmpDir, "jiaozfs")
19+
externalPath := block.BlockstoreTypeLocal + "://" + path.Join(tmpDir, "jiaozfs", "external")
2020
adapter, err := local.NewAdapter(localPath, local.WithRemoveEmptyDir(false))
2121
if err != nil {
2222
t.Fatal("Failed to create new adapter", err)
@@ -26,7 +26,7 @@ func TestLocalAdapter(t *testing.T) {
2626

2727
func TestAdapterNamespace(t *testing.T) {
2828
tmpDir := t.TempDir()
29-
localPath := path.Join(tmpDir, "lakefs")
29+
localPath := path.Join(tmpDir, "jiaozfs")
3030
adapter, err := local.NewAdapter(localPath, local.WithRemoveEmptyDir(false))
3131
require.NoError(t, err, "create new adapter")
3232
expr, err := regexp.Compile(adapter.GetStorageNamespaceInfo().ValidityRegex)

‎block/local/walker.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import (
1919
gonanoid "github.com/matoous/go-nanoid/v2"
2020
)
2121

22-
const cacheDirName = "_lakefs_cache"
22+
const cacheDirName = "_jiaozfs_cache"
2323

2424
type Walker struct {
2525
mark block.Mark

‎block/mem/adapter.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,18 @@ func (a *Adapter) Remove(_ context.Context, obj block.ObjectPointer) error {
176176
return nil
177177
}
178178

179+
func (a *Adapter) RemoveNameSpace(_ context.Context, storageNamespace string) error {
180+
if storageNamespace == "" {
181+
return fmt.Errorf("storageNamespace cannot be empty")
182+
}
183+
for key := range a.data {
184+
if strings.HasPrefix(key, storageNamespace+":") {
185+
delete(a.data, key)
186+
}
187+
}
188+
return nil
189+
}
190+
179191
func (a *Adapter) Copy(_ context.Context, sourceObj, destinationObj block.ObjectPointer) error {
180192
if err := verifyObjectPointer(sourceObj); err != nil {
181193
return err

0 commit comments

Comments
 (0)