Skip to content

Commit 033511c

Browse files
bk2204bk2204
committed
object_db: use storage abstraction
Now that we have a general storage abstraction, use that to read and write data from an arbitrary storage backend. Add a function that reads from an arbitrary backend; use this when updating the tests.
1 parent fedfac8 commit 033511c

File tree

2 files changed

+103
-74
lines changed

2 files changed

+103
-74
lines changed

‎object_db.go‎

Lines changed: 34 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,9 @@ import (
66
"io"
77
"io/ioutil"
88
"os"
9-
"strings"
109
"sync/atomic"
1110

12-
"github.com/git-lfs/gitobj/errors"
13-
"github.com/git-lfs/gitobj/pack"
11+
"github.com/git-lfs/gitobj/storage"
1412
)
1513

1614
// ObjectDatabase enables the reading and writing of objects against a storage
@@ -24,11 +22,10 @@ type ObjectDatabase struct {
2422
// and a value of 1 if it is closed.
2523
closed uint32
2624

27-
// s is the storage backend which opens/creates/reads/writes.
28-
s storer
29-
// packs are the set of packfiles which contain all packed objects
30-
// within this repository.
31-
packs *pack.Set
25+
// ro is the locations from which we can read objects.
26+
ro storage.Storage
27+
// rw is the location to which we write objects.
28+
rw storage.WritableStorage
3229

3330
// temp directory, defaults to os.TempDir
3431
tmp string
@@ -39,15 +36,24 @@ type ObjectDatabase struct {
3936
//
4037
// /absolute/repo/path/.git/objects
4138
func FromFilesystem(root, tmp string) (*ObjectDatabase, error) {
42-
packs, err := pack.NewSet(root)
39+
b, err := NewFilesystemBackend(root, tmp)
4340
if err != nil {
4441
return nil, err
4542
}
4643

44+
ro, rw := b.Storage()
4745
return &ObjectDatabase{
48-
tmp: tmp,
49-
s: newFileStorer(root, tmp),
50-
packs: packs,
46+
tmp: tmp,
47+
ro: ro,
48+
rw: rw,
49+
}, nil
50+
}
51+
52+
func FromBackend(b storage.Backend) (*ObjectDatabase, error) {
53+
ro, rw := b.Storage()
54+
return &ObjectDatabase{
55+
ro: ro,
56+
rw: rw,
5157
}, nil
5258
}
5359

@@ -61,7 +67,10 @@ func (o *ObjectDatabase) Close() error {
6167
return fmt.Errorf("gitobj: *ObjectDatabase already closed")
6268
}
6369

64-
if err := o.packs.Close(); err != nil {
70+
if err := o.ro.Close(); err != nil {
71+
return err
72+
}
73+
if err := o.rw.Close(); err != nil {
6574
return err
6675
}
6776
return nil
@@ -203,7 +212,7 @@ func (o *ObjectDatabase) Root() (string, bool) {
203212
Root() string
204213
}
205214

206-
if root, ok := o.s.(rooter); ok {
215+
if root, ok := o.rw.(rooter); ok {
207216
return root.Root(), true
208217
}
209218
return "", false
@@ -257,52 +266,26 @@ func (d *ObjectDatabase) encodeBuffer(object Object, buf io.ReadWriter) (sha []b
257266
// save writes the given buffer to the location given by the storer "o.s" as
258267
// identified by the sha []byte.
259268
func (o *ObjectDatabase) save(sha []byte, buf io.Reader) ([]byte, int64, error) {
260-
n, err := o.s.Store(sha, buf)
269+
n, err := o.rw.Store(sha, buf)
261270

262271
return sha, n, err
263272
}
264273

265274
// open gives an `*ObjectReader` for the given loose object keyed by the given
266275
// "sha" []byte, or an error.
267276
func (o *ObjectDatabase) open(sha []byte) (*ObjectReader, error) {
268-
f, err := o.s.Open(sha)
269-
if err != nil {
270-
if !errors.IsNoSuchObject(err) {
271-
// If there was some other issue beyond not being able
272-
// to find the object, return that immediately and don't
273-
// try and fallback to the *git.ObjectScanner.
274-
return nil, err
275-
}
276-
277-
// Otherwise, if the file simply couldn't be found, attempt to
278-
// load its contents from the *git.ObjectScanner by leveraging
279-
// `git-cat-file --batch`.
280-
if atomic.LoadUint32(&o.closed) == 1 {
281-
return nil, fmt.Errorf("gitobj: cannot use closed *pack.Set")
282-
}
283-
284-
packed, err := o.packs.Object(sha)
285-
if err != nil {
286-
return nil, err
287-
}
288-
289-
unpacked, err := packed.Unpack()
290-
if err != nil {
291-
return nil, err
292-
}
293-
294-
return NewUncompressedObjectReader(io.MultiReader(
295-
// Git object header:
296-
strings.NewReader(fmt.Sprintf("%s %d\x00",
297-
packed.Type(), len(unpacked),
298-
)),
299-
300-
// Git object (uncompressed) contents:
301-
bytes.NewReader(unpacked),
302-
))
277+
if atomic.LoadUint32(&o.closed) == 1 {
278+
return nil, fmt.Errorf("gitobj: cannot use closed *pack.Set")
303279
}
304280

305-
return NewObjectReadCloser(f)
281+
f, err := o.ro.Open(sha)
282+
if err != nil {
283+
return nil, err
284+
}
285+
if o.ro.IsCompressed() {
286+
return NewObjectReadCloser(f)
287+
}
288+
return NewUncompressedObjectReader(f)
306289
}
307290

308291
// openDecode calls decode (see: below) on the object named "sha" after openin

‎object_db_test.go‎

Lines changed: 69 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,13 @@ func TestDecodeObject(t *testing.T) {
2525
fmt.Fprintf(zw, "blob 14\x00%s", contents)
2626
zw.Close()
2727

28-
odb := &ObjectDatabase{s: newMemoryStorer(map[string]io.ReadWriter{
28+
b, err := NewMemoryBackend(map[string]io.ReadWriter{
2929
sha: &buf,
30-
})}
30+
})
31+
require.NoError(t, err)
32+
33+
odb, err := FromBackend(b)
34+
require.NoError(t, err)
3135

3236
shaHex, _ := hex.DecodeString(sha)
3337
obj, err := odb.Object(shaHex)
@@ -51,9 +55,13 @@ func TestDecodeBlob(t *testing.T) {
5155
fmt.Fprintf(zw, "blob 14\x00%s", contents)
5256
zw.Close()
5357

54-
odb := &ObjectDatabase{s: newMemoryStorer(map[string]io.ReadWriter{
58+
b, err := NewMemoryBackend(map[string]io.ReadWriter{
5559
sha: &buf,
56-
})}
60+
})
61+
require.NoError(t, err)
62+
63+
odb, err := FromBackend(b)
64+
require.NoError(t, err)
5765

5866
shaHex, _ := hex.DecodeString(sha)
5967
blob, err := odb.Blob(shaHex)
@@ -83,9 +91,13 @@ func TestDecodeTree(t *testing.T) {
8391
zw.Write(hexBlobSha)
8492
zw.Close()
8593

86-
odb := &ObjectDatabase{s: newMemoryStorer(map[string]io.ReadWriter{
94+
b, err := NewMemoryBackend(map[string]io.ReadWriter{
8795
sha: &buf,
88-
})}
96+
})
97+
require.NoError(t, err)
98+
99+
odb, err := FromBackend(b)
100+
require.NoError(t, err)
89101

90102
tree, err := odb.Tree(hexSha)
91103

@@ -113,9 +125,13 @@ func TestDecodeCommit(t *testing.T) {
113125
fmt.Fprintf(zw, "\ninitial commit\n")
114126
zw.Close()
115127

116-
odb := &ObjectDatabase{s: newMemoryStorer(map[string]io.ReadWriter{
128+
b, err := NewMemoryBackend(map[string]io.ReadWriter{
117129
sha: &buf,
118-
})}
130+
})
131+
require.NoError(t, err)
132+
133+
odb, err := FromBackend(b)
134+
require.NoError(t, err)
119135

120136
commit, err := odb.Commit(commitShaHex)
121137

@@ -128,8 +144,11 @@ func TestDecodeCommit(t *testing.T) {
128144
}
129145

130146
func TestWriteBlob(t *testing.T) {
131-
fs := newMemoryStorer(make(map[string]io.ReadWriter))
132-
odb := &ObjectDatabase{s: fs}
147+
b, err := NewMemoryBackend(nil)
148+
require.NoError(t, err)
149+
150+
odb, err := FromBackend(b)
151+
require.NoError(t, err)
133152

134153
sha, err := odb.WriteBlob(&Blob{
135154
Size: 14,
@@ -138,14 +157,19 @@ func TestWriteBlob(t *testing.T) {
138157

139158
expected := "af5626b4a114abcb82d63db7c8082c3c4756e51b"
140159

160+
_, s := b.Storage()
161+
141162
assert.Nil(t, err)
142163
assert.Equal(t, expected, hex.EncodeToString(sha))
143-
assert.NotNil(t, fs.fs[hex.EncodeToString(sha)])
164+
assert.NotNil(t, s.(*memoryStorer).fs[hex.EncodeToString(sha)])
144165
}
145166

146167
func TestWriteTree(t *testing.T) {
147-
fs := newMemoryStorer(make(map[string]io.ReadWriter))
148-
odb := &ObjectDatabase{s: fs}
168+
b, err := NewMemoryBackend(nil)
169+
require.NoError(t, err)
170+
171+
odb, err := FromBackend(b)
172+
require.NoError(t, err)
149173

150174
blobSha := "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
151175
hexBlobSha, err := hex.DecodeString(blobSha)
@@ -161,14 +185,19 @@ func TestWriteTree(t *testing.T) {
161185

162186
expected := "fcb545d5746547a597811b7441ed8eba307be1ff"
163187

188+
_, s := b.Storage()
189+
164190
assert.Nil(t, err)
165191
assert.Equal(t, expected, hex.EncodeToString(sha))
166-
assert.NotNil(t, fs.fs[hex.EncodeToString(sha)])
192+
assert.NotNil(t, s.(*memoryStorer).fs[hex.EncodeToString(sha)])
167193
}
168194

169195
func TestWriteCommit(t *testing.T) {
170-
fs := newMemoryStorer(make(map[string]io.ReadWriter))
171-
odb := &ObjectDatabase{s: fs}
196+
b, err := NewMemoryBackend(nil)
197+
require.NoError(t, err)
198+
199+
odb, err := FromBackend(b)
200+
require.NoError(t, err)
172201

173202
when := time.Unix(1257894000, 0).UTC()
174203
author := &Signature{Name: "John Doe", Email: "john@example.com", When: when}
@@ -187,9 +216,11 @@ func TestWriteCommit(t *testing.T) {
187216

188217
expected := "fee8a35c2890cd6e0e28d24cc457fcecbd460962"
189218

219+
_, s := b.Storage()
220+
190221
assert.Nil(t, err)
191222
assert.Equal(t, expected, hex.EncodeToString(sha))
192-
assert.NotNil(t, fs.fs[hex.EncodeToString(sha)])
223+
assert.NotNil(t, s.(*memoryStorer).fs[hex.EncodeToString(sha)])
193224
}
194225

195226
func TestDecodeTag(t *testing.T) {
@@ -208,9 +239,13 @@ func TestDecodeTag(t *testing.T) {
208239
fmt.Fprintf(zw, "The quick brown fox jumps over the lazy dog.\n")
209240
zw.Close()
210241

211-
odb := &ObjectDatabase{s: newMemoryStorer(map[string]io.ReadWriter{
242+
b, err := NewMemoryBackend(map[string]io.ReadWriter{
212243
sha: &buf,
213-
})}
244+
})
245+
require.NoError(t, err)
246+
247+
odb, err := FromBackend(b)
248+
require.NoError(t, err)
214249

215250
tag, err := odb.Tag(tagShaHex)
216251

@@ -224,8 +259,11 @@ func TestDecodeTag(t *testing.T) {
224259
}
225260

226261
func TestWriteTag(t *testing.T) {
227-
fs := newMemoryStorer(make(map[string]io.ReadWriter))
228-
odb := &ObjectDatabase{s: fs}
262+
b, err := NewMemoryBackend(nil)
263+
require.NoError(t, err)
264+
265+
odb, err := FromBackend(b)
266+
require.NoError(t, err)
229267

230268
sha, err := odb.WriteTag(&Tag{
231269
Object: []byte("aaaaaaaaaaaaaaaaaaaa"),
@@ -238,16 +276,24 @@ func TestWriteTag(t *testing.T) {
238276

239277
expected := "e614dda21829f4176d3db27fe62fb4aee2e2475d"
240278

279+
_, s := b.Storage()
280+
241281
assert.Nil(t, err)
242282
assert.Equal(t, expected, hex.EncodeToString(sha))
243-
assert.NotNil(t, fs.fs[hex.EncodeToString(sha)])
283+
assert.NotNil(t, s.(*memoryStorer).fs[hex.EncodeToString(sha)])
244284
}
245285

246286
func TestReadingAMissingObjectAfterClose(t *testing.T) {
247287
sha, _ := hex.DecodeString("af5626b4a114abcb82d63db7c8082c3c4756e51b")
248288

289+
b, err := NewMemoryBackend(nil)
290+
require.NoError(t, err)
291+
292+
ro, rw := b.Storage()
293+
249294
db := &ObjectDatabase{
250-
s: newMemoryStorer(nil),
295+
ro: ro,
296+
rw: rw,
251297
closed: 1,
252298
}
253299

0 commit comments

Comments
 (0)