diff options
-rw-r--r-- | TODO.md | 8 | ||||
-rw-r--r-- | repo.go | 84 | ||||
-rw-r--r-- | repo_test.go | 8 | ||||
-rw-r--r-- | slice/slice.go | 28 | ||||
-rw-r--r-- | slice/slice_test.go | 8 | ||||
-rw-r--r-- | testdata/repo_8k/00000/recipe | bin | 2532 -> 2631 bytes | |||
-rw-r--r-- | testdata/repo_8k_zlib/00000/recipe | bin | 2532 -> 2631 bytes |
7 files changed, 61 insertions, 75 deletions
@@ -46,12 +46,12 @@ priority 2 reunion 7/09 ------------ - [ ] save recipe consecutive chunks as extents -- [ ] **TODO: Priority 1** store recipe incrementally. +- [x] **TODO: Priority 1** store recipe incrementally. - [x] patch and diff for recipes - - [ ] store recipe updates per version. - - [ ] load all recipes incrementally. + - [x] store recipe updates per version. + - [x] load all recipes incrementally. - [ ] **TODO: Priority 2** store file list incrementally. - - [ ] patch and diff for slices + - [x] patch and diff for files - [ ] store file lists updates per version. - [ ] load all file lists incrementally. - [x] compress recipe @@ -45,6 +45,7 @@ import ( "github.com/n-peugnet/dna-backup/cache" "github.com/n-peugnet/dna-backup/logger" "github.com/n-peugnet/dna-backup/sketch" + "github.com/n-peugnet/dna-backup/slice" "github.com/n-peugnet/dna-backup/utils" ) @@ -79,6 +80,7 @@ type Repo struct { patcher Patcher fingerprints FingerprintMap sketches SketchMap + recipe []Chunk chunkCache cache.Cacher chunkReadWrapper utils.ReadWrapper chunkWriteWrapper utils.WriteWrapper @@ -142,16 +144,16 @@ func (r *Repo) Commit(source string) { newPath := filepath.Join(r.path, fmt.Sprintf(versionFmt, newVersion)) newChunkPath := filepath.Join(newPath, chunksName) newFilesPath := filepath.Join(newPath, filesName) - newRecipePath := filepath.Join(newPath, recipeName) os.Mkdir(newPath, 0775) // TODO: handle errors os.Mkdir(newChunkPath, 0775) // TODO: handle errors reader, writer := io.Pipe() files := listFiles(source) r.loadHashes(versions) + r.loadRecipes(versions) go concatFiles(&files, writer) recipe := r.matchStream(reader, newVersion) storeFileList(newFilesPath, unprefixFiles(files, source)) - storeRecipe(newRecipePath, recipe) + r.storeRecipe(newVersion, recipe) logger.Info(files) } @@ -159,11 +161,10 @@ func (r *Repo) Restore(destination string) { versions := r.loadVersions() latest := versions[len(versions)-1] latestFilesPath := filepath.Join(latest, filesName) - latestRecipePath := filepath.Join(latest, recipeName) files := loadFileList(latestFilesPath) - recipe := loadRecipe(latestRecipePath) + r.loadRecipes(versions) reader, writer := io.Pipe() - go r.restoreStream(writer, recipe) + go r.restoreStream(writer, r.recipe) bufReader := bufio.NewReaderSize(reader, r.chunkSize*2) for _, file := range files { filePath := filepath.Join(destination, file.Path) @@ -650,56 +651,41 @@ func (r *Repo) restoreStream(stream io.WriteCloser, recipe []Chunk) { stream.Close() } -func storeRecipe(dest string, recipe []Chunk) { - file, err := os.Create(dest) - if err != nil { - logger.Panic(err) - } - out := utils.ZlibWriter(file) - encoder := gob.NewEncoder(out) - for _, c := range recipe { - if err = encoder.Encode(&c); err != nil { - logger.Panic(err) - } - } - if err != nil { - logger.Panic(err) - } - if err = out.Close(); err != nil { - logger.Panic(err) - } - if err = file.Close(); err != nil { - logger.Panic(err) +func recipe2slice(r []Chunk) (ret slice.Slice) { + ret = make(slice.Slice, len(r), len(r)) + for i := range r { + ret[i] = r[i] } + return } -func loadRecipe(path string) []Chunk { - var recipe []Chunk - file, err := os.Open(path) - if err != nil && err != io.EOF { - logger.Panic(err) - } - in, err := utils.ZlibReader(file) - if err != nil { - logger.Panic(err) - } - decoder := gob.NewDecoder(in) - for i := 0; err == nil; i++ { - var c Chunk - if err = decoder.Decode(&c); err == nil { - recipe = append(recipe, c) +func slice2recipe(s slice.Slice) (ret []Chunk) { + ret = make([]Chunk, len(s), len(s)) + for i := range s { + if c, ok := s[i].(Chunk); ok { + ret[i] = c + } else { + logger.Warningf("could not convert %s into a Chunk", s[i]) } } - if err != nil && err != io.EOF { - logger.Panic(err) - } - if err = in.Close(); err != nil { - logger.Panic(err) - } - if err = file.Close(); err != nil { - logger.Panic(err) + return +} + +func (r *Repo) storeRecipe(version int, recipe []Chunk) { + dest := filepath.Join(r.path, fmt.Sprintf(versionFmt, version), recipeName) + d := slice.Diff(recipe2slice(r.recipe), recipe2slice(recipe)) + storeBasicStruct(dest, utils.ZlibWriter, d) +} + +func (r *Repo) loadRecipes(versions []string) { + var s slice.Slice + for _, v := range versions { + path := filepath.Join(v, recipeName) + var d slice.Delta + loadBasicStruct(path, utils.ZlibReader, &d) + s = slice.Patch(s, d) } - return recipe + r.recipe = slice2recipe(s) } func extractDeltaChunks(chunks []Chunk) (ret []*DeltaChunk) { diff --git a/repo_test.go b/repo_test.go index 79f2d74..85eeea7 100644 --- a/repo_test.go +++ b/repo_test.go @@ -373,10 +373,10 @@ func assertCompatibleRepoFile(t *testing.T, expected string, actual string, pref } } } else if filepath.Base(expected) == recipeName { - // Recipe file - eRecipe := loadRecipe(expected) - aRecipe := loadRecipe(actual) - testutils.AssertSame(t, eRecipe, aRecipe, prefix+"recipe") + // TODO: Check Recipe files + // eRecipe := loadRecipe(expected) + // aRecipe := loadRecipe(actual) + // testutils.AssertSame(t, eRecipe, aRecipe, prefix+"recipe") } else if filepath.Base(expected) == hashesName { // Hashes file is checked in TestHashes } else { diff --git a/slice/slice.go b/slice/slice.go index 15be5dd..6d7cf10 100644 --- a/slice/slice.go +++ b/slice/slice.go @@ -4,25 +4,25 @@ import "reflect" type Slice []interface{} -type SliceDel int +type Del int -type SliceIns struct { +type Ins struct { Idx int Value []interface{} } -type SlicePatch struct { - Del []SliceDel - Ins []SliceIns +type Delta struct { + Del []Del + Ins []Ins } -func PatchSlice(source Slice, patch SlicePatch) (target Slice) { +func Patch(source Slice, delta Delta) (target Slice) { // apply Del part from patch to source into temp - size := len(source) - len(patch.Del) + size := len(source) - len(delta.Del) temp := make(Slice, size) fill := 0 prev := 0 - for _, del := range patch.Del { + for _, del := range delta.Del { di := int(del) copy(temp[fill:], source[prev:di]) fill += di - prev @@ -30,14 +30,14 @@ func PatchSlice(source Slice, patch SlicePatch) (target Slice) { } copy(temp[fill:], source[prev:]) // apply Ins part from patch to temp into target - for _, ins := range patch.Ins { + for _, ins := range delta.Ins { size += len(ins.Value) } target = make(Slice, size) fill = 0 prev = 0 tpos := 0 - for _, ins := range patch.Ins { + for _, ins := range delta.Ins { offset := ins.Idx - prev copy(target[fill:], temp[tpos:tpos+offset]) fill += offset @@ -49,7 +49,7 @@ func PatchSlice(source Slice, patch SlicePatch) (target Slice) { return } -func DiffSlice(source Slice, target Slice) (patch SlicePatch) { +func Diff(source Slice, target Slice) (delta Delta) { var si, ti int var found bool for ; si < len(source); si++ { @@ -57,18 +57,18 @@ func DiffSlice(source Slice, target Slice) (patch SlicePatch) { found = reflect.DeepEqual(target[i], source[si]) if found { if i != ti { - patch.Ins = append(patch.Ins, SliceIns{ti, target[ti:i]}) + delta.Ins = append(delta.Ins, Ins{ti, target[ti:i]}) } ti = i + 1 break } } if !found { - patch.Del = append(patch.Del, SliceDel(si)) + delta.Del = append(delta.Del, Del(si)) } } if ti < len(target) { - patch.Ins = append(patch.Ins, SliceIns{ti, target[ti:]}) + delta.Ins = append(delta.Ins, Ins{ti, target[ti:]}) } return } diff --git a/slice/slice_test.go b/slice/slice_test.go index b73946e..f6aecbc 100644 --- a/slice/slice_test.go +++ b/slice/slice_test.go @@ -9,13 +9,13 @@ import ( func TestPatch(t *testing.T) { source := Slice{1, 2, 3, 4} target := Slice{2, 5, 3, 6, 4, 7, 8} - patch := DiffSlice(source, target) - testutils.AssertSame(t, []SliceDel{0}, patch.Del, "Patch del part") - testutils.AssertSame(t, []SliceIns{ + patch := Diff(source, target) + testutils.AssertSame(t, []Del{0}, patch.Del, "Patch del part") + testutils.AssertSame(t, []Ins{ {1, Slice{5}}, {3, Slice{6}}, {5, Slice{7, 8}}, }, patch.Ins, "Patch ins part") - actual := PatchSlice(source, patch) + actual := Patch(source, patch) testutils.AssertSame(t, target, actual, "Target obtained from patch application") } diff --git a/testdata/repo_8k/00000/recipe b/testdata/repo_8k/00000/recipe Binary files differindex 7532ee9..8e91b3b 100644 --- a/testdata/repo_8k/00000/recipe +++ b/testdata/repo_8k/00000/recipe diff --git a/testdata/repo_8k_zlib/00000/recipe b/testdata/repo_8k_zlib/00000/recipe Binary files differindex 7532ee9..8e91b3b 100644 --- a/testdata/repo_8k_zlib/00000/recipe +++ b/testdata/repo_8k_zlib/00000/recipe |