diff options
author | n-peugnet <n.peugnet@free.fr> | 2021-09-07 12:45:11 +0200 |
---|---|---|
committer | n-peugnet <n.peugnet@free.fr> | 2021-09-07 12:45:11 +0200 |
commit | 3d8e298b02dd09ca0ff896fbd8c1d4fc91be0b03 (patch) | |
tree | 40a19c594b72b9623378656507a26e462a84b603 | |
parent | 88ecfcb7b517fd3cbed1c683b0d8835a4d55b2fc (diff) | |
download | dna-backup-3d8e298b02dd09ca0ff896fbd8c1d4fc91be0b03.tar.gz dna-backup-3d8e298b02dd09ca0ff896fbd8c1d4fc91be0b03.zip |
prepare for recipe encoding by exporting fileds
-rw-r--r-- | chunk.go | 48 | ||||
-rw-r--r-- | repo.go | 96 | ||||
-rw-r--r-- | repo_test.go | 18 |
3 files changed, 99 insertions, 63 deletions
@@ -17,7 +17,7 @@ type Chunk interface { type IdentifiedChunk interface { Chunk - Id() *ChunkId + GetId() *ChunkId } type BufferedChunk interface { @@ -49,16 +49,16 @@ func (i *ChunkId) Reader(repo *Repo) io.ReadSeeker { } func NewLoadedChunk(id *ChunkId, value []byte) *LoadedChunk { - return &LoadedChunk{id: id, value: value} + return &LoadedChunk{Id: id, value: value} } type LoadedChunk struct { - id *ChunkId + Id *ChunkId value []byte } -func (c *LoadedChunk) Id() *ChunkId { - return c.id +func (c *LoadedChunk) GetId() *ChunkId { + return c.Id } func (c *LoadedChunk) Reader() io.ReadSeeker { @@ -75,29 +75,29 @@ func (c *LoadedChunk) Bytes() []byte { } func (c *LoadedChunk) Store(path string) error { - return storeChunk(c.Reader(), c.id.Path(path)) + return storeChunk(c.Reader(), c.Id.Path(path)) } -func NewStoredFile(repo *Repo, id *ChunkId) *StoredChunk { - return &StoredChunk{repo: repo, id: id} +func NewStoredChunk(repo *Repo, id *ChunkId) *StoredChunk { + return &StoredChunk{repo: repo, Id: id} } type StoredChunk struct { repo *Repo - id *ChunkId + Id *ChunkId } -func (c *StoredChunk) Id() *ChunkId { - return c.id +func (c *StoredChunk) GetId() *ChunkId { + return c.Id } func (c *StoredChunk) Reader() io.ReadSeeker { // log.Printf("Chunk %d: Reading from file\n", c.id) - return c.id.Reader(c.repo) + return c.Id.Reader(c.repo) } func (c *StoredChunk) Len() int { - path := c.id.Path(c.repo.path) + path := c.Id.Path(c.repo.path) info, err := os.Stat(path) if err != nil { log.Println("Chunk: could not stat file:", path) @@ -106,23 +106,23 @@ func (c *StoredChunk) Len() int { } func NewTempChunk(value []byte) *TempChunk { - return &TempChunk{value: value} + return &TempChunk{Value: value} } type TempChunk struct { - value []byte + Value []byte } func (c *TempChunk) Reader() io.ReadSeeker { - return bytes.NewReader(c.value) + return bytes.NewReader(c.Value) } func (c *TempChunk) Len() int { - return len(c.value) + return len(c.Value) } func (c *TempChunk) Bytes() []byte { - return c.value + return c.Value } func (c *TempChunk) AppendFrom(r io.Reader) { @@ -130,25 +130,25 @@ func (c *TempChunk) AppendFrom(r io.Reader) { if err != nil { println("Chunk: error appending to temp chunk:", err) } - c.value = append(c.value, buff...) + c.Value = append(c.Value, buff...) } type DeltaChunk struct { repo *Repo - source *ChunkId - patch []byte - size int + Source *ChunkId + Patch []byte + Size int } func (c *DeltaChunk) Reader() io.ReadSeeker { var buff bytes.Buffer - c.repo.Patcher().Patch(c.source.Reader(c.repo), &buff, bytes.NewReader(c.patch)) + c.repo.Patcher().Patch(c.Source.Reader(c.repo), &buff, bytes.NewReader(c.Patch)) return bytes.NewReader(buff.Bytes()) } // TODO: Maybe return the size of the patch instead ? func (c *DeltaChunk) Len() int { - return c.size + return c.Size } func storeChunk(r io.Reader, path string) error { @@ -193,17 +193,31 @@ func (r *Repo) chunkStream(stream io.Reader, chunks chan<- []byte) { } func storeFileList(dest string, files []File) { - err := writeFile(dest, files) + file, err := os.Create(dest) + if err == nil { + encoder := gob.NewEncoder(file) + err = encoder.Encode(files) + } if err != nil { - log.Println(err) + log.Panicln(err) + } + if err = file.Close(); err != nil { + log.Panicln(err) } } func loadFileList(path string) []File { var files []File - err := readFile(path, &files) + file, err := os.Open(path) + if err == nil { + decoder := gob.NewDecoder(file) + err = decoder.Decode(&files) + } if err != nil { - log.Println(err) + log.Panicln(err) + } + if err = file.Close(); err != nil { + log.Panicln(err) } return files } @@ -267,7 +281,7 @@ func (r *Repo) hashAndStoreChunk(chunk IdentifiedChunk, hasher hash.Hash64) { io.Copy(hasher, chunk.Reader()) fingerprint := hasher.Sum64() sketch, _ := SketchChunk(chunk, r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) - r.storeChunkId(chunk.Id(), fingerprint, sketch) + r.storeChunkId(chunk.GetId(), fingerprint, sketch) } func (r *Repo) storeChunkId(id *ChunkId, fingerprint uint64, sketch []uint64) { @@ -322,9 +336,9 @@ func (r *Repo) tryDeltaEncodeChunk(temp BufferedChunk) (Chunk, bool) { } else { return &DeltaChunk{ repo: r, - source: id, - patch: buff.Bytes(), - size: temp.Len(), + Source: id, + Patch: buff.Bytes(), + Size: temp.Len(), }, true } } @@ -404,7 +418,7 @@ func (r *Repo) matchStream(stream io.Reader, version int) []Chunk { prev = nil } log.Printf("Add existing chunk: %d\n", chunkId) - chunks = append(chunks, NewStoredFile(r, chunkId)) + chunks = append(chunks, NewStoredChunk(r, chunkId)) buff = make([]byte, 0, r.chunkSize*2) for i := 0; i < r.chunkSize && err == nil; i++ { b, err = bufStream.ReadByte() @@ -443,10 +457,50 @@ func (r *Repo) matchStream(stream io.Reader, version int) []Chunk { } func storeRecipe(dest string, recipe []Chunk) { - err := writeFile(dest, recipe) + gob.Register(&LoadedChunk{}) + gob.Register(&StoredChunk{}) + gob.Register(&TempChunk{}) + gob.Register(&DeltaChunk{}) + file, err := os.Create(dest) + if err == nil { + encoder := gob.NewEncoder(file) + for _, c := range recipe { + if err = encoder.Encode(&c); err != nil { + log.Panicln(err) + } + } + } if err != nil { - log.Println(err) + log.Panicln(err) + } + if err = file.Close(); err != nil { + log.Panicln(err) + } +} + +func loadRecipe(path string) []Chunk { + var recipe []Chunk + gob.Register(&LoadedChunk{}) + gob.Register(&StoredChunk{}) + gob.Register(&TempChunk{}) + gob.Register(&DeltaChunk{}) + file, err := os.Open(path) + if err == nil { + decoder := gob.NewDecoder(file) + for i := 0; err == nil; i++ { + var c Chunk + if err = decoder.Decode(&c); err == nil { + recipe = append(recipe, c) + } + } + } + if err != nil && err != io.EOF { + log.Panicln(err) + } + if err = file.Close(); err != nil { + log.Panicln(err) } + return recipe } // mergeTempChunks joins temporary partial chunks from an array of chunks if possible. @@ -499,23 +553,3 @@ func extractDeltaChunks(chunks []Chunk) (ret []*DeltaChunk) { } return } - -func writeFile(filePath string, object interface{}) error { - file, err := os.Create(filePath) - if err == nil { - encoder := gob.NewEncoder(file) - encoder.Encode(object) - } - file.Close() - return err -} - -func readFile(filePath string, object interface{}) error { - file, err := os.Open(filePath) - if err == nil { - decoder := gob.NewDecoder(file) - err = decoder.Decode(object) - } - file.Close() - return err -} diff --git a/repo_test.go b/repo_test.go index 244942e..dc04126 100644 --- a/repo_test.go +++ b/repo_test.go @@ -123,11 +123,11 @@ func TestLoadChunks(t *testing.T) { func TestExtractNewChunks(t *testing.T) { repo := NewRepo("") chunks := []Chunk{ - &TempChunk{value: []byte{'a'}}, - &LoadedChunk{id: &ChunkId{0, 0}}, - &TempChunk{value: []byte{'b'}}, - &TempChunk{value: []byte{'c'}}, - &LoadedChunk{id: &ChunkId{0, 1}}, + &TempChunk{Value: []byte{'a'}}, + &LoadedChunk{Id: &ChunkId{0, 0}}, + &TempChunk{Value: []byte{'b'}}, + &TempChunk{Value: []byte{'c'}}, + &LoadedChunk{Id: &ChunkId{0, 1}}, } newChunks := extractTempChunks(repo.mergeTempChunks(chunks)) assertLen(t, 2, newChunks, "New chunks:") @@ -197,9 +197,9 @@ func TestBsdiff(t *testing.T) { newChunks := extractDeltaChunks(repo.mergeTempChunks(recipe)) assertLen(t, 2, newChunks, "New delta chunks:") for _, c := range newChunks { - log.Println("Patch size:", len(c.patch)) - if len(c.patch) >= repo.chunkSize/10 { - t.Errorf("Bsdiff of chunk is too large: %d", len(c.patch)) + log.Println("Patch size:", len(c.Patch)) + if len(c.Patch) >= repo.chunkSize/10 { + t.Errorf("Bsdiff of chunk is too large: %d", len(c.Patch)) } } } @@ -209,6 +209,8 @@ func TestCommit(t *testing.T) { source := path.Join("test", "data") repo := NewRepo(dest) repo.Commit(source) + recipe := loadRecipe(path.Join(dest, "00000", recipeName)) + log.Println(recipe) } func assertLen(t *testing.T, expected int, actual interface{}, prefix string) { |