diff options
author | n-peugnet <n.peugnet@free.fr> | 2021-09-20 14:52:05 +0200 |
---|---|---|
committer | n-peugnet <n.peugnet@free.fr> | 2021-09-20 14:52:05 +0200 |
commit | a20f4f17708356150eb8335be1d0f5a767416c1d (patch) | |
tree | 3884d6b0c61ac5db8b7ffd994dee8d768c5799d6 | |
parent | 540426db79b6e49b79d58aef99ffee5005dc987d (diff) | |
download | dna-backup-a20f4f17708356150eb8335be1d0f5a767416c1d.tar.gz dna-backup-a20f4f17708356150eb8335be1d0f5a767416c1d.zip |
store recipe compressed
for now using zlib on a 556Mo dir:
recipe: 11668558 o -> 5540831 o
size divided by 2
-rw-r--r-- | TODO.md | 2 | ||||
-rw-r--r-- | repo.go | 39 | ||||
-rw-r--r-- | testdata/repo_8k/00000/recipe | bin | 6570 -> 2532 bytes | |||
-rw-r--r-- | testdata/repo_8k_zlib/00000/recipe | bin | 6570 -> 2532 bytes |
4 files changed, 27 insertions, 14 deletions
@@ -45,7 +45,7 @@ reunion 7/09 ------------ - [ ] save recipe consecutive chunks as extents - [ ] store recipe and files incrementally -- [ ] compress recipe +- [x] compress recipe - [x] compress file list - [ ] make size comparison between recipe and chunks with some datasets @@ -652,17 +652,22 @@ func (r *Repo) restoreStream(stream io.WriteCloser, recipe []Chunk) { func storeRecipe(dest string, recipe []Chunk) { file, err := os.Create(dest) - if err == nil { - encoder := gob.NewEncoder(file) - for _, c := range recipe { - if err = encoder.Encode(&c); err != nil { - logger.Panic(err) - } + if err != nil { + logger.Panic(err) + } + out := utils.ZlibWriter(file) + encoder := gob.NewEncoder(out) + for _, c := range recipe { + if err = encoder.Encode(&c); err != nil { + logger.Panic(err) } } if err != nil { logger.Panic(err) } + if err = out.Close(); err != nil { + logger.Panic(err) + } if err = file.Close(); err != nil { logger.Panic(err) } @@ -671,18 +676,26 @@ func storeRecipe(dest string, recipe []Chunk) { func loadRecipe(path string) []Chunk { var recipe []Chunk file, err := os.Open(path) - if err == nil { - decoder := gob.NewDecoder(file) - for i := 0; err == nil; i++ { - var c Chunk - if err = decoder.Decode(&c); err == nil { - recipe = append(recipe, c) - } + if err != nil && err != io.EOF { + logger.Panic(err) + } + in, err := utils.ZlibReader(file) + if err != nil { + logger.Panic(err) + } + decoder := gob.NewDecoder(in) + for i := 0; err == nil; i++ { + var c Chunk + if err = decoder.Decode(&c); err == nil { + recipe = append(recipe, c) } } if err != nil && err != io.EOF { logger.Panic(err) } + if err = in.Close(); err != nil { + logger.Panic(err) + } if err = file.Close(); err != nil { logger.Panic(err) } diff --git a/testdata/repo_8k/00000/recipe b/testdata/repo_8k/00000/recipe Binary files differindex 9a3962c..7532ee9 100644 --- a/testdata/repo_8k/00000/recipe +++ b/testdata/repo_8k/00000/recipe diff --git a/testdata/repo_8k_zlib/00000/recipe b/testdata/repo_8k_zlib/00000/recipe Binary files differindex 9a3962c..7532ee9 100644 --- a/testdata/repo_8k_zlib/00000/recipe +++ b/testdata/repo_8k_zlib/00000/recipe |