aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorn-peugnet <n.peugnet@free.fr>2021-09-20 11:06:47 +0200
committern-peugnet <n.peugnet@free.fr>2021-09-20 11:44:32 +0200
commit81f191fdcd347df0b7ae52e061c9ab39e88bdbb1 (patch)
tree489fff15739ba095f75223b460bec48d034c06a6
parent129032a13d19d500011fe23bcdd232b87eeb9b2f (diff)
downloaddna-backup-81f191fdcd347df0b7ae52e061c9ab39e88bdbb1.tar.gz
dna-backup-81f191fdcd347df0b7ae52e061c9ab39e88bdbb1.zip
store fileList compressed
for now using zlib on a 556Mo dir: files: 1469447 o -> 207289 o size divided by 7
-rw-r--r--TODO.md5
-rw-r--r--logger/logger.go23
-rw-r--r--repo.go37
-rw-r--r--repo_test.go24
-rw-r--r--testdata/repo_8k/00000/filesbin155 -> 147 bytes
-rw-r--r--testdata/repo_8k_zlib/00000/filesbin155 -> 147 bytes
-rw-r--r--utils/ioutils.go11
7 files changed, 59 insertions, 41 deletions
diff --git a/TODO.md b/TODO.md
index ea88781..60f7ba2 100644
--- a/TODO.md
+++ b/TODO.md
@@ -14,6 +14,9 @@ priority 1
- [ ] store compressed chunks into tracks of `trackSize` (1024o)
- [x] add chunk cache to uniquely store chunks in RAM
- [x] better tests for `(*Repo).Commit`
+- [ ] remove errored files from `fileList`
+- [ ] add superblock logic
+- [ ] add version blocks or journal logic
priority 2
----------
@@ -43,7 +46,7 @@ reunion 7/09
- [ ] save recipe consecutive chunks as extents
- [ ] store recipe and files incrementally
- [ ] compress recipe
-- [ ] TODO: compress file list
+- [x] compress file list
- [ ] make size comparison between recipe and chunks with some datasets
ideas
diff --git a/logger/logger.go b/logger/logger.go
index bc2b3c8..d6a8630 100644
--- a/logger/logger.go
+++ b/logger/logger.go
@@ -43,15 +43,19 @@ var (
defaultLogger *Logger
)
+func newLoggers() [4]logger {
+ return [4]logger{
+ log.New(os.Stderr, tagInfo, flags),
+ log.New(os.Stderr, tagWarning, flags),
+ log.New(os.Stderr, tagError, flags),
+ log.New(os.Stderr, tagFatal, flags),
+ }
+}
+
// initialize resets defaultLogger. Which allows tests to reset environment.
func initialize() {
defaultLogger = &Logger{
- loggers: [4]logger{
- log.New(os.Stderr, tagInfo, flags),
- log.New(os.Stderr, tagWarning, flags),
- log.New(os.Stderr, tagError, flags),
- log.New(os.Stderr, tagFatal, flags),
- },
+ loggers: newLoggers(),
minSeverity: 0,
}
}
@@ -68,12 +72,7 @@ func init() {
// logger.
func Init(level int) *Logger {
l := Logger{
- loggers: [4]logger{
- log.New(os.Stderr, tagInfo, flags),
- log.New(os.Stderr, tagWarning, flags),
- log.New(os.Stderr, tagError, flags),
- log.New(os.Stderr, tagFatal, flags),
- },
+ loggers: newLoggers(),
minSeverity: sFatal - severity(level),
initialized: true,
}
diff --git a/repo.go b/repo.go
index 3dab2b3..108ef5f 100644
--- a/repo.go
+++ b/repo.go
@@ -73,8 +73,8 @@ type Repo struct {
fingerprints FingerprintMap
sketches SketchMap
chunkCache cache.Cacher
- chunkReadWrapper func(r io.Reader) (io.ReadCloser, error)
- chunkWriteWrapper func(w io.Writer) io.WriteCloser
+ chunkReadWrapper utils.ReadWrapper
+ chunkWriteWrapper utils.WriteWrapper
}
type chunkHashes struct {
@@ -240,41 +240,54 @@ func concatFiles(files []File, stream io.WriteCloser) {
stream.Close()
}
-func storeBasicStruct(dest string, obj interface{}) {
+func storeBasicStruct(dest string, wrapper utils.WriteWrapper, obj interface{}) {
file, err := os.Create(dest)
- if err == nil {
- encoder := gob.NewEncoder(file)
- err = encoder.Encode(obj)
+ if err != nil {
+ logger.Panic(err)
}
+ out := wrapper(file)
+ encoder := gob.NewEncoder(out)
+ err = encoder.Encode(obj)
if err != nil {
logger.Panic(err)
}
+ if err = out.Close(); err != nil {
+ logger.Panic(err)
+ }
if err = file.Close(); err != nil {
logger.Panic(err)
}
}
-func loadBasicStruct(path string, obj interface{}) {
+func loadBasicStruct(path string, wrapper utils.ReadWrapper, obj interface{}) {
file, err := os.Open(path)
- if err == nil {
- decoder := gob.NewDecoder(file)
- err = decoder.Decode(obj)
+ if err != nil {
+ logger.Panic(err)
}
+ in, err := wrapper(file)
if err != nil {
logger.Panic(err)
}
+ decoder := gob.NewDecoder(in)
+ err = decoder.Decode(obj)
+ if err != nil {
+ logger.Panic(err)
+ }
+ if err = in.Close(); err != nil {
+ logger.Panic(err)
+ }
if err = file.Close(); err != nil {
logger.Panic(err)
}
}
func storeFileList(dest string, files []File) {
- storeBasicStruct(dest, files)
+ storeBasicStruct(dest, utils.ZlibWriter, files)
}
func loadFileList(path string) []File {
var files []File
- loadBasicStruct(path, &files)
+ loadBasicStruct(path, utils.ZlibReader, &files)
return files
}
diff --git a/repo_test.go b/repo_test.go
index cf88224..e1a80a3 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -131,8 +131,8 @@ func TestLoadChunks(t *testing.T) {
resultDir := t.TempDir()
dataDir := filepath.Join("testdata", "logs")
repo := NewRepo(resultDir)
- repo.chunkReadWrapper = dummyReader
- repo.chunkWriteWrapper = dummyWriter
+ repo.chunkReadWrapper = utils.NopReadWrapper
+ repo.chunkWriteWrapper = utils.NopWriteWrapper
resultVersion := filepath.Join(resultDir, "00000")
resultChunks := filepath.Join(resultVersion, chunksName)
os.MkdirAll(resultChunks, 0775)
@@ -200,14 +200,6 @@ func getDataStream(dataDir string, streamFunc func([]File, io.WriteCloser)) io.R
return reader
}
-func dummyReader(r io.Reader) (io.ReadCloser, error) {
- return io.NopCloser(r), nil
-}
-
-func dummyWriter(w io.Writer) io.WriteCloser {
- return utils.NopCloser(w)
-}
-
func TestBsdiff(t *testing.T) {
resultDir := t.TempDir()
repo := NewRepo(resultDir)
@@ -224,8 +216,8 @@ func TestBsdiff(t *testing.T) {
defer os.Remove(addedFile2)
// configure repo
- repo.chunkReadWrapper = dummyReader
- repo.chunkWriteWrapper = dummyWriter
+ repo.chunkReadWrapper = utils.NopReadWrapper
+ repo.chunkWriteWrapper = utils.NopWriteWrapper
// Load previously stored chunks
oldChunks := make(chan IdentifiedChunk, 16)
@@ -254,8 +246,8 @@ func TestCommit(t *testing.T) {
source := filepath.Join("testdata", "logs")
expected := filepath.Join("testdata", "repo_8k")
repo := NewRepo(dest)
- repo.chunkReadWrapper = dummyReader
- repo.chunkWriteWrapper = dummyWriter
+ repo.chunkReadWrapper = utils.NopReadWrapper
+ repo.chunkWriteWrapper = utils.NopWriteWrapper
repo.Commit(source)
assertSameTree(t, assertCompatibleRepoFile, expected, dest, "Commit")
@@ -278,8 +270,8 @@ func TestRestore(t *testing.T) {
source := filepath.Join("testdata", "repo_8k")
expected := filepath.Join("testdata", "logs")
repo := NewRepo(source)
- repo.chunkReadWrapper = dummyReader
- repo.chunkWriteWrapper = dummyWriter
+ repo.chunkReadWrapper = utils.NopReadWrapper
+ repo.chunkWriteWrapper = utils.NopWriteWrapper
repo.Restore(dest)
assertSameTree(t, assertSameFile, expected, dest, "Restore")
diff --git a/testdata/repo_8k/00000/files b/testdata/repo_8k/00000/files
index 5a7b1da..b12d67b 100644
--- a/testdata/repo_8k/00000/files
+++ b/testdata/repo_8k/00000/files
Binary files differ
diff --git a/testdata/repo_8k_zlib/00000/files b/testdata/repo_8k_zlib/00000/files
index 5a7b1da..b12d67b 100644
--- a/testdata/repo_8k_zlib/00000/files
+++ b/testdata/repo_8k_zlib/00000/files
Binary files differ
diff --git a/utils/ioutils.go b/utils/ioutils.go
index a861995..fcc341e 100644
--- a/utils/ioutils.go
+++ b/utils/ioutils.go
@@ -17,6 +17,9 @@ type nopCloser struct {
func (nopCloser) Close() error { return nil }
+type ReadWrapper func(r io.Reader) (io.ReadCloser, error)
+type WriteWrapper func(w io.Writer) io.WriteCloser
+
// ZlibReader wraps a reader with a new zlib.Reader.
func ZlibReader(r io.Reader) (io.ReadCloser, error) {
return zlib.NewReader(r)
@@ -26,3 +29,11 @@ func ZlibReader(r io.Reader) (io.ReadCloser, error) {
func ZlibWriter(w io.Writer) io.WriteCloser {
return zlib.NewWriter(w)
}
+
+func NopReadWrapper(r io.Reader) (io.ReadCloser, error) {
+ return io.NopCloser(r), nil
+}
+
+func NopWriteWrapper(w io.Writer) io.WriteCloser {
+ return NopCloser(w)
+}