diff options
| -rw-r--r-- | TODO.md | 5 | ||||
| -rw-r--r-- | logger/logger.go | 23 | ||||
| -rw-r--r-- | repo.go | 37 | ||||
| -rw-r--r-- | repo_test.go | 24 | ||||
| -rw-r--r-- | testdata/repo_8k/00000/files | bin | 155 -> 147 bytes | |||
| -rw-r--r-- | testdata/repo_8k_zlib/00000/files | bin | 155 -> 147 bytes | |||
| -rw-r--r-- | utils/ioutils.go | 11 |
7 files changed, 59 insertions, 41 deletions
@@ -14,6 +14,9 @@ priority 1 - [ ] store compressed chunks into tracks of `trackSize` (1024o) - [x] add chunk cache to uniquely store chunks in RAM - [x] better tests for `(*Repo).Commit` +- [ ] remove errored files from `fileList` +- [ ] add superblock logic +- [ ] add version blocks or journal logic priority 2 ---------- @@ -43,7 +46,7 @@ reunion 7/09 - [ ] save recipe consecutive chunks as extents - [ ] store recipe and files incrementally - [ ] compress recipe -- [ ] TODO: compress file list +- [x] compress file list - [ ] make size comparison between recipe and chunks with some datasets ideas diff --git a/logger/logger.go b/logger/logger.go index bc2b3c8..d6a8630 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -43,15 +43,19 @@ var ( defaultLogger *Logger ) +func newLoggers() [4]logger { + return [4]logger{ + log.New(os.Stderr, tagInfo, flags), + log.New(os.Stderr, tagWarning, flags), + log.New(os.Stderr, tagError, flags), + log.New(os.Stderr, tagFatal, flags), + } +} + // initialize resets defaultLogger. Which allows tests to reset environment. func initialize() { defaultLogger = &Logger{ - loggers: [4]logger{ - log.New(os.Stderr, tagInfo, flags), - log.New(os.Stderr, tagWarning, flags), - log.New(os.Stderr, tagError, flags), - log.New(os.Stderr, tagFatal, flags), - }, + loggers: newLoggers(), minSeverity: 0, } } @@ -68,12 +72,7 @@ func init() { // logger. func Init(level int) *Logger { l := Logger{ - loggers: [4]logger{ - log.New(os.Stderr, tagInfo, flags), - log.New(os.Stderr, tagWarning, flags), - log.New(os.Stderr, tagError, flags), - log.New(os.Stderr, tagFatal, flags), - }, + loggers: newLoggers(), minSeverity: sFatal - severity(level), initialized: true, } @@ -73,8 +73,8 @@ type Repo struct { fingerprints FingerprintMap sketches SketchMap chunkCache cache.Cacher - chunkReadWrapper func(r io.Reader) (io.ReadCloser, error) - chunkWriteWrapper func(w io.Writer) io.WriteCloser + chunkReadWrapper utils.ReadWrapper + chunkWriteWrapper utils.WriteWrapper } type chunkHashes struct { @@ -240,41 +240,54 @@ func concatFiles(files []File, stream io.WriteCloser) { stream.Close() } -func storeBasicStruct(dest string, obj interface{}) { +func storeBasicStruct(dest string, wrapper utils.WriteWrapper, obj interface{}) { file, err := os.Create(dest) - if err == nil { - encoder := gob.NewEncoder(file) - err = encoder.Encode(obj) + if err != nil { + logger.Panic(err) } + out := wrapper(file) + encoder := gob.NewEncoder(out) + err = encoder.Encode(obj) if err != nil { logger.Panic(err) } + if err = out.Close(); err != nil { + logger.Panic(err) + } if err = file.Close(); err != nil { logger.Panic(err) } } -func loadBasicStruct(path string, obj interface{}) { +func loadBasicStruct(path string, wrapper utils.ReadWrapper, obj interface{}) { file, err := os.Open(path) - if err == nil { - decoder := gob.NewDecoder(file) - err = decoder.Decode(obj) + if err != nil { + logger.Panic(err) } + in, err := wrapper(file) if err != nil { logger.Panic(err) } + decoder := gob.NewDecoder(in) + err = decoder.Decode(obj) + if err != nil { + logger.Panic(err) + } + if err = in.Close(); err != nil { + logger.Panic(err) + } if err = file.Close(); err != nil { logger.Panic(err) } } func storeFileList(dest string, files []File) { - storeBasicStruct(dest, files) + storeBasicStruct(dest, utils.ZlibWriter, files) } func loadFileList(path string) []File { var files []File - loadBasicStruct(path, &files) + loadBasicStruct(path, utils.ZlibReader, &files) return files } diff --git a/repo_test.go b/repo_test.go index cf88224..e1a80a3 100644 --- a/repo_test.go +++ b/repo_test.go @@ -131,8 +131,8 @@ func TestLoadChunks(t *testing.T) { resultDir := t.TempDir() dataDir := filepath.Join("testdata", "logs") repo := NewRepo(resultDir) - repo.chunkReadWrapper = dummyReader - repo.chunkWriteWrapper = dummyWriter + repo.chunkReadWrapper = utils.NopReadWrapper + repo.chunkWriteWrapper = utils.NopWriteWrapper resultVersion := filepath.Join(resultDir, "00000") resultChunks := filepath.Join(resultVersion, chunksName) os.MkdirAll(resultChunks, 0775) @@ -200,14 +200,6 @@ func getDataStream(dataDir string, streamFunc func([]File, io.WriteCloser)) io.R return reader } -func dummyReader(r io.Reader) (io.ReadCloser, error) { - return io.NopCloser(r), nil -} - -func dummyWriter(w io.Writer) io.WriteCloser { - return utils.NopCloser(w) -} - func TestBsdiff(t *testing.T) { resultDir := t.TempDir() repo := NewRepo(resultDir) @@ -224,8 +216,8 @@ func TestBsdiff(t *testing.T) { defer os.Remove(addedFile2) // configure repo - repo.chunkReadWrapper = dummyReader - repo.chunkWriteWrapper = dummyWriter + repo.chunkReadWrapper = utils.NopReadWrapper + repo.chunkWriteWrapper = utils.NopWriteWrapper // Load previously stored chunks oldChunks := make(chan IdentifiedChunk, 16) @@ -254,8 +246,8 @@ func TestCommit(t *testing.T) { source := filepath.Join("testdata", "logs") expected := filepath.Join("testdata", "repo_8k") repo := NewRepo(dest) - repo.chunkReadWrapper = dummyReader - repo.chunkWriteWrapper = dummyWriter + repo.chunkReadWrapper = utils.NopReadWrapper + repo.chunkWriteWrapper = utils.NopWriteWrapper repo.Commit(source) assertSameTree(t, assertCompatibleRepoFile, expected, dest, "Commit") @@ -278,8 +270,8 @@ func TestRestore(t *testing.T) { source := filepath.Join("testdata", "repo_8k") expected := filepath.Join("testdata", "logs") repo := NewRepo(source) - repo.chunkReadWrapper = dummyReader - repo.chunkWriteWrapper = dummyWriter + repo.chunkReadWrapper = utils.NopReadWrapper + repo.chunkWriteWrapper = utils.NopWriteWrapper repo.Restore(dest) assertSameTree(t, assertSameFile, expected, dest, "Restore") diff --git a/testdata/repo_8k/00000/files b/testdata/repo_8k/00000/files Binary files differindex 5a7b1da..b12d67b 100644 --- a/testdata/repo_8k/00000/files +++ b/testdata/repo_8k/00000/files diff --git a/testdata/repo_8k_zlib/00000/files b/testdata/repo_8k_zlib/00000/files Binary files differindex 5a7b1da..b12d67b 100644 --- a/testdata/repo_8k_zlib/00000/files +++ b/testdata/repo_8k_zlib/00000/files diff --git a/utils/ioutils.go b/utils/ioutils.go index a861995..fcc341e 100644 --- a/utils/ioutils.go +++ b/utils/ioutils.go @@ -17,6 +17,9 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } +type ReadWrapper func(r io.Reader) (io.ReadCloser, error) +type WriteWrapper func(w io.Writer) io.WriteCloser + // ZlibReader wraps a reader with a new zlib.Reader. func ZlibReader(r io.Reader) (io.ReadCloser, error) { return zlib.NewReader(r) @@ -26,3 +29,11 @@ func ZlibReader(r io.Reader) (io.ReadCloser, error) { func ZlibWriter(w io.Writer) io.WriteCloser { return zlib.NewWriter(w) } + +func NopReadWrapper(r io.Reader) (io.ReadCloser, error) { + return io.NopCloser(r), nil +} + +func NopWriteWrapper(w io.Writer) io.WriteCloser { + return NopCloser(w) +} |