store fileList compressed

for now using zlib on a 556Mo dir:
files: 1469447 o -> 207289 o
size divided by 7

7 files changed, 59 insertions, 41 deletions

diff --git a/TODO.md b/TODO.md
index ea88781..60f7ba2 100644
--- a/TODO.md
+++ b/TODO.md
@@ -14,6 +14,9 @@ priority 1
     - [ ] store compressed chunks into tracks of `trackSize` (1024o)
 - [x] add chunk cache to uniquely store chunks in RAM
 - [x] better tests for `(*Repo).Commit`
+- [ ] remove errored files from `fileList`
+- [ ] add superblock logic
+- [ ] add version blocks or journal logic
 
 priority 2
 ----------
@@ -43,7 +46,7 @@ reunion 7/09
 - [ ] save recipe consecutive chunks as extents
 - [ ] store recipe and files incrementally
 - [ ] compress recipe
-- [ ] TODO: compress file list
+- [x] compress file list
 - [ ] make size comparison between recipe and chunks with some datasets
 
 ideas
diff --git a/logger/logger.go b/logger/logger.go
index bc2b3c8..d6a8630 100644
--- a/logger/logger.go
+++ b/logger/logger.go
@@ -43,15 +43,19 @@ var (
 	defaultLogger *Logger
 )
 
+func newLoggers() [4]logger {
+	return [4]logger{
+		log.New(os.Stderr, tagInfo, flags),
+		log.New(os.Stderr, tagWarning, flags),
+		log.New(os.Stderr, tagError, flags),
+		log.New(os.Stderr, tagFatal, flags),
+	}
+}
+
 // initialize resets defaultLogger.  Which allows tests to reset environment.
 func initialize() {
 	defaultLogger = &Logger{
-		loggers: [4]logger{
-			log.New(os.Stderr, tagInfo, flags),
-			log.New(os.Stderr, tagWarning, flags),
-			log.New(os.Stderr, tagError, flags),
-			log.New(os.Stderr, tagFatal, flags),
-		},
+		loggers:     newLoggers(),
 		minSeverity: 0,
 	}
 }
@@ -68,12 +72,7 @@ func init() {
 // logger.
 func Init(level int) *Logger {
 	l := Logger{
-		loggers: [4]logger{
-			log.New(os.Stderr, tagInfo, flags),
-			log.New(os.Stderr, tagWarning, flags),
-			log.New(os.Stderr, tagError, flags),
-			log.New(os.Stderr, tagFatal, flags),
-		},
+		loggers:     newLoggers(),
 		minSeverity: sFatal - severity(level),
 		initialized: true,
 	}
diff --git a/repo.go b/repo.go
index 3dab2b3..108ef5f 100644
--- a/repo.go
+++ b/repo.go
@@ -73,8 +73,8 @@ type Repo struct {
 	fingerprints      FingerprintMap
 	sketches          SketchMap
 	chunkCache        cache.Cacher
-	chunkReadWrapper  func(r io.Reader) (io.ReadCloser, error)
-	chunkWriteWrapper func(w io.Writer) io.WriteCloser
+	chunkReadWrapper  utils.ReadWrapper
+	chunkWriteWrapper utils.WriteWrapper
 }
 
 type chunkHashes struct {
@@ -240,41 +240,54 @@ func concatFiles(files []File, stream io.WriteCloser) {
 	stream.Close()
 }
 
-func storeBasicStruct(dest string, obj interface{}) {
+func storeBasicStruct(dest string, wrapper utils.WriteWrapper, obj interface{}) {
 	file, err := os.Create(dest)
-	if err == nil {
-		encoder := gob.NewEncoder(file)
-		err = encoder.Encode(obj)
+	if err != nil {
+		logger.Panic(err)
 	}
+	out := wrapper(file)
+	encoder := gob.NewEncoder(out)
+	err = encoder.Encode(obj)
 	if err != nil {
 		logger.Panic(err)
 	}
+	if err = out.Close(); err != nil {
+		logger.Panic(err)
+	}
 	if err = file.Close(); err != nil {
 		logger.Panic(err)
 	}
 }
 
-func loadBasicStruct(path string, obj interface{}) {
+func loadBasicStruct(path string, wrapper utils.ReadWrapper, obj interface{}) {
 	file, err := os.Open(path)
-	if err == nil {
-		decoder := gob.NewDecoder(file)
-		err = decoder.Decode(obj)
+	if err != nil {
+		logger.Panic(err)
 	}
+	in, err := wrapper(file)
 	if err != nil {
 		logger.Panic(err)
 	}
+	decoder := gob.NewDecoder(in)
+	err = decoder.Decode(obj)
+	if err != nil {
+		logger.Panic(err)
+	}
+	if err = in.Close(); err != nil {
+		logger.Panic(err)
+	}
 	if err = file.Close(); err != nil {
 		logger.Panic(err)
 	}
 }
 
 func storeFileList(dest string, files []File) {
-	storeBasicStruct(dest, files)
+	storeBasicStruct(dest, utils.ZlibWriter, files)
 }
 
 func loadFileList(path string) []File {
 	var files []File
-	loadBasicStruct(path, &files)
+	loadBasicStruct(path, utils.ZlibReader, &files)
 	return files
 }
 
diff --git a/repo_test.go b/repo_test.go
index cf88224..e1a80a3 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -131,8 +131,8 @@ func TestLoadChunks(t *testing.T) {
 	resultDir := t.TempDir()
 	dataDir := filepath.Join("testdata", "logs")
 	repo := NewRepo(resultDir)
-	repo.chunkReadWrapper = dummyReader
-	repo.chunkWriteWrapper = dummyWriter
+	repo.chunkReadWrapper = utils.NopReadWrapper
+	repo.chunkWriteWrapper = utils.NopWriteWrapper
 	resultVersion := filepath.Join(resultDir, "00000")
 	resultChunks := filepath.Join(resultVersion, chunksName)
 	os.MkdirAll(resultChunks, 0775)
@@ -200,14 +200,6 @@ func getDataStream(dataDir string, streamFunc func([]File, io.WriteCloser)) io.R
 	return reader
 }
 
-func dummyReader(r io.Reader) (io.ReadCloser, error) {
-	return io.NopCloser(r), nil
-}
-
-func dummyWriter(w io.Writer) io.WriteCloser {
-	return utils.NopCloser(w)
-}
-
 func TestBsdiff(t *testing.T) {
 	resultDir := t.TempDir()
 	repo := NewRepo(resultDir)
@@ -224,8 +216,8 @@ func TestBsdiff(t *testing.T) {
 	defer os.Remove(addedFile2)
 
 	// configure repo
-	repo.chunkReadWrapper = dummyReader
-	repo.chunkWriteWrapper = dummyWriter
+	repo.chunkReadWrapper = utils.NopReadWrapper
+	repo.chunkWriteWrapper = utils.NopWriteWrapper
 
 	// Load previously stored chunks
 	oldChunks := make(chan IdentifiedChunk, 16)
@@ -254,8 +246,8 @@ func TestCommit(t *testing.T) {
 	source := filepath.Join("testdata", "logs")
 	expected := filepath.Join("testdata", "repo_8k")
 	repo := NewRepo(dest)
-	repo.chunkReadWrapper = dummyReader
-	repo.chunkWriteWrapper = dummyWriter
+	repo.chunkReadWrapper = utils.NopReadWrapper
+	repo.chunkWriteWrapper = utils.NopWriteWrapper
 
 	repo.Commit(source)
 	assertSameTree(t, assertCompatibleRepoFile, expected, dest, "Commit")
@@ -278,8 +270,8 @@ func TestRestore(t *testing.T) {
 	source := filepath.Join("testdata", "repo_8k")
 	expected := filepath.Join("testdata", "logs")
 	repo := NewRepo(source)
-	repo.chunkReadWrapper = dummyReader
-	repo.chunkWriteWrapper = dummyWriter
+	repo.chunkReadWrapper = utils.NopReadWrapper
+	repo.chunkWriteWrapper = utils.NopWriteWrapper
 
 	repo.Restore(dest)
 	assertSameTree(t, assertSameFile, expected, dest, "Restore")
diff --git a/testdata/repo_8k/00000/files b/testdata/repo_8k/00000/files
index 5a7b1da..b12d67b 100644
--- a/testdata/repo_8k/00000/files
+++ b/testdata/repo_8k/00000/files
diff --git a/testdata/repo_8k_zlib/00000/files b/testdata/repo_8k_zlib/00000/files
index 5a7b1da..b12d67b 100644
--- a/testdata/repo_8k_zlib/00000/files
+++ b/testdata/repo_8k_zlib/00000/files
diff --git a/utils/ioutils.go b/utils/ioutils.go
index a861995..fcc341e 100644
--- a/utils/ioutils.go
+++ b/utils/ioutils.go
@@ -17,6 +17,9 @@ type nopCloser struct {
 
 func (nopCloser) Close() error { return nil }
 
+type ReadWrapper func(r io.Reader) (io.ReadCloser, error)
+type WriteWrapper func(w io.Writer) io.WriteCloser
+
 // ZlibReader wraps a reader with a new zlib.Reader.
 func ZlibReader(r io.Reader) (io.ReadCloser, error) {
 	return zlib.NewReader(r)
@@ -26,3 +29,11 @@ func ZlibReader(r io.Reader) (io.ReadCloser, error) {
 func ZlibWriter(w io.Writer) io.WriteCloser {
 	return zlib.NewWriter(w)
 }
+
+func NopReadWrapper(r io.Reader) (io.ReadCloser, error) {
+	return io.NopCloser(r), nil
+}
+
+func NopWriteWrapper(w io.Writer) io.WriteCloser {
+	return NopCloser(w)
+}