start dir exporter

3 files changed, 127 insertions, 37 deletions

diff --git a/repo/export_dir.go b/repo/export_dir.go
new file mode 100644
index 0000000..8c63fdb
--- /dev/null
+++ b/repo/export_dir.go
@@ -0,0 +1,88 @@
+/* Copyright (C) 2021 Nicolas Peugnet <n.peugnet@free.fr>
+
+   This file is part of dna-backup.
+
+   dna-backup is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   dna-backup is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with dna-backup.  If not, see <https://www.gnu.org/licenses/>. */
+
+package repo
+
+import (
+	"bytes"
+	"compress/zlib"
+	"encoding/binary"
+	"io"
+
+	"github.com/n-peugnet/dna-backup/dna"
+	"github.com/n-peugnet/dna-backup/logger"
+	"github.com/n-peugnet/dna-backup/utils"
+)
+
+type Version struct {
+	Chunks uint64
+	Recipe uint64
+	Files  uint64
+}
+
+func (r *Repo) ExportDir(dest string, trackSize int) {
+	r.Init()
+	versions := make([]Version, len(r.versions))
+	chunks := r.loadChunks(r.versions)
+	for i := range versions {
+		var count int64
+		var content bytes.Buffer // replace with a reader capable of switching files
+		var recipe, fileList []byte
+		var err error
+		tracker := dna.NewWriter(&content, trackSize)
+		counter := utils.NewWriteCounter(tracker)
+		compressor := zlib.NewWriter(counter)
+		for _, c := range chunks[i] {
+			n, err := io.Copy(compressor, c.Reader())
+			if err != nil {
+				logger.Error(err)
+			}
+			count += n
+		}
+		compressor.Close()
+		tracker.Close()
+		readDelta(r.versions[i], recipeName, utils.NopReadWrapper, func(rc io.ReadCloser) {
+			recipe, err = io.ReadAll(rc)
+			if err != nil {
+				logger.Error("load recipe ", err)
+			}
+		})
+		readDelta(r.versions[i], filesName, utils.NopReadWrapper, func(rc io.ReadCloser) {
+			fileList, err = io.ReadAll(rc)
+			if err != nil {
+				logger.Error("load files ", err)
+			}
+		})
+		versions[i] = Version{
+			uint64(counter.Count()),
+			uint64(len(recipe)),
+			uint64(len(fileList)),
+		}
+		header := versions[i].createHeader()
+		logger.Info(header)
+	}
+}
+
+func (v Version) createHeader() []byte {
+	buf := make([]byte, binary.MaxVarintLen64*3)
+	i := 0
+	for _, x := range []uint64{v.Chunks, v.Recipe, v.Files} {
+		n := binary.PutUvarint(buf[i:], x)
+		i += n
+	}
+	return buf[:i]
+}
diff --git a/repo/repo.go b/repo/repo.go
index d3a5904..d12c72d 100644
--- a/repo/repo.go
+++ b/repo/repo.go
@@ -380,31 +380,36 @@ func storeDelta(prevRaw []byte, curr interface{}, dest string, differ delta.Diff
 	}
 }
 
+func readDelta(version string, name string, wrapper utils.ReadWrapper, callback func(io.ReadCloser)) {
+	path := filepath.Join(version, name)
+	file, err := os.Open(path)
+	if err != nil {
+		logger.Panic(err)
+	}
+	in, err := wrapper(file)
+	if err != nil {
+		logger.Panic(err)
+	}
+	callback(in)
+	if err = in.Close(); err != nil {
+		logger.Panic(err)
+	}
+	if err = file.Close(); err != nil {
+		logger.Panic(err)
+	}
+}
+
 func loadDeltas(target interface{}, versions []string, patcher delta.Patcher, wrapper utils.ReadWrapper, name string) (ret []byte) {
 	var prev bytes.Buffer
 	var err error
-
 	for _, v := range versions {
-		var curr bytes.Buffer
-		path := filepath.Join(v, name)
-		file, err := os.Open(path)
-		if err != nil {
-			logger.Panic(err)
-		}
-		in, err := wrapper(file)
-		if err != nil {
-			logger.Panic(err)
-		}
-		if err = patcher.Patch(&prev, &curr, in); err != nil {
-			logger.Panic(err)
-		}
-		prev = curr
-		if err = in.Close(); err != nil {
-			logger.Panic(err)
-		}
-		if err = file.Close(); err != nil {
-			logger.Panic(err)
-		}
+		readDelta(v, name, wrapper, func(in io.ReadCloser) {
+			var curr bytes.Buffer
+			if err = patcher.Patch(&prev, &curr, in); err != nil {
+				logger.Panic(err)
+			}
+			prev = curr
+		})
 	}
 	ret = prev.Bytes()
 	if len(ret) == 0 {
@@ -505,8 +510,9 @@ func (r *Repo) LoadChunkContent(id *ChunkId) *bytes.Reader {
 }
 
 // TODO: use atoi for chunkid ?
-func (r *Repo) LoadChunks(chunks chan<- IdentifiedChunk) {
-	for i, v := range r.versions {
+func (r *Repo) loadChunks(versions []string) (chunks [][]IdentifiedChunk) {
+	for i, v := range versions {
+		vc := make([]IdentifiedChunk, 0)
 		p := filepath.Join(v, chunksName)
 		entries, err := os.ReadDir(p)
 		if err != nil {
@@ -518,10 +524,11 @@ func (r *Repo) LoadChunks(chunks chan<- IdentifiedChunk) {
 			}
 			id := &ChunkId{Ver: i, Idx: uint64(j)}
 			c := NewStoredChunk(r, id)
-			chunks <- c
+			vc = append(vc, c)
 		}
+		chunks = append(chunks, vc)
 	}
-	close(chunks)
+	return chunks
 }
 
 // loadHashes loads and aggregates the hashes stored for each given version and
diff --git a/repo/repo_test.go b/repo/repo_test.go
index 38219e3..6338425 100644
--- a/repo/repo_test.go
+++ b/repo/repo_test.go
@@ -135,8 +135,8 @@ func storeChunks(dest string, chunks <-chan []byte) {
 // For each chunk, both a fingerprint (hash over the full content) and a sketch
 // (resemblance hash based on maximal values of regions) are calculated and
 // stored in an hashmap.
-func (r *Repo) hashChunks(chunks <-chan IdentifiedChunk) {
-	for c := range chunks {
+func (r *Repo) hashChunks(chunks []IdentifiedChunk) {
+	for _, c := range chunks {
 		r.hashChunk(c.GetId(), c.Reader())
 	}
 }
@@ -260,7 +260,6 @@ func TestLoadChunks(t *testing.T) {
 	reader2, writer2 := io.Pipe()
 	chunks1 := make(chan []byte, 16)
 	chunks2 := make(chan []byte, 16)
-	chunks3 := make(chan IdentifiedChunk, 16)
 	files := listFiles(dataDir)
 	go concatFiles(&files, writer1)
 	go concatFiles(&files, writer2)
@@ -268,11 +267,11 @@ func TestLoadChunks(t *testing.T) {
 	go repo.chunkStream(reader2, chunks2)
 	storeChunks(resultChunks, chunks1)
 	repo.versions = []string{resultVersion}
-	go repo.LoadChunks(chunks3)
+	chunks3 := repo.loadChunks(repo.versions)
 
 	i := 0
 	for c2 := range chunks2 {
-		c3 := <-chunks3
+		c3 := chunks3[0][i]
 		buff, err := io.ReadAll(c3.Reader())
 		if err != nil {
 			t.Errorf("Error reading from chunk %d: %s\n", c3, err)
@@ -285,7 +284,6 @@ func TestLoadChunks(t *testing.T) {
 		i++
 	}
 }
-
 func prepareChunks(dataDir string, repo *Repo, streamFunc func(*[]File, io.WriteCloser)) {
 	resultVersion := filepath.Join(repo.path, "00000")
 	resultChunks := filepath.Join(resultVersion, chunksName)
@@ -327,10 +325,8 @@ func TestBsdiff(t *testing.T) {
 	repo.chunkWriteWrapper = utils.NopWriteWrapper
 
 	// Load previously stored chunks
-	oldChunks := make(chan IdentifiedChunk, 16)
 	repo.loadVersions()
-	go repo.LoadChunks(oldChunks)
-	repo.hashChunks(oldChunks)
+	repo.hashChunks(repo.loadChunks(repo.versions)[0])
 
 	// Read new data
 	newVersion := len(repo.versions)
@@ -405,7 +401,6 @@ func TestHashes(t *testing.T) {
 	dest := t.TempDir()
 	source := filepath.Join("testdata", "repo_8k_zlib")
 
-	chunks := make(chan IdentifiedChunk, 16)
 	storeQueue := make(chan chunkData, 16)
 	storeEnd := make(chan bool)
 
@@ -413,8 +408,8 @@ func TestHashes(t *testing.T) {
 	repo1.chunkReadWrapper = utils.ZlibReader
 	repo1.chunkWriteWrapper = utils.ZlibWriter
 	repo1.versions = []string{filepath.Join(source, "00000")}
-	go repo1.LoadChunks(chunks)
-	for c := range chunks {
+	chunks := repo1.loadChunks(repo1.versions)
+	for _, c := range chunks[0] {
 		fp, sk := repo1.hashChunk(c.GetId(), c.Reader())
 		content, err := io.ReadAll(c.Reader())
 		if err != nil {