add extractNewChunks

8 files changed, 98 insertions, 34 deletions

diff --git a/.gitignore b/.gitignore
index 3a0ec3b..348605c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,2 @@
 ## Test generated
-test/result
 test/repo
diff --git a/chunk.go b/chunk.go
index 9f09e55..321f046 100644
--- a/chunk.go
+++ b/chunk.go
@@ -38,6 +38,10 @@ func (c *Chunk) Reader(repo string) (io.Reader, error) {
 	return nil, &ChunkError{"Uninitialized chunk"}
 }
 
+func (c *Chunk) isStored() bool {
+	return c.Id != nil
+}
+
 type ChunkError struct {
 	err string
 }
diff --git a/chunk_test.go b/chunk_test.go
new file mode 100644
index 0000000..4ea6b44
--- /dev/null
+++ b/chunk_test.go
@@ -0,0 +1,14 @@
+package main
+
+import "testing"
+
+func TestIsStored(t *testing.T) {
+	stored := Chunk{Id: &ChunkId{0, 0}}
+	if !stored.isStored() {
+		t.Error("Chunk ", stored, " should be stored")
+	}
+	unstored := Chunk{}
+	if unstored.isStored() {
+		t.Error("Chunk ", unstored, " should not be stored")
+	}
+}
diff --git a/go.mod b/go.mod
index 59fed50..faf604d 100644
--- a/go.mod
+++ b/go.mod
@@ -5,4 +5,5 @@ go 1.16
 require (
 	github.com/chmduquesne/rollinghash v4.0.0+incompatible
 	github.com/gabstv/go-bsdiff v1.0.5
+	github.com/google/go-cmp v0.5.6
 )
diff --git a/go.sum b/go.sum
index 24e690c..e450c22 100644
--- a/go.sum
+++ b/go.sum
@@ -4,3 +4,6 @@ github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 h1:eX+pdPPlD279OWgd
 github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76/go.mod h1:KjxHHirfLaw19iGT70HvVjHQsL1vq1SRQB4yOsAfy2s=
 github.com/gabstv/go-bsdiff v1.0.5 h1:g29MC/38Eaig+iAobW10/CiFvPtin8U3Jj4yNLcNG9k=
 github.com/gabstv/go-bsdiff v1.0.5/go.mod h1:/Zz6GK+/f/TMylRtVaW3uwZlb0FZITILfA0q12XKGwg=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/main_test.go b/main_test.go
new file mode 100644
index 0000000..57db053
--- /dev/null
+++ b/main_test.go
@@ -0,0 +1,20 @@
+package main
+
+import (
+	"log"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	setup()
+	code := m.Run()
+	shutdown()
+	os.Exit(code)
+}
+
+func setup() {
+	log.SetFlags(log.Lshortfile)
+}
+
+func shutdown() {}
diff --git a/repo.go b/repo.go
index 3bb60b9..ef61111 100644
--- a/repo.go
+++ b/repo.go
@@ -57,7 +57,7 @@ func (r *Repo) Commit(source string) {
 	newVersion := len(versions)
 	newPath := path.Join(r.path, fmt.Sprintf(versionFmt, newVersion))
 	newChunkPath := path.Join(newPath, chunksName)
-	newFilesPath := path.Join(newPath, filesName)
+	// newFilesPath := path.Join(newPath, filesName)
 	os.Mkdir(newPath, 0775)
 	os.Mkdir(newChunkPath, 0775)
 	newChunks := make(chan []byte, 16)
@@ -65,10 +65,11 @@ func (r *Repo) Commit(source string) {
 	files := listFiles(source)
 	go loadChunks(versions, oldChunks)
 	go readFiles(files, newChunks)
-	// hashes := HashChunks(oldChunks)
-	// MatchChunks(newChunks, hashes)
-	storeChunks(newChunkPath, newChunks)
-	storeFiles(newFilesPath, files)
+	hashes := hashChunks(oldChunks)
+	chunks := r.matchChunks(newChunks, hashes)
+	extractNewChunks(chunks)
+	// storeChunks(newChunkPath, newChunks)
+	// storeFiles(newFilesPath, files)
 	fmt.Println(files)
 }
 
@@ -254,6 +255,25 @@ func (r *Repo) matchChunks(chunks <-chan []byte, hashes map[uint64]ChunkId) []Ch
 	return recipe
 }
 
+func extractNewChunks(chunks []Chunk) (ret [][]Chunk) {
+	var i int
+	ret = append(ret, make([]Chunk, 0))
+	for _, c := range chunks {
+		if c.isStored() {
+			if len(ret[i]) != 0 {
+				i++
+				ret = append(ret, make([]Chunk, 0))
+			}
+		} else {
+			ret[i] = append(ret[i], c)
+		}
+	}
+	if len(ret[i]) == 0 {
+		ret = ret[:i]
+	}
+	return
+}
+
 func writeFile(filePath string, object interface{}) error {
 	file, err := os.Create(filePath)
 	if err == nil {
diff --git a/repo_test.go b/repo_test.go
index d25ab8a..6a7a526 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -3,36 +3,15 @@ package main
 import (
 	"bytes"
 	"io/ioutil"
-	"log"
 	"os"
 	"path"
 	"testing"
 
 	"github.com/gabstv/go-bsdiff/pkg/bsdiff"
+	"github.com/google/go-cmp/cmp"
 )
 
-func TestMain(m *testing.M) {
-	setup()
-	code := m.Run()
-	shutdown()
-	os.Exit(code)
-}
-
-func setup() {
-	log.SetFlags(log.Lshortfile)
-}
-
-func shutdown() {}
-
-func prepareResult() string {
-	result := path.Join("test", "result")
-	os.RemoveAll(result)
-	os.MkdirAll(result, 0775)
-	return result
-}
-
 func chunkCompare(t *testing.T, dataDir string, testFiles []string, chunkCount int) {
-
 	chunks := make(chan []byte)
 	files := listFiles(dataDir)
 	go readFiles(files, chunks)
@@ -59,7 +38,7 @@ func chunkCompare(t *testing.T, dataDir string, testFiles []string, chunkCount i
 		if bytes.Compare(c, content) != 0 {
 			t.Errorf("Chunk %d does not match file content", i)
 			t.Log("Expected: ", c[:10], "...")
-			t.Log("Result:", content)
+			t.Log("Actual:", content)
 		}
 		i++
 	}
@@ -95,7 +74,7 @@ func TestReadFiles3(t *testing.T) {
 }
 
 func TestLoadChunks(t *testing.T) {
-	resultDir := prepareResult()
+	resultDir := t.TempDir()
 	dataDir := path.Join("test", "data")
 	resultVersion := path.Join(resultDir, "00000")
 	resultChunks := path.Join(resultVersion, chunksName)
@@ -116,14 +95,38 @@ func TestLoadChunks(t *testing.T) {
 		if bytes.Compare(c2, c3.Value) != 0 {
 			t.Errorf("Chunk %d does not match file content", i)
 			t.Log("Expected: ", c2[:10], "...")
-			t.Log("Result:", c3.Value)
+			t.Log("Actual:", c3.Value)
 		}
 		i++
 	}
 }
 
+func TestExtractNewChunks(t *testing.T) {
+	chunks := []Chunk{
+		{Value: []byte{'a'}},
+		{Id: &ChunkId{0, 0}},
+		{Value: []byte{'b'}},
+		{Value: []byte{'c'}},
+		{Id: &ChunkId{0, 1}},
+	}
+	newChunks := extractNewChunks(chunks)
+	if len(newChunks) != 2 {
+		t.Error("New chunks should contain 2 slices")
+		t.Log("Actual: ", newChunks)
+	}
+	if len(newChunks[1]) != 2 {
+		t.Error("New chunks second slice should contain 2 chunks")
+		t.Log("Actual: ", newChunks[0])
+	}
+	if !cmp.Equal(newChunks[1][0], chunks[2]) {
+		t.Error("New chunks do not match")
+		t.Log("Expected: ", chunks[2])
+		t.Log("Actual: ", newChunks[1][0])
+	}
+}
+
 func TestStoreLoadFiles(t *testing.T) {
-	resultDir := prepareResult()
+	resultDir := t.TempDir()
 	dataDir := path.Join("test", "data")
 	resultFiles := path.Join(resultDir, filesName)
 	files1 := listFiles(dataDir)
@@ -133,13 +136,13 @@ func TestStoreLoadFiles(t *testing.T) {
 		if f != files2[i] {
 			t.Errorf("Loaded file data %d does not match stored one", i)
 			t.Log("Expected: ", f)
-			t.Log("Result: ", files2[i])
+			t.Log("Actual: ", files2[i])
 		}
 	}
 }
 
 func TestBsdiff(t *testing.T) {
-	resultDir := prepareResult()
+	resultDir := t.TempDir()
 	dataDir := path.Join("test", "data")
 	addedFile := path.Join(dataDir, "logs.2", "slogTest.log")
 	resultVersion := path.Join(resultDir, "00000")