initial chunk storage

4 files changed, 52 insertions, 5 deletions

diff --git a/chunk.go b/chunk.go
index d9833ea..670ed24 100644
--- a/chunk.go
+++ b/chunk.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -24,6 +25,11 @@ type BufferedChunk interface {
 	Bytes() []byte
 }
 
+type StorerChunk interface {
+	Chunk
+	Store(path string) error
+}
+
 type ChunkId struct {
 	Ver int
 	Idx uint64
@@ -68,6 +74,10 @@ func (c *LoadedChunk) Bytes() []byte {
 	return c.value
 }
 
+func (c *LoadedChunk) Store(path string) error {
+	return storeChunk(c.Reader(), c.id.Path(path))
+}
+
 func NewStoredFile(repo *Repo, id *ChunkId) *StoredChunk {
 	return &StoredChunk{repo: repo, id: id}
 }
@@ -140,3 +150,18 @@ func (c *DeltaChunk) Reader() io.ReadSeeker {
 func (c *DeltaChunk) Len() int {
 	return c.size
 }
+
+func storeChunk(r io.Reader, path string) error {
+	file, err := os.Create(path)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Error creating chunk for '%s'; %s\n", path, err))
+	}
+	n, err := io.Copy(file, r)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Error writing chunk content for '%s', written %d bytes: %s\n", path, n, err))
+	}
+	if err := file.Close(); err != nil {
+		return errors.New(fmt.Sprintf("Error closing chunk for '%s': %s\n", path, err))
+	}
+	return nil
+}
diff --git a/const.go b/const.go
index 71762db..2c58dd6 100644
--- a/const.go
+++ b/const.go
@@ -5,4 +5,5 @@ const (
 	chunkIdFmt = "%015d"
 	versionFmt = "%05d"
 	filesName  = "files"
+	recipeName = "recipe"
 )
diff --git a/repo.go b/repo.go
index 8bd069a..86760ed 100644
--- a/repo.go
+++ b/repo.go
@@ -99,7 +99,8 @@ func (r *Repo) Commit(source string) {
 	newVersion := len(versions)
 	newPath := path.Join(r.path, fmt.Sprintf(versionFmt, newVersion))
 	newChunkPath := path.Join(newPath, chunksName)
-	// newFilesPath := path.Join(newPath, filesName)
+	newFilesPath := path.Join(newPath, filesName)
+	newRecipePath := path.Join(newPath, recipeName)
 	os.Mkdir(newPath, 0775)
 	os.Mkdir(newChunkPath, 0775)
 	reader, writer := io.Pipe()
@@ -108,10 +109,9 @@ func (r *Repo) Commit(source string) {
 	go r.loadChunks(versions, oldChunks)
 	go concatFiles(files, writer)
 	r.hashChunks(oldChunks)
-	chunks := r.matchStream(reader, newVersion)
-	extractTempChunks(chunks)
-	// storeChunks(newChunkPath, newChunks)
-	// storeFiles(newFilesPath, files)
+	recipe := r.matchStream(reader, newVersion)
+	storeRecipe(newRecipePath, recipe)
+	storeFileList(newFilesPath, files)
 	fmt.Println(files)
 }
 
@@ -331,6 +331,8 @@ func (r *Repo) tryDeltaEncodeChunk(temp BufferedChunk) (Chunk, bool) {
 	return temp, false
 }
 
+// encodeTempChunk first tries to delta-encode the given chunk before attributing
+// it an Id and saving it into the fingerprints and sketches maps.
 func (r *Repo) encodeTempChunk(temp BufferedChunk, version int, last *uint64) (chunk Chunk, isDelta bool) {
 	chunk, isDelta = r.tryDeltaEncodeChunk(temp)
 	if isDelta {
@@ -343,6 +345,7 @@ func (r *Repo) encodeTempChunk(temp BufferedChunk, version int, last *uint64) (c
 		ic := NewLoadedChunk(id, temp.Bytes())
 		hasher := rabinkarp64.NewFromPol(r.pol)
 		r.hashAndStoreChunk(ic, hasher)
+		ic.Store(r.path)
 		log.Println("Add new chunk", id)
 		return ic, false
 	}
@@ -350,6 +353,10 @@ func (r *Repo) encodeTempChunk(temp BufferedChunk, version int, last *uint64) (c
 	return
 }
 
+// encodeTempChunks encodes the current temporary chunks based on the value of the previous one.
+// Temporary chunks can be partial. If the current chunk is smaller than the size of a
+// super-feature and there exists a previous chunk, then both are merged before attempting
+// to delta-encode them.
 func (r *Repo) encodeTempChunks(prev BufferedChunk, curr BufferedChunk, version int, last *uint64) []Chunk {
 	if reflect.ValueOf(prev).IsNil() {
 		c, _ := r.encodeTempChunk(curr, version, last)
@@ -435,6 +442,13 @@ func (r *Repo) matchStream(stream io.Reader, version int) []Chunk {
 	return chunks
 }
 
+func storeRecipe(dest string, recipe []Chunk) {
+	err := writeFile(dest, recipe)
+	if err != nil {
+		log.Println(err)
+	}
+}
+
 // mergeTempChunks joins temporary partial chunks from an array of chunks if possible.
 // If a chunk is smaller than the size required to calculate a super-feature,
 // it is then appended to the previous consecutive temporary chunk if it exists.
diff --git a/repo_test.go b/repo_test.go
index 21eccac..244942e 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -204,6 +204,13 @@ func TestBsdiff(t *testing.T) {
 	}
 }
 
+func TestCommit(t *testing.T) {
+	dest := t.TempDir()
+	source := path.Join("test", "data")
+	repo := NewRepo(dest)
+	repo.Commit(source)
+}
+
 func assertLen(t *testing.T, expected int, actual interface{}, prefix string) {
 	s := reflect.ValueOf(actual)
 	if s.Len() != expected {