diff options
| -rw-r--r-- | chunk.go | 25 | ||||
| -rw-r--r-- | const.go | 1 | ||||
| -rw-r--r-- | repo.go | 24 | ||||
| -rw-r--r-- | repo_test.go | 7 |
4 files changed, 52 insertions, 5 deletions
@@ -2,6 +2,7 @@ package main import ( "bytes" + "errors" "fmt" "io" "log" @@ -24,6 +25,11 @@ type BufferedChunk interface { Bytes() []byte } +type StorerChunk interface { + Chunk + Store(path string) error +} + type ChunkId struct { Ver int Idx uint64 @@ -68,6 +74,10 @@ func (c *LoadedChunk) Bytes() []byte { return c.value } +func (c *LoadedChunk) Store(path string) error { + return storeChunk(c.Reader(), c.id.Path(path)) +} + func NewStoredFile(repo *Repo, id *ChunkId) *StoredChunk { return &StoredChunk{repo: repo, id: id} } @@ -140,3 +150,18 @@ func (c *DeltaChunk) Reader() io.ReadSeeker { func (c *DeltaChunk) Len() int { return c.size } + +func storeChunk(r io.Reader, path string) error { + file, err := os.Create(path) + if err != nil { + return errors.New(fmt.Sprintf("Error creating chunk for '%s'; %s\n", path, err)) + } + n, err := io.Copy(file, r) + if err != nil { + return errors.New(fmt.Sprintf("Error writing chunk content for '%s', written %d bytes: %s\n", path, n, err)) + } + if err := file.Close(); err != nil { + return errors.New(fmt.Sprintf("Error closing chunk for '%s': %s\n", path, err)) + } + return nil +} @@ -5,4 +5,5 @@ const ( chunkIdFmt = "%015d" versionFmt = "%05d" filesName = "files" + recipeName = "recipe" ) @@ -99,7 +99,8 @@ func (r *Repo) Commit(source string) { newVersion := len(versions) newPath := path.Join(r.path, fmt.Sprintf(versionFmt, newVersion)) newChunkPath := path.Join(newPath, chunksName) - // newFilesPath := path.Join(newPath, filesName) + newFilesPath := path.Join(newPath, filesName) + newRecipePath := path.Join(newPath, recipeName) os.Mkdir(newPath, 0775) os.Mkdir(newChunkPath, 0775) reader, writer := io.Pipe() @@ -108,10 +109,9 @@ func (r *Repo) Commit(source string) { go r.loadChunks(versions, oldChunks) go concatFiles(files, writer) r.hashChunks(oldChunks) - chunks := r.matchStream(reader, newVersion) - extractTempChunks(chunks) - // storeChunks(newChunkPath, newChunks) - // storeFiles(newFilesPath, files) + recipe := r.matchStream(reader, newVersion) + storeRecipe(newRecipePath, recipe) + storeFileList(newFilesPath, files) fmt.Println(files) } @@ -331,6 +331,8 @@ func (r *Repo) tryDeltaEncodeChunk(temp BufferedChunk) (Chunk, bool) { return temp, false } +// encodeTempChunk first tries to delta-encode the given chunk before attributing +// it an Id and saving it into the fingerprints and sketches maps. func (r *Repo) encodeTempChunk(temp BufferedChunk, version int, last *uint64) (chunk Chunk, isDelta bool) { chunk, isDelta = r.tryDeltaEncodeChunk(temp) if isDelta { @@ -343,6 +345,7 @@ func (r *Repo) encodeTempChunk(temp BufferedChunk, version int, last *uint64) (c ic := NewLoadedChunk(id, temp.Bytes()) hasher := rabinkarp64.NewFromPol(r.pol) r.hashAndStoreChunk(ic, hasher) + ic.Store(r.path) log.Println("Add new chunk", id) return ic, false } @@ -350,6 +353,10 @@ func (r *Repo) encodeTempChunk(temp BufferedChunk, version int, last *uint64) (c return } +// encodeTempChunks encodes the current temporary chunks based on the value of the previous one. +// Temporary chunks can be partial. If the current chunk is smaller than the size of a +// super-feature and there exists a previous chunk, then both are merged before attempting +// to delta-encode them. func (r *Repo) encodeTempChunks(prev BufferedChunk, curr BufferedChunk, version int, last *uint64) []Chunk { if reflect.ValueOf(prev).IsNil() { c, _ := r.encodeTempChunk(curr, version, last) @@ -435,6 +442,13 @@ func (r *Repo) matchStream(stream io.Reader, version int) []Chunk { return chunks } +func storeRecipe(dest string, recipe []Chunk) { + err := writeFile(dest, recipe) + if err != nil { + log.Println(err) + } +} + // mergeTempChunks joins temporary partial chunks from an array of chunks if possible. // If a chunk is smaller than the size required to calculate a super-feature, // it is then appended to the previous consecutive temporary chunk if it exists. diff --git a/repo_test.go b/repo_test.go index 21eccac..244942e 100644 --- a/repo_test.go +++ b/repo_test.go @@ -204,6 +204,13 @@ func TestBsdiff(t *testing.T) { } } +func TestCommit(t *testing.T) { + dest := t.TempDir() + source := path.Join("test", "data") + repo := NewRepo(dest) + repo.Commit(source) +} + func assertLen(t *testing.T, expected int, actual interface{}, prefix string) { s := reflect.ValueOf(actual) if s.Len() != expected { |