diff options
| -rw-r--r-- | TODO.md | 2 | ||||
| -rw-r--r-- | repo.go | 40 | ||||
| -rw-r--r-- | repo_test.go | 39 |
3 files changed, 41 insertions, 40 deletions
@@ -27,7 +27,7 @@ priority 1 priority 2 ---------- -- [ ] use more the `Reader` API (which is analoguous to the `IOStream` in Java) +- [x] use more the `Reader` API (which is analoguous to the `IOStream` in Java) - [ ] refactor matchStream as right now it is quite complex - [ ] better test for `Repo.matchStream` - [ ] tail packing of PartialChunks (this Struct does not exist yet as it is in fact just `TempChunks` for now) @@ -171,32 +171,6 @@ func (r *Repo) chunkMinLen() int { return sketch.SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) } -func (r *Repo) chunkStream(stream io.Reader, chunks chan<- []byte) { - var buff []byte - var prev, read = r.chunkSize, 0 - var err error - - for err != io.EOF { - if prev == r.chunkSize { - buff = make([]byte, r.chunkSize) - prev, err = stream.Read(buff) - } else { - read, err = stream.Read(buff[prev:]) - prev += read - } - if err != nil && err != io.EOF { - log.Println(err) - } - if prev == r.chunkSize { - chunks <- buff - } - } - if prev != r.chunkSize { - chunks <- buff[:prev] - } - close(chunks) -} - func storeFileList(dest string, files []File) { file, err := os.Create(dest) if err == nil { @@ -245,7 +219,7 @@ func (r *Repo) StoreChunkContent(id *ChunkId, reader io.Reader) error { // LoadChunkContent loads a chunk from the repo. // If the chunk is in cache, get it from cache, else read it from drive. -func (r *Repo) LoadChunkContent(id *ChunkId) io.ReadSeeker { +func (r *Repo) LoadChunkContent(id *ChunkId) *bytes.Reader { value, exists := r.chunkCache.Get(id) if !exists { path := id.Path(r.path) @@ -262,18 +236,6 @@ func (r *Repo) LoadChunkContent(id *ChunkId) io.ReadSeeker { return bytes.NewReader(value) } -func storeChunks(dest string, chunks <-chan []byte) { - i := 0 - for c := range chunks { - path := path.Join(dest, fmt.Sprintf(chunkIdFmt, i)) - err := os.WriteFile(path, c, 0664) - if err != nil { - log.Println(err) - } - i++ - } -} - // TODO: use atoi for chunkid func (r *Repo) loadChunks(versions []string, chunks chan<- IdentifiedChunk) { for i, v := range versions { diff --git a/repo_test.go b/repo_test.go index 7d54ef5..07af682 100644 --- a/repo_test.go +++ b/repo_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "fmt" "io" "io/ioutil" "log" @@ -54,6 +55,44 @@ func chunkCompare(t *testing.T, dataDir string, repo *Repo, testFiles []string, } } +func (r *Repo) chunkStream(stream io.Reader, chunks chan<- []byte) { + var buff []byte + var prev, read = r.chunkSize, 0 + var err error + + for err != io.EOF { + if prev == r.chunkSize { + buff = make([]byte, r.chunkSize) + prev, err = stream.Read(buff) + } else { + read, err = stream.Read(buff[prev:]) + prev += read + } + if err != nil && err != io.EOF { + log.Println(err) + } + if prev == r.chunkSize { + chunks <- buff + } + } + if prev != r.chunkSize { + chunks <- buff[:prev] + } + close(chunks) +} + +func storeChunks(dest string, chunks <-chan []byte) { + i := 0 + for c := range chunks { + path := path.Join(dest, fmt.Sprintf(chunkIdFmt, i)) + err := os.WriteFile(path, c, 0664) + if err != nil { + log.Println(err) + } + i++ + } +} + func TestReadFiles1(t *testing.T) { repo := NewRepo("") chunkCount := 590/repo.chunkSize + 1 |