diff options
| -rw-r--r-- | .golangci.yaml | 2 | ||||
| -rw-r--r-- | TODO.md | 7 | ||||
| -rw-r--r-- | chunk.go | 26 | ||||
| -rw-r--r-- | delta.go | 2 | ||||
| -rw-r--r-- | repo.go | 14 | ||||
| -rw-r--r-- | repo_test.go | 2 |
6 files changed, 12 insertions, 41 deletions
diff --git a/.golangci.yaml b/.golangci.yaml index 7c47848..c618020 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -6,5 +6,5 @@ severity: rules: - severity: warning linters: - - errcheck # for now, make errcheck only a warning, remove it later + - errcheck # for now, make errcheck only a warning, TODO: remove it later - deadcode @@ -26,7 +26,7 @@ priority 1 - [x] compress before storing - [x] decompress before loading - [ ] store compressed chunks into tracks of `trackSize` (1024o) -- [x] add chunk cache... what was it for again ?? +- [x] add chunk cache to uniquely store chunks in RAM - [x] better tests for `(*Repo).Commit` priority 2 @@ -41,6 +41,11 @@ priority 2 - [ ] custom binary marshal and unmarshal for chunks - [ ] use `loadChunkContent` in `loadChunks` - [ ] store hashes for faster maps rebuild +- [ ] try [Fdelta](https://github.com/amlwwalker/fdelta) and + [Xdelta](https://github.com/nine-lives-later/go-xdelta) instead of Bsdiff +- [ ] maybe use an LRU cache instead of the current FIFO one. +- [x] remove `LoadedChunk` and only use `StoredChunk` instead now that the cache + is implemented reunion 7/09 ------------ @@ -36,32 +36,6 @@ func (i *ChunkId) Path(repo string) string { return filepath.Join(repo, fmt.Sprintf(versionFmt, i.Ver), chunksName, fmt.Sprintf(chunkIdFmt, i.Idx)) } -func NewLoadedChunk(id *ChunkId, value []byte) *LoadedChunk { - return &LoadedChunk{Id: id, value: value} -} - -type LoadedChunk struct { - Id *ChunkId - value []byte -} - -func (c *LoadedChunk) GetId() *ChunkId { - return c.Id -} - -func (c *LoadedChunk) Reader() io.ReadSeeker { - // log.Printf("Chunk %d: Reading from in-memory value\n", c.id) - return bytes.NewReader(c.value) -} - -func (c *LoadedChunk) Len() int { - return len(c.value) -} - -func (c *LoadedChunk) Bytes() []byte { - return c.value -} - func NewStoredChunk(repo *Repo, id *ChunkId) *StoredChunk { return &StoredChunk{repo: repo, Id: id} } @@ -20,7 +20,7 @@ type Patcher interface { Patch(source io.Reader, target io.Writer, patch io.Reader) error } -// TODO: maybe move this in it own file ? +// TODO: maybe move this in its own file ? type Bsdiff struct{} func (*Bsdiff) Diff(source io.Reader, target io.Reader, patch io.Writer) error { @@ -299,18 +299,8 @@ func (r *Repo) loadChunks(versions []string, chunks chan<- IdentifiedChunk) { if e.IsDir() { continue } - f := filepath.Join(p, e.Name()) - buff, err := os.ReadFile(f) - if err != nil { - log.Printf("Error reading chunk '%s': %s", f, err.Error()) - } - c := NewLoadedChunk( - &ChunkId{ - Ver: i, - Idx: uint64(j), - }, - buff, - ) + id := &ChunkId{Ver: i, Idx: uint64(j)} + c := NewStoredChunk(r, id) chunks <- c } } diff --git a/repo_test.go b/repo_test.go index 39556c6..8748d88 100644 --- a/repo_test.go +++ b/repo_test.go @@ -131,6 +131,8 @@ func TestLoadChunks(t *testing.T) { resultDir := t.TempDir() dataDir := filepath.Join("testdata", "logs") repo := NewRepo(resultDir) + repo.chunkReadWrapper = dummyReader + repo.chunkWriteWrapper = dummyWriter resultVersion := filepath.Join(resultDir, "00000") resultChunks := filepath.Join(resultVersion, chunksName) os.MkdirAll(resultChunks, 0775) |