From 3e0123b9d8a1097e74fcd500b72cbdaa00c7a49a Mon Sep 17 00:00:00 2001
From: n-peugnet <n.peugnet@free.fr>
Date: Mon, 13 Sep 2021 15:23:32 +0200
Subject: fix loadChunks by using cache instead of reading file
This way there is only one place where we read chunks and where the
read/write wrapper is used.
This also allows to remove LoadedChunk as it is not used anymore.
---
.golangci.yaml | 2 +-
TODO.md | 7 ++++++-
chunk.go | 26 --------------------------
delta.go | 2 +-
repo.go | 14 ++------------
repo_test.go | 2 ++
6 files changed, 12 insertions(+), 41 deletions(-)
diff --git a/.golangci.yaml b/.golangci.yaml
index 7c47848..c618020 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -6,5 +6,5 @@ severity:
rules:
- severity: warning
linters:
- - errcheck # for now, make errcheck only a warning, remove it later
+ - errcheck # for now, make errcheck only a warning, TODO: remove it later
- deadcode
diff --git a/TODO.md b/TODO.md
index 9df11d2..adff59b 100644
--- a/TODO.md
+++ b/TODO.md
@@ -26,7 +26,7 @@ priority 1
- [x] compress before storing
- [x] decompress before loading
- [ ] store compressed chunks into tracks of `trackSize` (1024o)
-- [x] add chunk cache... what was it for again ??
+- [x] add chunk cache to uniquely store chunks in RAM
- [x] better tests for `(*Repo).Commit`
priority 2
@@ -41,6 +41,11 @@ priority 2
- [ ] custom binary marshal and unmarshal for chunks
- [ ] use `loadChunkContent` in `loadChunks`
- [ ] store hashes for faster maps rebuild
+- [ ] try [Fdelta](https://github.com/amlwwalker/fdelta) and
+ [Xdelta](https://github.com/nine-lives-later/go-xdelta) instead of Bsdiff
+- [ ] maybe use an LRU cache instead of the current FIFO one.
+- [x] remove `LoadedChunk` and only use `StoredChunk` instead now that the cache
+ is implemented
reunion 7/09
------------
diff --git a/chunk.go b/chunk.go
index 0d84040..3efaf4a 100644
--- a/chunk.go
+++ b/chunk.go
@@ -36,32 +36,6 @@ func (i *ChunkId) Path(repo string) string {
return filepath.Join(repo, fmt.Sprintf(versionFmt, i.Ver), chunksName, fmt.Sprintf(chunkIdFmt, i.Idx))
}
-func NewLoadedChunk(id *ChunkId, value []byte) *LoadedChunk {
- return &LoadedChunk{Id: id, value: value}
-}
-
-type LoadedChunk struct {
- Id *ChunkId
- value []byte
-}
-
-func (c *LoadedChunk) GetId() *ChunkId {
- return c.Id
-}
-
-func (c *LoadedChunk) Reader() io.ReadSeeker {
- // log.Printf("Chunk %d: Reading from in-memory value\n", c.id)
- return bytes.NewReader(c.value)
-}
-
-func (c *LoadedChunk) Len() int {
- return len(c.value)
-}
-
-func (c *LoadedChunk) Bytes() []byte {
- return c.value
-}
-
func NewStoredChunk(repo *Repo, id *ChunkId) *StoredChunk {
return &StoredChunk{repo: repo, Id: id}
}
diff --git a/delta.go b/delta.go
index 423fb07..ff42e86 100644
--- a/delta.go
+++ b/delta.go
@@ -20,7 +20,7 @@ type Patcher interface {
Patch(source io.Reader, target io.Writer, patch io.Reader) error
}
-// TODO: maybe move this in it own file ?
+// TODO: maybe move this in its own file ?
type Bsdiff struct{}
func (*Bsdiff) Diff(source io.Reader, target io.Reader, patch io.Writer) error {
diff --git a/repo.go b/repo.go
index 50eeb25..f7aa5e9 100644
--- a/repo.go
+++ b/repo.go
@@ -299,18 +299,8 @@ func (r *Repo) loadChunks(versions []string, chunks chan<- IdentifiedChunk) {
if e.IsDir() {
continue
}
- f := filepath.Join(p, e.Name())
- buff, err := os.ReadFile(f)
- if err != nil {
- log.Printf("Error reading chunk '%s': %s", f, err.Error())
- }
- c := NewLoadedChunk(
- &ChunkId{
- Ver: i,
- Idx: uint64(j),
- },
- buff,
- )
+ id := &ChunkId{Ver: i, Idx: uint64(j)}
+ c := NewStoredChunk(r, id)
chunks <- c
}
}
diff --git a/repo_test.go b/repo_test.go
index 39556c6..8748d88 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -131,6 +131,8 @@ func TestLoadChunks(t *testing.T) {
resultDir := t.TempDir()
dataDir := filepath.Join("testdata", "logs")
repo := NewRepo(resultDir)
+ repo.chunkReadWrapper = dummyReader
+ repo.chunkWriteWrapper = dummyWriter
resultVersion := filepath.Join(resultDir, "00000")
resultChunks := filepath.Join(resultVersion, chunksName)
os.MkdirAll(resultChunks, 0775)
--
cgit v1.2.3