From 4ca1c3bbb76834c69b0d8e63c921ca3548f529ac Mon Sep 17 00:00:00 2001 From: n-peugnet Date: Tue, 14 Sep 2021 16:43:29 +0200 Subject: add load/store sketches+fingerprints & store them Fix corresponding Commit tests by adding sketches and fingerprints files in repo_8k and repo_8k_zlib. Also enhance CommitTests by checking recipe files --- const.go | 12 ++++--- repo.go | 52 ++++++++++++++++++++++++++----- repo_test.go | 18 ++++++++++- testdata/repo_8k/00000/fingerprints | Bin 0 -> 208 bytes testdata/repo_8k/00000/sketches | Bin 0 -> 538 bytes testdata/repo_8k_zlib/00000/fingerprints | Bin 0 -> 208 bytes testdata/repo_8k_zlib/00000/sketches | Bin 0 -> 538 bytes 7 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 testdata/repo_8k/00000/fingerprints create mode 100644 testdata/repo_8k/00000/sketches create mode 100644 testdata/repo_8k_zlib/00000/fingerprints create mode 100644 testdata/repo_8k_zlib/00000/sketches diff --git a/const.go b/const.go index 2c58dd6..d5ec7f0 100644 --- a/const.go +++ b/const.go @@ -1,9 +1,11 @@ package main const ( - chunksName = "chunks" - chunkIdFmt = "%015d" - versionFmt = "%05d" - filesName = "files" - recipeName = "recipe" + chunksName = "chunks" + chunkIdFmt = "%015d" + versionFmt = "%05d" + filesName = "files" + fingerprintsName = "fingerprints" + recipeName = "recipe" + sketchesName = "sketches" ) diff --git a/repo.go b/repo.go index cd07759..a3aee80 100644 --- a/repo.go +++ b/repo.go @@ -12,13 +12,17 @@ repo/ │ │ ├── 000000000000002 │ │ ├── 000000000000003 │ ├── files -│ └── recipe +│ ├── fingerprints +│ ├── recipe +│ └── sketches └── 00001/ ├── chunks/ │ ├── 000000000000000 │ ├── 000000000000001 ├── files - └── recipe +│ ├── fingerprints +│ ├── recipe +│ └── sketches ``` */ @@ -110,7 +114,9 @@ func (r *Repo) Commit(source string) { newPath := filepath.Join(r.path, fmt.Sprintf(versionFmt, newVersion)) newChunkPath := filepath.Join(newPath, chunksName) newFilesPath := filepath.Join(newPath, filesName) + newFingerprintsPath := filepath.Join(newPath, fingerprintsName) newRecipePath := filepath.Join(newPath, recipeName) + newSketchesPath := filepath.Join(newPath, sketchesName) os.Mkdir(newPath, 0775) // TODO: handle errors os.Mkdir(newChunkPath, 0775) // TODO: handle errors reader, writer := io.Pipe() @@ -120,8 +126,10 @@ func (r *Repo) Commit(source string) { go concatFiles(files, writer) r.hashChunks(oldChunks) recipe := r.matchStream(reader, newVersion) - storeRecipe(newRecipePath, recipe) storeFileList(newFilesPath, unprefixFiles(files, source)) + storeFingerprints(newFingerprintsPath, r.fingerprints) + storeRecipe(newRecipePath, recipe) + storeSketches(newSketchesPath, r.sketches) logger.Info(files) } @@ -211,11 +219,11 @@ func concatFiles(files []File, stream io.WriteCloser) { stream.Close() } -func storeFileList(dest string, files []File) { +func storeBasicStruct(dest string, obj interface{}) { file, err := os.Create(dest) if err == nil { encoder := gob.NewEncoder(file) - err = encoder.Encode(files) + err = encoder.Encode(obj) } if err != nil { logger.Panic(err) @@ -225,12 +233,11 @@ func storeFileList(dest string, files []File) { } } -func loadFileList(path string) []File { - var files []File +func loadBasicStruct(path string, obj interface{}) { file, err := os.Open(path) if err == nil { decoder := gob.NewDecoder(file) - err = decoder.Decode(&files) + err = decoder.Decode(obj) } if err != nil { logger.Panic(err) @@ -238,6 +245,15 @@ func loadFileList(path string) []File { if err = file.Close(); err != nil { logger.Panic(err) } +} + +func storeFileList(dest string, files []File) { + storeBasicStruct(dest, files) +} + +func loadFileList(path string) []File { + var files []File + loadBasicStruct(path, &files) return files } @@ -574,6 +590,26 @@ func loadRecipe(path string) []Chunk { return recipe } +func storeFingerprints(dest string, fingerprints FingerprintMap) { + storeBasicStruct(dest, fingerprints) +} + +func loadFingerprints(path string) FingerprintMap { + var fingerprints FingerprintMap + loadBasicStruct(path, &fingerprints) + return fingerprints +} + +func storeSketches(dest string, sketches SketchMap) { + storeBasicStruct(dest, sketches) +} + +func loadSketches(path string) SketchMap { + var sketches SketchMap + loadBasicStruct(path, &sketches) + return sketches +} + func extractDeltaChunks(chunks []Chunk) (ret []*DeltaChunk) { for _, c := range chunks { tmp, isDelta := c.(*DeltaChunk) diff --git a/repo_test.go b/repo_test.go index b16ddc6..f6f6edd 100644 --- a/repo_test.go +++ b/repo_test.go @@ -319,6 +319,7 @@ func assertSameTree(t *testing.T, apply func(t *testing.T, expected string, actu func assertCompatibleRepoFile(t *testing.T, expected string, actual string, prefix string) { if filepath.Base(expected) == filesName { + // Filelist file eFiles := loadFileList(expected) aFiles := loadFileList(actual) assertLen(t, len(eFiles), aFiles, prefix) @@ -329,8 +330,23 @@ func assertCompatibleRepoFile(t *testing.T, expected string, actual string, pref } } } else if filepath.Base(expected) == recipeName { - // TODO: check recipies equality + // Recipe file + eRecipe := loadRecipe(expected) + aRecipe := loadRecipe(actual) + assertLen(t, len(eRecipe), aRecipe, prefix) + for i, eChunk := range eRecipe { + if reflect.DeepEqual(eChunk, aRecipe[i]) { + continue + } else { + t.Fatal(prefix, "chunk do not match:", aRecipe[i], ", expected", eChunk) + } + } + } else if filepath.Base(expected) == sketchesName { + // TODO: check Sketches file + } else if filepath.Base(expected) == fingerprintsName { + // TODO: check Fingerprints file } else { + // Chunk content file assertSameFile(t, expected, actual, prefix) } } diff --git a/testdata/repo_8k/00000/fingerprints b/testdata/repo_8k/00000/fingerprints new file mode 100644 index 0000000..d015949 Binary files /dev/null and b/testdata/repo_8k/00000/fingerprints differ diff --git a/testdata/repo_8k/00000/sketches b/testdata/repo_8k/00000/sketches new file mode 100644 index 0000000..83572df Binary files /dev/null and b/testdata/repo_8k/00000/sketches differ diff --git a/testdata/repo_8k_zlib/00000/fingerprints b/testdata/repo_8k_zlib/00000/fingerprints new file mode 100644 index 0000000..d015949 Binary files /dev/null and b/testdata/repo_8k_zlib/00000/fingerprints differ diff --git a/testdata/repo_8k_zlib/00000/sketches b/testdata/repo_8k_zlib/00000/sketches new file mode 100644 index 0000000..83572df Binary files /dev/null and b/testdata/repo_8k_zlib/00000/sketches differ -- cgit v1.2.3