aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--README.md2
-rw-r--r--repo.go9
-rw-r--r--repo_test.go14
-rw-r--r--sketch/sketch.go (renamed from sketch.go)15
-rw-r--r--sketch/sketch_test.go43
-rw-r--r--sketch/testdata/000000000000000 (renamed from test/data/repo_8k/00000/chunks/000000000000000)0
-rw-r--r--sketch/testdata/000000000000014 (renamed from test/data/repo_8k/00000/chunks/000000000000014)0
-rw-r--r--sketch_test.go39
-rw-r--r--testdata/logs/1/logTest.log (renamed from test/data/logs/1/logTest.log)0
-rw-r--r--testdata/logs/2/csvParserTest.log (renamed from test/data/logs/2/csvParserTest.log)0
-rw-r--r--testdata/logs/2/slipdb.log (renamed from test/data/logs/2/slipdb.log)0
-rw-r--r--testdata/logs/3/indexingTreeTest.log (renamed from test/data/logs/3/indexingTreeTest.log)0
-rw-r--r--testdata/repo_8k/00000/chunks/00000000000000085
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000001 (renamed from test/data/repo_8k/00000/chunks/000000000000001)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000002 (renamed from test/data/repo_8k/00000/chunks/000000000000002)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000003 (renamed from test/data/repo_8k/00000/chunks/000000000000003)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000004 (renamed from test/data/repo_8k/00000/chunks/000000000000004)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000005 (renamed from test/data/repo_8k/00000/chunks/000000000000005)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000006 (renamed from test/data/repo_8k/00000/chunks/000000000000006)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000007 (renamed from test/data/repo_8k/00000/chunks/000000000000007)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000008 (renamed from test/data/repo_8k/00000/chunks/000000000000008)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000009 (renamed from test/data/repo_8k/00000/chunks/000000000000009)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000010 (renamed from test/data/repo_8k/00000/chunks/000000000000010)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000011 (renamed from test/data/repo_8k/00000/chunks/000000000000011)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000012 (renamed from test/data/repo_8k/00000/chunks/000000000000012)0
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000013 (renamed from test/data/repo_8k/00000/chunks/000000000000013)0
-rw-r--r--testdata/repo_8k/00000/chunks/00000000000001447
27 files changed, 198 insertions, 56 deletions
diff --git a/README.md b/README.md
index cfea121..b7808cc 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ _Classical go_
go build
# Test
-go test -v
+go test ./...
# Run
go run . <source-dir> <repository>
diff --git a/repo.go b/repo.go
index 2be457d..908429d 100644
--- a/repo.go
+++ b/repo.go
@@ -41,6 +41,7 @@ import (
"github.com/chmduquesne/rollinghash/rabinkarp64"
"github.com/n-peugnet/dna-backup/cache"
+ "github.com/n-peugnet/dna-backup/sketch"
)
type FingerprintMap map[uint64]*ChunkId
@@ -167,7 +168,7 @@ func concatFiles(files []File, stream io.WriteCloser) {
}
func (r *Repo) chunkMinLen() int {
- return SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount)
+ return sketch.SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount)
}
func (r *Repo) chunkStream(stream io.Reader, chunks chan<- []byte) {
@@ -319,7 +320,7 @@ func (r *Repo) hashAndStoreChunk(chunk IdentifiedChunk, hasher hash.Hash64) {
hasher.Reset()
io.Copy(hasher, chunk.Reader())
fingerprint := hasher.Sum64()
- sketch, _ := SketchChunk(chunk, r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount)
+ sketch, _ := sketch.SketchChunk(chunk.Reader(), r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount)
r.storeChunkId(chunk.GetId(), fingerprint, sketch)
}
@@ -347,7 +348,7 @@ func (r *Repo) findSimilarChunk(chunk Chunk) (*ChunkId, bool) {
var similarChunks = make(map[ChunkId]int)
var max int
var similarChunk *ChunkId
- sketch, _ := SketchChunk(chunk, r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount)
+ sketch, _ := sketch.SketchChunk(chunk.Reader(), r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount)
for _, s := range sketch {
chunkIds, exists := r.sketches[s]
if !exists {
@@ -557,7 +558,7 @@ func (r *Repo) mergeTempChunks(chunks []Chunk) (ret []Chunk) {
for _, c := range chunks {
tmp, isTmp := c.(*TempChunk)
if !isTmp {
- if prev != nil && curr.Len() <= SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) {
+ if prev != nil && curr.Len() <= sketch.SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) {
prev.AppendFrom(curr.Reader())
} else if curr != nil {
ret = append(ret, curr)
diff --git a/repo_test.go b/repo_test.go
index dc04126..7d54ef5 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -57,7 +57,7 @@ func chunkCompare(t *testing.T, dataDir string, repo *Repo, testFiles []string,
func TestReadFiles1(t *testing.T) {
repo := NewRepo("")
chunkCount := 590/repo.chunkSize + 1
- dataDir := path.Join("test", "data", "logs", "1")
+ dataDir := path.Join("testdata", "logs", "1")
files := []string{"logTest.log"}
chunkCompare(t, dataDir, repo, files, chunkCount)
}
@@ -65,7 +65,7 @@ func TestReadFiles1(t *testing.T) {
func TestReadFiles2(t *testing.T) {
repo := NewRepo("")
chunkCount := 22899/repo.chunkSize + 1
- dataDir := path.Join("test", "data", "logs", "2")
+ dataDir := path.Join("testdata", "logs", "2")
files := []string{"csvParserTest.log", "slipdb.log"}
chunkCompare(t, dataDir, repo, files, chunkCount)
}
@@ -73,7 +73,7 @@ func TestReadFiles2(t *testing.T) {
func TestReadFiles3(t *testing.T) {
repo := NewRepo("")
chunkCount := 119398/repo.chunkSize + 1
- dataDir := path.Join("test", "data", "logs")
+ dataDir := path.Join("testdata", "logs")
files := []string{
path.Join("1", "logTest.log"),
path.Join("2", "csvParserTest.log"),
@@ -85,7 +85,7 @@ func TestReadFiles3(t *testing.T) {
func TestLoadChunks(t *testing.T) {
resultDir := t.TempDir()
- dataDir := path.Join("test", "data", "logs")
+ dataDir := path.Join("testdata", "logs")
repo := NewRepo(resultDir)
resultVersion := path.Join(resultDir, "00000")
resultChunks := path.Join(resultVersion, chunksName)
@@ -137,7 +137,7 @@ func TestExtractNewChunks(t *testing.T) {
func TestStoreLoadFiles(t *testing.T) {
resultDir := t.TempDir()
- dataDir := path.Join("test", "data", "logs")
+ dataDir := path.Join("testdata", "logs")
resultFiles := path.Join(resultDir, filesName)
files1 := listFiles(dataDir)
storeFileList(resultFiles, files1)
@@ -172,7 +172,7 @@ func getDataStream(dataDir string, streamFunc func([]File, io.WriteCloser)) io.R
func TestBsdiff(t *testing.T) {
resultDir := t.TempDir()
repo := NewRepo(resultDir)
- dataDir := path.Join("test", "data", "logs")
+ dataDir := path.Join("testdata", "logs")
addedFile1 := path.Join(dataDir, "2", "slogTest.log")
addedFile2 := path.Join(dataDir, "3", "slogTest.log")
// Store initial chunks
@@ -206,7 +206,7 @@ func TestBsdiff(t *testing.T) {
func TestCommit(t *testing.T) {
dest := t.TempDir()
- source := path.Join("test", "data")
+ source := "testdata"
repo := NewRepo(dest)
repo.Commit(source)
recipe := loadRecipe(path.Join(dest, "00000", recipeName))
diff --git a/sketch.go b/sketch/sketch.go
index dca813d..12e62fa 100644
--- a/sketch.go
+++ b/sketch/sketch.go
@@ -1,4 +1,4 @@
-package main
+package sketch
import (
"bytes"
@@ -22,18 +22,23 @@ const fBytes = 8
// SketchChunk produces a sketch for a chunk based on wSize: the window size,
// sfCount: the number of super-features, and fCount: the number of feature
// per super-feature
-func SketchChunk(chunk Chunk, pol rabinkarp64.Pol, chunkSize int, wSize int, sfCount int, fCount int) (Sketch, error) {
+func SketchChunk(r io.Reader, pol rabinkarp64.Pol, chunkSize int, wSize int, sfCount int, fCount int) (Sketch, error) {
var wg sync.WaitGroup
var fSize = FeatureSize(chunkSize, sfCount, fCount)
+ var chunk bytes.Buffer
superfeatures := make([]uint64, 0, sfCount)
features := make([]uint64, 0, fCount*sfCount)
sfBuff := make([]byte, fBytes*fCount)
- r := chunk.Reader()
- for f := 0; f < chunk.Len()/fSize; f++ {
+ chunkLen, err := chunk.ReadFrom(r)
+ if err != nil {
+ log.Panicln(chunkLen, err)
+ }
+ for f := 0; f < int(chunkLen)/fSize; f++ {
var fBuff bytes.Buffer
- n, err := io.CopyN(&fBuff, r, int64(fSize))
+ n, err := io.CopyN(&fBuff, &chunk, int64(fSize))
if err != nil {
log.Println(n, err)
+ continue
}
features = append(features, 0)
wg.Add(1)
diff --git a/sketch/sketch_test.go b/sketch/sketch_test.go
new file mode 100644
index 0000000..df35514
--- /dev/null
+++ b/sketch/sketch_test.go
@@ -0,0 +1,43 @@
+package sketch
+
+import (
+ "os"
+ "path"
+ "reflect"
+ "testing"
+
+ "github.com/chmduquesne/rollinghash/rabinkarp64"
+)
+
+func TestSketchChunk(t *testing.T) {
+ var sketch, expected Sketch
+ var err error
+ dataDir := "testdata"
+ pol, err := rabinkarp64.RandomPolynomial(1)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ c0, err := os.Open(path.Join(dataDir, "000000000000000"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ sketch, err = SketchChunk(c0, pol, 8<<10, 32, 3, 4)
+ if err != nil {
+ t.Fatal(err)
+ }
+ expected = Sketch{429857165471867, 6595034117354675, 8697818304802825}
+ if !reflect.DeepEqual(sketch, expected) {
+ t.Errorf("Sketch does not match, expected: %d, actual: %d", expected, sketch)
+ }
+
+ c14, err := os.Open(path.Join(dataDir, "000000000000014"))
+ sketch, err = SketchChunk(c14, pol, 8<<10, 32, 3, 4)
+ if err != nil {
+ t.Error(err)
+ }
+ expected = Sketch{658454504014104}
+ if !reflect.DeepEqual(sketch, expected) {
+ t.Errorf("Sketch does not match, expected: %d, actual: %d", expected, sketch)
+ }
+}
diff --git a/test/data/repo_8k/00000/chunks/000000000000000 b/sketch/testdata/000000000000000
index ac78620..ac78620 100644
--- a/test/data/repo_8k/00000/chunks/000000000000000
+++ b/sketch/testdata/000000000000000
diff --git a/test/data/repo_8k/00000/chunks/000000000000014 b/sketch/testdata/000000000000014
index ab7db22..ab7db22 100644
--- a/test/data/repo_8k/00000/chunks/000000000000014
+++ b/sketch/testdata/000000000000014
diff --git a/sketch_test.go b/sketch_test.go
deleted file mode 100644
index 3dce9b5..0000000
--- a/sketch_test.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package main
-
-import (
- "path"
- "reflect"
- "testing"
-)
-
-func TestSketchChunk(t *testing.T) {
- dataDir := path.Join("test", "data", "repo_8k")
- chunks := make(chan IdentifiedChunk, 16)
- repo := NewRepo(dataDir)
- versions := repo.loadVersions()
- go repo.loadChunks(versions, chunks)
- var i int
- for c := range chunks {
- if i < 1 {
- sketch, err := SketchChunk(c, repo.pol, 8<<10, 32, 3, 4)
- if err != nil {
- t.Error(err)
- }
- expected := Sketch{429857165471867, 6595034117354675, 8697818304802825}
- if !reflect.DeepEqual(sketch, expected) {
- t.Errorf("Sketch does not match, expected: %d, actual: %d", expected, sketch)
- }
- }
- if i == 14 {
- sketch, err := SketchChunk(c, repo.pol, 8<<10, 32, 3, 4)
- if err != nil {
- t.Error(err)
- }
- expected := Sketch{658454504014104}
- if !reflect.DeepEqual(sketch, expected) {
- t.Errorf("Sketch does not match, expected: %d, actual: %d", expected, sketch)
- }
- }
- i++
- }
-}
diff --git a/test/data/logs/1/logTest.log b/testdata/logs/1/logTest.log
index 6bf31c3..6bf31c3 100644
--- a/test/data/logs/1/logTest.log
+++ b/testdata/logs/1/logTest.log
diff --git a/test/data/logs/2/csvParserTest.log b/testdata/logs/2/csvParserTest.log
index 328720a..328720a 100644
--- a/test/data/logs/2/csvParserTest.log
+++ b/testdata/logs/2/csvParserTest.log
diff --git a/test/data/logs/2/slipdb.log b/testdata/logs/2/slipdb.log
index 1f0e3b4..1f0e3b4 100644
--- a/test/data/logs/2/slipdb.log
+++ b/testdata/logs/2/slipdb.log
diff --git a/test/data/logs/3/indexingTreeTest.log b/testdata/logs/3/indexingTreeTest.log
index b115b45..b115b45 100644
--- a/test/data/logs/3/indexingTreeTest.log
+++ b/testdata/logs/3/indexingTreeTest.log
diff --git a/testdata/repo_8k/00000/chunks/000000000000000 b/testdata/repo_8k/00000/chunks/000000000000000
new file mode 100644
index 0000000..ac78620
--- /dev/null
+++ b/testdata/repo_8k/00000/chunks/000000000000000
@@ -0,0 +1,85 @@
+2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52)
+2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57)
+2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52)
+2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57)
+2019-06-05T20:51 [INFO] testInfoString (LogTest.java:52)
+2019-06-05T20:51 [INFO][YO] testInfoString (LogTest.java:57)
+2019-06-05T22:41 [INFO] testInfoString (LogTest.java:52)
+2019-06-05T22:41 [INFO][YO] testInfoString (LogTest.java:57)
+2019-06-05T23:02 [INFO] testInfoString (LogTest.java:52)
+2019-06-05T23:02 [INFO][YO] testInfoString (LogTest.java:57)
+2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.52 millis (Loader.java:116)
+2019-06-05T20:13 [INFO][MEMORY] 160,128 (Loader.java:117)
+2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-05T20:13 [INFO][MEMORY] 231,367 (Loader.java:138)
+2019-06-05T20:13 [INFO][TIMER] Temps pris par le parsing: 0.898 s (LoaderTest.java:69)
+2019-06-05T20:13 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-05T20:14 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.46 millis (Loader.java:116)
+2019-06-05T20:14 [INFO][MEMORY] 267,348 (Loader.java:117)
+2019-06-05T20:14 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-05T20:14 [INFO][MEMORY] 231,359 (Loader.java:138)
+2019-06-05T20:14 [INFO][TIMER] Temps pris par le parsing: 0.839 s (LoaderTest.java:69)
+2019-06-05T20:14 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-05T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.61 millis (Loader.java:116)
+2019-06-05T20:51 [INFO][MEMORY] 189,561 (Loader.java:117)
+2019-06-05T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-05T20:51 [INFO][MEMORY] 168,686 (Loader.java:138)
+2019-06-05T20:51 [INFO][TIMER] Temps pris par le parsing: 0.930 s (LoaderTest.java:69)
+2019-06-05T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-05T22:41 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.83 millis (Loader.java:116)
+2019-06-05T22:41 [INFO][MEMORY] 189,494 (Loader.java:117)
+2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-05T22:41 [INFO][MEMORY] 168,619 (Loader.java:138)
+2019-06-05T22:41 [INFO][TIMER] Temps pris par le parsing: 0.882 s (LoaderTest.java:69)
+2019-06-05T22:41 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116)
+2019-06-05T23:02 [INFO][MEMORY] 189,551 (Loader.java:117)
+2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-05T23:02 [INFO][MEMORY] 168,676 (Loader.java:138)
+2019-06-05T23:02 [INFO][TIMER] Temps pris par le parsing: 1.04 s (LoaderTest.java:69)
+2019-06-05T23:02 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.82 millis (Loader.java:116)
+2019-06-10T20:34 [INFO][MEMORY] 82,958 (Loader.java:117)
+2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T20:34 [INFO][MEMORY] 163,350 (Loader.java:138)
+2019-06-10T20:34 [INFO][TIMER] Temps pris par le parsing: 0.938 s (LoaderTest.java:69)
+2019-06-10T20:34 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116)
+2019-06-10T20:50 [INFO][MEMORY] 82,957 (Loader.java:117)
+2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T20:50 [INFO][MEMORY] 163,349 (Loader.java:138)
+2019-06-10T20:50 [INFO][TIMER] Temps pris par le parsing: 0.853 s (LoaderTest.java:69)
+2019-06-10T20:50 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.51 millis (Loader.java:116)
+2019-06-10T20:51 [INFO][MEMORY] 189,538 (Loader.java:117)
+2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T20:51 [INFO][MEMORY] 168,663 (Loader.java:138)
+2019-06-10T20:51 [INFO][TIMER] Temps pris par le parsing: 1.27 s (LoaderTest.java:69)
+2019-06-10T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.58 millis (Loader.java:116)
+2019-06-10T21:24 [INFO][MEMORY] 189,539 (Loader.java:117)
+2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T21:24 [INFO][MEMORY] 168,664 (Loader.java:138)
+2019-06-10T21:24 [INFO][TIMER] Temps pris par le parsing: 1.86 s (LoaderTest.java:69)
+2019-06-10T21:24 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97)
+2019-06-05T14:47 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71)
+2019-06-05T14:49 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
+2019-06-05T14:49 [INFO][MEMORY] 229,457 (ParallelLoader.java:114)
+2019-06-05T14:49 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.522 sactiveThreadNb = 0 (ParallelLoader.java:150)
+2019-06-05T14:49 [ERROR] debugCount = 0 (ParallelLoader.java:203)
+2019-06-05T14:49 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
+2019-06-05T14:49 [INFO] Parsing terminé !! temps écoulé = 2.78 s (ParallelLoader.java:217)
+2019-06-05T14:49 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
+2019-06-05T14:53 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71)
+2019-06-05T14:53 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
+2019-06-05T14:53 [INFO][MEMORY] 227,095 (ParallelLoader.java:114)
+2019-06-05T14:53 [INFO] Loader : nombre de résulta \ No newline at end of file
diff --git a/test/data/repo_8k/00000/chunks/000000000000001 b/testdata/repo_8k/00000/chunks/000000000000001
index 3e728e2..3e728e2 100644
--- a/test/data/repo_8k/00000/chunks/000000000000001
+++ b/testdata/repo_8k/00000/chunks/000000000000001
diff --git a/test/data/repo_8k/00000/chunks/000000000000002 b/testdata/repo_8k/00000/chunks/000000000000002
index 079ca90..079ca90 100644
--- a/test/data/repo_8k/00000/chunks/000000000000002
+++ b/testdata/repo_8k/00000/chunks/000000000000002
diff --git a/test/data/repo_8k/00000/chunks/000000000000003 b/testdata/repo_8k/00000/chunks/000000000000003
index 8720f6b..8720f6b 100644
--- a/test/data/repo_8k/00000/chunks/000000000000003
+++ b/testdata/repo_8k/00000/chunks/000000000000003
diff --git a/test/data/repo_8k/00000/chunks/000000000000004 b/testdata/repo_8k/00000/chunks/000000000000004
index f05dda7..f05dda7 100644
--- a/test/data/repo_8k/00000/chunks/000000000000004
+++ b/testdata/repo_8k/00000/chunks/000000000000004
diff --git a/test/data/repo_8k/00000/chunks/000000000000005 b/testdata/repo_8k/00000/chunks/000000000000005
index 0134101..0134101 100644
--- a/test/data/repo_8k/00000/chunks/000000000000005
+++ b/testdata/repo_8k/00000/chunks/000000000000005
diff --git a/test/data/repo_8k/00000/chunks/000000000000006 b/testdata/repo_8k/00000/chunks/000000000000006
index 91c72a3..91c72a3 100644
--- a/test/data/repo_8k/00000/chunks/000000000000006
+++ b/testdata/repo_8k/00000/chunks/000000000000006
diff --git a/test/data/repo_8k/00000/chunks/000000000000007 b/testdata/repo_8k/00000/chunks/000000000000007
index f9f54db..f9f54db 100644
--- a/test/data/repo_8k/00000/chunks/000000000000007
+++ b/testdata/repo_8k/00000/chunks/000000000000007
diff --git a/test/data/repo_8k/00000/chunks/000000000000008 b/testdata/repo_8k/00000/chunks/000000000000008
index 5995b3c..5995b3c 100644
--- a/test/data/repo_8k/00000/chunks/000000000000008
+++ b/testdata/repo_8k/00000/chunks/000000000000008
diff --git a/test/data/repo_8k/00000/chunks/000000000000009 b/testdata/repo_8k/00000/chunks/000000000000009
index 8a0d0af..8a0d0af 100644
--- a/test/data/repo_8k/00000/chunks/000000000000009
+++ b/testdata/repo_8k/00000/chunks/000000000000009
diff --git a/test/data/repo_8k/00000/chunks/000000000000010 b/testdata/repo_8k/00000/chunks/000000000000010
index 97a481a..97a481a 100644
--- a/test/data/repo_8k/00000/chunks/000000000000010
+++ b/testdata/repo_8k/00000/chunks/000000000000010
diff --git a/test/data/repo_8k/00000/chunks/000000000000011 b/testdata/repo_8k/00000/chunks/000000000000011
index 574a9a6..574a9a6 100644
--- a/test/data/repo_8k/00000/chunks/000000000000011
+++ b/testdata/repo_8k/00000/chunks/000000000000011
diff --git a/test/data/repo_8k/00000/chunks/000000000000012 b/testdata/repo_8k/00000/chunks/000000000000012
index 5a30329..5a30329 100644
--- a/test/data/repo_8k/00000/chunks/000000000000012
+++ b/testdata/repo_8k/00000/chunks/000000000000012
diff --git a/test/data/repo_8k/00000/chunks/000000000000013 b/testdata/repo_8k/00000/chunks/000000000000013
index 612b2ee..612b2ee 100644
--- a/test/data/repo_8k/00000/chunks/000000000000013
+++ b/testdata/repo_8k/00000/chunks/000000000000013
diff --git a/testdata/repo_8k/00000/chunks/000000000000014 b/testdata/repo_8k/00000/chunks/000000000000014
new file mode 100644
index 0000000..ab7db22
--- /dev/null
+++ b/testdata/repo_8k/00000/chunks/000000000000014
@@ -0,0 +1,47 @@
+shAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338)
+2019-06-10T21:24 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56)
+2019-06-10T21:24 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57)
+2019-06-10T21:24 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73)
+2019-06-10T21:24 [ERROR] null (SerialStructure.java:74)
+2019-06-10T21:24 [INFO] Lancé (IndexTreeMessyTest.java:366)
+2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 37 millis (IndexTreeDic.java:1164)
+2019-06-10T21:24 [INFO][TIMER] Temps total recherche: 0.044 s (IndexTreeMessyTest.java:408)
+2019-06-10T21:24 [INFO][TIMER] Temps parcours des résultats: 0.59 millis (IndexTreeMessyTest.java:416)
+2019-06-10T21:24 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417)
+2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164)
+2019-06-10T21:24 [INFO][TIMER] Temps total recherche: 7.06 millis (IndexTreeMessyTest.java:425)
+2019-06-10T21:24 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.071 s (IndexTreeMessyTest.java:439)
+2019-06-10T21:24 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449)
+2019-06-10T21:24 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450)
+2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 8 millis (IndexTreeDic.java:1164)
+2019-06-10T21:24 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456)
+2019-06-10T21:24 [INFO] 40 (HashMapTests.java:27)
+2019-06-10T21:24 [INFO][MEMORY] Mem usage début - 88,272 (ParallelLoaderTest.java:56)
+2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65)
+2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
+2019-06-10T21:24 [INFO][MEMORY] 90,320 (ParallelLoader.java:114)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.046 sactiveThreadNb = 0 (ParallelLoader.java:150)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
+2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.898 s (ParallelLoader.java:217)
+2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
+2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65)
+2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
+2019-06-10T21:24 [INFO][MEMORY] 232,213 (ParallelLoader.java:114)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.047 sactiveThreadNb = 0 (ParallelLoader.java:150)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
+2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.709 s (ParallelLoader.java:217)
+2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
+2019-06-10T21:24 [INFO][MEMORY] Mem usage fin - 189,539 (ParallelLoaderTest.java:114)
+2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.68 s (ParallelLoaderTest.java:115)
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-24 23:15:32 +0000