diff options
-rw-r--r-- | README.md | 2 | ||||
-rw-r--r-- | repo.go | 9 | ||||
-rw-r--r-- | repo_test.go | 14 | ||||
-rw-r--r-- | sketch/sketch.go (renamed from sketch.go) | 15 | ||||
-rw-r--r-- | sketch/sketch_test.go | 43 | ||||
-rw-r--r-- | sketch/testdata/000000000000000 (renamed from test/data/repo_8k/00000/chunks/000000000000000) | 0 | ||||
-rw-r--r-- | sketch/testdata/000000000000014 (renamed from test/data/repo_8k/00000/chunks/000000000000014) | 0 | ||||
-rw-r--r-- | sketch_test.go | 39 | ||||
-rw-r--r-- | testdata/logs/1/logTest.log (renamed from test/data/logs/1/logTest.log) | 0 | ||||
-rw-r--r-- | testdata/logs/2/csvParserTest.log (renamed from test/data/logs/2/csvParserTest.log) | 0 | ||||
-rw-r--r-- | testdata/logs/2/slipdb.log (renamed from test/data/logs/2/slipdb.log) | 0 | ||||
-rw-r--r-- | testdata/logs/3/indexingTreeTest.log (renamed from test/data/logs/3/indexingTreeTest.log) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000000 | 85 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000001 (renamed from test/data/repo_8k/00000/chunks/000000000000001) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000002 (renamed from test/data/repo_8k/00000/chunks/000000000000002) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000003 (renamed from test/data/repo_8k/00000/chunks/000000000000003) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000004 (renamed from test/data/repo_8k/00000/chunks/000000000000004) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000005 (renamed from test/data/repo_8k/00000/chunks/000000000000005) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000006 (renamed from test/data/repo_8k/00000/chunks/000000000000006) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000007 (renamed from test/data/repo_8k/00000/chunks/000000000000007) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000008 (renamed from test/data/repo_8k/00000/chunks/000000000000008) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000009 (renamed from test/data/repo_8k/00000/chunks/000000000000009) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000010 (renamed from test/data/repo_8k/00000/chunks/000000000000010) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000011 (renamed from test/data/repo_8k/00000/chunks/000000000000011) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000012 (renamed from test/data/repo_8k/00000/chunks/000000000000012) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000013 (renamed from test/data/repo_8k/00000/chunks/000000000000013) | 0 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000014 | 47 |
27 files changed, 198 insertions, 56 deletions
@@ -15,7 +15,7 @@ _Classical go_ go build # Test -go test -v +go test ./... # Run go run . <source-dir> <repository> @@ -41,6 +41,7 @@ import ( "github.com/chmduquesne/rollinghash/rabinkarp64" "github.com/n-peugnet/dna-backup/cache" + "github.com/n-peugnet/dna-backup/sketch" ) type FingerprintMap map[uint64]*ChunkId @@ -167,7 +168,7 @@ func concatFiles(files []File, stream io.WriteCloser) { } func (r *Repo) chunkMinLen() int { - return SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) + return sketch.SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) } func (r *Repo) chunkStream(stream io.Reader, chunks chan<- []byte) { @@ -319,7 +320,7 @@ func (r *Repo) hashAndStoreChunk(chunk IdentifiedChunk, hasher hash.Hash64) { hasher.Reset() io.Copy(hasher, chunk.Reader()) fingerprint := hasher.Sum64() - sketch, _ := SketchChunk(chunk, r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) + sketch, _ := sketch.SketchChunk(chunk.Reader(), r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) r.storeChunkId(chunk.GetId(), fingerprint, sketch) } @@ -347,7 +348,7 @@ func (r *Repo) findSimilarChunk(chunk Chunk) (*ChunkId, bool) { var similarChunks = make(map[ChunkId]int) var max int var similarChunk *ChunkId - sketch, _ := SketchChunk(chunk, r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) + sketch, _ := sketch.SketchChunk(chunk.Reader(), r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) for _, s := range sketch { chunkIds, exists := r.sketches[s] if !exists { @@ -557,7 +558,7 @@ func (r *Repo) mergeTempChunks(chunks []Chunk) (ret []Chunk) { for _, c := range chunks { tmp, isTmp := c.(*TempChunk) if !isTmp { - if prev != nil && curr.Len() <= SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) { + if prev != nil && curr.Len() <= sketch.SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) { prev.AppendFrom(curr.Reader()) } else if curr != nil { ret = append(ret, curr) diff --git a/repo_test.go b/repo_test.go index dc04126..7d54ef5 100644 --- a/repo_test.go +++ b/repo_test.go @@ -57,7 +57,7 @@ func chunkCompare(t *testing.T, dataDir string, repo *Repo, testFiles []string, func TestReadFiles1(t *testing.T) { repo := NewRepo("") chunkCount := 590/repo.chunkSize + 1 - dataDir := path.Join("test", "data", "logs", "1") + dataDir := path.Join("testdata", "logs", "1") files := []string{"logTest.log"} chunkCompare(t, dataDir, repo, files, chunkCount) } @@ -65,7 +65,7 @@ func TestReadFiles1(t *testing.T) { func TestReadFiles2(t *testing.T) { repo := NewRepo("") chunkCount := 22899/repo.chunkSize + 1 - dataDir := path.Join("test", "data", "logs", "2") + dataDir := path.Join("testdata", "logs", "2") files := []string{"csvParserTest.log", "slipdb.log"} chunkCompare(t, dataDir, repo, files, chunkCount) } @@ -73,7 +73,7 @@ func TestReadFiles2(t *testing.T) { func TestReadFiles3(t *testing.T) { repo := NewRepo("") chunkCount := 119398/repo.chunkSize + 1 - dataDir := path.Join("test", "data", "logs") + dataDir := path.Join("testdata", "logs") files := []string{ path.Join("1", "logTest.log"), path.Join("2", "csvParserTest.log"), @@ -85,7 +85,7 @@ func TestReadFiles3(t *testing.T) { func TestLoadChunks(t *testing.T) { resultDir := t.TempDir() - dataDir := path.Join("test", "data", "logs") + dataDir := path.Join("testdata", "logs") repo := NewRepo(resultDir) resultVersion := path.Join(resultDir, "00000") resultChunks := path.Join(resultVersion, chunksName) @@ -137,7 +137,7 @@ func TestExtractNewChunks(t *testing.T) { func TestStoreLoadFiles(t *testing.T) { resultDir := t.TempDir() - dataDir := path.Join("test", "data", "logs") + dataDir := path.Join("testdata", "logs") resultFiles := path.Join(resultDir, filesName) files1 := listFiles(dataDir) storeFileList(resultFiles, files1) @@ -172,7 +172,7 @@ func getDataStream(dataDir string, streamFunc func([]File, io.WriteCloser)) io.R func TestBsdiff(t *testing.T) { resultDir := t.TempDir() repo := NewRepo(resultDir) - dataDir := path.Join("test", "data", "logs") + dataDir := path.Join("testdata", "logs") addedFile1 := path.Join(dataDir, "2", "slogTest.log") addedFile2 := path.Join(dataDir, "3", "slogTest.log") // Store initial chunks @@ -206,7 +206,7 @@ func TestBsdiff(t *testing.T) { func TestCommit(t *testing.T) { dest := t.TempDir() - source := path.Join("test", "data") + source := "testdata" repo := NewRepo(dest) repo.Commit(source) recipe := loadRecipe(path.Join(dest, "00000", recipeName)) diff --git a/sketch.go b/sketch/sketch.go index dca813d..12e62fa 100644 --- a/sketch.go +++ b/sketch/sketch.go @@ -1,4 +1,4 @@ -package main +package sketch import ( "bytes" @@ -22,18 +22,23 @@ const fBytes = 8 // SketchChunk produces a sketch for a chunk based on wSize: the window size, // sfCount: the number of super-features, and fCount: the number of feature // per super-feature -func SketchChunk(chunk Chunk, pol rabinkarp64.Pol, chunkSize int, wSize int, sfCount int, fCount int) (Sketch, error) { +func SketchChunk(r io.Reader, pol rabinkarp64.Pol, chunkSize int, wSize int, sfCount int, fCount int) (Sketch, error) { var wg sync.WaitGroup var fSize = FeatureSize(chunkSize, sfCount, fCount) + var chunk bytes.Buffer superfeatures := make([]uint64, 0, sfCount) features := make([]uint64, 0, fCount*sfCount) sfBuff := make([]byte, fBytes*fCount) - r := chunk.Reader() - for f := 0; f < chunk.Len()/fSize; f++ { + chunkLen, err := chunk.ReadFrom(r) + if err != nil { + log.Panicln(chunkLen, err) + } + for f := 0; f < int(chunkLen)/fSize; f++ { var fBuff bytes.Buffer - n, err := io.CopyN(&fBuff, r, int64(fSize)) + n, err := io.CopyN(&fBuff, &chunk, int64(fSize)) if err != nil { log.Println(n, err) + continue } features = append(features, 0) wg.Add(1) diff --git a/sketch/sketch_test.go b/sketch/sketch_test.go new file mode 100644 index 0000000..df35514 --- /dev/null +++ b/sketch/sketch_test.go @@ -0,0 +1,43 @@ +package sketch + +import ( + "os" + "path" + "reflect" + "testing" + + "github.com/chmduquesne/rollinghash/rabinkarp64" +) + +func TestSketchChunk(t *testing.T) { + var sketch, expected Sketch + var err error + dataDir := "testdata" + pol, err := rabinkarp64.RandomPolynomial(1) + if err != nil { + t.Fatal(err) + } + + c0, err := os.Open(path.Join(dataDir, "000000000000000")) + if err != nil { + t.Fatal(err) + } + sketch, err = SketchChunk(c0, pol, 8<<10, 32, 3, 4) + if err != nil { + t.Fatal(err) + } + expected = Sketch{429857165471867, 6595034117354675, 8697818304802825} + if !reflect.DeepEqual(sketch, expected) { + t.Errorf("Sketch does not match, expected: %d, actual: %d", expected, sketch) + } + + c14, err := os.Open(path.Join(dataDir, "000000000000014")) + sketch, err = SketchChunk(c14, pol, 8<<10, 32, 3, 4) + if err != nil { + t.Error(err) + } + expected = Sketch{658454504014104} + if !reflect.DeepEqual(sketch, expected) { + t.Errorf("Sketch does not match, expected: %d, actual: %d", expected, sketch) + } +} diff --git a/test/data/repo_8k/00000/chunks/000000000000000 b/sketch/testdata/000000000000000 index ac78620..ac78620 100644 --- a/test/data/repo_8k/00000/chunks/000000000000000 +++ b/sketch/testdata/000000000000000 diff --git a/test/data/repo_8k/00000/chunks/000000000000014 b/sketch/testdata/000000000000014 index ab7db22..ab7db22 100644 --- a/test/data/repo_8k/00000/chunks/000000000000014 +++ b/sketch/testdata/000000000000014 diff --git a/sketch_test.go b/sketch_test.go deleted file mode 100644 index 3dce9b5..0000000 --- a/sketch_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "path" - "reflect" - "testing" -) - -func TestSketchChunk(t *testing.T) { - dataDir := path.Join("test", "data", "repo_8k") - chunks := make(chan IdentifiedChunk, 16) - repo := NewRepo(dataDir) - versions := repo.loadVersions() - go repo.loadChunks(versions, chunks) - var i int - for c := range chunks { - if i < 1 { - sketch, err := SketchChunk(c, repo.pol, 8<<10, 32, 3, 4) - if err != nil { - t.Error(err) - } - expected := Sketch{429857165471867, 6595034117354675, 8697818304802825} - if !reflect.DeepEqual(sketch, expected) { - t.Errorf("Sketch does not match, expected: %d, actual: %d", expected, sketch) - } - } - if i == 14 { - sketch, err := SketchChunk(c, repo.pol, 8<<10, 32, 3, 4) - if err != nil { - t.Error(err) - } - expected := Sketch{658454504014104} - if !reflect.DeepEqual(sketch, expected) { - t.Errorf("Sketch does not match, expected: %d, actual: %d", expected, sketch) - } - } - i++ - } -} diff --git a/test/data/logs/1/logTest.log b/testdata/logs/1/logTest.log index 6bf31c3..6bf31c3 100644 --- a/test/data/logs/1/logTest.log +++ b/testdata/logs/1/logTest.log diff --git a/test/data/logs/2/csvParserTest.log b/testdata/logs/2/csvParserTest.log index 328720a..328720a 100644 --- a/test/data/logs/2/csvParserTest.log +++ b/testdata/logs/2/csvParserTest.log diff --git a/test/data/logs/2/slipdb.log b/testdata/logs/2/slipdb.log index 1f0e3b4..1f0e3b4 100644 --- a/test/data/logs/2/slipdb.log +++ b/testdata/logs/2/slipdb.log diff --git a/test/data/logs/3/indexingTreeTest.log b/testdata/logs/3/indexingTreeTest.log index b115b45..b115b45 100644 --- a/test/data/logs/3/indexingTreeTest.log +++ b/testdata/logs/3/indexingTreeTest.log diff --git a/testdata/repo_8k/00000/chunks/000000000000000 b/testdata/repo_8k/00000/chunks/000000000000000 new file mode 100644 index 0000000..ac78620 --- /dev/null +++ b/testdata/repo_8k/00000/chunks/000000000000000 @@ -0,0 +1,85 @@ +2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T20:51 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:51 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T22:41 [INFO] testInfoString (LogTest.java:52) +2019-06-05T22:41 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T23:02 [INFO] testInfoString (LogTest.java:52) +2019-06-05T23:02 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.52 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 160,128 (Loader.java:117) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 231,367 (Loader.java:138) +2019-06-05T20:13 [INFO][TIMER] Temps pris par le parsing: 0.898 s (LoaderTest.java:69) +2019-06-05T20:13 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:14 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.46 millis (Loader.java:116) +2019-06-05T20:14 [INFO][MEMORY] 267,348 (Loader.java:117) +2019-06-05T20:14 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:14 [INFO][MEMORY] 231,359 (Loader.java:138) +2019-06-05T20:14 [INFO][TIMER] Temps pris par le parsing: 0.839 s (LoaderTest.java:69) +2019-06-05T20:14 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.61 millis (Loader.java:116) +2019-06-05T20:51 [INFO][MEMORY] 189,561 (Loader.java:117) +2019-06-05T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:51 [INFO][MEMORY] 168,686 (Loader.java:138) +2019-06-05T20:51 [INFO][TIMER] Temps pris par le parsing: 0.930 s (LoaderTest.java:69) +2019-06-05T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T22:41 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.83 millis (Loader.java:116) +2019-06-05T22:41 [INFO][MEMORY] 189,494 (Loader.java:117) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 168,619 (Loader.java:138) +2019-06-05T22:41 [INFO][TIMER] Temps pris par le parsing: 0.882 s (LoaderTest.java:69) +2019-06-05T22:41 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) +2019-06-05T23:02 [INFO][MEMORY] 189,551 (Loader.java:117) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 168,676 (Loader.java:138) +2019-06-05T23:02 [INFO][TIMER] Temps pris par le parsing: 1.04 s (LoaderTest.java:69) +2019-06-05T23:02 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.82 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 82,958 (Loader.java:117) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 163,350 (Loader.java:138) +2019-06-10T20:34 [INFO][TIMER] Temps pris par le parsing: 0.938 s (LoaderTest.java:69) +2019-06-10T20:34 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) +2019-06-10T20:50 [INFO][MEMORY] 82,957 (Loader.java:117) +2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:50 [INFO][MEMORY] 163,349 (Loader.java:138) +2019-06-10T20:50 [INFO][TIMER] Temps pris par le parsing: 0.853 s (LoaderTest.java:69) +2019-06-10T20:50 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.51 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 189,538 (Loader.java:117) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 168,663 (Loader.java:138) +2019-06-10T20:51 [INFO][TIMER] Temps pris par le parsing: 1.27 s (LoaderTest.java:69) +2019-06-10T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.58 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 189,539 (Loader.java:117) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 168,664 (Loader.java:138) +2019-06-10T21:24 [INFO][TIMER] Temps pris par le parsing: 1.86 s (LoaderTest.java:69) +2019-06-10T21:24 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T14:47 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T14:49 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T14:49 [INFO][MEMORY] 229,457 (ParallelLoader.java:114) +2019-06-05T14:49 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.522 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T14:49 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T14:49 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T14:49 [INFO] Parsing terminé !! temps écoulé = 2.78 s (ParallelLoader.java:217) +2019-06-05T14:49 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T14:53 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T14:53 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T14:53 [INFO][MEMORY] 227,095 (ParallelLoader.java:114) +2019-06-05T14:53 [INFO] Loader : nombre de résulta
\ No newline at end of file diff --git a/test/data/repo_8k/00000/chunks/000000000000001 b/testdata/repo_8k/00000/chunks/000000000000001 index 3e728e2..3e728e2 100644 --- a/test/data/repo_8k/00000/chunks/000000000000001 +++ b/testdata/repo_8k/00000/chunks/000000000000001 diff --git a/test/data/repo_8k/00000/chunks/000000000000002 b/testdata/repo_8k/00000/chunks/000000000000002 index 079ca90..079ca90 100644 --- a/test/data/repo_8k/00000/chunks/000000000000002 +++ b/testdata/repo_8k/00000/chunks/000000000000002 diff --git a/test/data/repo_8k/00000/chunks/000000000000003 b/testdata/repo_8k/00000/chunks/000000000000003 index 8720f6b..8720f6b 100644 --- a/test/data/repo_8k/00000/chunks/000000000000003 +++ b/testdata/repo_8k/00000/chunks/000000000000003 diff --git a/test/data/repo_8k/00000/chunks/000000000000004 b/testdata/repo_8k/00000/chunks/000000000000004 index f05dda7..f05dda7 100644 --- a/test/data/repo_8k/00000/chunks/000000000000004 +++ b/testdata/repo_8k/00000/chunks/000000000000004 diff --git a/test/data/repo_8k/00000/chunks/000000000000005 b/testdata/repo_8k/00000/chunks/000000000000005 index 0134101..0134101 100644 --- a/test/data/repo_8k/00000/chunks/000000000000005 +++ b/testdata/repo_8k/00000/chunks/000000000000005 diff --git a/test/data/repo_8k/00000/chunks/000000000000006 b/testdata/repo_8k/00000/chunks/000000000000006 index 91c72a3..91c72a3 100644 --- a/test/data/repo_8k/00000/chunks/000000000000006 +++ b/testdata/repo_8k/00000/chunks/000000000000006 diff --git a/test/data/repo_8k/00000/chunks/000000000000007 b/testdata/repo_8k/00000/chunks/000000000000007 index f9f54db..f9f54db 100644 --- a/test/data/repo_8k/00000/chunks/000000000000007 +++ b/testdata/repo_8k/00000/chunks/000000000000007 diff --git a/test/data/repo_8k/00000/chunks/000000000000008 b/testdata/repo_8k/00000/chunks/000000000000008 index 5995b3c..5995b3c 100644 --- a/test/data/repo_8k/00000/chunks/000000000000008 +++ b/testdata/repo_8k/00000/chunks/000000000000008 diff --git a/test/data/repo_8k/00000/chunks/000000000000009 b/testdata/repo_8k/00000/chunks/000000000000009 index 8a0d0af..8a0d0af 100644 --- a/test/data/repo_8k/00000/chunks/000000000000009 +++ b/testdata/repo_8k/00000/chunks/000000000000009 diff --git a/test/data/repo_8k/00000/chunks/000000000000010 b/testdata/repo_8k/00000/chunks/000000000000010 index 97a481a..97a481a 100644 --- a/test/data/repo_8k/00000/chunks/000000000000010 +++ b/testdata/repo_8k/00000/chunks/000000000000010 diff --git a/test/data/repo_8k/00000/chunks/000000000000011 b/testdata/repo_8k/00000/chunks/000000000000011 index 574a9a6..574a9a6 100644 --- a/test/data/repo_8k/00000/chunks/000000000000011 +++ b/testdata/repo_8k/00000/chunks/000000000000011 diff --git a/test/data/repo_8k/00000/chunks/000000000000012 b/testdata/repo_8k/00000/chunks/000000000000012 index 5a30329..5a30329 100644 --- a/test/data/repo_8k/00000/chunks/000000000000012 +++ b/testdata/repo_8k/00000/chunks/000000000000012 diff --git a/test/data/repo_8k/00000/chunks/000000000000013 b/testdata/repo_8k/00000/chunks/000000000000013 index 612b2ee..612b2ee 100644 --- a/test/data/repo_8k/00000/chunks/000000000000013 +++ b/testdata/repo_8k/00000/chunks/000000000000013 diff --git a/testdata/repo_8k/00000/chunks/000000000000014 b/testdata/repo_8k/00000/chunks/000000000000014 new file mode 100644 index 0000000..ab7db22 --- /dev/null +++ b/testdata/repo_8k/00000/chunks/000000000000014 @@ -0,0 +1,47 @@ +shAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) +2019-06-10T21:24 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-10T21:24 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-10T21:24 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-10T21:24 [ERROR] null (SerialStructure.java:74) +2019-06-10T21:24 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 37 millis (IndexTreeDic.java:1164) +2019-06-10T21:24 [INFO][TIMER] Temps total recherche: 0.044 s (IndexTreeMessyTest.java:408) +2019-06-10T21:24 [INFO][TIMER] Temps parcours des résultats: 0.59 millis (IndexTreeMessyTest.java:416) +2019-06-10T21:24 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) +2019-06-10T21:24 [INFO][TIMER] Temps total recherche: 7.06 millis (IndexTreeMessyTest.java:425) +2019-06-10T21:24 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.071 s (IndexTreeMessyTest.java:439) +2019-06-10T21:24 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) +2019-06-10T21:24 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 8 millis (IndexTreeDic.java:1164) +2019-06-10T21:24 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-10T21:24 [INFO] 40 (HashMapTests.java:27) +2019-06-10T21:24 [INFO][MEMORY] Mem usage début - 88,272 (ParallelLoaderTest.java:56) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 90,320 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.046 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.898 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 232,213 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.047 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.709 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO][MEMORY] Mem usage fin - 189,539 (ParallelLoaderTest.java:114) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.68 s (ParallelLoaderTest.java:115) |