aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--TODO.md4
-rw-r--r--chunk.go13
-rw-r--r--repo.go42
-rw-r--r--repo_test.go33
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000011168
-rw-r--r--testdata/repo_8k/00000/chunks/000000000000012168
-rw-r--r--testdata/repo_8k/00000/chunks/00000000000001387
-rw-r--r--testdata/repo_8k/00000/filesbin0 -> 208 bytes
-rw-r--r--testdata/repo_8k/00000/recipe (renamed from testdata/repo_8k/00000/chunks/000000000000014)bin4710 -> 6570 bytes
9 files changed, 253 insertions, 262 deletions
diff --git a/TODO.md b/TODO.md
index e194839..bb95b39 100644
--- a/TODO.md
+++ b/TODO.md
@@ -12,10 +12,10 @@ priority 1
store in fingerprintMap
store in sketchMap
```
-- [ ] read from repo
+- [ ] read from repo (Restore function)
- [x] store recipe
- [x] load recipe
- - [ ] read chunks in-order into a stream
+ - [x] read chunks in-order into a stream
- [ ] read individual files
- [ ] properly store informations to be DNA encoded
- [ ] tar source to keep files metadata ?
diff --git a/chunk.go b/chunk.go
index 196152e..a1cf3cd 100644
--- a/chunk.go
+++ b/chunk.go
@@ -24,6 +24,11 @@ type BufferedChunk interface {
Bytes() []byte
}
+type RepoChunk interface {
+ Chunk
+ SetRepo(r *Repo)
+}
+
type ChunkId struct {
Ver int
Idx uint64
@@ -72,6 +77,10 @@ func (c *StoredChunk) GetId() *ChunkId {
return c.Id
}
+func (c *StoredChunk) SetRepo(r *Repo) {
+ c.repo = r
+}
+
func (c *StoredChunk) Reader() io.ReadSeeker {
// log.Printf("Chunk %d: Reading from file\n", c.id)
return c.repo.LoadChunkContent(c.Id)
@@ -121,6 +130,10 @@ type DeltaChunk struct {
Size int
}
+func (c *DeltaChunk) SetRepo(r *Repo) {
+ c.repo = r
+}
+
func (c *DeltaChunk) Reader() io.ReadSeeker {
var buff bytes.Buffer
c.repo.Patcher().Patch(c.repo.LoadChunkContent(c.Source), &buff, bytes.NewReader(c.Patch))
diff --git a/repo.go b/repo.go
index 467567d..585e505 100644
--- a/repo.go
+++ b/repo.go
@@ -101,13 +101,13 @@ func (r *Repo) Patcher() Patcher {
func (r *Repo) Commit(source string) {
versions := r.loadVersions()
- newVersion := len(versions)
+ newVersion := len(versions) // TODO: add newVersion functino
newPath := path.Join(r.path, fmt.Sprintf(versionFmt, newVersion))
newChunkPath := path.Join(newPath, chunksName)
newFilesPath := path.Join(newPath, filesName)
newRecipePath := path.Join(newPath, recipeName)
- os.Mkdir(newPath, 0775)
- os.Mkdir(newChunkPath, 0775)
+ os.Mkdir(newPath, 0775) // TODO: handle errors
+ os.Mkdir(newChunkPath, 0775) // TODO: handle errors
reader, writer := io.Pipe()
oldChunks := make(chan IdentifiedChunk, 16)
files := listFiles(source)
@@ -120,6 +120,28 @@ func (r *Repo) Commit(source string) {
fmt.Println(files)
}
+func (r *Repo) Restore(destination string) {
+ versions := r.loadVersions()
+ latest := versions[len(versions)-1]
+ latestFilesPath := path.Join(latest, filesName)
+ latestRecipePath := path.Join(latest, recipeName)
+ files := loadFileList(latestFilesPath)
+ recipe := loadRecipe(latestRecipePath)
+ reader, writer := io.Pipe()
+ go r.restoreStream(writer, recipe)
+ bufReader := bufio.NewReaderSize(reader, r.chunkSize*2)
+ for _, file := range files {
+ filePath := path.Join(destination, file.Path)
+ dir := filepath.Dir(filePath)
+ os.MkdirAll(dir, 0775) // TODO: handle errors
+ f, _ := os.Create(filePath) // TODO: handle errors
+ n, err := io.CopyN(f, bufReader, file.Size)
+ if err != nil {
+ log.Printf("Error storing file content for '%s', written %d/%d bytes: %s\n", filePath, n, file.Size, err)
+ }
+ }
+}
+
func (r *Repo) loadVersions() []string {
var versions []string
files, err := os.ReadDir(r.path)
@@ -468,8 +490,19 @@ func (r *Repo) matchStream(stream io.Reader, version int) []Chunk {
return chunks
}
+func (r *Repo) restoreStream(stream io.WriteCloser, recipe []Chunk) {
+ for _, c := range recipe {
+ if rc, isRepo := c.(RepoChunk); isRepo {
+ rc.SetRepo(r)
+ }
+ if n, err := io.Copy(stream, c.Reader()); err != nil {
+ log.Printf("Error copying to stream, read %d bytes from chunk: %s\n", n, err)
+ }
+ }
+ stream.Close()
+}
+
func storeRecipe(dest string, recipe []Chunk) {
- gob.Register(&LoadedChunk{})
gob.Register(&StoredChunk{})
gob.Register(&TempChunk{})
gob.Register(&DeltaChunk{})
@@ -492,7 +525,6 @@ func storeRecipe(dest string, recipe []Chunk) {
func loadRecipe(path string) []Chunk {
var recipe []Chunk
- gob.Register(&LoadedChunk{})
gob.Register(&StoredChunk{})
gob.Register(&TempChunk{})
gob.Register(&DeltaChunk{})
diff --git a/repo_test.go b/repo_test.go
index 29282bc..5127263 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -230,13 +230,44 @@ func TestBsdiff(t *testing.T) {
func TestCommit(t *testing.T) {
dest := t.TempDir()
- source := "testdata"
+ source := path.Join("testdata", "logs")
repo := NewRepo(dest)
repo.Commit(source)
recipe := loadRecipe(path.Join(dest, "00000", recipeName))
log.Println(recipe)
}
+func TestRestore(t *testing.T) {
+ dest := t.TempDir()
+ source := path.Join("testdata", "repo_8k")
+ repo := NewRepo(source)
+ repo.Restore(dest)
+ destFiles := listFiles(dest)
+ sourceFiles := listFiles(path.Join("testdata", "logs"))
+ sfCount := len(sourceFiles)
+ if sfCount <= 0 {
+ t.Fatalf("No source files: %d", sfCount)
+ }
+ dfCount := len(destFiles)
+ if sfCount != dfCount {
+ t.Fatalf("Incorrect number for destination files: %d, should be %d", dfCount, sfCount)
+ }
+ for i, sf := range sourceFiles {
+ sfContent, err := os.ReadFile(sf.Path)
+ if err != nil {
+ t.Fatalf("Error reading from source file '%s': %s", sf.Path, err)
+ }
+ df := destFiles[i]
+ dfContent, err := os.ReadFile(df.Path)
+ if err != nil {
+ t.Fatalf("Error reading from source file '%s': %s", df.Path, err)
+ }
+ if bytes.Compare(sfContent, dfContent) != 0 {
+ t.Errorf("File content of '%s' does not match '%s'", df.Path, sf.Path)
+ }
+ }
+}
+
func assertLen(t *testing.T, expected int, actual interface{}, prefix string) {
s := reflect.ValueOf(actual)
if s.Len() != expected {
diff --git a/testdata/repo_8k/00000/chunks/000000000000011 b/testdata/repo_8k/00000/chunks/000000000000011
index 574a9a6..5a30329 100644
--- a/testdata/repo_8k/00000/chunks/000000000000011
+++ b/testdata/repo_8k/00000/chunks/000000000000011
@@ -1,85 +1,87 @@
-a:136)
-2019-06-10T20:50 [INFO][MEMORY] 150,362 (Loader.java:138)
-2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T20:50 [INFO][MEMORY] 150,428 (Loader.java:138)
-2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T20:50 [INFO][MEMORY] 150,268 (Loader.java:138)
-2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T20:50 [INFO][MEMORY] 150,648 (Loader.java:138)
-2019-06-10T20:50 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.96 s (IndexTreeMessyTest.java:314)
-2019-06-10T20:50 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322)
-2019-06-10T20:50 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,382 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b083826 (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,382 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@105fece7 (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,758 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ec300f1 (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,758 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@482cd91f (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,807 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@123f1134 (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7d68ef40 (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5b0abc94 (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@75c072cb (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1f1c7bf6 (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25b485ba (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,406 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2b546384 (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,406 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5d740a0f (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@214b199c (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@20d3d15a (Table.java:338)
-2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517)
-2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2893de87 (Table.java:338)
-2019-06-10T20:50 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56)
-2019-06-10T20:50 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57)
-2019-06-10T20:50 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73)
-2019-06-10T20:50 [ERROR] null (SerialStructure.java:74)
-2019-06-10T20:50 [INFO] Lancé (IndexTreeMessyTest.java:366)
-2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 36 millis (IndexTreeDic.java:1164)
-2019-06-10T20:50 [INFO][TIMER] Temps total recherche: 0.044 s (IndexTreeMessyTest.java:408)
-2019-06-10T20:50 [INFO][TIMER] Temps parcours des résultats: 0.73 millis (IndexTreeMessyTest.java:416)
-2019-06-10T20:50 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417)
-2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164)
-2019-06-10T20:50 [INFO][TIMER] Temps total recherche: 7.60 millis (IndexTreeMessyTest.java:425)
-2019-06-10T20:50 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.070 s (IndexTreeMessyTest.java:439)
-2019-06-10T20:50 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.043 s (IndexTreeMessyTest.java:449)
-2019-06-10T20:50 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450)
-2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 3 millis (IndexTreeDic.java:1164)
-2019-06-10T20:50 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456)
-2019-06-10T20:50 [INFO] 40 (HashMapTests.java:27)
-2019-06-10T20:50 [INFO][MEMORY] Mem usage début - 82,957 (ParallelLoaderTest.java:56)
-2019-06-10T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180)
-2019-06-10T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
-2019-06-10T20:50 [INFO][MEMORY] 11,889 (ParallelLoader.java:114)
-2019-06-10T20:50 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.114 sactiveThreadNb = 0 (ParallelLoader.java:150)
-2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
-2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 1.68 s (ParallelLoader.java:217)
-2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
-2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.68 s (IndexMemDicStorageTests.java:183)
-2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217)
-2019-06-10T20:51 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.208 s (IndexMemDicStorageTests.java:83)
-2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217)
-2019-06-10T20:51 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.176 s (IndexMemDicStorageTests.java:86)
-2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108)
-2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111)
-2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114)
-2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117)
-2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170)
+va:217)
+2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71)
+2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74)
+2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77)
+2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80)
+2019-06-10T20:51 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101)
+2019-06-10T20:51 [INFO] totalLength = 66666 (IndexMemDic.java:217)
+2019-06-10T20:51 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109)
+2019-06-10T20:51 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73)
+2019-06-10T20:51 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186)
+2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.67 millis (Loader.java:116)
+2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.13 millis (Loader.java:116)
+2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117)
+2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117)
+2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.14 millis (Loader.java:116)
+2019-06-10T20:51 [INFO][MEMORY] 178,179 (Loader.java:117)
+2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.71 millis (Loader.java:116)
+2019-06-10T20:51 [INFO][MEMORY] 199,412 (Loader.java:117)
+2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T20:51 [INFO][MEMORY] 155,755 (Loader.java:138)
+2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T20:51 [INFO][MEMORY] 155,569 (Loader.java:138)
+2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T20:51 [INFO][MEMORY] 156,154 (Loader.java:138)
+2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T20:51 [INFO][MEMORY] 155,614 (Loader.java:138)
+2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 4.79 s (IndexTreeMessyTest.java:314)
+2019-06-10T20:51 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322)
+2019-06-10T20:51 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,063 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338)
+2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
+2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338)
+2019-06-10T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56)
+2019-06-10T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57)
+2019-06-10T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73)
+2019-06-10T20:51 [ERROR] null (SerialStructure.java:74)
+2019-06-10T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366)
+2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 41 millis (IndexTreeDic.java:1164)
+2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 0.049 s (IndexTreeMessyTest.java:408)
+2019-06-10T20:51 [INFO][TIMER] Temps parcours des résultats: 1.53 millis (IndexTreeMessyTest.java:416)
+2019-06-10T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417)
+2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164)
+2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 7.95 millis (IndexTreeMessyTest.java:425)
+2019-06-10T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.079 s (IndexTreeMessyTest.java:439)
+2019-06-10T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449)
+2019-06-10T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450)
+2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164)
+2019-06-10T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456)
+2019-06-10T20:51 [INFO] 40 (HashMapTests.java:27)
+2019-06-10T20:51 [INFO][MEMORY] Mem usage début - 88,271 (ParallelLoaderTest.java:56)
+2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65)
2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
-2019-06-10T20:51 [INFO][MEMORY] 108,834 (ParallelLoader.java:114)
-2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.141 sactiveThreadNb = 0 (ParallelLoader.java:150)
+2019-06-10T20:51 [INFO][MEMORY] 90,319 (ParallelLoader.java:114)
+2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150)
2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
-2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.890 s (ParallelLoader.java:217)
-2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
-2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.900 s (NewLoaderAndIndexTest.java:189)
-2019-06-10T20:51 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193)
-2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.ja \ No newline at end of file
+2019-06-10T20 \ No newline at end of file
diff --git a/testdata/repo_8k/00000/chunks/000000000000012 b/testdata/repo_8k/00000/chunks/000000000000012
index 5a30329..612b2ee 100644
--- a/testdata/repo_8k/00000/chunks/000000000000012
+++ b/testdata/repo_8k/00000/chunks/000000000000012
@@ -1,87 +1,87 @@
-va:217)
-2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71)
-2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74)
-2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77)
-2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80)
-2019-06-10T20:51 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101)
-2019-06-10T20:51 [INFO] totalLength = 66666 (IndexMemDic.java:217)
-2019-06-10T20:51 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109)
-2019-06-10T20:51 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73)
-2019-06-10T20:51 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186)
-2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
-2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.67 millis (Loader.java:116)
-2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
-2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.13 millis (Loader.java:116)
-2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117)
-2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117)
-2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
-2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.14 millis (Loader.java:116)
-2019-06-10T20:51 [INFO][MEMORY] 178,179 (Loader.java:117)
-2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
-2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.71 millis (Loader.java:116)
-2019-06-10T20:51 [INFO][MEMORY] 199,412 (Loader.java:117)
-2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T20:51 [INFO][MEMORY] 155,755 (Loader.java:138)
-2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T20:51 [INFO][MEMORY] 155,569 (Loader.java:138)
-2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T20:51 [INFO][MEMORY] 156,154 (Loader.java:138)
-2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T20:51 [INFO][MEMORY] 155,614 (Loader.java:138)
-2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 4.79 s (IndexTreeMessyTest.java:314)
-2019-06-10T20:51 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322)
-2019-06-10T20:51 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,063 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338)
-2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517)
-2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338)
-2019-06-10T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56)
-2019-06-10T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57)
-2019-06-10T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73)
-2019-06-10T20:51 [ERROR] null (SerialStructure.java:74)
-2019-06-10T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366)
-2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 41 millis (IndexTreeDic.java:1164)
-2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 0.049 s (IndexTreeMessyTest.java:408)
-2019-06-10T20:51 [INFO][TIMER] Temps parcours des résultats: 1.53 millis (IndexTreeMessyTest.java:416)
-2019-06-10T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417)
-2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164)
-2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 7.95 millis (IndexTreeMessyTest.java:425)
-2019-06-10T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.079 s (IndexTreeMessyTest.java:439)
-2019-06-10T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449)
-2019-06-10T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450)
-2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164)
-2019-06-10T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456)
-2019-06-10T20:51 [INFO] 40 (HashMapTests.java:27)
-2019-06-10T20:51 [INFO][MEMORY] Mem usage début - 88,271 (ParallelLoaderTest.java:56)
+:51 [INFO] Parsing terminé !! temps écoulé = 0.842 s (ParallelLoader.java:217)
+2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65)
2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
-2019-06-10T20:51 [INFO][MEMORY] 90,319 (ParallelLoader.java:114)
-2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150)
+2019-06-10T20:51 [INFO][MEMORY] 209,650 (ParallelLoader.java:114)
+2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.056 sactiveThreadNb = 0 (ParallelLoader.java:150)
2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
-2019-06-10T20 \ No newline at end of file
+2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.637 s (ParallelLoader.java:217)
+2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
+2019-06-10T20:51 [INFO][MEMORY] Mem usage fin - 189,538 (ParallelLoaderTest.java:114)
+2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (ParallelLoaderTest.java:115)
+2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180)
+2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
+2019-06-10T21:24 [INFO][MEMORY] 12,384 (ParallelLoader.java:114)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.122 sactiveThreadNb = 0 (ParallelLoader.java:150)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
+2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 1.54 s (ParallelLoader.java:217)
+2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
+2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (IndexMemDicStorageTests.java:183)
+2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217)
+2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.312 s (IndexMemDicStorageTests.java:83)
+2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217)
+2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.172 s (IndexMemDicStorageTests.java:86)
+2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108)
+2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111)
+2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114)
+2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117)
+2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170)
+2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
+2019-06-10T21:24 [INFO][MEMORY] 108,772 (ParallelLoader.java:114)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.168 sactiveThreadNb = 0 (ParallelLoader.java:150)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
+2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.889 s (ParallelLoader.java:217)
+2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
+2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.904 s (NewLoaderAndIndexTest.java:189)
+2019-06-10T21:24 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193)
+2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217)
+2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71)
+2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74)
+2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77)
+2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80)
+2019-06-10T21:24 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101)
+2019-06-10T21:24 [INFO] totalLength = 66666 (IndexMemDic.java:217)
+2019-06-10T21:24 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109)
+2019-06-10T21:24 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73)
+2019-06-10T21:24 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.99 millis (Loader.java:116)
+2019-06-10T21:24 [INFO][MEMORY] 234,123 (Loader.java:117)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116)
+2019-06-10T21:24 [INFO][MEMORY] 154,808 (Loader.java:117)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.013 s (Loader.java:116)
+2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.63 millis (Loader.java:116)
+2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117)
+2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117)
+2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T21:24 [INFO][MEMORY] 154,918 (Loader.java:138)
+2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T21:24 [INFO][MEMORY] 155,713 (Loader.java:138)
+2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T21:24 [INFO][MEMORY] 155,525 (Loader.java:138)
+2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
+2019-06-10T21:24 [INFO][MEMORY] 155,614 (Loader.java:138)
+2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 12.07 s (IndexTreeMessyTest.java:314)
+2019-06-10T21:24 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322)
+2019-06-10T21:24 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,064 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,576 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338)
+2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517)
+2019-06-10T21:24 [INFO] Table.flu \ No newline at end of file
diff --git a/testdata/repo_8k/00000/chunks/000000000000013 b/testdata/repo_8k/00000/chunks/000000000000013
deleted file mode 100644
index 612b2ee..0000000
--- a/testdata/repo_8k/00000/chunks/000000000000013
+++ /dev/null
@@ -1,87 +0,0 @@
-:51 [INFO] Parsing terminé !! temps écoulé = 0.842 s (ParallelLoader.java:217)
-2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
-2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65)
-2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
-2019-06-10T20:51 [INFO][MEMORY] 209,650 (ParallelLoader.java:114)
-2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.056 sactiveThreadNb = 0 (ParallelLoader.java:150)
-2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
-2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.637 s (ParallelLoader.java:217)
-2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
-2019-06-10T20:51 [INFO][MEMORY] Mem usage fin - 189,538 (ParallelLoaderTest.java:114)
-2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (ParallelLoaderTest.java:115)
-2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180)
-2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
-2019-06-10T21:24 [INFO][MEMORY] 12,384 (ParallelLoader.java:114)
-2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.122 sactiveThreadNb = 0 (ParallelLoader.java:150)
-2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
-2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 1.54 s (ParallelLoader.java:217)
-2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
-2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (IndexMemDicStorageTests.java:183)
-2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217)
-2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.312 s (IndexMemDicStorageTests.java:83)
-2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217)
-2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.172 s (IndexMemDicStorageTests.java:86)
-2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108)
-2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111)
-2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114)
-2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117)
-2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170)
-2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112)
-2019-06-10T21:24 [INFO][MEMORY] 108,772 (ParallelLoader.java:114)
-2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.168 sactiveThreadNb = 0 (ParallelLoader.java:150)
-2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287)
-2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.889 s (ParallelLoader.java:217)
-2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218)
-2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.904 s (NewLoaderAndIndexTest.java:189)
-2019-06-10T21:24 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193)
-2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217)
-2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71)
-2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74)
-2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77)
-2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80)
-2019-06-10T21:24 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101)
-2019-06-10T21:24 [INFO] totalLength = 66666 (IndexMemDic.java:217)
-2019-06-10T21:24 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109)
-2019-06-10T21:24 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73)
-2019-06-10T21:24 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186)
-2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
-2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.99 millis (Loader.java:116)
-2019-06-10T21:24 [INFO][MEMORY] 234,123 (Loader.java:117)
-2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
-2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116)
-2019-06-10T21:24 [INFO][MEMORY] 154,808 (Loader.java:117)
-2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
-2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107)
-2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.013 s (Loader.java:116)
-2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.63 millis (Loader.java:116)
-2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117)
-2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117)
-2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T21:24 [INFO][MEMORY] 154,918 (Loader.java:138)
-2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T21:24 [INFO][MEMORY] 155,713 (Loader.java:138)
-2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T21:24 [INFO][MEMORY] 155,525 (Loader.java:138)
-2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136)
-2019-06-10T21:24 [INFO][MEMORY] 155,614 (Loader.java:138)
-2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 12.07 s (IndexTreeMessyTest.java:314)
-2019-06-10T21:24 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322)
-2019-06-10T21:24 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,064 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,576 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338)
-2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517)
-2019-06-10T21:24 [INFO] Table.flu \ No newline at end of file
diff --git a/testdata/repo_8k/00000/files b/testdata/repo_8k/00000/files
new file mode 100644
index 0000000..20d5905
--- /dev/null
+++ b/testdata/repo_8k/00000/files
Binary files differ
diff --git a/testdata/repo_8k/00000/chunks/000000000000014 b/testdata/repo_8k/00000/recipe
index ab7db22..1d94f3a 100644
--- a/testdata/repo_8k/00000/chunks/000000000000014
+++ b/testdata/repo_8k/00000/recipe
Binary files differ
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-14 17:27:11 +0000