diff options
author | n-peugnet <n.peugnet@free.fr> | 2021-09-09 16:21:20 +0200 |
---|---|---|
committer | n-peugnet <n.peugnet@free.fr> | 2021-09-09 16:21:20 +0200 |
commit | 3e866db2accc6ed5aa65befdf21474dbc7b1ec18 (patch) | |
tree | 62670e092cbc41817edc76317a0ac2482b9480af | |
parent | 11404879f2e7ee02118852b6b42e19f99cb2edd9 (diff) | |
download | dna-backup-3e866db2accc6ed5aa65befdf21474dbc7b1ec18.tar.gz dna-backup-3e866db2accc6ed5aa65befdf21474dbc7b1ec18.zip |
first add of restore function to Repo
-rw-r--r-- | TODO.md | 4 | ||||
-rw-r--r-- | chunk.go | 13 | ||||
-rw-r--r-- | repo.go | 42 | ||||
-rw-r--r-- | repo_test.go | 33 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000011 | 168 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000012 | 168 | ||||
-rw-r--r-- | testdata/repo_8k/00000/chunks/000000000000013 | 87 | ||||
-rw-r--r-- | testdata/repo_8k/00000/files | bin | 0 -> 208 bytes | |||
-rw-r--r-- | testdata/repo_8k/00000/recipe (renamed from testdata/repo_8k/00000/chunks/000000000000014) | bin | 4710 -> 6570 bytes |
9 files changed, 253 insertions, 262 deletions
@@ -12,10 +12,10 @@ priority 1 store in fingerprintMap store in sketchMap ``` -- [ ] read from repo +- [ ] read from repo (Restore function) - [x] store recipe - [x] load recipe - - [ ] read chunks in-order into a stream + - [x] read chunks in-order into a stream - [ ] read individual files - [ ] properly store informations to be DNA encoded - [ ] tar source to keep files metadata ? @@ -24,6 +24,11 @@ type BufferedChunk interface { Bytes() []byte } +type RepoChunk interface { + Chunk + SetRepo(r *Repo) +} + type ChunkId struct { Ver int Idx uint64 @@ -72,6 +77,10 @@ func (c *StoredChunk) GetId() *ChunkId { return c.Id } +func (c *StoredChunk) SetRepo(r *Repo) { + c.repo = r +} + func (c *StoredChunk) Reader() io.ReadSeeker { // log.Printf("Chunk %d: Reading from file\n", c.id) return c.repo.LoadChunkContent(c.Id) @@ -121,6 +130,10 @@ type DeltaChunk struct { Size int } +func (c *DeltaChunk) SetRepo(r *Repo) { + c.repo = r +} + func (c *DeltaChunk) Reader() io.ReadSeeker { var buff bytes.Buffer c.repo.Patcher().Patch(c.repo.LoadChunkContent(c.Source), &buff, bytes.NewReader(c.Patch)) @@ -101,13 +101,13 @@ func (r *Repo) Patcher() Patcher { func (r *Repo) Commit(source string) { versions := r.loadVersions() - newVersion := len(versions) + newVersion := len(versions) // TODO: add newVersion functino newPath := path.Join(r.path, fmt.Sprintf(versionFmt, newVersion)) newChunkPath := path.Join(newPath, chunksName) newFilesPath := path.Join(newPath, filesName) newRecipePath := path.Join(newPath, recipeName) - os.Mkdir(newPath, 0775) - os.Mkdir(newChunkPath, 0775) + os.Mkdir(newPath, 0775) // TODO: handle errors + os.Mkdir(newChunkPath, 0775) // TODO: handle errors reader, writer := io.Pipe() oldChunks := make(chan IdentifiedChunk, 16) files := listFiles(source) @@ -120,6 +120,28 @@ func (r *Repo) Commit(source string) { fmt.Println(files) } +func (r *Repo) Restore(destination string) { + versions := r.loadVersions() + latest := versions[len(versions)-1] + latestFilesPath := path.Join(latest, filesName) + latestRecipePath := path.Join(latest, recipeName) + files := loadFileList(latestFilesPath) + recipe := loadRecipe(latestRecipePath) + reader, writer := io.Pipe() + go r.restoreStream(writer, recipe) + bufReader := bufio.NewReaderSize(reader, r.chunkSize*2) + for _, file := range files { + filePath := path.Join(destination, file.Path) + dir := filepath.Dir(filePath) + os.MkdirAll(dir, 0775) // TODO: handle errors + f, _ := os.Create(filePath) // TODO: handle errors + n, err := io.CopyN(f, bufReader, file.Size) + if err != nil { + log.Printf("Error storing file content for '%s', written %d/%d bytes: %s\n", filePath, n, file.Size, err) + } + } +} + func (r *Repo) loadVersions() []string { var versions []string files, err := os.ReadDir(r.path) @@ -468,8 +490,19 @@ func (r *Repo) matchStream(stream io.Reader, version int) []Chunk { return chunks } +func (r *Repo) restoreStream(stream io.WriteCloser, recipe []Chunk) { + for _, c := range recipe { + if rc, isRepo := c.(RepoChunk); isRepo { + rc.SetRepo(r) + } + if n, err := io.Copy(stream, c.Reader()); err != nil { + log.Printf("Error copying to stream, read %d bytes from chunk: %s\n", n, err) + } + } + stream.Close() +} + func storeRecipe(dest string, recipe []Chunk) { - gob.Register(&LoadedChunk{}) gob.Register(&StoredChunk{}) gob.Register(&TempChunk{}) gob.Register(&DeltaChunk{}) @@ -492,7 +525,6 @@ func storeRecipe(dest string, recipe []Chunk) { func loadRecipe(path string) []Chunk { var recipe []Chunk - gob.Register(&LoadedChunk{}) gob.Register(&StoredChunk{}) gob.Register(&TempChunk{}) gob.Register(&DeltaChunk{}) diff --git a/repo_test.go b/repo_test.go index 29282bc..5127263 100644 --- a/repo_test.go +++ b/repo_test.go @@ -230,13 +230,44 @@ func TestBsdiff(t *testing.T) { func TestCommit(t *testing.T) { dest := t.TempDir() - source := "testdata" + source := path.Join("testdata", "logs") repo := NewRepo(dest) repo.Commit(source) recipe := loadRecipe(path.Join(dest, "00000", recipeName)) log.Println(recipe) } +func TestRestore(t *testing.T) { + dest := t.TempDir() + source := path.Join("testdata", "repo_8k") + repo := NewRepo(source) + repo.Restore(dest) + destFiles := listFiles(dest) + sourceFiles := listFiles(path.Join("testdata", "logs")) + sfCount := len(sourceFiles) + if sfCount <= 0 { + t.Fatalf("No source files: %d", sfCount) + } + dfCount := len(destFiles) + if sfCount != dfCount { + t.Fatalf("Incorrect number for destination files: %d, should be %d", dfCount, sfCount) + } + for i, sf := range sourceFiles { + sfContent, err := os.ReadFile(sf.Path) + if err != nil { + t.Fatalf("Error reading from source file '%s': %s", sf.Path, err) + } + df := destFiles[i] + dfContent, err := os.ReadFile(df.Path) + if err != nil { + t.Fatalf("Error reading from source file '%s': %s", df.Path, err) + } + if bytes.Compare(sfContent, dfContent) != 0 { + t.Errorf("File content of '%s' does not match '%s'", df.Path, sf.Path) + } + } +} + func assertLen(t *testing.T, expected int, actual interface{}, prefix string) { s := reflect.ValueOf(actual) if s.Len() != expected { diff --git a/testdata/repo_8k/00000/chunks/000000000000011 b/testdata/repo_8k/00000/chunks/000000000000011 index 574a9a6..5a30329 100644 --- a/testdata/repo_8k/00000/chunks/000000000000011 +++ b/testdata/repo_8k/00000/chunks/000000000000011 @@ -1,85 +1,87 @@ -a:136) -2019-06-10T20:50 [INFO][MEMORY] 150,362 (Loader.java:138) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 150,428 (Loader.java:138) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 150,268 (Loader.java:138) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 150,648 (Loader.java:138) -2019-06-10T20:50 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.96 s (IndexTreeMessyTest.java:314) -2019-06-10T20:50 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:50 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,382 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b083826 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,382 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@105fece7 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,758 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ec300f1 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,758 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@482cd91f (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,807 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@123f1134 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7d68ef40 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5b0abc94 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@75c072cb (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1f1c7bf6 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25b485ba (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,406 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2b546384 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,406 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5d740a0f (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@214b199c (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@20d3d15a (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2893de87 (Table.java:338) -2019-06-10T20:50 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-10T20:50 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-10T20:50 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-10T20:50 [ERROR] null (SerialStructure.java:74) -2019-06-10T20:50 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 36 millis (IndexTreeDic.java:1164) -2019-06-10T20:50 [INFO][TIMER] Temps total recherche: 0.044 s (IndexTreeMessyTest.java:408) -2019-06-10T20:50 [INFO][TIMER] Temps parcours des résultats: 0.73 millis (IndexTreeMessyTest.java:416) -2019-06-10T20:50 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) -2019-06-10T20:50 [INFO][TIMER] Temps total recherche: 7.60 millis (IndexTreeMessyTest.java:425) -2019-06-10T20:50 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.070 s (IndexTreeMessyTest.java:439) -2019-06-10T20:50 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.043 s (IndexTreeMessyTest.java:449) -2019-06-10T20:50 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 3 millis (IndexTreeDic.java:1164) -2019-06-10T20:50 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-10T20:50 [INFO] 40 (HashMapTests.java:27) -2019-06-10T20:50 [INFO][MEMORY] Mem usage début - 82,957 (ParallelLoaderTest.java:56) -2019-06-10T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-10T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:50 [INFO][MEMORY] 11,889 (ParallelLoader.java:114) -2019-06-10T20:50 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.114 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 1.68 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.68 s (IndexMemDicStorageTests.java:183) -2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T20:51 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.208 s (IndexMemDicStorageTests.java:83) -2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T20:51 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.176 s (IndexMemDicStorageTests.java:86) -2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +va:217) +2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-10T20:51 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-10T20:51 [INFO] totalLength = 66666 (IndexMemDic.java:217) +2019-06-10T20:51 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) +2019-06-10T20:51 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:51 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.67 millis (Loader.java:116) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.13 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) +2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.14 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 178,179 (Loader.java:117) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.71 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 199,412 (Loader.java:117) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,755 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,569 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 156,154 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,614 (Loader.java:138) +2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 4.79 s (IndexTreeMessyTest.java:314) +2019-06-10T20:51 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T20:51 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,063 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) +2019-06-10T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-10T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-10T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-10T20:51 [ERROR] null (SerialStructure.java:74) +2019-06-10T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 41 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 0.049 s (IndexTreeMessyTest.java:408) +2019-06-10T20:51 [INFO][TIMER] Temps parcours des résultats: 1.53 millis (IndexTreeMessyTest.java:416) +2019-06-10T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 7.95 millis (IndexTreeMessyTest.java:425) +2019-06-10T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.079 s (IndexTreeMessyTest.java:439) +2019-06-10T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) +2019-06-10T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-10T20:51 [INFO] 40 (HashMapTests.java:27) +2019-06-10T20:51 [INFO][MEMORY] Mem usage début - 88,271 (ParallelLoaderTest.java:56) +2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) 2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:51 [INFO][MEMORY] 108,834 (ParallelLoader.java:114) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.141 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T20:51 [INFO][MEMORY] 90,319 (ParallelLoader.java:114) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150) 2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.890 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.900 s (NewLoaderAndIndexTest.java:189) -2019-06-10T20:51 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.ja
\ No newline at end of file +2019-06-10T20
\ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000012 b/testdata/repo_8k/00000/chunks/000000000000012 index 5a30329..612b2ee 100644 --- a/testdata/repo_8k/00000/chunks/000000000000012 +++ b/testdata/repo_8k/00000/chunks/000000000000012 @@ -1,87 +1,87 @@ -va:217) -2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-10T20:51 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-10T20:51 [INFO] totalLength = 66666 (IndexMemDic.java:217) -2019-06-10T20:51 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) -2019-06-10T20:51 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:51 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.67 millis (Loader.java:116) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.13 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) -2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.14 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 178,179 (Loader.java:117) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.71 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 199,412 (Loader.java:117) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,755 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,569 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 156,154 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,614 (Loader.java:138) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 4.79 s (IndexTreeMessyTest.java:314) -2019-06-10T20:51 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:51 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,063 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) -2019-06-10T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-10T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-10T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-10T20:51 [ERROR] null (SerialStructure.java:74) -2019-06-10T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 41 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 0.049 s (IndexTreeMessyTest.java:408) -2019-06-10T20:51 [INFO][TIMER] Temps parcours des résultats: 1.53 millis (IndexTreeMessyTest.java:416) -2019-06-10T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 7.95 millis (IndexTreeMessyTest.java:425) -2019-06-10T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.079 s (IndexTreeMessyTest.java:439) -2019-06-10T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) -2019-06-10T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-10T20:51 [INFO] 40 (HashMapTests.java:27) -2019-06-10T20:51 [INFO][MEMORY] Mem usage début - 88,271 (ParallelLoaderTest.java:56) +:51 [INFO] Parsing terminé !! temps écoulé = 0.842 s (ParallelLoader.java:217) +2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) 2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) 2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:51 [INFO][MEMORY] 90,319 (ParallelLoader.java:114) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T20:51 [INFO][MEMORY] 209,650 (ParallelLoader.java:114) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.056 sactiveThreadNb = 0 (ParallelLoader.java:150) 2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20
\ No newline at end of file +2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.637 s (ParallelLoader.java:217) +2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T20:51 [INFO][MEMORY] Mem usage fin - 189,538 (ParallelLoaderTest.java:114) +2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (ParallelLoaderTest.java:115) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 12,384 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.122 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 1.54 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (IndexMemDicStorageTests.java:183) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.312 s (IndexMemDicStorageTests.java:83) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.172 s (IndexMemDicStorageTests.java:86) +2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 108,772 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.168 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.889 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.904 s (NewLoaderAndIndexTest.java:189) +2019-06-10T21:24 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-10T21:24 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-10T21:24 [INFO] totalLength = 66666 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) +2019-06-10T21:24 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T21:24 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.99 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 234,123 (Loader.java:117) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 154,808 (Loader.java:117) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.013 s (Loader.java:116) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.63 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) +2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 154,918 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,713 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,525 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,614 (Loader.java:138) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 12.07 s (IndexTreeMessyTest.java:314) +2019-06-10T21:24 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T21:24 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,064 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,576 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flu
\ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000013 b/testdata/repo_8k/00000/chunks/000000000000013 deleted file mode 100644 index 612b2ee..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000013 +++ /dev/null @@ -1,87 +0,0 @@ -:51 [INFO] Parsing terminé !! temps écoulé = 0.842 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:51 [INFO][MEMORY] 209,650 (ParallelLoader.java:114) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.056 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.637 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO][MEMORY] Mem usage fin - 189,538 (ParallelLoaderTest.java:114) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (ParallelLoaderTest.java:115) -2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T21:24 [INFO][MEMORY] 12,384 (ParallelLoader.java:114) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.122 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 1.54 s (ParallelLoader.java:217) -2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (IndexMemDicStorageTests.java:183) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.312 s (IndexMemDicStorageTests.java:83) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.172 s (IndexMemDicStorageTests.java:86) -2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T21:24 [INFO][MEMORY] 108,772 (ParallelLoader.java:114) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.168 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.889 s (ParallelLoader.java:217) -2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.904 s (NewLoaderAndIndexTest.java:189) -2019-06-10T21:24 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-10T21:24 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-10T21:24 [INFO] totalLength = 66666 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) -2019-06-10T21:24 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T21:24 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.99 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 234,123 (Loader.java:117) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 154,808 (Loader.java:117) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.013 s (Loader.java:116) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.63 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) -2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 154,918 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,713 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,525 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,614 (Loader.java:138) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 12.07 s (IndexTreeMessyTest.java:314) -2019-06-10T21:24 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T21:24 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,064 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,576 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flu
\ No newline at end of file diff --git a/testdata/repo_8k/00000/files b/testdata/repo_8k/00000/files Binary files differnew file mode 100644 index 0000000..20d5905 --- /dev/null +++ b/testdata/repo_8k/00000/files diff --git a/testdata/repo_8k/00000/chunks/000000000000014 b/testdata/repo_8k/00000/recipe Binary files differindex ab7db22..1d94f3a 100644 --- a/testdata/repo_8k/00000/chunks/000000000000014 +++ b/testdata/repo_8k/00000/recipe |