From fccc89de47a3ed5fc576f28f7f02b1111a59c0c4 Mon Sep 17 00:00:00 2001 From: n-peugnet Date: Wed, 6 Oct 2021 11:34:14 +0200 Subject: refactor: move repo and delta in their own package --- chunk.go | 113 --- const.go | 10 - delta.go | 59 -- delta/delta.go | 59 ++ main.go | 5 +- main_test.go | 21 - nowindows_test.go | 38 - repo.go | 783 ---------------- repo/chunk.go | 113 +++ repo/const.go | 10 + repo/nowindows_test.go | 38 + repo/repo.go | 784 ++++++++++++++++ repo/repo_test.go | 509 +++++++++++ repo/testdata/.gitattributes | 1 + repo/testdata/logs/1/logTest.log | 10 + repo/testdata/logs/2/csvParserTest.log | 63 ++ repo/testdata/logs/2/slipdb.log | 185 ++++ repo/testdata/logs/3/indexingTreeTest.log | 999 +++++++++++++++++++++ repo/testdata/repo_8k/00000/chunks/000000000000000 | 85 ++ repo/testdata/repo_8k/00000/chunks/000000000000001 | 89 ++ repo/testdata/repo_8k/00000/chunks/000000000000002 | 98 ++ repo/testdata/repo_8k/00000/chunks/000000000000003 | 85 ++ repo/testdata/repo_8k/00000/chunks/000000000000004 | 87 ++ repo/testdata/repo_8k/00000/chunks/000000000000005 | 87 ++ repo/testdata/repo_8k/00000/chunks/000000000000006 | 84 ++ repo/testdata/repo_8k/00000/chunks/000000000000007 | 87 ++ repo/testdata/repo_8k/00000/chunks/000000000000008 | 87 ++ repo/testdata/repo_8k/00000/chunks/000000000000009 | 88 ++ repo/testdata/repo_8k/00000/chunks/000000000000010 | 88 ++ repo/testdata/repo_8k/00000/chunks/000000000000011 | 87 ++ repo/testdata/repo_8k/00000/chunks/000000000000012 | 87 ++ repo/testdata/repo_8k/00000/files | Bin 0 -> 379 bytes repo/testdata/repo_8k/00000/hashes | Bin 0 -> 570 bytes repo/testdata/repo_8k/00000/recipe | Bin 0 -> 6783 bytes .../repo_8k_zlib/00000/chunks/000000000000000 | Bin 0 -> 1077 bytes .../repo_8k_zlib/00000/chunks/000000000000001 | Bin 0 -> 985 bytes .../repo_8k_zlib/00000/chunks/000000000000002 | Bin 0 -> 1118 bytes .../repo_8k_zlib/00000/chunks/000000000000003 | Bin 0 -> 1343 bytes .../repo_8k_zlib/00000/chunks/000000000000004 | Bin 0 -> 1366 bytes .../repo_8k_zlib/00000/chunks/000000000000005 | Bin 0 -> 1350 bytes .../repo_8k_zlib/00000/chunks/000000000000006 | Bin 0 -> 1360 bytes .../repo_8k_zlib/00000/chunks/000000000000007 | Bin 0 -> 1373 bytes .../repo_8k_zlib/00000/chunks/000000000000008 | Bin 0 -> 1254 bytes .../repo_8k_zlib/00000/chunks/000000000000009 | Bin 0 -> 1350 bytes .../repo_8k_zlib/00000/chunks/000000000000010 | Bin 0 -> 1173 bytes .../repo_8k_zlib/00000/chunks/000000000000011 | Bin 0 -> 1356 bytes .../repo_8k_zlib/00000/chunks/000000000000012 | Bin 0 -> 1252 bytes repo/testdata/repo_8k_zlib/00000/files | Bin 0 -> 275 bytes repo/testdata/repo_8k_zlib/00000/hashes | Bin 0 -> 570 bytes repo/testdata/repo_8k_zlib/00000/recipe | Bin 0 -> 2638 bytes repo_test.go | 494 ---------- tar.go | 3 +- testdata/.gitattributes | 1 - testdata/logs/1/logTest.log | 10 - testdata/logs/2/csvParserTest.log | 63 -- testdata/logs/2/slipdb.log | 185 ---- testdata/logs/3/indexingTreeTest.log | 999 --------------------- testdata/repo_8k/00000/chunks/000000000000000 | 85 -- testdata/repo_8k/00000/chunks/000000000000001 | 89 -- testdata/repo_8k/00000/chunks/000000000000002 | 98 -- testdata/repo_8k/00000/chunks/000000000000003 | 85 -- testdata/repo_8k/00000/chunks/000000000000004 | 87 -- testdata/repo_8k/00000/chunks/000000000000005 | 87 -- testdata/repo_8k/00000/chunks/000000000000006 | 84 -- testdata/repo_8k/00000/chunks/000000000000007 | 87 -- testdata/repo_8k/00000/chunks/000000000000008 | 87 -- testdata/repo_8k/00000/chunks/000000000000009 | 88 -- testdata/repo_8k/00000/chunks/000000000000010 | 88 -- testdata/repo_8k/00000/chunks/000000000000011 | 87 -- testdata/repo_8k/00000/chunks/000000000000012 | 87 -- testdata/repo_8k/00000/files | Bin 379 -> 0 bytes testdata/repo_8k/00000/hashes | Bin 570 -> 0 bytes testdata/repo_8k/00000/recipe | Bin 6783 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000000 | Bin 1077 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000001 | Bin 985 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000002 | Bin 1118 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000003 | Bin 1343 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000004 | Bin 1366 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000005 | Bin 1350 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000006 | Bin 1360 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000007 | Bin 1373 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000008 | Bin 1254 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000009 | Bin 1350 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000010 | Bin 1173 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000011 | Bin 1356 -> 0 bytes testdata/repo_8k_zlib/00000/chunks/000000000000012 | Bin 1252 -> 0 bytes testdata/repo_8k_zlib/00000/files | Bin 275 -> 0 bytes testdata/repo_8k_zlib/00000/hashes | Bin 570 -> 0 bytes testdata/repo_8k_zlib/00000/recipe | Bin 2638 -> 0 bytes 89 files changed, 3915 insertions(+), 3918 deletions(-) delete mode 100644 chunk.go delete mode 100644 const.go delete mode 100644 delta.go create mode 100644 delta/delta.go delete mode 100644 nowindows_test.go delete mode 100644 repo.go create mode 100644 repo/chunk.go create mode 100644 repo/const.go create mode 100644 repo/nowindows_test.go create mode 100644 repo/repo.go create mode 100644 repo/repo_test.go create mode 100644 repo/testdata/.gitattributes create mode 100644 repo/testdata/logs/1/logTest.log create mode 100644 repo/testdata/logs/2/csvParserTest.log create mode 100644 repo/testdata/logs/2/slipdb.log create mode 100644 repo/testdata/logs/3/indexingTreeTest.log create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000000 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000001 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000002 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000003 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000004 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000005 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000006 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000007 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000008 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000009 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000010 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000011 create mode 100644 repo/testdata/repo_8k/00000/chunks/000000000000012 create mode 100644 repo/testdata/repo_8k/00000/files create mode 100644 repo/testdata/repo_8k/00000/hashes create mode 100644 repo/testdata/repo_8k/00000/recipe create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000000 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000001 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000002 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000003 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000004 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000005 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000006 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000007 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000008 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000009 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000010 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000011 create mode 100644 repo/testdata/repo_8k_zlib/00000/chunks/000000000000012 create mode 100644 repo/testdata/repo_8k_zlib/00000/files create mode 100644 repo/testdata/repo_8k_zlib/00000/hashes create mode 100644 repo/testdata/repo_8k_zlib/00000/recipe delete mode 100644 repo_test.go delete mode 100644 testdata/.gitattributes delete mode 100644 testdata/logs/1/logTest.log delete mode 100644 testdata/logs/2/csvParserTest.log delete mode 100644 testdata/logs/2/slipdb.log delete mode 100644 testdata/logs/3/indexingTreeTest.log delete mode 100644 testdata/repo_8k/00000/chunks/000000000000000 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000001 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000002 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000003 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000004 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000005 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000006 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000007 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000008 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000009 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000010 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000011 delete mode 100644 testdata/repo_8k/00000/chunks/000000000000012 delete mode 100644 testdata/repo_8k/00000/files delete mode 100644 testdata/repo_8k/00000/hashes delete mode 100644 testdata/repo_8k/00000/recipe delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000000 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000001 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000002 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000003 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000004 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000005 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000006 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000007 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000008 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000009 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000010 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000011 delete mode 100644 testdata/repo_8k_zlib/00000/chunks/000000000000012 delete mode 100644 testdata/repo_8k_zlib/00000/files delete mode 100644 testdata/repo_8k_zlib/00000/hashes delete mode 100644 testdata/repo_8k_zlib/00000/recipe diff --git a/chunk.go b/chunk.go deleted file mode 100644 index 3efaf4a..0000000 --- a/chunk.go +++ /dev/null @@ -1,113 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io" - "path/filepath" -) - -type Chunk interface { - Reader() io.ReadSeeker - Len() int -} - -type IdentifiedChunk interface { - Chunk - GetId() *ChunkId -} - -type BufferedChunk interface { - Chunk - Bytes() []byte -} - -type RepoChunk interface { - Chunk - SetRepo(r *Repo) -} - -type ChunkId struct { - Ver int - Idx uint64 -} - -func (i *ChunkId) Path(repo string) string { - return filepath.Join(repo, fmt.Sprintf(versionFmt, i.Ver), chunksName, fmt.Sprintf(chunkIdFmt, i.Idx)) -} - -func NewStoredChunk(repo *Repo, id *ChunkId) *StoredChunk { - return &StoredChunk{repo: repo, Id: id} -} - -type StoredChunk struct { - repo *Repo - Id *ChunkId -} - -func (c *StoredChunk) GetId() *ChunkId { - return c.Id -} - -func (c *StoredChunk) SetRepo(r *Repo) { - c.repo = r -} - -func (c *StoredChunk) Reader() io.ReadSeeker { - // log.Printf("Chunk %d: Reading from file\n", c.id) - return c.repo.LoadChunkContent(c.Id) -} - -func (c *StoredChunk) Len() int { - return c.repo.chunkSize -} - -func NewTempChunk(value []byte) *TempChunk { - return &TempChunk{Value: value} -} - -type TempChunk struct { - Value []byte -} - -func (c *TempChunk) Reader() io.ReadSeeker { - return bytes.NewReader(c.Value) -} - -func (c *TempChunk) Len() int { - return len(c.Value) -} - -func (c *TempChunk) Bytes() []byte { - return c.Value -} - -func (c *TempChunk) AppendFrom(r io.Reader) { - buff, err := io.ReadAll(r) - if err != nil { - println("Chunk: error appending to temp chunk:", err) - } - c.Value = append(c.Value, buff...) -} - -type DeltaChunk struct { - repo *Repo - Source *ChunkId - Patch []byte - Size int -} - -func (c *DeltaChunk) SetRepo(r *Repo) { - c.repo = r -} - -func (c *DeltaChunk) Reader() io.ReadSeeker { - var buff bytes.Buffer - c.repo.Patcher().Patch(c.repo.LoadChunkContent(c.Source), &buff, bytes.NewReader(c.Patch)) - return bytes.NewReader(buff.Bytes()) -} - -// TODO: Maybe return the size of the patch instead ? -func (c *DeltaChunk) Len() int { - return c.Size -} diff --git a/const.go b/const.go deleted file mode 100644 index c40cb97..0000000 --- a/const.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -const ( - chunksName = "chunks" - chunkIdFmt = "%015d" - versionFmt = "%05d" - filesName = "files" - hashesName = "hashes" - recipeName = "recipe" -) diff --git a/delta.go b/delta.go deleted file mode 100644 index a127580..0000000 --- a/delta.go +++ /dev/null @@ -1,59 +0,0 @@ -package main - -import ( - "io" - - "github.com/gabstv/go-bsdiff/pkg/bsdiff" - "github.com/gabstv/go-bsdiff/pkg/bspatch" - "github.com/mdvan/fdelta" -) - -type Differ interface { - Diff(source io.Reader, target io.Reader, patch io.Writer) error -} - -type Patcher interface { - Patch(source io.Reader, target io.Writer, patch io.Reader) error -} - -type Bsdiff struct{} - -func (Bsdiff) Diff(source io.Reader, target io.Reader, patch io.Writer) error { - return bsdiff.Reader(source, target, patch) -} - -func (Bsdiff) Patch(source io.Reader, target io.Writer, patch io.Reader) error { - return bspatch.Reader(source, target, patch) -} - -type Fdelta struct{} - -func (Fdelta) Diff(source io.Reader, target io.Reader, patch io.Writer) error { - sourceBuf, err := io.ReadAll(source) - if err != nil { - return err - } - targetBuf, err := io.ReadAll(target) - if err != nil { - return err - } - _, err = patch.Write(fdelta.Create(sourceBuf, targetBuf)) - return err -} - -func (Fdelta) Patch(source io.Reader, target io.Writer, patch io.Reader) error { - sourceBuf, err := io.ReadAll(source) - if err != nil { - return err - } - patchBuf, err := io.ReadAll(patch) - if err != nil { - return err - } - targetBuf, err := fdelta.Apply(sourceBuf, patchBuf) - if err != nil { - return err - } - _, err = target.Write(targetBuf) - return err -} diff --git a/delta/delta.go b/delta/delta.go new file mode 100644 index 0000000..439da1d --- /dev/null +++ b/delta/delta.go @@ -0,0 +1,59 @@ +package delta + +import ( + "io" + + "github.com/gabstv/go-bsdiff/pkg/bsdiff" + "github.com/gabstv/go-bsdiff/pkg/bspatch" + "github.com/mdvan/fdelta" +) + +type Differ interface { + Diff(source io.Reader, target io.Reader, patch io.Writer) error +} + +type Patcher interface { + Patch(source io.Reader, target io.Writer, patch io.Reader) error +} + +type Bsdiff struct{} + +func (Bsdiff) Diff(source io.Reader, target io.Reader, patch io.Writer) error { + return bsdiff.Reader(source, target, patch) +} + +func (Bsdiff) Patch(source io.Reader, target io.Writer, patch io.Reader) error { + return bspatch.Reader(source, target, patch) +} + +type Fdelta struct{} + +func (Fdelta) Diff(source io.Reader, target io.Reader, patch io.Writer) error { + sourceBuf, err := io.ReadAll(source) + if err != nil { + return err + } + targetBuf, err := io.ReadAll(target) + if err != nil { + return err + } + _, err = patch.Write(fdelta.Create(sourceBuf, targetBuf)) + return err +} + +func (Fdelta) Patch(source io.Reader, target io.Writer, patch io.Reader) error { + sourceBuf, err := io.ReadAll(source) + if err != nil { + return err + } + patchBuf, err := io.ReadAll(patch) + if err != nil { + return err + } + targetBuf, err := fdelta.Apply(sourceBuf, patchBuf) + if err != nil { + return err + } + _, err = target.Write(targetBuf) + return err +} diff --git a/main.go b/main.go index 51cfac2..f1bb1d8 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "os" "github.com/n-peugnet/dna-backup/logger" + "github.com/n-peugnet/dna-backup/repo" ) type command struct { @@ -84,7 +85,7 @@ func commitMain(args []string) error { } source := args[0] dest := args[1] - repo := NewRepo(dest) + repo := repo.NewRepo(dest) repo.Commit(source) return nil } @@ -95,7 +96,7 @@ func restoreMain(args []string) error { } source := args[0] dest := args[1] - repo := NewRepo(source) + repo := repo.NewRepo(source) repo.Restore(dest) return nil } diff --git a/main_test.go b/main_test.go index 27a3d65..06ab7d0 100644 --- a/main_test.go +++ b/main_test.go @@ -1,22 +1 @@ package main - -import ( - "log" - "os" - "testing" - - "github.com/n-peugnet/dna-backup/logger" -) - -func TestMain(m *testing.M) { - setup() - code := m.Run() - shutdown() - os.Exit(code) -} - -func setup() { - logger.SetFlags(log.Lshortfile) -} - -func shutdown() {} diff --git a/nowindows_test.go b/nowindows_test.go deleted file mode 100644 index 2f2879c..0000000 --- a/nowindows_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build !windows - -package main - -import ( - "bytes" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/n-peugnet/dna-backup/logger" - "github.com/n-peugnet/dna-backup/testutils" - "github.com/n-peugnet/dna-backup/utils" -) - -func TestNotReadable(t *testing.T) { - var output bytes.Buffer - logger.SetOutput(&output) - defer logger.SetOutput(os.Stderr) - tmpDir := t.TempDir() - f, err := os.OpenFile(filepath.Join(tmpDir, "notreadable"), os.O_CREATE, 0000) - if err != nil { - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } - var buff bytes.Buffer - files := listFiles(tmpDir) - testutils.AssertLen(t, 1, files, "Files") - concatFiles(&files, utils.NopCloser(&buff)) - testutils.AssertLen(t, 0, files, "Files") - testutils.AssertLen(t, 0, buff.Bytes(), "Buffer") - if !strings.Contains(output.String(), "notreadable") { - t.Errorf("log should contain a warning for notreadable, actual %q", &output) - } -} diff --git a/repo.go b/repo.go deleted file mode 100644 index 543f6c3..0000000 --- a/repo.go +++ /dev/null @@ -1,783 +0,0 @@ -/* -Manage a deduplicated versionned backups repository. - -Sample repository: - -``` -repo/ -├── 00000/ -│ ├── chunks/ -│ │ ├── 000000000000000 -│ │ ├── 000000000000001 -│ │ ├── 000000000000002 -│ │ └── 000000000000003 -│ ├── files -│ ├── hashes -│ └── recipe -└── 00001/ - ├── chunks/ - │ ├── 000000000000000 - │ └── 000000000000001 - ├── files - ├── hashes - └── recipe -``` -*/ - -package main - -import ( - "bufio" - "bytes" - "encoding/gob" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "reflect" - "strings" - - "github.com/chmduquesne/rollinghash/rabinkarp64" - "github.com/n-peugnet/dna-backup/cache" - "github.com/n-peugnet/dna-backup/logger" - "github.com/n-peugnet/dna-backup/sketch" - "github.com/n-peugnet/dna-backup/slice" - "github.com/n-peugnet/dna-backup/utils" -) - -func init() { - // register chunk structs for encoding/decoding using gob - gob.RegisterName("*dna-backup.StoredChunk", &StoredChunk{}) - gob.RegisterName("*dna-backup.TempChunk", &TempChunk{}) - gob.RegisterName("*dna-backup.DeltaChunk", &DeltaChunk{}) - gob.RegisterName("dna-backup.File", File{}) -} - -type FingerprintMap map[uint64]*ChunkId -type SketchMap map[uint64][]*ChunkId - -func (m SketchMap) Set(key []uint64, value *ChunkId) { - for _, s := range key { - prev := m[s] - if contains(prev, value) { - continue - } - m[s] = append(prev, value) - } -} - -type Repo struct { - path string - chunkSize int - sketchWSize int - sketchSfCount int - sketchFCount int - pol rabinkarp64.Pol - differ Differ - patcher Patcher - fingerprints FingerprintMap - sketches SketchMap - recipe []Chunk - files []File - chunkCache cache.Cacher - chunkReadWrapper utils.ReadWrapper - chunkWriteWrapper utils.WriteWrapper -} - -type chunkHashes struct { - Fp uint64 - Sk []uint64 -} - -type chunkData struct { - hashes chunkHashes - content []byte - id *ChunkId -} - -type File struct { - Path string - Size int64 - Link string -} - -func NewRepo(path string) *Repo { - var err error - path, err = filepath.Abs(path) - if err != nil { - logger.Fatal(err) - } - err = os.MkdirAll(path, 0775) - if err != nil { - logger.Panic(err) - } - var seed int64 = 1 - p, err := rabinkarp64.RandomPolynomial(seed) - if err != nil { - logger.Panic(err) - } - return &Repo{ - path: path, - chunkSize: 8 << 10, - sketchWSize: 32, - sketchSfCount: 3, - sketchFCount: 4, - pol: p, - differ: Fdelta{}, - patcher: Fdelta{}, - fingerprints: make(FingerprintMap), - sketches: make(SketchMap), - chunkCache: cache.NewFifoCache(10000), - chunkReadWrapper: utils.ZlibReader, - chunkWriteWrapper: utils.ZlibWriter, - } -} - -func (r *Repo) Differ() Differ { - return r.differ -} - -func (r *Repo) Patcher() Patcher { - return r.patcher -} - -func (r *Repo) Commit(source string) { - source, err := filepath.Abs(source) - if err != nil { - logger.Fatal(err) - } - versions := r.loadVersions() - newVersion := len(versions) // TODO: add newVersion functino - newPath := filepath.Join(r.path, fmt.Sprintf(versionFmt, newVersion)) - newChunkPath := filepath.Join(newPath, chunksName) - os.Mkdir(newPath, 0775) // TODO: handle errors - os.Mkdir(newChunkPath, 0775) // TODO: handle errors - logger.Info("listing files") - files := listFiles(source) - logger.Info("loading previous hashes") - r.loadHashes(versions) - logger.Info("loading previous file lists") - r.loadFileLists(versions) - logger.Info("loading previous recipies") - r.loadRecipes(versions) - storeQueue := make(chan chunkData, 32) - storeEnd := make(chan bool) - go r.storageWorker(newVersion, storeQueue, storeEnd) - var last, nlast, pass uint64 - var recipe []Chunk - for ; nlast > last || pass == 0; pass++ { - logger.Infof("matcher pass number %d", pass+1) - last = nlast - reader, writer := io.Pipe() - go concatFiles(&files, writer) - recipe, nlast = r.matchStream(reader, storeQueue, newVersion, last) - } - close(storeQueue) - <-storeEnd - r.storeFileList(newVersion, unprefixFiles(files, source)) - r.storeRecipe(newVersion, recipe) -} - -func (r *Repo) Restore(destination string) { - versions := r.loadVersions() - logger.Info("loading previous file lists") - r.loadFileLists(versions) - logger.Info("loading previous recipies") - r.loadRecipes(versions) - reader, writer := io.Pipe() - logger.Info("restoring latest version") - go r.restoreStream(writer, r.recipe) - bufReader := bufio.NewReaderSize(reader, r.chunkSize*2) - for _, file := range r.files { - filePath := filepath.Join(destination, file.Path) - dir := filepath.Dir(filePath) - os.MkdirAll(dir, 0775) // TODO: handle errors - if file.Link != "" { - link := file.Link - if filepath.IsAbs(link) { - filepath.Join(destination, file.Link) - } - err := os.Symlink(link, filePath) - if err != nil { - logger.Errorf("restored symlink ", err) - } - } else { - f, _ := os.Create(filePath) // TODO: handle errors - n, err := io.CopyN(f, bufReader, file.Size) - if err != nil { - logger.Errorf("restored file, written %d/%d bytes: %s", filePath, n, file.Size, err) - } - if err := f.Close(); err != nil { - logger.Errorf("restored file ", err) - } - } - } -} - -func (r *Repo) loadVersions() []string { - var versions []string - files, err := os.ReadDir(r.path) - if err != nil { - logger.Fatal(err) - } - for _, f := range files { - if !f.IsDir() { - continue - } - versions = append(versions, filepath.Join(r.path, f.Name())) - } - return versions -} - -func listFiles(path string) []File { - var files []File - err := filepath.Walk(path, func(p string, i fs.FileInfo, err error) error { - if err != nil { - logger.Warning(err) - return nil - } - if i.IsDir() { - return nil - } - var file = File{Path: p, Size: i.Size()} - if i.Mode()&fs.ModeSymlink != 0 { - file, err = cleanSymlink(path, p, i) - if err != nil { - logger.Warning("skipping symlink ", err) - return nil - } - } - files = append(files, file) - return nil - }) - if err != nil { - logger.Error(err) - } - return files -} - -func cleanSymlink(root string, p string, i fs.FileInfo) (f File, err error) { - dir := filepath.Dir(p) - target, err := os.Readlink(p) - if err != nil { - return - } - isAbs := filepath.IsAbs(target) - cleaned := target - if !isAbs { - cleaned = filepath.Join(dir, cleaned) - } - cleaned = filepath.Clean(cleaned) - if !strings.HasPrefix(cleaned, root) { - err = fmt.Errorf("external %s -> %s", p, cleaned) - return - } - if isAbs { - f.Link, err = utils.Unprefix(cleaned, root) - } else { - f.Link, err = filepath.Rel(dir, filepath.Join(dir, target)) - } - if err != nil { - return - } - if f.Link == "" { - err = fmt.Errorf("empty %s", p) - return - } - f.Path = p - f.Size = 0 - return f, nil -} - -func unprefixFiles(files []File, prefix string) (ret []File) { - var err error - ret = make([]File, len(files)) - for i, f := range files { - if f.Path, err = utils.Unprefix(f.Path, prefix); err != nil { - logger.Warning(err) - } else { - ret[i] = f - } - } - return -} - -// concatFiles reads the content of all the listed files into a continuous stream. -// If any errors are encoutered while opening a file, it is then removed from the -// list. -// -// If read is incomplete, then the actual read size is used. -func concatFiles(files *[]File, stream io.WriteCloser) { - actual := make([]File, 0, len(*files)) - for _, f := range *files { - if f.Link != "" { - actual = append(actual, f) - continue - } - file, err := os.Open(f.Path) - if err != nil { - logger.Warning(err) - continue - } - af := f - if n, err := io.Copy(stream, file); err != nil { - logger.Error("read ", n, " bytes, ", err) - af.Size = n - } - actual = append(actual, af) - if err = file.Close(); err != nil { - logger.Panic(err) - } - } - stream.Close() - *files = actual -} - -func storeBasicStruct(dest string, wrapper utils.WriteWrapper, obj interface{}) { - file, err := os.Create(dest) - if err != nil { - logger.Panic(err) - } - out := wrapper(file) - encoder := gob.NewEncoder(out) - err = encoder.Encode(obj) - if err != nil { - logger.Panic(err) - } - if err = out.Close(); err != nil { - logger.Panic(err) - } - if err = file.Close(); err != nil { - logger.Panic(err) - } -} - -func loadBasicStruct(path string, wrapper utils.ReadWrapper, obj interface{}) { - file, err := os.Open(path) - if err != nil { - logger.Panic(err) - } - in, err := wrapper(file) - if err != nil { - logger.Panic(err) - } - decoder := gob.NewDecoder(in) - err = decoder.Decode(obj) - if err != nil { - logger.Panic(err) - } - if err = in.Close(); err != nil { - logger.Panic(err) - } - if err = file.Close(); err != nil { - logger.Panic(err) - } -} - -func (r *Repo) loadDeltas(versions []string, wrapper utils.ReadWrapper, name string) (ret slice.Slice) { - for _, v := range versions { - path := filepath.Join(v, name) - var delta slice.Delta - loadBasicStruct(path, wrapper, &delta) - ret = slice.Patch(ret, delta) - } - return -} - -func fileList2slice(l []File) (ret slice.Slice) { - ret = make(slice.Slice, len(l)) - for i := range l { - ret[i] = l[i] - } - return -} - -func slice2fileList(s slice.Slice) (ret []File) { - ret = make([]File, len(s)) - for i := range s { - if f, ok := s[i].(File); ok { - ret[i] = f - } else { - logger.Warningf("could not convert %s into a File", s[i]) - } - } - return -} - -// storeFileList stores the given list in the repo dir as a delta against the -// previous version's one. -func (r *Repo) storeFileList(version int, list []File) { - dest := filepath.Join(r.path, fmt.Sprintf(versionFmt, version), filesName) - delta := slice.Diff(fileList2slice(r.files), fileList2slice(list)) - logger.Infof("files delta %s", delta.String()) - storeBasicStruct(dest, r.chunkWriteWrapper, delta) -} - -// loadFileLists loads incrementally the file lists' delta of each given version. -func (r *Repo) loadFileLists(versions []string) { - r.files = slice2fileList(r.loadDeltas(versions, r.chunkReadWrapper, filesName)) -} - -// storageWorker is meant to be started in a goroutine and stores each new chunk's -// data in the repo directory until the store queue channel is closed. -// -// it will put true in the end channel once everything is stored. -func (r *Repo) storageWorker(version int, storeQueue <-chan chunkData, end chan<- bool) { - hashesFile := filepath.Join(r.path, fmt.Sprintf(versionFmt, version), hashesName) - file, err := os.Create(hashesFile) - if err != nil { - logger.Panic(err) - } - encoder := gob.NewEncoder(file) - for data := range storeQueue { - err = encoder.Encode(data.hashes) - r.StoreChunkContent(data.id, bytes.NewReader(data.content)) - // logger.Debug("stored ", data.id) - } - if err = file.Close(); err != nil { - logger.Panic(err) - } - end <- true -} - -func (r *Repo) StoreChunkContent(id *ChunkId, reader io.Reader) { - path := id.Path(r.path) - file, err := os.Create(path) - if err != nil { - logger.Panic("chunk store ", err) - } - wrapper := r.chunkWriteWrapper(file) - n, err := io.Copy(wrapper, reader) - if err != nil { - logger.Errorf("chunk store, %d written, %s", n, err) - } - if err := wrapper.Close(); err != nil { - logger.Warning("chunk store wrapper ", err) - } - if err := file.Close(); err != nil { - logger.Warning("chunk store ", err) - } -} - -// LoadChunkContent loads a chunk from the repo directory. -// If the chunk is in cache, get it from cache, else read it from drive. -func (r *Repo) LoadChunkContent(id *ChunkId) *bytes.Reader { - value, exists := r.chunkCache.Get(id) - if !exists { - path := id.Path(r.path) - f, err := os.Open(path) - if err != nil { - logger.Panic("chunk load ", err) - } - wrapper, err := r.chunkReadWrapper(f) - if err != nil { - logger.Error("chunk load wrapper ", err) - } - value, err = io.ReadAll(wrapper) - if err != nil { - logger.Error("chunk load ", err) - } - if err = wrapper.Close(); err != nil { - logger.Warning("chunk load wrapper", err) - } - if err = f.Close(); err != nil { - logger.Warning("chunk load ", err) - } - r.chunkCache.Set(id, value) - } - return bytes.NewReader(value) -} - -// TODO: use atoi for chunkid ? -func (r *Repo) loadChunks(versions []string, chunks chan<- IdentifiedChunk) { - for i, v := range versions { - p := filepath.Join(v, chunksName) - entries, err := os.ReadDir(p) - if err != nil { - logger.Error("version dir ", err) - } - for j, e := range entries { - if e.IsDir() { - continue - } - id := &ChunkId{Ver: i, Idx: uint64(j)} - c := NewStoredChunk(r, id) - chunks <- c - } - } - close(chunks) -} - -// loadHashes loads and aggregates the hashes stored for each given version and -// stores them in the repo maps. -func (r *Repo) loadHashes(versions []string) { - for i, v := range versions { - path := filepath.Join(v, hashesName) - file, err := os.Open(path) - if err != nil { - logger.Error("hashes ", err) - } - decoder := gob.NewDecoder(file) - for j := 0; err == nil; j++ { - var h chunkHashes - if err = decoder.Decode(&h); err == nil { - id := &ChunkId{i, uint64(j)} - r.fingerprints[h.Fp] = id - r.sketches.Set(h.Sk, id) - } - } - if err != nil && err != io.EOF { - logger.Panic(err) - } - if err = file.Close(); err != nil { - logger.Warning(err) - } - } -} - -func (r *Repo) chunkMinLen() int { - return sketch.SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) -} - -func contains(s []*ChunkId, id *ChunkId) bool { - for _, v := range s { - if v == id { - return true - } - } - return false -} - -// findSimilarChunk looks in the repo sketch map for a match of the given sketch. -// -// There can be multiple matches but only the best one is returned. Indeed, the -// more superfeature matches, the better the quality of the match. For now we -// consider that a single superfeature match is enough to count it as valid. -func (r *Repo) findSimilarChunk(sketch []uint64) (*ChunkId, bool) { - var similarChunks = make(map[ChunkId]int) - var max int - var similarChunk *ChunkId - for _, s := range sketch { - chunkIds, exists := r.sketches[s] - if !exists { - continue - } - for _, id := range chunkIds { - count := similarChunks[*id] - count += 1 - logger.Debugf("found %d %d time(s)", id, count) - if count > max { - max = count - similarChunk = id - } - similarChunks[*id] = count - } - } - return similarChunk, max > 0 -} - -// encodeTempChunk first tries to delta-encode the given chunk before attributing -// it an Id and saving it into the fingerprints and sketches maps. -func (r *Repo) encodeTempChunk(temp BufferedChunk, version int, last *uint64, storeQueue chan<- chunkData) (Chunk, bool) { - sk, _ := sketch.SketchChunk(temp.Reader(), r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) - id, found := r.findSimilarChunk(sk) - if found { - var buff bytes.Buffer - if err := r.differ.Diff(r.LoadChunkContent(id), temp.Reader(), &buff); err != nil { - logger.Error("trying delta encode chunk:", temp, "with source:", id, ":", err) - } else { - logger.Debugf("add new delta chunk of size %d", len(buff.Bytes())) - return &DeltaChunk{ - repo: r, - Source: id, - Patch: buff.Bytes(), - Size: temp.Len(), - }, true - } - } - if temp.Len() == r.chunkSize { - id := &ChunkId{Ver: version, Idx: *last} - *last++ - hasher := rabinkarp64.NewFromPol(r.pol) - io.Copy(hasher, temp.Reader()) - fp := hasher.Sum64() - r.fingerprints[fp] = id - r.sketches.Set(sk, id) - storeQueue <- chunkData{ - hashes: chunkHashes{fp, sk}, - content: temp.Bytes(), - id: id, - } - r.chunkCache.Set(id, temp.Bytes()) - logger.Debug("add new chunk ", id) - return NewStoredChunk(r, id), false - } - logger.Debug("add new partial chunk of size: ", temp.Len()) - return temp, false -} - -// encodeTempChunks encodes the current temporary chunks based on the value of the previous one. -// Temporary chunks can be partial. If the current chunk is smaller than the size of a -// super-feature and there exists a previous chunk, then both are merged before attempting -// to delta-encode them. -func (r *Repo) encodeTempChunks(prev BufferedChunk, curr BufferedChunk, version int, last *uint64, storeQueue chan<- chunkData) []Chunk { - if reflect.ValueOf(prev).IsNil() { - c, _ := r.encodeTempChunk(curr, version, last, storeQueue) - return []Chunk{c} - } else if curr.Len() < r.chunkMinLen() { - tmp := NewTempChunk(append(prev.Bytes(), curr.Bytes()...)) - c, success := r.encodeTempChunk(tmp, version, last, storeQueue) - if success { - return []Chunk{c} - } - } - prevD, _ := r.encodeTempChunk(prev, version, last, storeQueue) - currD, _ := r.encodeTempChunk(curr, version, last, storeQueue) - return []Chunk{prevD, currD} -} - -// matchStream is the heart of DNA-backup. Thus, it sounded rude not to add some comment to it. -// -// It applies a rolling hash on the content of a given stream to look for matching fingerprints -// in the repo. If no match is found after the equivalent of three chunks of data are processed, -// then the first unmatched chunk sketch is checked to see if it could be delta-encoded. -// If not, the chunk is then stored as a new chunk for this version and its fingerprint and -// sketch are added to the repo maps. -// -// If a match happens during the processing of the third chunk, then, if possible, the remaining -// of the second chunk is merged with the first one to try to delta encode it at once. -// -// Each time a new chunk is added it is sent to the store worker through the store queue. -func (r *Repo) matchStream(stream io.Reader, storeQueue chan<- chunkData, version int, last uint64) ([]Chunk, uint64) { - var b byte - var chunks []Chunk - var prev *TempChunk - var err error - bufStream := bufio.NewReaderSize(stream, r.chunkSize*2) - buff := make([]byte, r.chunkSize, r.chunkSize*2) - if n, err := io.ReadFull(stream, buff); n < r.chunkSize { - if err == io.ErrUnexpectedEOF { - c, _ := r.encodeTempChunk(NewTempChunk(buff[:n]), version, &last, storeQueue) - chunks = append(chunks, c) - return chunks, last - } else { - logger.Panicf("matching stream, read only %d bytes with error '%s'", n, err) - } - } - hasher := rabinkarp64.NewFromPol(r.pol) - hasher.Write(buff) - for err != io.EOF { - h := hasher.Sum64() - chunkId, exists := r.fingerprints[h] - if exists { - if len(buff) > r.chunkSize && len(buff) <= r.chunkSize*2 { - size := len(buff) - r.chunkSize - temp := NewTempChunk(buff[:size]) - chunks = append(chunks, r.encodeTempChunks(prev, temp, version, &last, storeQueue)...) - prev = nil - } else if prev != nil { - c, _ := r.encodeTempChunk(prev, version, &last, storeQueue) - chunks = append(chunks, c) - prev = nil - } - logger.Debugf("add existing chunk: %d", chunkId) - chunks = append(chunks, NewStoredChunk(r, chunkId)) - buff = make([]byte, 0, r.chunkSize*2) - for i := 0; i < r.chunkSize && err == nil; i++ { - b, err = bufStream.ReadByte() - if err != io.EOF { - hasher.Roll(b) - buff = append(buff, b) - } - } - continue - } - if len(buff) == r.chunkSize*2 { - if prev != nil { - chunk, _ := r.encodeTempChunk(prev, version, &last, storeQueue) - chunks = append(chunks, chunk) - } - prev = NewTempChunk(buff[:r.chunkSize]) - tmp := buff[r.chunkSize:] - buff = make([]byte, r.chunkSize, r.chunkSize*2) - copy(buff, tmp) - } - b, err = bufStream.ReadByte() - if err != io.EOF { - hasher.Roll(b) - buff = append(buff, b) - } - } - if len(buff) > 0 { - var temp *TempChunk - if len(buff) > r.chunkSize { - if prev != nil { - chunk, _ := r.encodeTempChunk(prev, version, &last, storeQueue) - chunks = append(chunks, chunk) - } - prev = NewTempChunk(buff[:r.chunkSize]) - temp = NewTempChunk(buff[r.chunkSize:]) - } else { - temp = NewTempChunk(buff) - } - chunks = append(chunks, r.encodeTempChunks(prev, temp, version, &last, storeQueue)...) - } - return chunks, last -} - -func (r *Repo) restoreStream(stream io.WriteCloser, recipe []Chunk) { - for _, c := range recipe { - if n, err := io.Copy(stream, c.Reader()); err != nil { - logger.Errorf("copying to stream, read %d bytes from chunk: %s", n, err) - } - } - stream.Close() -} - -func recipe2slice(r []Chunk) (ret slice.Slice) { - ret = make(slice.Slice, len(r)) - for i := range r { - ret[i] = r[i] - } - return -} - -func slice2recipe(s slice.Slice) (ret []Chunk) { - ret = make([]Chunk, len(s)) - for i := range s { - if c, ok := s[i].(Chunk); ok { - ret[i] = c - } else { - logger.Warningf("could not convert %s into a Chunk", s[i]) - } - } - return -} - -func (r *Repo) storeRecipe(version int, recipe []Chunk) { - dest := filepath.Join(r.path, fmt.Sprintf(versionFmt, version), recipeName) - delta := slice.Diff(recipe2slice(r.recipe), recipe2slice(recipe)) - logger.Infof("recipe delta %s", delta.String()) - storeBasicStruct(dest, r.chunkWriteWrapper, delta) -} - -func (r *Repo) loadRecipes(versions []string) { - recipe := slice2recipe(r.loadDeltas(versions, r.chunkReadWrapper, recipeName)) - for _, c := range recipe { - if rc, isRepo := c.(RepoChunk); isRepo { - rc.SetRepo(r) - } - } - r.recipe = recipe -} - -func extractDeltaChunks(chunks []Chunk) (ret []*DeltaChunk) { - for _, c := range chunks { - tmp, isDelta := c.(*DeltaChunk) - if isDelta { - ret = append(ret, tmp) - } - } - return -} diff --git a/repo/chunk.go b/repo/chunk.go new file mode 100644 index 0000000..618cc9b --- /dev/null +++ b/repo/chunk.go @@ -0,0 +1,113 @@ +package repo + +import ( + "bytes" + "fmt" + "io" + "path/filepath" +) + +type Chunk interface { + Reader() io.ReadSeeker + Len() int +} + +type IdentifiedChunk interface { + Chunk + GetId() *ChunkId +} + +type BufferedChunk interface { + Chunk + Bytes() []byte +} + +type RepoChunk interface { + Chunk + SetRepo(r *Repo) +} + +type ChunkId struct { + Ver int + Idx uint64 +} + +func (i *ChunkId) Path(repo string) string { + return filepath.Join(repo, fmt.Sprintf(versionFmt, i.Ver), chunksName, fmt.Sprintf(chunkIdFmt, i.Idx)) +} + +func NewStoredChunk(repo *Repo, id *ChunkId) *StoredChunk { + return &StoredChunk{repo: repo, Id: id} +} + +type StoredChunk struct { + repo *Repo + Id *ChunkId +} + +func (c *StoredChunk) GetId() *ChunkId { + return c.Id +} + +func (c *StoredChunk) SetRepo(r *Repo) { + c.repo = r +} + +func (c *StoredChunk) Reader() io.ReadSeeker { + // log.Printf("Chunk %d: Reading from file\n", c.id) + return c.repo.LoadChunkContent(c.Id) +} + +func (c *StoredChunk) Len() int { + return c.repo.chunkSize +} + +func NewTempChunk(value []byte) *TempChunk { + return &TempChunk{Value: value} +} + +type TempChunk struct { + Value []byte +} + +func (c *TempChunk) Reader() io.ReadSeeker { + return bytes.NewReader(c.Value) +} + +func (c *TempChunk) Len() int { + return len(c.Value) +} + +func (c *TempChunk) Bytes() []byte { + return c.Value +} + +func (c *TempChunk) AppendFrom(r io.Reader) { + buff, err := io.ReadAll(r) + if err != nil { + println("Chunk: error appending to temp chunk:", err) + } + c.Value = append(c.Value, buff...) +} + +type DeltaChunk struct { + repo *Repo + Source *ChunkId + Patch []byte + Size int +} + +func (c *DeltaChunk) SetRepo(r *Repo) { + c.repo = r +} + +func (c *DeltaChunk) Reader() io.ReadSeeker { + var buff bytes.Buffer + c.repo.Patcher().Patch(c.repo.LoadChunkContent(c.Source), &buff, bytes.NewReader(c.Patch)) + return bytes.NewReader(buff.Bytes()) +} + +// TODO: Maybe return the size of the patch instead ? +func (c *DeltaChunk) Len() int { + return c.Size +} diff --git a/repo/const.go b/repo/const.go new file mode 100644 index 0000000..f4d79ae --- /dev/null +++ b/repo/const.go @@ -0,0 +1,10 @@ +package repo + +const ( + chunksName = "chunks" + chunkIdFmt = "%015d" + versionFmt = "%05d" + filesName = "files" + hashesName = "hashes" + recipeName = "recipe" +) diff --git a/repo/nowindows_test.go b/repo/nowindows_test.go new file mode 100644 index 0000000..28752ba --- /dev/null +++ b/repo/nowindows_test.go @@ -0,0 +1,38 @@ +// +build !windows + +package repo + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/n-peugnet/dna-backup/logger" + "github.com/n-peugnet/dna-backup/testutils" + "github.com/n-peugnet/dna-backup/utils" +) + +func TestNotReadable(t *testing.T) { + var output bytes.Buffer + logger.SetOutput(&output) + defer logger.SetOutput(os.Stderr) + tmpDir := t.TempDir() + f, err := os.OpenFile(filepath.Join(tmpDir, "notreadable"), os.O_CREATE, 0000) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + var buff bytes.Buffer + files := listFiles(tmpDir) + testutils.AssertLen(t, 1, files, "Files") + concatFiles(&files, utils.NopCloser(&buff)) + testutils.AssertLen(t, 0, files, "Files") + testutils.AssertLen(t, 0, buff.Bytes(), "Buffer") + if !strings.Contains(output.String(), "notreadable") { + t.Errorf("log should contain a warning for notreadable, actual %q", &output) + } +} diff --git a/repo/repo.go b/repo/repo.go new file mode 100644 index 0000000..97d0b5c --- /dev/null +++ b/repo/repo.go @@ -0,0 +1,784 @@ +/* +Manage a deduplicated versionned backups repository. + +Sample repository: + +``` +repo/ +├── 00000/ +│ ├── chunks/ +│ │ ├── 000000000000000 +│ │ ├── 000000000000001 +│ │ ├── 000000000000002 +│ │ └── 000000000000003 +│ ├── files +│ ├── hashes +│ └── recipe +└── 00001/ + ├── chunks/ + │ ├── 000000000000000 + │ └── 000000000000001 + ├── files + ├── hashes + └── recipe +``` +*/ + +package repo + +import ( + "bufio" + "bytes" + "encoding/gob" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "reflect" + "strings" + + "github.com/chmduquesne/rollinghash/rabinkarp64" + "github.com/n-peugnet/dna-backup/cache" + "github.com/n-peugnet/dna-backup/delta" + "github.com/n-peugnet/dna-backup/logger" + "github.com/n-peugnet/dna-backup/sketch" + "github.com/n-peugnet/dna-backup/slice" + "github.com/n-peugnet/dna-backup/utils" +) + +func init() { + // register chunk structs for encoding/decoding using gob + gob.RegisterName("*dna-backup.StoredChunk", &StoredChunk{}) + gob.RegisterName("*dna-backup.TempChunk", &TempChunk{}) + gob.RegisterName("*dna-backup.DeltaChunk", &DeltaChunk{}) + gob.RegisterName("dna-backup.File", File{}) +} + +type FingerprintMap map[uint64]*ChunkId +type SketchMap map[uint64][]*ChunkId + +func (m SketchMap) Set(key []uint64, value *ChunkId) { + for _, s := range key { + prev := m[s] + if contains(prev, value) { + continue + } + m[s] = append(prev, value) + } +} + +type Repo struct { + path string + chunkSize int + sketchWSize int + sketchSfCount int + sketchFCount int + pol rabinkarp64.Pol + differ delta.Differ + patcher delta.Patcher + fingerprints FingerprintMap + sketches SketchMap + recipe []Chunk + files []File + chunkCache cache.Cacher + chunkReadWrapper utils.ReadWrapper + chunkWriteWrapper utils.WriteWrapper +} + +type chunkHashes struct { + Fp uint64 + Sk []uint64 +} + +type chunkData struct { + hashes chunkHashes + content []byte + id *ChunkId +} + +type File struct { + Path string + Size int64 + Link string +} + +func NewRepo(path string) *Repo { + var err error + path, err = filepath.Abs(path) + if err != nil { + logger.Fatal(err) + } + err = os.MkdirAll(path, 0775) + if err != nil { + logger.Panic(err) + } + var seed int64 = 1 + p, err := rabinkarp64.RandomPolynomial(seed) + if err != nil { + logger.Panic(err) + } + return &Repo{ + path: path, + chunkSize: 8 << 10, + sketchWSize: 32, + sketchSfCount: 3, + sketchFCount: 4, + pol: p, + differ: delta.Fdelta{}, + patcher: delta.Fdelta{}, + fingerprints: make(FingerprintMap), + sketches: make(SketchMap), + chunkCache: cache.NewFifoCache(10000), + chunkReadWrapper: utils.ZlibReader, + chunkWriteWrapper: utils.ZlibWriter, + } +} + +func (r *Repo) Differ() delta.Differ { + return r.differ +} + +func (r *Repo) Patcher() delta.Patcher { + return r.patcher +} + +func (r *Repo) Commit(source string) { + source, err := filepath.Abs(source) + if err != nil { + logger.Fatal(err) + } + versions := r.loadVersions() + newVersion := len(versions) // TODO: add newVersion functino + newPath := filepath.Join(r.path, fmt.Sprintf(versionFmt, newVersion)) + newChunkPath := filepath.Join(newPath, chunksName) + os.Mkdir(newPath, 0775) // TODO: handle errors + os.Mkdir(newChunkPath, 0775) // TODO: handle errors + logger.Info("listing files") + files := listFiles(source) + logger.Info("loading previous hashes") + r.loadHashes(versions) + logger.Info("loading previous file lists") + r.loadFileLists(versions) + logger.Info("loading previous recipies") + r.loadRecipes(versions) + storeQueue := make(chan chunkData, 32) + storeEnd := make(chan bool) + go r.storageWorker(newVersion, storeQueue, storeEnd) + var last, nlast, pass uint64 + var recipe []Chunk + for ; nlast > last || pass == 0; pass++ { + logger.Infof("matcher pass number %d", pass+1) + last = nlast + reader, writer := io.Pipe() + go concatFiles(&files, writer) + recipe, nlast = r.matchStream(reader, storeQueue, newVersion, last) + } + close(storeQueue) + <-storeEnd + r.storeFileList(newVersion, unprefixFiles(files, source)) + r.storeRecipe(newVersion, recipe) +} + +func (r *Repo) Restore(destination string) { + versions := r.loadVersions() + logger.Info("loading previous file lists") + r.loadFileLists(versions) + logger.Info("loading previous recipies") + r.loadRecipes(versions) + reader, writer := io.Pipe() + logger.Info("restoring latest version") + go r.restoreStream(writer, r.recipe) + bufReader := bufio.NewReaderSize(reader, r.chunkSize*2) + for _, file := range r.files { + filePath := filepath.Join(destination, file.Path) + dir := filepath.Dir(filePath) + os.MkdirAll(dir, 0775) // TODO: handle errors + if file.Link != "" { + link := file.Link + if filepath.IsAbs(link) { + filepath.Join(destination, file.Link) + } + err := os.Symlink(link, filePath) + if err != nil { + logger.Errorf("restored symlink ", err) + } + } else { + f, _ := os.Create(filePath) // TODO: handle errors + n, err := io.CopyN(f, bufReader, file.Size) + if err != nil { + logger.Errorf("restored file, written %d/%d bytes: %s", filePath, n, file.Size, err) + } + if err := f.Close(); err != nil { + logger.Errorf("restored file ", err) + } + } + } +} + +func (r *Repo) loadVersions() []string { + var versions []string + files, err := os.ReadDir(r.path) + if err != nil { + logger.Fatal(err) + } + for _, f := range files { + if !f.IsDir() { + continue + } + versions = append(versions, filepath.Join(r.path, f.Name())) + } + return versions +} + +func listFiles(path string) []File { + var files []File + err := filepath.Walk(path, func(p string, i fs.FileInfo, err error) error { + if err != nil { + logger.Warning(err) + return nil + } + if i.IsDir() { + return nil + } + var file = File{Path: p, Size: i.Size()} + if i.Mode()&fs.ModeSymlink != 0 { + file, err = cleanSymlink(path, p, i) + if err != nil { + logger.Warning("skipping symlink ", err) + return nil + } + } + files = append(files, file) + return nil + }) + if err != nil { + logger.Error(err) + } + return files +} + +func cleanSymlink(root string, p string, i fs.FileInfo) (f File, err error) { + dir := filepath.Dir(p) + target, err := os.Readlink(p) + if err != nil { + return + } + isAbs := filepath.IsAbs(target) + cleaned := target + if !isAbs { + cleaned = filepath.Join(dir, cleaned) + } + cleaned = filepath.Clean(cleaned) + if !strings.HasPrefix(cleaned, root) { + err = fmt.Errorf("external %s -> %s", p, cleaned) + return + } + if isAbs { + f.Link, err = utils.Unprefix(cleaned, root) + } else { + f.Link, err = filepath.Rel(dir, filepath.Join(dir, target)) + } + if err != nil { + return + } + if f.Link == "" { + err = fmt.Errorf("empty %s", p) + return + } + f.Path = p + f.Size = 0 + return f, nil +} + +func unprefixFiles(files []File, prefix string) (ret []File) { + var err error + ret = make([]File, len(files)) + for i, f := range files { + if f.Path, err = utils.Unprefix(f.Path, prefix); err != nil { + logger.Warning(err) + } else { + ret[i] = f + } + } + return +} + +// concatFiles reads the content of all the listed files into a continuous stream. +// If any errors are encoutered while opening a file, it is then removed from the +// list. +// +// If read is incomplete, then the actual read size is used. +func concatFiles(files *[]File, stream io.WriteCloser) { + actual := make([]File, 0, len(*files)) + for _, f := range *files { + if f.Link != "" { + actual = append(actual, f) + continue + } + file, err := os.Open(f.Path) + if err != nil { + logger.Warning(err) + continue + } + af := f + if n, err := io.Copy(stream, file); err != nil { + logger.Error("read ", n, " bytes, ", err) + af.Size = n + } + actual = append(actual, af) + if err = file.Close(); err != nil { + logger.Panic(err) + } + } + stream.Close() + *files = actual +} + +func storeBasicStruct(dest string, wrapper utils.WriteWrapper, obj interface{}) { + file, err := os.Create(dest) + if err != nil { + logger.Panic(err) + } + out := wrapper(file) + encoder := gob.NewEncoder(out) + err = encoder.Encode(obj) + if err != nil { + logger.Panic(err) + } + if err = out.Close(); err != nil { + logger.Panic(err) + } + if err = file.Close(); err != nil { + logger.Panic(err) + } +} + +func loadBasicStruct(path string, wrapper utils.ReadWrapper, obj interface{}) { + file, err := os.Open(path) + if err != nil { + logger.Panic(err) + } + in, err := wrapper(file) + if err != nil { + logger.Panic(err) + } + decoder := gob.NewDecoder(in) + err = decoder.Decode(obj) + if err != nil { + logger.Panic(err) + } + if err = in.Close(); err != nil { + logger.Panic(err) + } + if err = file.Close(); err != nil { + logger.Panic(err) + } +} + +func (r *Repo) loadDeltas(versions []string, wrapper utils.ReadWrapper, name string) (ret slice.Slice) { + for _, v := range versions { + path := filepath.Join(v, name) + var delta slice.Delta + loadBasicStruct(path, wrapper, &delta) + ret = slice.Patch(ret, delta) + } + return +} + +func fileList2slice(l []File) (ret slice.Slice) { + ret = make(slice.Slice, len(l)) + for i := range l { + ret[i] = l[i] + } + return +} + +func slice2fileList(s slice.Slice) (ret []File) { + ret = make([]File, len(s)) + for i := range s { + if f, ok := s[i].(File); ok { + ret[i] = f + } else { + logger.Warningf("could not convert %s into a File", s[i]) + } + } + return +} + +// storeFileList stores the given list in the repo dir as a delta against the +// previous version's one. +func (r *Repo) storeFileList(version int, list []File) { + dest := filepath.Join(r.path, fmt.Sprintf(versionFmt, version), filesName) + delta := slice.Diff(fileList2slice(r.files), fileList2slice(list)) + logger.Infof("files delta %s", delta.String()) + storeBasicStruct(dest, r.chunkWriteWrapper, delta) +} + +// loadFileLists loads incrementally the file lists' delta of each given version. +func (r *Repo) loadFileLists(versions []string) { + r.files = slice2fileList(r.loadDeltas(versions, r.chunkReadWrapper, filesName)) +} + +// storageWorker is meant to be started in a goroutine and stores each new chunk's +// data in the repo directory until the store queue channel is closed. +// +// it will put true in the end channel once everything is stored. +func (r *Repo) storageWorker(version int, storeQueue <-chan chunkData, end chan<- bool) { + hashesFile := filepath.Join(r.path, fmt.Sprintf(versionFmt, version), hashesName) + file, err := os.Create(hashesFile) + if err != nil { + logger.Panic(err) + } + encoder := gob.NewEncoder(file) + for data := range storeQueue { + err = encoder.Encode(data.hashes) + r.StoreChunkContent(data.id, bytes.NewReader(data.content)) + // logger.Debug("stored ", data.id) + } + if err = file.Close(); err != nil { + logger.Panic(err) + } + end <- true +} + +func (r *Repo) StoreChunkContent(id *ChunkId, reader io.Reader) { + path := id.Path(r.path) + file, err := os.Create(path) + if err != nil { + logger.Panic("chunk store ", err) + } + wrapper := r.chunkWriteWrapper(file) + n, err := io.Copy(wrapper, reader) + if err != nil { + logger.Errorf("chunk store, %d written, %s", n, err) + } + if err := wrapper.Close(); err != nil { + logger.Warning("chunk store wrapper ", err) + } + if err := file.Close(); err != nil { + logger.Warning("chunk store ", err) + } +} + +// LoadChunkContent loads a chunk from the repo directory. +// If the chunk is in cache, get it from cache, else read it from drive. +func (r *Repo) LoadChunkContent(id *ChunkId) *bytes.Reader { + value, exists := r.chunkCache.Get(id) + if !exists { + path := id.Path(r.path) + f, err := os.Open(path) + if err != nil { + logger.Panic("chunk load ", err) + } + wrapper, err := r.chunkReadWrapper(f) + if err != nil { + logger.Error("chunk load wrapper ", err) + } + value, err = io.ReadAll(wrapper) + if err != nil { + logger.Error("chunk load ", err) + } + if err = wrapper.Close(); err != nil { + logger.Warning("chunk load wrapper", err) + } + if err = f.Close(); err != nil { + logger.Warning("chunk load ", err) + } + r.chunkCache.Set(id, value) + } + return bytes.NewReader(value) +} + +// TODO: use atoi for chunkid ? +func (r *Repo) loadChunks(versions []string, chunks chan<- IdentifiedChunk) { + for i, v := range versions { + p := filepath.Join(v, chunksName) + entries, err := os.ReadDir(p) + if err != nil { + logger.Error("version dir ", err) + } + for j, e := range entries { + if e.IsDir() { + continue + } + id := &ChunkId{Ver: i, Idx: uint64(j)} + c := NewStoredChunk(r, id) + chunks <- c + } + } + close(chunks) +} + +// loadHashes loads and aggregates the hashes stored for each given version and +// stores them in the repo maps. +func (r *Repo) loadHashes(versions []string) { + for i, v := range versions { + path := filepath.Join(v, hashesName) + file, err := os.Open(path) + if err != nil { + logger.Error("hashes ", err) + } + decoder := gob.NewDecoder(file) + for j := 0; err == nil; j++ { + var h chunkHashes + if err = decoder.Decode(&h); err == nil { + id := &ChunkId{i, uint64(j)} + r.fingerprints[h.Fp] = id + r.sketches.Set(h.Sk, id) + } + } + if err != nil && err != io.EOF { + logger.Panic(err) + } + if err = file.Close(); err != nil { + logger.Warning(err) + } + } +} + +func (r *Repo) chunkMinLen() int { + return sketch.SuperFeatureSize(r.chunkSize, r.sketchSfCount, r.sketchFCount) +} + +func contains(s []*ChunkId, id *ChunkId) bool { + for _, v := range s { + if v == id { + return true + } + } + return false +} + +// findSimilarChunk looks in the repo sketch map for a match of the given sketch. +// +// There can be multiple matches but only the best one is returned. Indeed, the +// more superfeature matches, the better the quality of the match. For now we +// consider that a single superfeature match is enough to count it as valid. +func (r *Repo) findSimilarChunk(sketch []uint64) (*ChunkId, bool) { + var similarChunks = make(map[ChunkId]int) + var max int + var similarChunk *ChunkId + for _, s := range sketch { + chunkIds, exists := r.sketches[s] + if !exists { + continue + } + for _, id := range chunkIds { + count := similarChunks[*id] + count += 1 + logger.Debugf("found %d %d time(s)", id, count) + if count > max { + max = count + similarChunk = id + } + similarChunks[*id] = count + } + } + return similarChunk, max > 0 +} + +// encodeTempChunk first tries to delta-encode the given chunk before attributing +// it an Id and saving it into the fingerprints and sketches maps. +func (r *Repo) encodeTempChunk(temp BufferedChunk, version int, last *uint64, storeQueue chan<- chunkData) (Chunk, bool) { + sk, _ := sketch.SketchChunk(temp.Reader(), r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) + id, found := r.findSimilarChunk(sk) + if found { + var buff bytes.Buffer + if err := r.differ.Diff(r.LoadChunkContent(id), temp.Reader(), &buff); err != nil { + logger.Error("trying delta encode chunk:", temp, "with source:", id, ":", err) + } else { + logger.Debugf("add new delta chunk of size %d", len(buff.Bytes())) + return &DeltaChunk{ + repo: r, + Source: id, + Patch: buff.Bytes(), + Size: temp.Len(), + }, true + } + } + if temp.Len() == r.chunkSize { + id := &ChunkId{Ver: version, Idx: *last} + *last++ + hasher := rabinkarp64.NewFromPol(r.pol) + io.Copy(hasher, temp.Reader()) + fp := hasher.Sum64() + r.fingerprints[fp] = id + r.sketches.Set(sk, id) + storeQueue <- chunkData{ + hashes: chunkHashes{fp, sk}, + content: temp.Bytes(), + id: id, + } + r.chunkCache.Set(id, temp.Bytes()) + logger.Debug("add new chunk ", id) + return NewStoredChunk(r, id), false + } + logger.Debug("add new partial chunk of size: ", temp.Len()) + return temp, false +} + +// encodeTempChunks encodes the current temporary chunks based on the value of the previous one. +// Temporary chunks can be partial. If the current chunk is smaller than the size of a +// super-feature and there exists a previous chunk, then both are merged before attempting +// to delta-encode them. +func (r *Repo) encodeTempChunks(prev BufferedChunk, curr BufferedChunk, version int, last *uint64, storeQueue chan<- chunkData) []Chunk { + if reflect.ValueOf(prev).IsNil() { + c, _ := r.encodeTempChunk(curr, version, last, storeQueue) + return []Chunk{c} + } else if curr.Len() < r.chunkMinLen() { + tmp := NewTempChunk(append(prev.Bytes(), curr.Bytes()...)) + c, success := r.encodeTempChunk(tmp, version, last, storeQueue) + if success { + return []Chunk{c} + } + } + prevD, _ := r.encodeTempChunk(prev, version, last, storeQueue) + currD, _ := r.encodeTempChunk(curr, version, last, storeQueue) + return []Chunk{prevD, currD} +} + +// matchStream is the heart of DNA-backup. Thus, it sounded rude not to add some comment to it. +// +// It applies a rolling hash on the content of a given stream to look for matching fingerprints +// in the repo. If no match is found after the equivalent of three chunks of data are processed, +// then the first unmatched chunk sketch is checked to see if it could be delta-encoded. +// If not, the chunk is then stored as a new chunk for this version and its fingerprint and +// sketch are added to the repo maps. +// +// If a match happens during the processing of the third chunk, then, if possible, the remaining +// of the second chunk is merged with the first one to try to delta encode it at once. +// +// Each time a new chunk is added it is sent to the store worker through the store queue. +func (r *Repo) matchStream(stream io.Reader, storeQueue chan<- chunkData, version int, last uint64) ([]Chunk, uint64) { + var b byte + var chunks []Chunk + var prev *TempChunk + var err error + bufStream := bufio.NewReaderSize(stream, r.chunkSize*2) + buff := make([]byte, r.chunkSize, r.chunkSize*2) + if n, err := io.ReadFull(stream, buff); n < r.chunkSize { + if err == io.ErrUnexpectedEOF { + c, _ := r.encodeTempChunk(NewTempChunk(buff[:n]), version, &last, storeQueue) + chunks = append(chunks, c) + return chunks, last + } else { + logger.Panicf("matching stream, read only %d bytes with error '%s'", n, err) + } + } + hasher := rabinkarp64.NewFromPol(r.pol) + hasher.Write(buff) + for err != io.EOF { + h := hasher.Sum64() + chunkId, exists := r.fingerprints[h] + if exists { + if len(buff) > r.chunkSize && len(buff) <= r.chunkSize*2 { + size := len(buff) - r.chunkSize + temp := NewTempChunk(buff[:size]) + chunks = append(chunks, r.encodeTempChunks(prev, temp, version, &last, storeQueue)...) + prev = nil + } else if prev != nil { + c, _ := r.encodeTempChunk(prev, version, &last, storeQueue) + chunks = append(chunks, c) + prev = nil + } + logger.Debugf("add existing chunk: %d", chunkId) + chunks = append(chunks, NewStoredChunk(r, chunkId)) + buff = make([]byte, 0, r.chunkSize*2) + for i := 0; i < r.chunkSize && err == nil; i++ { + b, err = bufStream.ReadByte() + if err != io.EOF { + hasher.Roll(b) + buff = append(buff, b) + } + } + continue + } + if len(buff) == r.chunkSize*2 { + if prev != nil { + chunk, _ := r.encodeTempChunk(prev, version, &last, storeQueue) + chunks = append(chunks, chunk) + } + prev = NewTempChunk(buff[:r.chunkSize]) + tmp := buff[r.chunkSize:] + buff = make([]byte, r.chunkSize, r.chunkSize*2) + copy(buff, tmp) + } + b, err = bufStream.ReadByte() + if err != io.EOF { + hasher.Roll(b) + buff = append(buff, b) + } + } + if len(buff) > 0 { + var temp *TempChunk + if len(buff) > r.chunkSize { + if prev != nil { + chunk, _ := r.encodeTempChunk(prev, version, &last, storeQueue) + chunks = append(chunks, chunk) + } + prev = NewTempChunk(buff[:r.chunkSize]) + temp = NewTempChunk(buff[r.chunkSize:]) + } else { + temp = NewTempChunk(buff) + } + chunks = append(chunks, r.encodeTempChunks(prev, temp, version, &last, storeQueue)...) + } + return chunks, last +} + +func (r *Repo) restoreStream(stream io.WriteCloser, recipe []Chunk) { + for _, c := range recipe { + if n, err := io.Copy(stream, c.Reader()); err != nil { + logger.Errorf("copying to stream, read %d bytes from chunk: %s", n, err) + } + } + stream.Close() +} + +func recipe2slice(r []Chunk) (ret slice.Slice) { + ret = make(slice.Slice, len(r)) + for i := range r { + ret[i] = r[i] + } + return +} + +func slice2recipe(s slice.Slice) (ret []Chunk) { + ret = make([]Chunk, len(s)) + for i := range s { + if c, ok := s[i].(Chunk); ok { + ret[i] = c + } else { + logger.Warningf("could not convert %s into a Chunk", s[i]) + } + } + return +} + +func (r *Repo) storeRecipe(version int, recipe []Chunk) { + dest := filepath.Join(r.path, fmt.Sprintf(versionFmt, version), recipeName) + delta := slice.Diff(recipe2slice(r.recipe), recipe2slice(recipe)) + logger.Infof("recipe delta %s", delta.String()) + storeBasicStruct(dest, r.chunkWriteWrapper, delta) +} + +func (r *Repo) loadRecipes(versions []string) { + recipe := slice2recipe(r.loadDeltas(versions, r.chunkReadWrapper, recipeName)) + for _, c := range recipe { + if rc, isRepo := c.(RepoChunk); isRepo { + rc.SetRepo(r) + } + } + r.recipe = recipe +} + +func extractDeltaChunks(chunks []Chunk) (ret []*DeltaChunk) { + for _, c := range chunks { + tmp, isDelta := c.(*DeltaChunk) + if isDelta { + ret = append(ret, tmp) + } + } + return +} diff --git a/repo/repo_test.go b/repo/repo_test.go new file mode 100644 index 0000000..0d26ccd --- /dev/null +++ b/repo/repo_test.go @@ -0,0 +1,509 @@ +package repo + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/chmduquesne/rollinghash/rabinkarp64" + "github.com/n-peugnet/dna-backup/delta" + "github.com/n-peugnet/dna-backup/logger" + "github.com/n-peugnet/dna-backup/sketch" + "github.com/n-peugnet/dna-backup/testutils" + "github.com/n-peugnet/dna-backup/utils" +) + +func TestMain(m *testing.M) { + setup() + code := m.Run() + shutdown() + os.Exit(code) +} + +func setup() { + logger.SetFlags(log.Lshortfile) +} + +func shutdown() {} + +func chunkCompare(t *testing.T, dataDir string, repo *Repo, testFiles []string, chunkCount int) { + reader, writer := io.Pipe() + chunks := make(chan []byte) + files := listFiles(dataDir) + go concatFiles(&files, writer) + go repo.chunkStream(reader, chunks) + + offset := 0 + buff := make([]byte, repo.chunkSize*chunkCount) + for _, f := range testFiles { + content, err := os.ReadFile(filepath.Join(dataDir, f)) + if err != nil { + t.Error("Error reading test data file") + } + for i := range content { + buff[offset+i] = content[i] + } + offset += len(content) + } + + i := 0 + for c := range chunks { + start := i * repo.chunkSize + end := (i + 1) * repo.chunkSize + if end > offset { + end = offset + } + content := buff[start:end] + if bytes.Compare(c, content) != 0 { + t.Errorf("Chunk %d does not match file content", i) + // for i, b := range c { + // fmt.Printf("E: %d, A: %d\n", b, content[i]) + // } + t.Log("Expected: ", c[:10], "...", c[end%repo.chunkSize-10:]) + t.Log("Actual:", content) + } + i++ + } + if i != chunkCount { + t.Errorf("Incorrect number of chunks: %d, should be: %d", i, chunkCount) + } +} + +func (r *Repo) chunkStream(stream io.Reader, chunks chan<- []byte) { + var buff []byte + var prev, read = r.chunkSize, 0 + var err error + + for err != io.EOF { + if prev == r.chunkSize { + buff = make([]byte, r.chunkSize) + prev, err = stream.Read(buff) + } else { + read, err = stream.Read(buff[prev:]) + prev += read + } + if err != nil && err != io.EOF { + logger.Error(err) + } + if prev == r.chunkSize { + chunks <- buff + } + } + if prev != r.chunkSize { + chunks <- buff[:prev] + } + close(chunks) +} + +func storeChunks(dest string, chunks <-chan []byte) { + i := 0 + for c := range chunks { + path := filepath.Join(dest, fmt.Sprintf(chunkIdFmt, i)) + err := os.WriteFile(path, c, 0664) + if err != nil { + logger.Error(err) + } + i++ + } +} + +// hashChunks calculates the hashes for a channel of chunks. +// For each chunk, both a fingerprint (hash over the full content) and a sketch +// (resemblance hash based on maximal values of regions) are calculated and +// stored in an hashmap. +func (r *Repo) hashChunks(chunks <-chan IdentifiedChunk) { + for c := range chunks { + r.hashChunk(c.GetId(), c.Reader()) + } +} + +// hashChunk calculates the hashes for a chunk and store them in th repo hashmaps. +func (r *Repo) hashChunk(id *ChunkId, reader io.Reader) (fp uint64, sk []uint64) { + var buffSk bytes.Buffer + var buffFp bytes.Buffer + var wg sync.WaitGroup + reader = io.TeeReader(reader, &buffSk) + io.Copy(&buffFp, reader) + wg.Add(2) + go r.makeFingerprint(id, &buffFp, &wg, &fp) + go r.makeSketch(id, &buffSk, &wg, &sk) + wg.Wait() + if _, e := r.fingerprints[fp]; e { + logger.Error(fp, " already exists in fingerprints map") + } + r.fingerprints[fp] = id + r.sketches.Set(sk, id) + return +} + +func (r *Repo) makeFingerprint(id *ChunkId, reader io.Reader, wg *sync.WaitGroup, ret *uint64) { + defer wg.Done() + hasher := rabinkarp64.NewFromPol(r.pol) + io.Copy(hasher, reader) + *ret = hasher.Sum64() +} + +func (r *Repo) makeSketch(id *ChunkId, reader io.Reader, wg *sync.WaitGroup, ret *[]uint64) { + defer wg.Done() + *ret, _ = sketch.SketchChunk(reader, r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) +} + +func TestReadFiles1(t *testing.T) { + tmpDir := t.TempDir() + repo := NewRepo(tmpDir) + chunkCount := 590/repo.chunkSize + 1 + dataDir := filepath.Join("testdata", "logs", "1") + files := []string{"logTest.log"} + chunkCompare(t, dataDir, repo, files, chunkCount) +} + +func TestReadFiles2(t *testing.T) { + tmpDir := t.TempDir() + repo := NewRepo(tmpDir) + chunkCount := 22899/repo.chunkSize + 1 + dataDir := filepath.Join("testdata", "logs", "2") + files := []string{"csvParserTest.log", "slipdb.log"} + chunkCompare(t, dataDir, repo, files, chunkCount) +} + +func TestReadFiles3(t *testing.T) { + tmpDir := t.TempDir() + repo := NewRepo(tmpDir) + chunkCount := 119398/repo.chunkSize + 1 + dataDir := filepath.Join("testdata", "logs") + files := []string{ + filepath.Join("1", "logTest.log"), + filepath.Join("2", "csvParserTest.log"), + filepath.Join("2", "slipdb.log"), + filepath.Join("3", "indexingTreeTest.log"), + } + chunkCompare(t, dataDir, repo, files, chunkCount) +} + +func TestSymlinks(t *testing.T) { + var output bytes.Buffer + multi := io.MultiWriter(&output, os.Stderr) + logger.SetOutput(multi) + defer logger.SetOutput(os.Stderr) + tmpDir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + extDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, "existing")) + if err != nil { + t.Fatal(err) + } + if n, err := f.Write([]byte("\n")); err != nil { + t.Fatal(n, err) + } + if err = f.Close(); err != nil { + t.Fatal(err) + } + if err = os.Symlink(extDir, filepath.Join(tmpDir, "linkexternal")); err != nil { + t.Fatal(err) + } + if err = os.Symlink(filepath.Join(tmpDir, "notexisting"), filepath.Join(tmpDir, "linknotexisting")); err != nil { + t.Fatal(err) + } + if err = os.Symlink(filepath.Join(tmpDir, "existing"), filepath.Join(tmpDir, "linkexisting")); err != nil { + t.Fatal(err) + } + files := listFiles(tmpDir) + fmt.Println(files) + testutils.AssertLen(t, 3, files, "Files") + if files[0].Link != "" { + t.Error("existing should not be a link, actual:", files[0].Link) + } + expected := string(filepath.Separator) + "existing" + if files[1].Link != expected { + t.Error("linkexisting should point to", expected, "actual:", files[1].Link) + } + if !strings.Contains(output.String(), "linkexternal") { + t.Errorf("log should contain a warning for linkexternal, actual %q", &output) + } +} + +func TestLoadChunks(t *testing.T) { + resultDir := t.TempDir() + dataDir := filepath.Join("testdata", "logs") + repo := NewRepo(resultDir) + repo.chunkReadWrapper = utils.NopReadWrapper + repo.chunkWriteWrapper = utils.NopWriteWrapper + resultVersion := filepath.Join(resultDir, "00000") + resultChunks := filepath.Join(resultVersion, chunksName) + os.MkdirAll(resultChunks, 0775) + reader1, writer1 := io.Pipe() + reader2, writer2 := io.Pipe() + chunks1 := make(chan []byte, 16) + chunks2 := make(chan []byte, 16) + chunks3 := make(chan IdentifiedChunk, 16) + files := listFiles(dataDir) + go concatFiles(&files, writer1) + go concatFiles(&files, writer2) + go repo.chunkStream(reader1, chunks1) + go repo.chunkStream(reader2, chunks2) + storeChunks(resultChunks, chunks1) + versions := []string{resultVersion} + go repo.loadChunks(versions, chunks3) + + i := 0 + for c2 := range chunks2 { + c3 := <-chunks3 + buff, err := io.ReadAll(c3.Reader()) + if err != nil { + t.Errorf("Error reading from chunk %d: %s\n", c3, err) + } + if bytes.Compare(c2, buff) != 0 { + t.Errorf("Chunk %d does not match file content", i) + t.Log("Expected: ", c2[:10], "...") + t.Log("Actual:", buff) + } + i++ + } +} + +func prepareChunks(dataDir string, repo *Repo, streamFunc func(*[]File, io.WriteCloser)) { + resultVersion := filepath.Join(repo.path, "00000") + resultChunks := filepath.Join(resultVersion, chunksName) + os.MkdirAll(resultChunks, 0775) + reader := getDataStream(dataDir, streamFunc) + chunks := make(chan []byte, 16) + go repo.chunkStream(reader, chunks) + storeChunks(resultChunks, chunks) +} + +func getDataStream(dataDir string, streamFunc func(*[]File, io.WriteCloser)) io.Reader { + reader, writer := io.Pipe() + files := listFiles(dataDir) + go streamFunc(&files, writer) + return reader +} + +func TestBsdiff(t *testing.T) { + logger.SetLevel(3) + defer logger.SetLevel(4) + resultDir := t.TempDir() + repo := NewRepo(resultDir) + dataDir := filepath.Join("testdata", "logs") + addedFile1 := filepath.Join(dataDir, "2", "slogTest.log") + addedFile2 := filepath.Join(dataDir, "3", "slogTest.log") + // Store initial chunks + prepareChunks(dataDir, repo, concatFiles) + + // Modify data + ioutil.WriteFile(addedFile1, []byte("hello"), 0664) + defer os.Remove(addedFile1) + ioutil.WriteFile(addedFile2, make([]byte, 4000), 0664) + defer os.Remove(addedFile2) + + // configure repo + repo.patcher = delta.Bsdiff{} + repo.differ = delta.Bsdiff{} + repo.chunkReadWrapper = utils.NopReadWrapper + repo.chunkWriteWrapper = utils.NopWriteWrapper + + // Load previously stored chunks + oldChunks := make(chan IdentifiedChunk, 16) + versions := repo.loadVersions() + go repo.loadChunks(versions, oldChunks) + repo.hashChunks(oldChunks) + + // Read new data + newVersion := len(versions) + newPath := filepath.Join(repo.path, fmt.Sprintf(versionFmt, newVersion)) + os.MkdirAll(newPath, 0775) + reader := getDataStream(dataDir, concatFiles) + storeQueue := make(chan chunkData, 10) + storeEnd := make(chan bool) + go repo.storageWorker(newVersion, storeQueue, storeEnd) + recipe, _ := repo.matchStream(reader, storeQueue, newVersion, 0) + close(storeQueue) + <-storeEnd + newChunks := extractDeltaChunks(recipe) + testutils.AssertLen(t, 2, newChunks, "New delta chunks:") + for _, c := range newChunks { + logger.Info("Patch size:", len(c.Patch)) + if len(c.Patch) >= repo.chunkSize/10 { + t.Errorf("Bsdiff of chunk is too large: %d", len(c.Patch)) + } + } +} + +func TestCommit(t *testing.T) { + dest := t.TempDir() + source := filepath.Join("testdata", "logs") + expected := filepath.Join("testdata", "repo_8k") + repo := NewRepo(dest) + repo.patcher = delta.Bsdiff{} + repo.differ = delta.Bsdiff{} + repo.chunkReadWrapper = utils.NopReadWrapper + repo.chunkWriteWrapper = utils.NopWriteWrapper + + repo.Commit(source) + assertSameTree(t, assertCompatibleRepoFile, expected, dest, "Commit") +} + +func TestCommitZlib(t *testing.T) { + dest := t.TempDir() + source := filepath.Join("testdata", "logs") + expected := filepath.Join("testdata", "repo_8k_zlib") + repo := NewRepo(dest) + repo.patcher = delta.Bsdiff{} + repo.differ = delta.Bsdiff{} + repo.chunkReadWrapper = utils.ZlibReader + repo.chunkWriteWrapper = utils.ZlibWriter + + repo.Commit(source) + assertSameTree(t, assertCompatibleRepoFile, expected, dest, "Commit") +} + +func TestRestore(t *testing.T) { + logger.SetLevel(2) + defer logger.SetLevel(4) + dest := t.TempDir() + source := filepath.Join("testdata", "repo_8k") + expected := filepath.Join("testdata", "logs") + repo := NewRepo(source) + repo.patcher = delta.Bsdiff{} + repo.differ = delta.Bsdiff{} + repo.chunkReadWrapper = utils.NopReadWrapper + repo.chunkWriteWrapper = utils.NopWriteWrapper + + repo.Restore(dest) + assertSameTree(t, testutils.AssertSameFile, expected, dest, "Restore") +} + +func TestRestoreZlib(t *testing.T) { + logger.SetLevel(2) + defer logger.SetLevel(4) + dest := t.TempDir() + source := filepath.Join("testdata", "repo_8k_zlib") + expected := filepath.Join("testdata", "logs") + repo := NewRepo(source) + repo.patcher = delta.Bsdiff{} + repo.differ = delta.Bsdiff{} + repo.chunkReadWrapper = utils.ZlibReader + repo.chunkWriteWrapper = utils.ZlibWriter + + repo.Restore(dest) + assertSameTree(t, testutils.AssertSameFile, expected, dest, "Restore") +} + +func TestRoundtrip(t *testing.T) { + logger.SetLevel(2) + defer logger.SetLevel(4) + temp := t.TempDir() + dest := t.TempDir() + source := filepath.Join("testdata", "logs") + repo1 := NewRepo(temp) + repo2 := NewRepo(temp) + + repo1.Commit(source) + // Commit a second version, just to see if it does not destroy everything + // TODO: check that the second version is indeed empty + repo1.Commit(source) + repo2.Restore(dest) + + assertSameTree(t, assertCompatibleRepoFile, source, dest, "Commit") +} + +func TestHashes(t *testing.T) { + dest := t.TempDir() + source := filepath.Join("testdata", "repo_8k") + + chunks := make(chan IdentifiedChunk, 16) + storeQueue := make(chan chunkData, 16) + storeEnd := make(chan bool) + + repo1 := NewRepo(source) + repo1.chunkReadWrapper = utils.NopReadWrapper + repo1.chunkWriteWrapper = utils.NopWriteWrapper + go repo1.loadChunks([]string{filepath.Join(source, "00000")}, chunks) + for c := range chunks { + fp, sk := repo1.hashChunk(c.GetId(), c.Reader()) + content, err := io.ReadAll(c.Reader()) + if err != nil { + t.Error(err) + } + storeQueue <- chunkData{ + hashes: chunkHashes{fp, sk}, + content: content, + id: c.GetId(), + } + } + repo2 := NewRepo(dest) + repo2.chunkReadWrapper = utils.NopReadWrapper + repo2.chunkWriteWrapper = utils.NopWriteWrapper + os.MkdirAll(filepath.Join(dest, "00000", chunksName), 0775) + go repo2.storageWorker(0, storeQueue, storeEnd) + close(storeQueue) + <-storeEnd + testutils.AssertLen(t, 0, repo2.fingerprints, "Fingerprints") + testutils.AssertLen(t, 0, repo2.sketches, "Sketches") + repo2.loadHashes([]string{filepath.Join(dest, "00000")}) + testutils.AssertSame(t, repo1.fingerprints, repo2.fingerprints, "Fingerprint maps") + testutils.AssertSame(t, repo1.sketches, repo2.sketches, "Sketches maps") +} + +func assertSameTree(t *testing.T, apply func(t *testing.T, expected string, actual string, prefix string), expected string, actual string, prefix string) { + actualFiles := listFiles(actual) + expectedFiles := listFiles(expected) + efCount := len(expectedFiles) + if efCount <= 0 { + t.Fatalf("No expected files: %d", efCount) + } + afCount := len(actualFiles) + if efCount != afCount { + t.Fatalf("Incorrect number of files: %d, should be %d", afCount, efCount) + } + for i, ef := range expectedFiles { + af := actualFiles[i] + efRelPath := ef.Path[len(expected):] + afRelPath := af.Path[len(actual):] + if efRelPath != afRelPath { + t.Fatalf("File path '%s' does not match '%s'", afRelPath, efRelPath) + } + apply(t, ef.Path, af.Path, prefix) + } +} + +func assertCompatibleRepoFile(t *testing.T, expected string, actual string, prefix string) { + if filepath.Base(expected) == filesName { + // TODO: Check Filelist file + // eFiles := loadFileList(expected) + // aFiles := loadFileList(actual) + // testutils.AssertLen(t, len(eFiles), aFiles, prefix) + // for i, eFile := range eFiles { + // eFile.Path = filepath.FromSlash(eFile.Path) + // if eFile != aFiles[i] { + // t.Fatal(prefix, "file entry do not match:", aFiles[i], ", expected:", eFile) + // } + // } + } else if filepath.Base(expected) == recipeName { + // TODO: Check Recipe files + // eRecipe := loadRecipe(expected) + // aRecipe := loadRecipe(actual) + // testutils.AssertSame(t, eRecipe, aRecipe, prefix+"recipe") + } else if filepath.Base(expected) == hashesName { + // Hashes file is checked in TestHashes + } else { + // Chunk content file + testutils.AssertSameFile(t, expected, actual, prefix) + } +} + +func assertChunkContent(t *testing.T, expected []byte, c Chunk, prefix string) { + buf, err := io.ReadAll(c.Reader()) + if err != nil { + t.Fatal(err) + } + testutils.AssertSame(t, expected, buf, prefix+" Chunk content") +} diff --git a/repo/testdata/.gitattributes b/repo/testdata/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/repo/testdata/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/repo/testdata/logs/1/logTest.log b/repo/testdata/logs/1/logTest.log new file mode 100644 index 0000000..6bf31c3 --- /dev/null +++ b/repo/testdata/logs/1/logTest.log @@ -0,0 +1,10 @@ +2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T20:51 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:51 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T22:41 [INFO] testInfoString (LogTest.java:52) +2019-06-05T22:41 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T23:02 [INFO] testInfoString (LogTest.java:52) +2019-06-05T23:02 [INFO][YO] testInfoString (LogTest.java:57) diff --git a/repo/testdata/logs/2/csvParserTest.log b/repo/testdata/logs/2/csvParserTest.log new file mode 100644 index 0000000..328720a --- /dev/null +++ b/repo/testdata/logs/2/csvParserTest.log @@ -0,0 +1,63 @@ +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.52 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 160,128 (Loader.java:117) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 231,367 (Loader.java:138) +2019-06-05T20:13 [INFO][TIMER] Temps pris par le parsing: 0.898 s (LoaderTest.java:69) +2019-06-05T20:13 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:14 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.46 millis (Loader.java:116) +2019-06-05T20:14 [INFO][MEMORY] 267,348 (Loader.java:117) +2019-06-05T20:14 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:14 [INFO][MEMORY] 231,359 (Loader.java:138) +2019-06-05T20:14 [INFO][TIMER] Temps pris par le parsing: 0.839 s (LoaderTest.java:69) +2019-06-05T20:14 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.61 millis (Loader.java:116) +2019-06-05T20:51 [INFO][MEMORY] 189,561 (Loader.java:117) +2019-06-05T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:51 [INFO][MEMORY] 168,686 (Loader.java:138) +2019-06-05T20:51 [INFO][TIMER] Temps pris par le parsing: 0.930 s (LoaderTest.java:69) +2019-06-05T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T22:41 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.83 millis (Loader.java:116) +2019-06-05T22:41 [INFO][MEMORY] 189,494 (Loader.java:117) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 168,619 (Loader.java:138) +2019-06-05T22:41 [INFO][TIMER] Temps pris par le parsing: 0.882 s (LoaderTest.java:69) +2019-06-05T22:41 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) +2019-06-05T23:02 [INFO][MEMORY] 189,551 (Loader.java:117) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 168,676 (Loader.java:138) +2019-06-05T23:02 [INFO][TIMER] Temps pris par le parsing: 1.04 s (LoaderTest.java:69) +2019-06-05T23:02 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.82 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 82,958 (Loader.java:117) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 163,350 (Loader.java:138) +2019-06-10T20:34 [INFO][TIMER] Temps pris par le parsing: 0.938 s (LoaderTest.java:69) +2019-06-10T20:34 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) +2019-06-10T20:50 [INFO][MEMORY] 82,957 (Loader.java:117) +2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:50 [INFO][MEMORY] 163,349 (Loader.java:138) +2019-06-10T20:50 [INFO][TIMER] Temps pris par le parsing: 0.853 s (LoaderTest.java:69) +2019-06-10T20:50 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.51 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 189,538 (Loader.java:117) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 168,663 (Loader.java:138) +2019-06-10T20:51 [INFO][TIMER] Temps pris par le parsing: 1.27 s (LoaderTest.java:69) +2019-06-10T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.58 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 189,539 (Loader.java:117) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 168,664 (Loader.java:138) +2019-06-10T21:24 [INFO][TIMER] Temps pris par le parsing: 1.86 s (LoaderTest.java:69) +2019-06-10T21:24 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) diff --git a/repo/testdata/logs/2/slipdb.log b/repo/testdata/logs/2/slipdb.log new file mode 100644 index 0000000..1f0e3b4 --- /dev/null +++ b/repo/testdata/logs/2/slipdb.log @@ -0,0 +1,185 @@ +2019-06-05T14:47 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T14:49 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T14:49 [INFO][MEMORY] 229,457 (ParallelLoader.java:114) +2019-06-05T14:49 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.522 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T14:49 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T14:49 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T14:49 [INFO] Parsing terminé !! temps écoulé = 2.78 s (ParallelLoader.java:217) +2019-06-05T14:49 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T14:53 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T14:53 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T14:53 [INFO][MEMORY] 227,095 (ParallelLoader.java:114) +2019-06-05T14:53 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.433 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T14:53 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T14:53 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T14:53 [INFO] Parsing terminé !! temps écoulé = 2.37 s (ParallelLoader.java:217) +2019-06-05T14:53 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T14:54 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) +2019-06-05T14:59 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) +2019-06-05T14:59 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) +2019-06-05T15:08 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:09 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:09 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) +2019-06-05T15:09 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:09 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) +2019-06-05T15:10 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:10 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) +2019-06-05T15:12 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:12 [INFO][MEMORY] 178,759 (ParallelLoader.java:114) +2019-06-05T15:12 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.220 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:12 [INFO] Parsing terminé !! temps écoulé = 2.25 s (ParallelLoader.java:217) +2019-06-05T15:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:13 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:14 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:328) +2019-06-05T15:14 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T15:14 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:328) +2019-06-05T15:14 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T15:14 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:14 [INFO][MEMORY] 308,168 (ParallelLoader.java:114) +2019-06-05T15:14 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.052 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:14 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:14 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:14 [INFO] Parsing terminé !! temps écoulé = 1.32 s (ParallelLoader.java:217) +2019-06-05T15:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:14 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:15 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) +2019-06-05T15:15 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:17 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:26 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:27 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:27 [INFO][MEMORY] 179,237 (ParallelLoader.java:114) +2019-06-05T15:27 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.098 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:27 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:27 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:27 [INFO] Parsing terminé !! temps écoulé = 2.22 s (ParallelLoader.java:217) +2019-06-05T15:27 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:31 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:31 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:31 [INFO][MEMORY] 178,254 (ParallelLoader.java:114) +2019-06-05T15:31 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.118 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:31 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:31 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:31 [INFO] Parsing terminé !! temps écoulé = 2.11 s (ParallelLoader.java:217) +2019-06-05T15:31 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:41 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:41 [INFO][MEMORY] 180,287 (ParallelLoader.java:114) +2019-06-05T15:41 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.195 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:41 [INFO] Parsing terminé !! temps écoulé = 1.92 s (ParallelLoader.java:217) +2019-06-05T15:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:42 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:43 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:43 [INFO][MEMORY] 178,731 (ParallelLoader.java:114) +2019-06-05T15:43 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.141 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:43 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:43 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:43 [INFO] Parsing terminé !! temps écoulé = 2.03 s (ParallelLoader.java:217) +2019-06-05T15:43 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:35 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T18:35 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:35 [INFO][MEMORY] 229,265 (ParallelLoader.java:114) +2019-06-05T18:35 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.641 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:35 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:35 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:35 [INFO] Parsing terminé !! temps écoulé = 3.04 s (ParallelLoader.java:217) +2019-06-05T18:35 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:36 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:36 [INFO][MEMORY] 326,934 (ParallelLoader.java:114) +2019-06-05T18:36 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.072 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:36 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:36 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:36 [INFO] Parsing terminé !! temps écoulé = 1.17 s (ParallelLoader.java:217) +2019-06-05T18:36 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:37 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:37 [INFO][MEMORY] 537,997 (ParallelLoader.java:114) +2019-06-05T18:37 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:37 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:37 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:37 [INFO] Parsing terminé !! temps écoulé = 1.23 s (ParallelLoader.java:217) +2019-06-05T18:37 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:38 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) +2019-06-05T18:38 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T18:38 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:38 [INFO][MEMORY] 498,154 (ParallelLoader.java:114) +2019-06-05T18:38 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.031 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:38 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:38 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:38 [INFO] Parsing terminé !! temps écoulé = 1.10 s (ParallelLoader.java:217) +2019-06-05T18:38 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:39 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) +2019-06-05T18:39 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T18:40 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:40 [INFO][MEMORY] 493,228 (ParallelLoader.java:114) +2019-06-05T18:40 [INFO] Loader : nombre de résultats (total) lus = 500000 temps écoulé = 0.032 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:40 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:40 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:40 [INFO] Parsing terminé !! temps écoulé = 1.02 s (ParallelLoader.java:217) +2019-06-05T18:40 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:14 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T20:14 [ERROR] null (SerialStructure.java:74) +2019-06-05T20:15 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) +2019-06-05T20:15 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T20:15 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:15 [INFO][MEMORY] 188,768 (ParallelLoader.java:114) +2019-06-05T20:15 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.301 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:15 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:15 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:15 [INFO] Parsing terminé !! temps écoulé = 2.42 s (ParallelLoader.java:217) +2019-06-05T20:15 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:15 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) +2019-06-05T20:15 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T20:16 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:16 [INFO][MEMORY] 457,918 (ParallelLoader.java:114) +2019-06-05T20:16 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.049 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:16 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:16 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:16 [INFO] Parsing terminé !! temps écoulé = 1.34 s (ParallelLoader.java:217) +2019-06-05T20:16 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:19 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:19 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:19 [INFO][MEMORY] 180,323 (ParallelLoader.java:114) +2019-06-05T20:19 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.178 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:19 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:19 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:19 [INFO] Parsing terminé !! temps écoulé = 1.92 s (ParallelLoader.java:217) +2019-06-05T20:19 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:24 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:25 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:43 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:43 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:43 [INFO][MEMORY] 179,894 (ParallelLoader.java:114) +2019-06-05T20:43 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.133 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:43 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:43 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:43 [INFO] Parsing terminé !! temps écoulé = 2.14 s (ParallelLoader.java:217) +2019-06-05T20:43 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:50 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:50 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T21:56 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T21:56 [ERROR] null +2019-06-05T22:41 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T22:41 [ERROR] null +2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T22:42 [ERROR] null +2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T22:42 [ERROR] null +2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T22:42 [ERROR] null +2019-06-05T23:03 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T23:03 [ERROR] null +2019-06-05T23:04 [INFO] SerialStructure.writeStructure : OK ! +2019-06-05T23:04 [INFO] totalLength = 0 +2019-06-05T23:04 [INFO] totalLength = 0 +2019-06-05T23:04 [INFO] PARSE : memusage init = +2019-06-05T23:04 [INFO][MEMORY] 111,311 +2019-06-05T23:04 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.256 sactiveThreadNb = 0 +2019-06-05T23:04 [WARNING] Fail to save entry number 0: incorrect data +2019-06-05T23:04 [INFO] Parsing terminé !! temps écoulé = 2.57 s +2019-06-05T23:04 [INFO] PARSE : Nombre de lignes (local) = 100000 +2019-06-05T23:04 [INFO] totalLength = 99999 +2019-06-05T23:04 [INFO] totalLength = 99999 diff --git a/repo/testdata/logs/3/indexingTreeTest.log b/repo/testdata/logs/3/indexingTreeTest.log new file mode 100644 index 0000000..b115b45 --- /dev/null +++ b/repo/testdata/logs/3/indexingTreeTest.log @@ -0,0 +1,999 @@ +2019-06-05T00:03 [ERROR] config.json (No such file or directory) (Database.java:33) +2019-06-05T00:03 [INFO][MEMORY] Mem usage début - 2,459 (ParallelLoaderTest.java:56) +2019-06-05T20:12 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T20:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:12 [INFO][MEMORY] 27,443 (ParallelLoader.java:114) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.214 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:12 [INFO] Parsing terminé !! temps écoulé = 1.44 s (ParallelLoader.java:217) +2019-06-05T20:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:12 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.45 s (IndexMemDicStorageTests.java:183) +2019-06-05T20:12 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.477 s (IndexMemDicStorageTests.java:83) +2019-06-05T20:12 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.054 s (IndexMemDicStorageTests.java:86) +2019-06-05T20:12 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-05T20:12 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-05T20:12 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-05T20:12 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-05T20:12 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T20:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:12 [INFO][MEMORY] 190,485 (ParallelLoader.java:114) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.064 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:12 [INFO] Parsing terminé !! temps écoulé = 0.908 s (ParallelLoader.java:217) +2019-06-05T20:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:12 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.917 s (NewLoaderAndIndexTest.java:189) +2019-06-05T20:12 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T20:12 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-05T20:12 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-05T20:12 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-05T20:12 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-05T20:12 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-05T20:12 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 66666 (NewLoaderAndIndexTest.java:109) +2019-06-05T20:12 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T20:12 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.57 millis (Loader.java:116) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) +2019-06-05T20:12 [INFO][MEMORY] 401,153 (Loader.java:117) +2019-06-05T20:12 [INFO][MEMORY] 401,153 (Loader.java:117) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.33 millis (Loader.java:116) +2019-06-05T20:12 [INFO][MEMORY] 273,252 (Loader.java:117) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.97 millis (Loader.java:116) +2019-06-05T20:12 [INFO][MEMORY] 273,252 (Loader.java:117) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 216,262 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 221,186 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 218,771 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 216,694 (Loader.java:138) +2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.57 s (IndexTreeMessyTest.java:314) +2019-06-05T20:13 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T20:13 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,432 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2bef09c0 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@62ce72ff (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@58a63629 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7de843ef (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2e1af49b (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3fd2322d (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@56ac5c80 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69e49a81 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1023150a (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ece1e79 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@ab20d7c (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@474fae39 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4f4c789f (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13f9ad9 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b7cae6f (Table.java:338) +2019-06-05T20:13 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T20:13 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T20:13 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T20:13 [ERROR] null (SerialStructure.java:74) +2019-06-05T20:13 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 48 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.051 s (IndexTreeMessyTest.java:408) +2019-06-05T20:13 [INFO][TIMER] Temps parcours des résultats: 1.72 millis (IndexTreeMessyTest.java:416) +2019-06-05T20:13 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 9 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.011 s (IndexTreeMessyTest.java:425) +2019-06-05T20:13 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.125 s (IndexTreeMessyTest.java:439) +2019-06-05T20:13 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.041 s (IndexTreeMessyTest.java:449) +2019-06-05T20:13 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T20:13 [INFO] 40 (HashMapTests.java:27) +2019-06-05T20:13 [INFO][MEMORY] Mem usage début - 157,206 (ParallelLoaderTest.java:56) +2019-06-05T20:13 [INFO] Parsing de csvName = E:/csv/yellow_tripdata_2015-01.csv (ParallelLoaderTest.java:64) +2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:13 [INFO][MEMORY] 28,086 (ParallelLoader.java:114) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.182 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:13 [INFO] Parsing terminé !! temps écoulé = 1.43 s (ParallelLoader.java:217) +2019-06-05T20:13 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.44 s (IndexMemDicStorageTests.java:183) +2019-06-05T20:13 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.356 s (IndexMemDicStorageTests.java:83) +2019-06-05T20:13 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.032 s (IndexMemDicStorageTests.java:86) +2019-06-05T20:13 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-05T20:13 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-05T20:13 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-05T20:13 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:13 [INFO][MEMORY] 184,068 (ParallelLoader.java:114) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.052 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:13 [INFO] Parsing terminé !! temps écoulé = 0.941 s (ParallelLoader.java:217) +2019-06-05T20:13 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.945 s (NewLoaderAndIndexTest.java:189) +2019-06-05T20:13 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T20:13 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-05T20:13 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-05T20:13 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-05T20:13 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-05T20:13 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-05T20:13 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 66666 (NewLoaderAndIndexTest.java:109) +2019-06-05T20:13 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T20:13 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.70 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 276,783 (Loader.java:117) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.69 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 341,077 (Loader.java:117) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.09 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 358,336 (Loader.java:117) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 5.74 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 358,336 (Loader.java:117) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 218,799 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 218,840 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 223,379 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 216,694 (Loader.java:138) +2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.18 s (IndexTreeMessyTest.java:314) +2019-06-05T20:13 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T20:13 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2881ad47 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2bef09c0 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@62ce72ff (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@58a63629 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7de843ef (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2e1af49b (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3fd2322d (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@56ac5c80 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69e49a81 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1023150a (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ece1e79 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@ab20d7c (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@474fae39 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4f4c789f (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13f9ad9 (Table.java:338) +2019-06-05T20:13 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T20:13 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T20:13 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T20:13 [ERROR] null (SerialStructure.java:74) +2019-06-05T20:13 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 58 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.061 s (IndexTreeMessyTest.java:408) +2019-06-05T20:13 [INFO][TIMER] Temps parcours des résultats: 1.25 millis (IndexTreeMessyTest.java:416) +2019-06-05T20:13 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 8.20 millis (IndexTreeMessyTest.java:425) +2019-06-05T20:13 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.072 s (IndexTreeMessyTest.java:439) +2019-06-05T20:13 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.040 s (IndexTreeMessyTest.java:449) +2019-06-05T20:13 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 3 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T20:13 [INFO] 40 (HashMapTests.java:27) +2019-06-05T20:13 [INFO][MEMORY] Mem usage début - 157,225 (ParallelLoaderTest.java:56) +2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:13 [INFO][MEMORY] 166,226 (ParallelLoader.java:114) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.041 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:14 [INFO] Parsing terminé !! temps écoulé = 0.638 s (ParallelLoader.java:217) +2019-06-05T20:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:14 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T20:14 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:14 [INFO][MEMORY] 422,898 (ParallelLoader.java:114) +2019-06-05T20:14 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.040 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:14 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:14 [INFO] Parsing terminé !! temps écoulé = 0.624 s (ParallelLoader.java:217) +2019-06-05T20:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:14 [INFO][MEMORY] Mem usage fin - 267,348 (ParallelLoaderTest.java:113) +2019-06-05T20:14 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.46 s (ParallelLoaderTest.java:114) +2019-06-05T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:50 [INFO][MEMORY] 12,904 (ParallelLoader.java:114) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.175 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:50 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:50 [INFO] Parsing terminé !! temps écoulé = 2.03 s (ParallelLoader.java:217) +2019-06-05T20:50 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 2.04 s (IndexMemDicStorageTests.java:183) +2019-06-05T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:50 [INFO][MEMORY] 157,253 (ParallelLoader.java:114) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.111 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:50 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:50 [INFO] Parsing terminé !! temps écoulé = 0.781 s (ParallelLoader.java:217) +2019-06-05T20:50 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.787 s (NewLoaderAndIndexTest.java:189) +2019-06-05T20:50 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T20:50 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T20:50 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 5.82 millis (Loader.java:116) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.66 millis (Loader.java:116) +2019-06-05T20:50 [INFO][MEMORY] 134,560 (Loader.java:117) +2019-06-05T20:50 [INFO][MEMORY] 134,560 (Loader.java:117) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.11 millis (Loader.java:116) +2019-06-05T20:50 [INFO][MEMORY] 152,063 (Loader.java:117) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.011 s (Loader.java:116) +2019-06-05T20:50 [INFO][MEMORY] 168,447 (Loader.java:117) +2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:50 [INFO][MEMORY] 155,685 (Loader.java:138) +2019-06-05T20:50 [INFO][MEMORY] 156,197 (Loader.java:138) +2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:50 [INFO][MEMORY] 155,691 (Loader.java:138) +2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:50 [INFO][MEMORY] 155,633 (Loader.java:138) +2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.14 s (IndexTreeMessyTest.java:314) +2019-06-05T20:50 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T20:50 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,657 (IndexTreeDic.java:517) +2019-06-05T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b0b0854 (Table.java:338) +2019-06-05T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,657 (IndexTreeDic.java:517) +2019-06-05T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@19bb07ed (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,033 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@10e41621 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,033 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@353d0772 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,082 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2667f029 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,594 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@67a20f67 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@57c758ac (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@a9cd3b1 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13e39c73 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,915 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@64cd705f (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@9225652 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@654f0d9c (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6a400542 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6580cfdd (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7e0b85f9 (Table.java:338) +2019-06-05T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T20:51 [ERROR] null (SerialStructure.java:74) +2019-06-05T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 31 millis (IndexTreeDic.java:1164) +2019-06-05T20:51 [INFO][TIMER] Temps total recherche: 0.037 s (IndexTreeMessyTest.java:408) +2019-06-05T20:51 [INFO][TIMER] Temps parcours des résultats: 1.96 millis (IndexTreeMessyTest.java:416) +2019-06-05T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) +2019-06-05T20:51 [INFO][TIMER] Temps total recherche: 7.48 millis (IndexTreeMessyTest.java:425) +2019-06-05T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.119 s (IndexTreeMessyTest.java:439) +2019-06-05T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.040 s (IndexTreeMessyTest.java:449) +2019-06-05T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 1 millis (IndexTreeDic.java:1164) +2019-06-05T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T20:51 [INFO] 40 (HashMapTests.java:27) +2019-06-05T20:51 [INFO][MEMORY] Mem usage début - 88,295 (ParallelLoaderTest.java:56) +2019-06-05T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:51 [INFO][MEMORY] 90,343 (ParallelLoader.java:114) +2019-06-05T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.064 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:51 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:51 [INFO] Parsing terminé !! temps écoulé = 0.862 s (ParallelLoader.java:217) +2019-06-05T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:51 [INFO][MEMORY] 227,179 (ParallelLoader.java:114) +2019-06-05T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.055 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:51 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:51 [INFO] Parsing terminé !! temps écoulé = 0.676 s (ParallelLoader.java:217) +2019-06-05T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:51 [INFO][MEMORY] Mem usage fin - 189,561 (ParallelLoaderTest.java:113) +2019-06-05T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.63 s (ParallelLoaderTest.java:114) +2019-06-05T21:56 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T21:56 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T21:56 [INFO][MEMORY] 13,925 (ParallelLoader.java:114) +2019-06-05T21:56 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.226 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T21:56 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T21:56 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T21:57 [INFO] Parsing terminé !! temps écoulé = 33.57 s (ParallelLoader.java:217) +2019-06-05T21:57 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T21:57 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 33.66 s (IndexMemDicStorageTests.java:183) +2019-06-05T21:57 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T21:57 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T21:57 [INFO][MEMORY] 58,123 (ParallelLoader.java:114) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.206 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T21:57 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T21:57 [INFO] Parsing terminé !! temps écoulé = 15.87 s (ParallelLoader.java:217) +2019-06-05T21:57 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T21:57 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 16.28 s (NewLoaderAndIndexTest.java:189) +2019-06-05T21:57 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T21:57 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T21:57 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 9.32 millis (Loader.java:116) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 7.80 millis (Loader.java:116) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 8.95 millis (Loader.java:116) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 8.32 millis (Loader.java:116) +2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) +2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) +2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) +2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 155,396 (Loader.java:138) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 155,945 (Loader.java:138) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 155,523 (Loader.java:138) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 155,565 (Loader.java:138) +2019-06-05T22:41 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 35.40 s (IndexTreeMessyTest.java:314) +2019-06-05T22:41 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T22:41 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,457 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@e056f20 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,457 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b0b0854 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,834 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@19bb07ed (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,834 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@10e41621 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,883 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@353d0772 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,067 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2667f029 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,067 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@67a20f67 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,579 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@57c758ac (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,579 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@a9cd3b1 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13e39c73 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@64cd705f (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@9225652 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@654f0d9c (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6a400542 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,139 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6580cfdd (Table.java:338) +2019-06-05T22:41 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T22:41 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T22:41 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T22:41 [ERROR] null (SerialStructure.java:74) +2019-06-05T22:41 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 50 millis (IndexTreeDic.java:1164) +2019-06-05T22:41 [INFO][TIMER] Temps total recherche: 0.058 s (IndexTreeMessyTest.java:408) +2019-06-05T22:41 [INFO][TIMER] Temps parcours des résultats: 1.04 millis (IndexTreeMessyTest.java:416) +2019-06-05T22:41 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 8 millis (IndexTreeDic.java:1164) +2019-06-05T22:41 [INFO][TIMER] Temps total recherche: 8.60 millis (IndexTreeMessyTest.java:425) +2019-06-05T22:41 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.122 s (IndexTreeMessyTest.java:439) +2019-06-05T22:41 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.072 s (IndexTreeMessyTest.java:449) +2019-06-05T22:41 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 22 millis (IndexTreeDic.java:1164) +2019-06-05T22:41 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T22:41 [INFO] 40 (HashMapTests.java:27) +2019-06-05T22:41 [INFO][MEMORY] Mem usage début - 88,230 (ParallelLoaderTest.java:56) +2019-06-05T22:41 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T22:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T22:41 [INFO][MEMORY] 90,316 (ParallelLoader.java:114) +2019-06-05T22:41 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.068 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T22:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T22:41 [INFO] Parsing terminé !! temps écoulé = 1.09 s (ParallelLoader.java:217) +2019-06-05T22:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T22:41 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T22:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T22:41 [INFO][MEMORY] 415,931 (ParallelLoader.java:114) +2019-06-05T22:41 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.070 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T22:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T22:41 [INFO] Parsing terminé !! temps écoulé = 0.968 s (ParallelLoader.java:217) +2019-06-05T22:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T22:41 [INFO][MEMORY] Mem usage fin - 189,494 (ParallelLoaderTest.java:113) +2019-06-05T22:41 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 2.17 s (ParallelLoaderTest.java:114) +2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T23:02 [INFO][MEMORY] 12,404 (ParallelLoader.java:114) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.154 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.61 s (ParallelLoader.java:217) +2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.62 s (IndexMemDicStorageTests.java:183) +2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-05T23:02 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.266 s (IndexMemDicStorageTests.java:83) +2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-05T23:02 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.188 s (IndexMemDicStorageTests.java:86) +2019-06-05T23:02 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-05T23:02 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-05T23:02 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-05T23:02 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T23:02 [INFO][MEMORY] 103,147 (ParallelLoader.java:114) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.174 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.01 s (ParallelLoader.java:217) +2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.02 s (NewLoaderAndIndexTest.java:189) +2019-06-05T23:02 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-05T23:02 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-05T23:02 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-05T23:02 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-05T23:02 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-05T23:02 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-05T23:02 [INFO] totalLength = 66666 (IndexMemDic.java:217) +2019-06-05T23:02 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) +2019-06-05T23:02 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T23:02 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.57 millis (Loader.java:116) +2019-06-05T23:02 [INFO][MEMORY] 223,892 (Loader.java:117) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.56 millis (Loader.java:116) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.00 millis (Loader.java:116) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.58 millis (Loader.java:116) +2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) +2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) +2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 155,525 (Loader.java:138) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 155,474 (Loader.java:138) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 156,042 (Loader.java:138) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 155,626 (Loader.java:138) +2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.03 s (IndexTreeMessyTest.java:314) +2019-06-05T23:02 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T23:02 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,650 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,038 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,414 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,414 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,464 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,674 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,674 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5ae50ce6 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6f96c77 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@be64738 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ba9ad43 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49d904ec (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@48e4374 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3d680b5a (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b5d6a01 (Table.java:338) +2019-06-05T23:02 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T23:02 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T23:02 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T23:02 [ERROR] null (SerialStructure.java:74) +2019-06-05T23:02 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 43 millis (IndexTreeDic.java:1164) +2019-06-05T23:02 [INFO][TIMER] Temps total recherche: 0.051 s (IndexTreeMessyTest.java:408) +2019-06-05T23:02 [INFO][TIMER] Temps parcours des résultats: 0.73 millis (IndexTreeMessyTest.java:416) +2019-06-05T23:02 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) +2019-06-05T23:02 [INFO][TIMER] Temps total recherche: 6.60 millis (IndexTreeMessyTest.java:425) +2019-06-05T23:02 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.096 s (IndexTreeMessyTest.java:439) +2019-06-05T23:02 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.069 s (IndexTreeMessyTest.java:449) +2019-06-05T23:02 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 10 millis (IndexTreeDic.java:1164) +2019-06-05T23:02 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T23:02 [INFO] 40 (HashMapTests.java:27) +2019-06-05T23:02 [INFO][MEMORY] Mem usage début - 88,284 (ParallelLoaderTest.java:56) +2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T23:02 [INFO][MEMORY] 90,332 (ParallelLoader.java:114) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.051 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.11 s (ParallelLoader.java:217) +2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T23:02 [INFO][MEMORY] 275,258 (ParallelLoader.java:114) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.076 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 0.696 s (ParallelLoader.java:217) +2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T23:02 [INFO][MEMORY] Mem usage fin - 189,551 (ParallelLoaderTest.java:114) +2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.87 s (ParallelLoaderTest.java:115) +2019-06-10T20:19 [ERROR] config.json (No such file or directory) (Database.java:33) +2019-06-10T20:19 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:19 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.37 millis (Loader.java:116) +2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.93 millis (Loader.java:116) +2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.37 millis (Loader.java:116) +2019-06-10T20:19 [INFO][MEMORY] 10,178 (Loader.java:117) +2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.010 s (Loader.java:116) +2019-06-10T20:19 [INFO][MEMORY] 10,178 (Loader.java:117) +2019-06-10T20:19 [INFO][MEMORY] 19,394 (Loader.java:117) +2019-06-10T20:19 [INFO][MEMORY] 19,394 (Loader.java:117) +2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:19 [INFO][MEMORY] 150,062 (Loader.java:138) +2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:19 [INFO][MEMORY] 150,767 (Loader.java:138) +2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:19 [INFO][MEMORY] 150,245 (Loader.java:138) +2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:19 [INFO][MEMORY] 150,333 (Loader.java:138) +2019-06-10T20:19 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.79 s (IndexTreeMessyTest.java:314) +2019-06-10T20:19 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T20:19 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,359 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b083826 (Table.java:338) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,871 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@105fece7 (Table.java:338) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,871 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ec300f1 (Table.java:338) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,381 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@482cd91f (Table.java:338) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,381 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@123f1134 (Table.java:338) +2019-06-10T20:34 [ERROR] config.json (No such file or directory) (Database.java:33) +2019-06-10T20:34 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:34 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.70 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 9,153 (Loader.java:117) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.74 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 14,616 (Loader.java:117) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.42 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 20,072 (Loader.java:117) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.10 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 32,705 (Loader.java:117) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 149,780 (Loader.java:138) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 150,170 (Loader.java:138) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 150,367 (Loader.java:138) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 150,299 (Loader.java:138) +2019-06-10T20:34 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.04 s (IndexTreeMessyTest.java:314) +2019-06-10T20:34 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T20:34 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,323 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7d68ef40 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,323 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5b0abc94 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,699 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@75c072cb (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,699 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1f1c7bf6 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,748 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25b485ba (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2b546384 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5d740a0f (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@214b199c (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@20d3d15a (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,349 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2893de87 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@55ca8de8 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2c34f934 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@12d3a4e9 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@240237d2 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25a65b77 (Table.java:338) +2019-06-10T20:34 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-10T20:34 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-10T20:34 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-10T20:34 [ERROR] null (SerialStructure.java:74) +2019-06-10T20:34 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 44 millis (IndexTreeDic.java:1164) +2019-06-10T20:34 [INFO][TIMER] Temps total recherche: 0.053 s (IndexTreeMessyTest.java:408) +2019-06-10T20:34 [INFO][TIMER] Temps parcours des résultats: 1.23 millis (IndexTreeMessyTest.java:416) +2019-06-10T20:34 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 14 millis (IndexTreeDic.java:1164) +2019-06-10T20:34 [INFO][TIMER] Temps total recherche: 0.015 s (IndexTreeMessyTest.java:425) +2019-06-10T20:34 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.119 s (IndexTreeMessyTest.java:439) +2019-06-10T20:34 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.045 s (IndexTreeMessyTest.java:449) +2019-06-10T20:34 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 2 millis (IndexTreeDic.java:1164) +2019-06-10T20:34 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-10T20:34 [INFO] 40 (HashMapTests.java:27) +2019-06-10T20:34 [INFO][MEMORY] Mem usage début - 82,958 (ParallelLoaderTest.java:56) +2019-06-10T20:49 [ERROR] config.json (No such file or directory) (Database.java:33) +2019-06-10T20:49 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:49 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.20 millis (Loader.java:116) +2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.69 millis (Loader.java:116) +2019-06-10T20:49 [INFO][MEMORY] 9,820 (Loader.java:117) +2019-06-10T20:49 [INFO][MEMORY] 9,820 (Loader.java:117) +2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.41 millis (Loader.java:116) +2019-06-10T20:49 [INFO][MEMORY] 19,381 (Loader.java:117) +2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.16 millis (Loader.java:116) +2019-06-10T20:49 [INFO][MEMORY] 24,844 (Loader.java:117) +2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:50 [INFO][MEMORY] 150,362 (Loader.java:138) +2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:50 [INFO][MEMORY] 150,428 (Loader.java:138) +2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:50 [INFO][MEMORY] 150,268 (Loader.java:138) +2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:50 [INFO][MEMORY] 150,648 (Loader.java:138) +2019-06-10T20:50 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.96 s (IndexTreeMessyTest.java:314) +2019-06-10T20:50 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T20:50 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,382 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b083826 (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,382 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@105fece7 (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,758 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ec300f1 (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,758 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@482cd91f (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,807 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@123f1134 (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7d68ef40 (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5b0abc94 (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@75c072cb (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1f1c7bf6 (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25b485ba (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,406 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2b546384 (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,406 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5d740a0f (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@214b199c (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@20d3d15a (Table.java:338) +2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) +2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2893de87 (Table.java:338) +2019-06-10T20:50 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-10T20:50 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-10T20:50 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-10T20:50 [ERROR] null (SerialStructure.java:74) +2019-06-10T20:50 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 36 millis (IndexTreeDic.java:1164) +2019-06-10T20:50 [INFO][TIMER] Temps total recherche: 0.044 s (IndexTreeMessyTest.java:408) +2019-06-10T20:50 [INFO][TIMER] Temps parcours des résultats: 0.73 millis (IndexTreeMessyTest.java:416) +2019-06-10T20:50 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) +2019-06-10T20:50 [INFO][TIMER] Temps total recherche: 7.60 millis (IndexTreeMessyTest.java:425) +2019-06-10T20:50 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.070 s (IndexTreeMessyTest.java:439) +2019-06-10T20:50 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.043 s (IndexTreeMessyTest.java:449) +2019-06-10T20:50 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 3 millis (IndexTreeDic.java:1164) +2019-06-10T20:50 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-10T20:50 [INFO] 40 (HashMapTests.java:27) +2019-06-10T20:50 [INFO][MEMORY] Mem usage début - 82,957 (ParallelLoaderTest.java:56) +2019-06-10T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-10T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T20:50 [INFO][MEMORY] 11,889 (ParallelLoader.java:114) +2019-06-10T20:50 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.114 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 1.68 s (ParallelLoader.java:217) +2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.68 s (IndexMemDicStorageTests.java:183) +2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T20:51 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.208 s (IndexMemDicStorageTests.java:83) +2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T20:51 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.176 s (IndexMemDicStorageTests.java:86) +2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T20:51 [INFO][MEMORY] 108,834 (ParallelLoader.java:114) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.141 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.890 s (ParallelLoader.java:217) +2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.900 s (NewLoaderAndIndexTest.java:189) +2019-06-10T20:51 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-10T20:51 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-10T20:51 [INFO] totalLength = 66666 (IndexMemDic.java:217) +2019-06-10T20:51 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) +2019-06-10T20:51 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:51 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.67 millis (Loader.java:116) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.13 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) +2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.14 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 178,179 (Loader.java:117) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.71 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 199,412 (Loader.java:117) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,755 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,569 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 156,154 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,614 (Loader.java:138) +2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 4.79 s (IndexTreeMessyTest.java:314) +2019-06-10T20:51 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T20:51 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,063 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) +2019-06-10T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-10T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-10T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-10T20:51 [ERROR] null (SerialStructure.java:74) +2019-06-10T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 41 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 0.049 s (IndexTreeMessyTest.java:408) +2019-06-10T20:51 [INFO][TIMER] Temps parcours des résultats: 1.53 millis (IndexTreeMessyTest.java:416) +2019-06-10T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 7.95 millis (IndexTreeMessyTest.java:425) +2019-06-10T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.079 s (IndexTreeMessyTest.java:439) +2019-06-10T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) +2019-06-10T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-10T20:51 [INFO] 40 (HashMapTests.java:27) +2019-06-10T20:51 [INFO][MEMORY] Mem usage début - 88,271 (ParallelLoaderTest.java:56) +2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T20:51 [INFO][MEMORY] 90,319 (ParallelLoader.java:114) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.842 s (ParallelLoader.java:217) +2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T20:51 [INFO][MEMORY] 209,650 (ParallelLoader.java:114) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.056 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.637 s (ParallelLoader.java:217) +2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T20:51 [INFO][MEMORY] Mem usage fin - 189,538 (ParallelLoaderTest.java:114) +2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (ParallelLoaderTest.java:115) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 12,384 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.122 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 1.54 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (IndexMemDicStorageTests.java:183) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.312 s (IndexMemDicStorageTests.java:83) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.172 s (IndexMemDicStorageTests.java:86) +2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 108,772 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.168 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.889 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.904 s (NewLoaderAndIndexTest.java:189) +2019-06-10T21:24 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-10T21:24 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-10T21:24 [INFO] totalLength = 66666 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) +2019-06-10T21:24 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T21:24 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.99 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 234,123 (Loader.java:117) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 154,808 (Loader.java:117) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.013 s (Loader.java:116) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.63 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) +2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 154,918 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,713 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,525 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,614 (Loader.java:138) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 12.07 s (IndexTreeMessyTest.java:314) +2019-06-10T21:24 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T21:24 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,064 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,576 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) +2019-06-10T21:24 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-10T21:24 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-10T21:24 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-10T21:24 [ERROR] null (SerialStructure.java:74) +2019-06-10T21:24 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 37 millis (IndexTreeDic.java:1164) +2019-06-10T21:24 [INFO][TIMER] Temps total recherche: 0.044 s (IndexTreeMessyTest.java:408) +2019-06-10T21:24 [INFO][TIMER] Temps parcours des résultats: 0.59 millis (IndexTreeMessyTest.java:416) +2019-06-10T21:24 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) +2019-06-10T21:24 [INFO][TIMER] Temps total recherche: 7.06 millis (IndexTreeMessyTest.java:425) +2019-06-10T21:24 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.071 s (IndexTreeMessyTest.java:439) +2019-06-10T21:24 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) +2019-06-10T21:24 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 8 millis (IndexTreeDic.java:1164) +2019-06-10T21:24 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-10T21:24 [INFO] 40 (HashMapTests.java:27) +2019-06-10T21:24 [INFO][MEMORY] Mem usage début - 88,272 (ParallelLoaderTest.java:56) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 90,320 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.046 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.898 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 232,213 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.047 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.709 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO][MEMORY] Mem usage fin - 189,539 (ParallelLoaderTest.java:114) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.68 s (ParallelLoaderTest.java:115) diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000000 b/repo/testdata/repo_8k/00000/chunks/000000000000000 new file mode 100644 index 0000000..ac78620 --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000000 @@ -0,0 +1,85 @@ +2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T20:51 [INFO] testInfoString (LogTest.java:52) +2019-06-05T20:51 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T22:41 [INFO] testInfoString (LogTest.java:52) +2019-06-05T22:41 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T23:02 [INFO] testInfoString (LogTest.java:52) +2019-06-05T23:02 [INFO][YO] testInfoString (LogTest.java:57) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.52 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 160,128 (Loader.java:117) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 231,367 (Loader.java:138) +2019-06-05T20:13 [INFO][TIMER] Temps pris par le parsing: 0.898 s (LoaderTest.java:69) +2019-06-05T20:13 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:14 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.46 millis (Loader.java:116) +2019-06-05T20:14 [INFO][MEMORY] 267,348 (Loader.java:117) +2019-06-05T20:14 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:14 [INFO][MEMORY] 231,359 (Loader.java:138) +2019-06-05T20:14 [INFO][TIMER] Temps pris par le parsing: 0.839 s (LoaderTest.java:69) +2019-06-05T20:14 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.61 millis (Loader.java:116) +2019-06-05T20:51 [INFO][MEMORY] 189,561 (Loader.java:117) +2019-06-05T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:51 [INFO][MEMORY] 168,686 (Loader.java:138) +2019-06-05T20:51 [INFO][TIMER] Temps pris par le parsing: 0.930 s (LoaderTest.java:69) +2019-06-05T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T22:41 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.83 millis (Loader.java:116) +2019-06-05T22:41 [INFO][MEMORY] 189,494 (Loader.java:117) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 168,619 (Loader.java:138) +2019-06-05T22:41 [INFO][TIMER] Temps pris par le parsing: 0.882 s (LoaderTest.java:69) +2019-06-05T22:41 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) +2019-06-05T23:02 [INFO][MEMORY] 189,551 (Loader.java:117) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 168,676 (Loader.java:138) +2019-06-05T23:02 [INFO][TIMER] Temps pris par le parsing: 1.04 s (LoaderTest.java:69) +2019-06-05T23:02 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.82 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 82,958 (Loader.java:117) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 163,350 (Loader.java:138) +2019-06-10T20:34 [INFO][TIMER] Temps pris par le parsing: 0.938 s (LoaderTest.java:69) +2019-06-10T20:34 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) +2019-06-10T20:50 [INFO][MEMORY] 82,957 (Loader.java:117) +2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:50 [INFO][MEMORY] 163,349 (Loader.java:138) +2019-06-10T20:50 [INFO][TIMER] Temps pris par le parsing: 0.853 s (LoaderTest.java:69) +2019-06-10T20:50 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.51 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 189,538 (Loader.java:117) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 168,663 (Loader.java:138) +2019-06-10T20:51 [INFO][TIMER] Temps pris par le parsing: 1.27 s (LoaderTest.java:69) +2019-06-10T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.58 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 189,539 (Loader.java:117) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 168,664 (Loader.java:138) +2019-06-10T21:24 [INFO][TIMER] Temps pris par le parsing: 1.86 s (LoaderTest.java:69) +2019-06-10T21:24 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) +2019-06-05T14:47 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T14:49 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T14:49 [INFO][MEMORY] 229,457 (ParallelLoader.java:114) +2019-06-05T14:49 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.522 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T14:49 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T14:49 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T14:49 [INFO] Parsing terminé !! temps écoulé = 2.78 s (ParallelLoader.java:217) +2019-06-05T14:49 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T14:53 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T14:53 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T14:53 [INFO][MEMORY] 227,095 (ParallelLoader.java:114) +2019-06-05T14:53 [INFO] Loader : nombre de résulta \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000001 b/repo/testdata/repo_8k/00000/chunks/000000000000001 new file mode 100644 index 0000000..3e728e2 --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000001 @@ -0,0 +1,89 @@ +ts (total) lus = 100000 temps écoulé = 0.433 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T14:53 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T14:53 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T14:53 [INFO] Parsing terminé !! temps écoulé = 2.37 s (ParallelLoader.java:217) +2019-06-05T14:53 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T14:54 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) +2019-06-05T14:59 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) +2019-06-05T14:59 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) +2019-06-05T15:08 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:09 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:09 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) +2019-06-05T15:09 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:09 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) +2019-06-05T15:10 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:10 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) +2019-06-05T15:12 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:12 [INFO][MEMORY] 178,759 (ParallelLoader.java:114) +2019-06-05T15:12 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.220 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:12 [INFO] Parsing terminé !! temps écoulé = 2.25 s (ParallelLoader.java:217) +2019-06-05T15:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:13 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:14 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:328) +2019-06-05T15:14 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T15:14 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:328) +2019-06-05T15:14 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T15:14 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:14 [INFO][MEMORY] 308,168 (ParallelLoader.java:114) +2019-06-05T15:14 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.052 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:14 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:14 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:14 [INFO] Parsing terminé !! temps écoulé = 1.32 s (ParallelLoader.java:217) +2019-06-05T15:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:14 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:15 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) +2019-06-05T15:15 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) +2019-06-05T15:17 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:26 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:27 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:27 [INFO][MEMORY] 179,237 (ParallelLoader.java:114) +2019-06-05T15:27 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.098 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:27 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:27 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:27 [INFO] Parsing terminé !! temps écoulé = 2.22 s (ParallelLoader.java:217) +2019-06-05T15:27 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:31 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:31 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:31 [INFO][MEMORY] 178,254 (ParallelLoader.java:114) +2019-06-05T15:31 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.118 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:31 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:31 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:31 [INFO] Parsing terminé !! temps écoulé = 2.11 s (ParallelLoader.java:217) +2019-06-05T15:31 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:41 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:41 [INFO][MEMORY] 180,287 (ParallelLoader.java:114) +2019-06-05T15:41 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.195 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:41 [INFO] Parsing terminé !! temps écoulé = 1.92 s (ParallelLoader.java:217) +2019-06-05T15:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T15:42 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T15:43 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T15:43 [INFO][MEMORY] 178,731 (ParallelLoader.java:114) +2019-06-05T15:43 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.141 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T15:43 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T15:43 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T15:43 [INFO] Parsing terminé !! temps écoulé = 2.03 s (ParallelLoader.java:217) +2019-06-05T15:43 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:35 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T18:35 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:35 [INFO][MEMORY] 229,265 (ParallelLoader.java:114) +2019-06-05T18:35 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.641 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:35 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:35 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:35 [INFO] Parsing terminé !! temps écoulé = 3.04 s (ParallelLoader.java:217) +2019-06-05T18:35 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:36 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:36 [INFO][MEMORY] 326,934 (ParallelLoader.java:114) +2019-06-05T18:36 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.072 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:36 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:36 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:36 [INFO] Parsing terminé !! temps écoulé = 1.17 s (ParallelLoader.java:217) +2019-06-05T18:36 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:37 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:37 [INFO][MEMORY] 537,997 (ParallelLoader.java:114) +2019-06-05T18:37 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadN \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000002 b/repo/testdata/repo_8k/00000/chunks/000000000000002 new file mode 100644 index 0000000..079ca90 --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000002 @@ -0,0 +1,98 @@ +b = 0 (ParallelLoader.java:150) +2019-06-05T18:37 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:37 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:37 [INFO] Parsing terminé !! temps écoulé = 1.23 s (ParallelLoader.java:217) +2019-06-05T18:37 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:38 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) +2019-06-05T18:38 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T18:38 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:38 [INFO][MEMORY] 498,154 (ParallelLoader.java:114) +2019-06-05T18:38 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.031 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:38 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:38 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:38 [INFO] Parsing terminé !! temps écoulé = 1.10 s (ParallelLoader.java:217) +2019-06-05T18:38 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T18:39 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) +2019-06-05T18:39 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T18:40 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T18:40 [INFO][MEMORY] 493,228 (ParallelLoader.java:114) +2019-06-05T18:40 [INFO] Loader : nombre de résultats (total) lus = 500000 temps écoulé = 0.032 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T18:40 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T18:40 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T18:40 [INFO] Parsing terminé !! temps écoulé = 1.02 s (ParallelLoader.java:217) +2019-06-05T18:40 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:14 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T20:14 [ERROR] null (SerialStructure.java:74) +2019-06-05T20:15 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) +2019-06-05T20:15 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T20:15 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:15 [INFO][MEMORY] 188,768 (ParallelLoader.java:114) +2019-06-05T20:15 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.301 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:15 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:15 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:15 [INFO] Parsing terminé !! temps écoulé = 2.42 s (ParallelLoader.java:217) +2019-06-05T20:15 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:15 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) +2019-06-05T20:15 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) +2019-06-05T20:16 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:16 [INFO][MEMORY] 457,918 (ParallelLoader.java:114) +2019-06-05T20:16 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.049 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:16 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:16 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:16 [INFO] Parsing terminé !! temps écoulé = 1.34 s (ParallelLoader.java:217) +2019-06-05T20:16 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:19 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:19 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:19 [INFO][MEMORY] 180,323 (ParallelLoader.java:114) +2019-06-05T20:19 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.178 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:19 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:19 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:19 [INFO] Parsing terminé !! temps écoulé = 1.92 s (ParallelLoader.java:217) +2019-06-05T20:19 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:24 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:25 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:43 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:43 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:43 [INFO][MEMORY] 179,894 (ParallelLoader.java:114) +2019-06-05T20:43 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.133 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:43 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:43 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:43 [INFO] Parsing terminé !! temps écoulé = 2.14 s (ParallelLoader.java:217) +2019-06-05T20:43 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:50 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T20:50 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) +2019-06-05T21:56 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T21:56 [ERROR] null +2019-06-05T22:41 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T22:41 [ERROR] null +2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T22:42 [ERROR] null +2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T22:42 [ERROR] null +2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T22:42 [ERROR] null +2019-06-05T23:03 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. +2019-06-05T23:03 [ERROR] null +2019-06-05T23:04 [INFO] SerialStructure.writeStructure : OK ! +2019-06-05T23:04 [INFO] totalLength = 0 +2019-06-05T23:04 [INFO] totalLength = 0 +2019-06-05T23:04 [INFO] PARSE : memusage init = +2019-06-05T23:04 [INFO][MEMORY] 111,311 +2019-06-05T23:04 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.256 sactiveThreadNb = 0 +2019-06-05T23:04 [WARNING] Fail to save entry number 0: incorrect data +2019-06-05T23:04 [INFO] Parsing terminé !! temps écoulé = 2.57 s +2019-06-05T23:04 [INFO] PARSE : Nombre de lignes (local) = 100000 +2019-06-05T23:04 [INFO] totalLength = 99999 +2019-06-05T23:04 [INFO] totalLength = 99999 +2019-06-05T00:03 [ERROR] config.json (No such file or directory) (Database.java:33) +2019-06-05T00:03 [INFO][MEMORY] Mem usage début - 2,459 (ParallelLoaderTest.java:56) +2019-06-05T20:12 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T20:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:12 [INFO][MEMORY] 27,443 (ParallelLoader.java:114) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.214 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:12 [INFO] Parsing terminé !! temps écoulé = 1.44 s (ParallelLoader.java:217) +2019-06-05T20:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:12 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.45 s (IndexMemDicStorageTests.java:183) +2019-06-05T20:12 [INFO][TIME \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000003 b/repo/testdata/repo_8k/00000/chunks/000000000000003 new file mode 100644 index 0000000..8720f6b --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000003 @@ -0,0 +1,85 @@ +R] Temps pris pour classer la 1ère fois: 0.477 s (IndexMemDicStorageTests.java:83) +2019-06-05T20:12 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.054 s (IndexMemDicStorageTests.java:86) +2019-06-05T20:12 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-05T20:12 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-05T20:12 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-05T20:12 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-05T20:12 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T20:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:12 [INFO][MEMORY] 190,485 (ParallelLoader.java:114) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.064 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:12 [INFO] Parsing terminé !! temps écoulé = 0.908 s (ParallelLoader.java:217) +2019-06-05T20:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:12 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.917 s (NewLoaderAndIndexTest.java:189) +2019-06-05T20:12 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T20:12 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-05T20:12 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-05T20:12 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-05T20:12 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-05T20:12 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-05T20:12 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 66666 (NewLoaderAndIndexTest.java:109) +2019-06-05T20:12 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T20:12 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.57 millis (Loader.java:116) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) +2019-06-05T20:12 [INFO][MEMORY] 401,153 (Loader.java:117) +2019-06-05T20:12 [INFO][MEMORY] 401,153 (Loader.java:117) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.33 millis (Loader.java:116) +2019-06-05T20:12 [INFO][MEMORY] 273,252 (Loader.java:117) +2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.97 millis (Loader.java:116) +2019-06-05T20:12 [INFO][MEMORY] 273,252 (Loader.java:117) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 216,262 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 221,186 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 218,771 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 216,694 (Loader.java:138) +2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.57 s (IndexTreeMessyTest.java:314) +2019-06-05T20:13 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T20:13 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,432 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2bef09c0 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@62ce72ff (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@58a63629 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7de843ef (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2e1af49b (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3fd2322d (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@56ac5c80 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69e49a81 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1023150a (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ece1e79 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@ab20d7c (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@474fae39 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4f4c789f (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13f9ad9 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b7cae6f (Table.java:338) +2019-06-05T20:13 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T20:13 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T20:13 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T20:13 [ERROR] null (SerialStructure.java:74) +2019-06-05T20:13 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 48 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.051 s (IndexTreeMessyTest.java:408) +2019-06-05T20:13 [INFO][TIMER] Temps parcours des résultats: 1.72 millis (IndexTreeMessyTest.ja \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000004 b/repo/testdata/repo_8k/00000/chunks/000000000000004 new file mode 100644 index 0000000..f05dda7 --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000004 @@ -0,0 +1,87 @@ +va:416) +2019-06-05T20:13 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 9 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.011 s (IndexTreeMessyTest.java:425) +2019-06-05T20:13 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.125 s (IndexTreeMessyTest.java:439) +2019-06-05T20:13 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.041 s (IndexTreeMessyTest.java:449) +2019-06-05T20:13 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T20:13 [INFO] 40 (HashMapTests.java:27) +2019-06-05T20:13 [INFO][MEMORY] Mem usage début - 157,206 (ParallelLoaderTest.java:56) +2019-06-05T20:13 [INFO] Parsing de csvName = E:/csv/yellow_tripdata_2015-01.csv (ParallelLoaderTest.java:64) +2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:13 [INFO][MEMORY] 28,086 (ParallelLoader.java:114) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.182 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:13 [INFO] Parsing terminé !! temps écoulé = 1.43 s (ParallelLoader.java:217) +2019-06-05T20:13 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.44 s (IndexMemDicStorageTests.java:183) +2019-06-05T20:13 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.356 s (IndexMemDicStorageTests.java:83) +2019-06-05T20:13 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.032 s (IndexMemDicStorageTests.java:86) +2019-06-05T20:13 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-05T20:13 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-05T20:13 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-05T20:13 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:13 [INFO][MEMORY] 184,068 (ParallelLoader.java:114) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.052 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:13 [INFO] Parsing terminé !! temps écoulé = 0.941 s (ParallelLoader.java:217) +2019-06-05T20:13 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.945 s (NewLoaderAndIndexTest.java:189) +2019-06-05T20:13 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T20:13 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-05T20:13 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-05T20:13 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-05T20:13 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-05T20:13 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-05T20:13 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 66666 (NewLoaderAndIndexTest.java:109) +2019-06-05T20:13 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T20:13 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.70 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 276,783 (Loader.java:117) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.69 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 341,077 (Loader.java:117) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.09 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 358,336 (Loader.java:117) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 5.74 millis (Loader.java:116) +2019-06-05T20:13 [INFO][MEMORY] 358,336 (Loader.java:117) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 218,799 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 218,840 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 223,379 (Loader.java:138) +2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:13 [INFO][MEMORY] 216,694 (Loader.java:138) +2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.18 s (IndexTreeMessyTest.java:314) +2019-06-05T20:13 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T20:13 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2881ad47 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2bef09c0 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@62ce72ff (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@58a63629 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7de843ef (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2e1af49b (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3fd2322d (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@56ac5c80 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69e49a81 (Table.java:338) +2019-06-05T20:13 [INFO] \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000005 b/repo/testdata/repo_8k/00000/chunks/000000000000005 new file mode 100644 index 0000000..0134101 --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000005 @@ -0,0 +1,87 @@ +[MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1023150a (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ece1e79 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@ab20d7c (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@474fae39 (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4f4c789f (Table.java:338) +2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) +2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13f9ad9 (Table.java:338) +2019-06-05T20:13 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T20:13 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T20:13 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T20:13 [ERROR] null (SerialStructure.java:74) +2019-06-05T20:13 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 58 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.061 s (IndexTreeMessyTest.java:408) +2019-06-05T20:13 [INFO][TIMER] Temps parcours des résultats: 1.25 millis (IndexTreeMessyTest.java:416) +2019-06-05T20:13 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 8.20 millis (IndexTreeMessyTest.java:425) +2019-06-05T20:13 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.072 s (IndexTreeMessyTest.java:439) +2019-06-05T20:13 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.040 s (IndexTreeMessyTest.java:449) +2019-06-05T20:13 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 3 millis (IndexTreeDic.java:1164) +2019-06-05T20:13 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T20:13 [INFO] 40 (HashMapTests.java:27) +2019-06-05T20:13 [INFO][MEMORY] Mem usage début - 157,225 (ParallelLoaderTest.java:56) +2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:13 [INFO][MEMORY] 166,226 (ParallelLoader.java:114) +2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.041 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:14 [INFO] Parsing terminé !! temps écoulé = 0.638 s (ParallelLoader.java:217) +2019-06-05T20:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:14 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T20:14 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:14 [INFO][MEMORY] 422,898 (ParallelLoader.java:114) +2019-06-05T20:14 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.040 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:14 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:14 [INFO] Parsing terminé !! temps écoulé = 0.624 s (ParallelLoader.java:217) +2019-06-05T20:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:14 [INFO][MEMORY] Mem usage fin - 267,348 (ParallelLoaderTest.java:113) +2019-06-05T20:14 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.46 s (ParallelLoaderTest.java:114) +2019-06-05T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:50 [INFO][MEMORY] 12,904 (ParallelLoader.java:114) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.175 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:50 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:50 [INFO] Parsing terminé !! temps écoulé = 2.03 s (ParallelLoader.java:217) +2019-06-05T20:50 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 2.04 s (IndexMemDicStorageTests.java:183) +2019-06-05T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:50 [INFO][MEMORY] 157,253 (ParallelLoader.java:114) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.111 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:50 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:50 [INFO] Parsing terminé !! temps écoulé = 0.781 s (ParallelLoader.java:217) +2019-06-05T20:50 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.787 s (NewLoaderAndIndexTest.java:189) +2019-06-05T20:50 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T20:50 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T20:50 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 5.82 millis (Loader.java:116) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.66 millis (Loader.java:116) +2019-06-05T20:50 [INFO][MEMORY] 134,560 (Loader.java:117) +2019-06-05T20:50 [INFO][MEMORY] 134,560 (Loader.java:117) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.11 millis (Loader.java:116) +2019-06-05T20:50 [INFO][MEMORY] 152,063 (Loader.java:117) +2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.011 s (Loader.java:116) +2019-06-05T20:50 [INFO][MEMORY] 168,447 (Loader.java:117) +2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:50 [INFO][MEMORY] 155,685 (Loader.java:138) +2019-06-05T20:50 [INFO][MEMORY] 156,197 (Loader.java:138) +2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:50 [INFO][MEMORY] 155,691 (Lo \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000006 b/repo/testdata/repo_8k/00000/chunks/000000000000006 new file mode 100644 index 0000000..91c72a3 --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000006 @@ -0,0 +1,84 @@ +ader.java:138) +2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T20:50 [INFO][MEMORY] 155,633 (Loader.java:138) +2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.14 s (IndexTreeMessyTest.java:314) +2019-06-05T20:50 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T20:50 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,657 (IndexTreeDic.java:517) +2019-06-05T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b0b0854 (Table.java:338) +2019-06-05T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,657 (IndexTreeDic.java:517) +2019-06-05T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@19bb07ed (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,033 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@10e41621 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,033 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@353d0772 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,082 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2667f029 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,594 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@67a20f67 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@57c758ac (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@a9cd3b1 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13e39c73 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,915 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@64cd705f (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@9225652 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@654f0d9c (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6a400542 (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6580cfdd (Table.java:338) +2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) +2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7e0b85f9 (Table.java:338) +2019-06-05T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T20:51 [ERROR] null (SerialStructure.java:74) +2019-06-05T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 31 millis (IndexTreeDic.java:1164) +2019-06-05T20:51 [INFO][TIMER] Temps total recherche: 0.037 s (IndexTreeMessyTest.java:408) +2019-06-05T20:51 [INFO][TIMER] Temps parcours des résultats: 1.96 millis (IndexTreeMessyTest.java:416) +2019-06-05T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) +2019-06-05T20:51 [INFO][TIMER] Temps total recherche: 7.48 millis (IndexTreeMessyTest.java:425) +2019-06-05T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.119 s (IndexTreeMessyTest.java:439) +2019-06-05T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.040 s (IndexTreeMessyTest.java:449) +2019-06-05T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 1 millis (IndexTreeDic.java:1164) +2019-06-05T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T20:51 [INFO] 40 (HashMapTests.java:27) +2019-06-05T20:51 [INFO][MEMORY] Mem usage début - 88,295 (ParallelLoaderTest.java:56) +2019-06-05T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:51 [INFO][MEMORY] 90,343 (ParallelLoader.java:114) +2019-06-05T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.064 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:51 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:51 [INFO] Parsing terminé !! temps écoulé = 0.862 s (ParallelLoader.java:217) +2019-06-05T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T20:51 [INFO][MEMORY] 227,179 (ParallelLoader.java:114) +2019-06-05T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.055 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T20:51 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T20:51 [INFO] Parsing terminé !! temps écoulé = 0.676 s (ParallelLoader.java:217) +2019-06-05T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T20:51 [INFO][MEMORY] Mem usage fin - 189,561 (ParallelLoaderTest.java:113) +2019-06-05T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.63 s (ParallelLoaderTest.java:114) +2019-06-05T21:56 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T21:56 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T21:56 [INFO][MEMORY] 13,925 (ParallelLoader.java:114) +2019-06-05T21:56 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.226 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T21:56 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T21:56 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T21:57 [INFO] Parsing terminé !! temps écoulé = 33.57 s (ParallelLoader.java:217) +2019-06-05T21:57 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T21:57 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 33.66 s (IndexMemDicStorageTests.java:183) +2019-06-05T21:57 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T21:57 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T21:57 [INFO][MEMOR \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000007 b/repo/testdata/repo_8k/00000/chunks/000000000000007 new file mode 100644 index 0000000..f9f54db --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000007 @@ -0,0 +1,87 @@ +Y] 58,123 (ParallelLoader.java:114) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.206 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T21:57 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T21:57 [INFO] Parsing terminé !! temps écoulé = 15.87 s (ParallelLoader.java:217) +2019-06-05T21:57 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T21:57 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 16.28 s (NewLoaderAndIndexTest.java:189) +2019-06-05T21:57 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T21:57 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T21:57 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 9.32 millis (Loader.java:116) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 7.80 millis (Loader.java:116) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 8.95 millis (Loader.java:116) +2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 8.32 millis (Loader.java:116) +2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) +2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) +2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) +2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 155,396 (Loader.java:138) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 155,945 (Loader.java:138) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 155,523 (Loader.java:138) +2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T22:41 [INFO][MEMORY] 155,565 (Loader.java:138) +2019-06-05T22:41 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 35.40 s (IndexTreeMessyTest.java:314) +2019-06-05T22:41 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T22:41 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,457 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@e056f20 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,457 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b0b0854 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,834 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@19bb07ed (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,834 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@10e41621 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,883 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@353d0772 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,067 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2667f029 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,067 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@67a20f67 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,579 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@57c758ac (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,579 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@a9cd3b1 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13e39c73 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@64cd705f (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@9225652 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@654f0d9c (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6a400542 (Table.java:338) +2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,139 (IndexTreeDic.java:517) +2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6580cfdd (Table.java:338) +2019-06-05T22:41 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T22:41 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T22:41 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T22:41 [ERROR] null (SerialStructure.java:74) +2019-06-05T22:41 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 50 millis (IndexTreeDic.java:1164) +2019-06-05T22:41 [INFO][TIMER] Temps total recherche: 0.058 s (IndexTreeMessyTest.java:408) +2019-06-05T22:41 [INFO][TIMER] Temps parcours des résultats: 1.04 millis (IndexTreeMessyTest.java:416) +2019-06-05T22:41 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 8 millis (IndexTreeDic.java:1164) +2019-06-05T22:41 [INFO][TIMER] Temps total recherche: 8.60 millis (IndexTreeMessyTest.java:425) +2019-06-05T22:41 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.122 s (IndexTreeMessyTest.java:439) +2019-06-05T22:41 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.072 s (IndexTreeMessyTest.java:449) +2019-06-05T22:41 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 22 millis (IndexTreeDic.java:1164) +2019-06-05T22:41 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T22:41 [INFO] 40 (HashMapTests.java:27) +2019-06-05T22:41 [INFO][MEMORY] Mem usage début - 88,230 (ParallelLoaderTest.java:56) +2019-06-05T22:41 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T22:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T22:41 [INFO][MEMORY] 90,316 (ParallelLoader.java:114) +2019-06-05T22:41 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.068 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T22:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T22:41 [WARNING] Fail \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000008 b/repo/testdata/repo_8k/00000/chunks/000000000000008 new file mode 100644 index 0000000..5995b3c --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000008 @@ -0,0 +1,87 @@ +to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T22:41 [INFO] Parsing terminé !! temps écoulé = 1.09 s (ParallelLoader.java:217) +2019-06-05T22:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T22:41 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) +2019-06-05T22:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T22:41 [INFO][MEMORY] 415,931 (ParallelLoader.java:114) +2019-06-05T22:41 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.070 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T22:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) +2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T22:41 [INFO] Parsing terminé !! temps écoulé = 0.968 s (ParallelLoader.java:217) +2019-06-05T22:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T22:41 [INFO][MEMORY] Mem usage fin - 189,494 (ParallelLoaderTest.java:113) +2019-06-05T22:41 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 2.17 s (ParallelLoaderTest.java:114) +2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T23:02 [INFO][MEMORY] 12,404 (ParallelLoader.java:114) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.154 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.61 s (ParallelLoader.java:217) +2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.62 s (IndexMemDicStorageTests.java:183) +2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-05T23:02 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.266 s (IndexMemDicStorageTests.java:83) +2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-05T23:02 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.188 s (IndexMemDicStorageTests.java:86) +2019-06-05T23:02 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-05T23:02 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-05T23:02 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-05T23:02 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T23:02 [INFO][MEMORY] 103,147 (ParallelLoader.java:114) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.174 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.01 s (ParallelLoader.java:217) +2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.02 s (NewLoaderAndIndexTest.java:189) +2019-06-05T23:02 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-05T23:02 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-05T23:02 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-05T23:02 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-05T23:02 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-05T23:02 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-05T23:02 [INFO] totalLength = 66666 (IndexMemDic.java:217) +2019-06-05T23:02 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) +2019-06-05T23:02 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-05T23:02 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.57 millis (Loader.java:116) +2019-06-05T23:02 [INFO][MEMORY] 223,892 (Loader.java:117) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.56 millis (Loader.java:116) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.00 millis (Loader.java:116) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.58 millis (Loader.java:116) +2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) +2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) +2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 155,525 (Loader.java:138) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 155,474 (Loader.java:138) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 156,042 (Loader.java:138) +2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-05T23:02 [INFO][MEMORY] 155,626 (Loader.java:138) +2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.03 s (IndexTreeMessyTest.java:314) +2019-06-05T23:02 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-05T23:02 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,650 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,038 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,414 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,414 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,464 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,674 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,674 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTree \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000009 b/repo/testdata/repo_8k/00000/chunks/000000000000009 new file mode 100644 index 0000000..8a0d0af --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000009 @@ -0,0 +1,88 @@ +Dic@5ae50ce6 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6f96c77 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@be64738 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ba9ad43 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49d904ec (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@48e4374 (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3d680b5a (Table.java:338) +2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) +2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b5d6a01 (Table.java:338) +2019-06-05T23:02 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-05T23:02 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-05T23:02 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-05T23:02 [ERROR] null (SerialStructure.java:74) +2019-06-05T23:02 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 43 millis (IndexTreeDic.java:1164) +2019-06-05T23:02 [INFO][TIMER] Temps total recherche: 0.051 s (IndexTreeMessyTest.java:408) +2019-06-05T23:02 [INFO][TIMER] Temps parcours des résultats: 0.73 millis (IndexTreeMessyTest.java:416) +2019-06-05T23:02 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) +2019-06-05T23:02 [INFO][TIMER] Temps total recherche: 6.60 millis (IndexTreeMessyTest.java:425) +2019-06-05T23:02 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.096 s (IndexTreeMessyTest.java:439) +2019-06-05T23:02 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.069 s (IndexTreeMessyTest.java:449) +2019-06-05T23:02 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 10 millis (IndexTreeDic.java:1164) +2019-06-05T23:02 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-05T23:02 [INFO] 40 (HashMapTests.java:27) +2019-06-05T23:02 [INFO][MEMORY] Mem usage début - 88,284 (ParallelLoaderTest.java:56) +2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T23:02 [INFO][MEMORY] 90,332 (ParallelLoader.java:114) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.051 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.11 s (ParallelLoader.java:217) +2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-05T23:02 [INFO][MEMORY] 275,258 (ParallelLoader.java:114) +2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.076 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 0.696 s (ParallelLoader.java:217) +2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-05T23:02 [INFO][MEMORY] Mem usage fin - 189,551 (ParallelLoaderTest.java:114) +2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.87 s (ParallelLoaderTest.java:115) +2019-06-10T20:19 [ERROR] config.json (No such file or directory) (Database.java:33) +2019-06-10T20:19 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:19 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.37 millis (Loader.java:116) +2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.93 millis (Loader.java:116) +2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.37 millis (Loader.java:116) +2019-06-10T20:19 [INFO][MEMORY] 10,178 (Loader.java:117) +2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.010 s (Loader.java:116) +2019-06-10T20:19 [INFO][MEMORY] 10,178 (Loader.java:117) +2019-06-10T20:19 [INFO][MEMORY] 19,394 (Loader.java:117) +2019-06-10T20:19 [INFO][MEMORY] 19,394 (Loader.java:117) +2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:19 [INFO][MEMORY] 150,062 (Loader.java:138) +2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:19 [INFO][MEMORY] 150,767 (Loader.java:138) +2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:19 [INFO][MEMORY] 150,245 (Loader.java:138) +2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:19 [INFO][MEMORY] 150,333 (Loader.java:138) +2019-06-10T20:19 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.79 s (IndexTreeMessyTest.java:314) +2019-06-10T20:19 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T20:19 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,359 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b083826 (Table.java:338) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,871 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@105fece7 (Table.java:338) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,871 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ec300f1 (Table.java:338) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,381 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@482cd91f (Table.java:338) +2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,381 (IndexTreeDic.java:517) +2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@123f1134 (Table.java:338) +2019-06-10T20:34 [ERROR] config.json (No such file or directory) (Database.java:33) +2019-06-10T20:34 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:34 [INFO] Nombre de résultats/ \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000010 b/repo/testdata/repo_8k/00000/chunks/000000000000010 new file mode 100644 index 0000000..97a481a --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000010 @@ -0,0 +1,88 @@ +entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.70 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 9,153 (Loader.java:117) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.74 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 14,616 (Loader.java:117) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.42 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 20,072 (Loader.java:117) +2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.10 millis (Loader.java:116) +2019-06-10T20:34 [INFO][MEMORY] 32,705 (Loader.java:117) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 149,780 (Loader.java:138) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 150,170 (Loader.java:138) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 150,367 (Loader.java:138) +2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:34 [INFO][MEMORY] 150,299 (Loader.java:138) +2019-06-10T20:34 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.04 s (IndexTreeMessyTest.java:314) +2019-06-10T20:34 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T20:34 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,323 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7d68ef40 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,323 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5b0abc94 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,699 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@75c072cb (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,699 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1f1c7bf6 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,748 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25b485ba (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2b546384 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5d740a0f (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@214b199c (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@20d3d15a (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,349 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2893de87 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@55ca8de8 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2c34f934 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@12d3a4e9 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@240237d2 (Table.java:338) +2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) +2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25a65b77 (Table.java:338) +2019-06-10T20:34 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-10T20:34 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-10T20:34 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-10T20:34 [ERROR] null (SerialStructure.java:74) +2019-06-10T20:34 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 44 millis (IndexTreeDic.java:1164) +2019-06-10T20:34 [INFO][TIMER] Temps total recherche: 0.053 s (IndexTreeMessyTest.java:408) +2019-06-10T20:34 [INFO][TIMER] Temps parcours des résultats: 1.23 millis (IndexTreeMessyTest.java:416) +2019-06-10T20:34 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 14 millis (IndexTreeDic.java:1164) +2019-06-10T20:34 [INFO][TIMER] Temps total recherche: 0.015 s (IndexTreeMessyTest.java:425) +2019-06-10T20:34 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.119 s (IndexTreeMessyTest.java:439) +2019-06-10T20:34 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.045 s (IndexTreeMessyTest.java:449) +2019-06-10T20:34 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 2 millis (IndexTreeDic.java:1164) +2019-06-10T20:34 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-10T20:34 [INFO] 40 (HashMapTests.java:27) +2019-06-10T20:34 [INFO][MEMORY] Mem usage début - 82,958 (ParallelLoaderTest.java:56) +2019-06-10T20:49 [ERROR] config.json (No such file or directory) (Database.java:33) +2019-06-10T20:49 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:49 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.20 millis (Loader.java:116) +2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.69 millis (Loader.java:116) +2019-06-10T20:49 [INFO][MEMORY] 9,820 (Loader.java:117) +2019-06-10T20:49 [INFO][MEMORY] 9,820 (Loader.java:117) +2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.41 millis (Loader.java:116) +2019-06-10T20:49 [INFO][MEMORY] 19,381 (Loader.java:117) +2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.16 millis (Loader.java:116) +2019-06-10T20:49 [INFO][MEMORY] 24,844 (Loader.java:117) +2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.jav \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000011 b/repo/testdata/repo_8k/00000/chunks/000000000000011 new file mode 100644 index 0000000..5a30329 --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000011 @@ -0,0 +1,87 @@ +va:217) +2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-10T20:51 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-10T20:51 [INFO] totalLength = 66666 (IndexMemDic.java:217) +2019-06-10T20:51 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) +2019-06-10T20:51 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T20:51 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.67 millis (Loader.java:116) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.13 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) +2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.14 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 178,179 (Loader.java:117) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.71 millis (Loader.java:116) +2019-06-10T20:51 [INFO][MEMORY] 199,412 (Loader.java:117) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,755 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,569 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 156,154 (Loader.java:138) +2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T20:51 [INFO][MEMORY] 155,614 (Loader.java:138) +2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 4.79 s (IndexTreeMessyTest.java:314) +2019-06-10T20:51 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T20:51 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,063 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) +2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) +2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) +2019-06-10T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) +2019-06-10T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) +2019-06-10T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) +2019-06-10T20:51 [ERROR] null (SerialStructure.java:74) +2019-06-10T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 41 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 0.049 s (IndexTreeMessyTest.java:408) +2019-06-10T20:51 [INFO][TIMER] Temps parcours des résultats: 1.53 millis (IndexTreeMessyTest.java:416) +2019-06-10T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 7.95 millis (IndexTreeMessyTest.java:425) +2019-06-10T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.079 s (IndexTreeMessyTest.java:439) +2019-06-10T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) +2019-06-10T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) +2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) +2019-06-10T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) +2019-06-10T20:51 [INFO] 40 (HashMapTests.java:27) +2019-06-10T20:51 [INFO][MEMORY] Mem usage début - 88,271 (ParallelLoaderTest.java:56) +2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T20:51 [INFO][MEMORY] 90,319 (ParallelLoader.java:114) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T20 \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/chunks/000000000000012 b/repo/testdata/repo_8k/00000/chunks/000000000000012 new file mode 100644 index 0000000..612b2ee --- /dev/null +++ b/repo/testdata/repo_8k/00000/chunks/000000000000012 @@ -0,0 +1,87 @@ +:51 [INFO] Parsing terminé !! temps écoulé = 0.842 s (ParallelLoader.java:217) +2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) +2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T20:51 [INFO][MEMORY] 209,650 (ParallelLoader.java:114) +2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.056 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.637 s (ParallelLoader.java:217) +2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T20:51 [INFO][MEMORY] Mem usage fin - 189,538 (ParallelLoaderTest.java:114) +2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (ParallelLoaderTest.java:115) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 12,384 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.122 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 1.54 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (IndexMemDicStorageTests.java:183) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.312 s (IndexMemDicStorageTests.java:83) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.172 s (IndexMemDicStorageTests.java:86) +2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) +2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) +2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) +2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) +2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) +2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) +2019-06-10T21:24 [INFO][MEMORY] 108,772 (ParallelLoader.java:114) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.168 sactiveThreadNb = 0 (ParallelLoader.java:150) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) +2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.889 s (ParallelLoader.java:217) +2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.904 s (NewLoaderAndIndexTest.java:189) +2019-06-10T21:24 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) +2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) +2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) +2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) +2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) +2019-06-10T21:24 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) +2019-06-10T21:24 [INFO] totalLength = 66666 (IndexMemDic.java:217) +2019-06-10T21:24 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) +2019-06-10T21:24 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) +2019-06-10T21:24 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.99 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 234,123 (Loader.java:117) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 154,808 (Loader.java:117) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.013 s (Loader.java:116) +2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.63 millis (Loader.java:116) +2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) +2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 154,918 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,713 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,525 (Loader.java:138) +2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) +2019-06-10T21:24 [INFO][MEMORY] 155,614 (Loader.java:138) +2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 12.07 s (IndexTreeMessyTest.java:314) +2019-06-10T21:24 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) +2019-06-10T21:24 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,064 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,576 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) +2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) +2019-06-10T21:24 [INFO] Table.flu \ No newline at end of file diff --git a/repo/testdata/repo_8k/00000/files b/repo/testdata/repo_8k/00000/files new file mode 100644 index 0000000..56431d4 Binary files /dev/null and b/repo/testdata/repo_8k/00000/files differ diff --git a/repo/testdata/repo_8k/00000/hashes b/repo/testdata/repo_8k/00000/hashes new file mode 100644 index 0000000..ec622b2 Binary files /dev/null and b/repo/testdata/repo_8k/00000/hashes differ diff --git a/repo/testdata/repo_8k/00000/recipe b/repo/testdata/repo_8k/00000/recipe new file mode 100644 index 0000000..66bde3d Binary files /dev/null and b/repo/testdata/repo_8k/00000/recipe differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000000 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000000 new file mode 100644 index 0000000..11c1cab Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000000 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000001 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000001 new file mode 100644 index 0000000..b212097 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000001 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000002 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000002 new file mode 100644 index 0000000..15d2a66 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000002 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000003 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000003 new file mode 100644 index 0000000..d3c7997 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000003 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000004 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000004 new file mode 100644 index 0000000..be13110 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000004 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000005 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000005 new file mode 100644 index 0000000..a92e0a6 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000005 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000006 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000006 new file mode 100644 index 0000000..47e1d74 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000006 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000007 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000007 new file mode 100644 index 0000000..2428c10 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000007 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000008 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000008 new file mode 100644 index 0000000..0524676 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000008 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000009 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000009 new file mode 100644 index 0000000..e89cb70 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000009 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000010 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000010 new file mode 100644 index 0000000..140ca5f Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000010 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000011 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000011 new file mode 100644 index 0000000..b61efae Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000011 differ diff --git a/repo/testdata/repo_8k_zlib/00000/chunks/000000000000012 b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000012 new file mode 100644 index 0000000..677a3a8 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/chunks/000000000000012 differ diff --git a/repo/testdata/repo_8k_zlib/00000/files b/repo/testdata/repo_8k_zlib/00000/files new file mode 100644 index 0000000..70ca423 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/files differ diff --git a/repo/testdata/repo_8k_zlib/00000/hashes b/repo/testdata/repo_8k_zlib/00000/hashes new file mode 100644 index 0000000..ec622b2 Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/hashes differ diff --git a/repo/testdata/repo_8k_zlib/00000/recipe b/repo/testdata/repo_8k_zlib/00000/recipe new file mode 100644 index 0000000..d8c788b Binary files /dev/null and b/repo/testdata/repo_8k_zlib/00000/recipe differ diff --git a/repo_test.go b/repo_test.go deleted file mode 100644 index 9e7b09b..0000000 --- a/repo_test.go +++ /dev/null @@ -1,494 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "sync" - "testing" - - "github.com/chmduquesne/rollinghash/rabinkarp64" - "github.com/n-peugnet/dna-backup/logger" - "github.com/n-peugnet/dna-backup/sketch" - "github.com/n-peugnet/dna-backup/testutils" - "github.com/n-peugnet/dna-backup/utils" -) - -func chunkCompare(t *testing.T, dataDir string, repo *Repo, testFiles []string, chunkCount int) { - reader, writer := io.Pipe() - chunks := make(chan []byte) - files := listFiles(dataDir) - go concatFiles(&files, writer) - go repo.chunkStream(reader, chunks) - - offset := 0 - buff := make([]byte, repo.chunkSize*chunkCount) - for _, f := range testFiles { - content, err := os.ReadFile(filepath.Join(dataDir, f)) - if err != nil { - t.Error("Error reading test data file") - } - for i := range content { - buff[offset+i] = content[i] - } - offset += len(content) - } - - i := 0 - for c := range chunks { - start := i * repo.chunkSize - end := (i + 1) * repo.chunkSize - if end > offset { - end = offset - } - content := buff[start:end] - if bytes.Compare(c, content) != 0 { - t.Errorf("Chunk %d does not match file content", i) - // for i, b := range c { - // fmt.Printf("E: %d, A: %d\n", b, content[i]) - // } - t.Log("Expected: ", c[:10], "...", c[end%repo.chunkSize-10:]) - t.Log("Actual:", content) - } - i++ - } - if i != chunkCount { - t.Errorf("Incorrect number of chunks: %d, should be: %d", i, chunkCount) - } -} - -func (r *Repo) chunkStream(stream io.Reader, chunks chan<- []byte) { - var buff []byte - var prev, read = r.chunkSize, 0 - var err error - - for err != io.EOF { - if prev == r.chunkSize { - buff = make([]byte, r.chunkSize) - prev, err = stream.Read(buff) - } else { - read, err = stream.Read(buff[prev:]) - prev += read - } - if err != nil && err != io.EOF { - logger.Error(err) - } - if prev == r.chunkSize { - chunks <- buff - } - } - if prev != r.chunkSize { - chunks <- buff[:prev] - } - close(chunks) -} - -func storeChunks(dest string, chunks <-chan []byte) { - i := 0 - for c := range chunks { - path := filepath.Join(dest, fmt.Sprintf(chunkIdFmt, i)) - err := os.WriteFile(path, c, 0664) - if err != nil { - logger.Error(err) - } - i++ - } -} - -// hashChunks calculates the hashes for a channel of chunks. -// For each chunk, both a fingerprint (hash over the full content) and a sketch -// (resemblance hash based on maximal values of regions) are calculated and -// stored in an hashmap. -func (r *Repo) hashChunks(chunks <-chan IdentifiedChunk) { - for c := range chunks { - r.hashChunk(c.GetId(), c.Reader()) - } -} - -// hashChunk calculates the hashes for a chunk and store them in th repo hashmaps. -func (r *Repo) hashChunk(id *ChunkId, reader io.Reader) (fp uint64, sk []uint64) { - var buffSk bytes.Buffer - var buffFp bytes.Buffer - var wg sync.WaitGroup - reader = io.TeeReader(reader, &buffSk) - io.Copy(&buffFp, reader) - wg.Add(2) - go r.makeFingerprint(id, &buffFp, &wg, &fp) - go r.makeSketch(id, &buffSk, &wg, &sk) - wg.Wait() - if _, e := r.fingerprints[fp]; e { - logger.Error(fp, " already exists in fingerprints map") - } - r.fingerprints[fp] = id - r.sketches.Set(sk, id) - return -} - -func (r *Repo) makeFingerprint(id *ChunkId, reader io.Reader, wg *sync.WaitGroup, ret *uint64) { - defer wg.Done() - hasher := rabinkarp64.NewFromPol(r.pol) - io.Copy(hasher, reader) - *ret = hasher.Sum64() -} - -func (r *Repo) makeSketch(id *ChunkId, reader io.Reader, wg *sync.WaitGroup, ret *[]uint64) { - defer wg.Done() - *ret, _ = sketch.SketchChunk(reader, r.pol, r.chunkSize, r.sketchWSize, r.sketchSfCount, r.sketchFCount) -} - -func TestReadFiles1(t *testing.T) { - tmpDir := t.TempDir() - repo := NewRepo(tmpDir) - chunkCount := 590/repo.chunkSize + 1 - dataDir := filepath.Join("testdata", "logs", "1") - files := []string{"logTest.log"} - chunkCompare(t, dataDir, repo, files, chunkCount) -} - -func TestReadFiles2(t *testing.T) { - tmpDir := t.TempDir() - repo := NewRepo(tmpDir) - chunkCount := 22899/repo.chunkSize + 1 - dataDir := filepath.Join("testdata", "logs", "2") - files := []string{"csvParserTest.log", "slipdb.log"} - chunkCompare(t, dataDir, repo, files, chunkCount) -} - -func TestReadFiles3(t *testing.T) { - tmpDir := t.TempDir() - repo := NewRepo(tmpDir) - chunkCount := 119398/repo.chunkSize + 1 - dataDir := filepath.Join("testdata", "logs") - files := []string{ - filepath.Join("1", "logTest.log"), - filepath.Join("2", "csvParserTest.log"), - filepath.Join("2", "slipdb.log"), - filepath.Join("3", "indexingTreeTest.log"), - } - chunkCompare(t, dataDir, repo, files, chunkCount) -} - -func TestSymlinks(t *testing.T) { - var output bytes.Buffer - multi := io.MultiWriter(&output, os.Stderr) - logger.SetOutput(multi) - defer logger.SetOutput(os.Stderr) - tmpDir, err := filepath.EvalSymlinks(t.TempDir()) - if err != nil { - t.Fatal(err) - } - extDir := t.TempDir() - f, err := os.Create(filepath.Join(tmpDir, "existing")) - if err != nil { - t.Fatal(err) - } - if n, err := f.Write([]byte("\n")); err != nil { - t.Fatal(n, err) - } - if err = f.Close(); err != nil { - t.Fatal(err) - } - if err = os.Symlink(extDir, filepath.Join(tmpDir, "linkexternal")); err != nil { - t.Fatal(err) - } - if err = os.Symlink(filepath.Join(tmpDir, "notexisting"), filepath.Join(tmpDir, "linknotexisting")); err != nil { - t.Fatal(err) - } - if err = os.Symlink(filepath.Join(tmpDir, "existing"), filepath.Join(tmpDir, "linkexisting")); err != nil { - t.Fatal(err) - } - files := listFiles(tmpDir) - fmt.Println(files) - testutils.AssertLen(t, 3, files, "Files") - if files[0].Link != "" { - t.Error("existing should not be a link, actual:", files[0].Link) - } - expected := string(filepath.Separator) + "existing" - if files[1].Link != expected { - t.Error("linkexisting should point to", expected, "actual:", files[1].Link) - } - if !strings.Contains(output.String(), "linkexternal") { - t.Errorf("log should contain a warning for linkexternal, actual %q", &output) - } -} - -func TestLoadChunks(t *testing.T) { - resultDir := t.TempDir() - dataDir := filepath.Join("testdata", "logs") - repo := NewRepo(resultDir) - repo.chunkReadWrapper = utils.NopReadWrapper - repo.chunkWriteWrapper = utils.NopWriteWrapper - resultVersion := filepath.Join(resultDir, "00000") - resultChunks := filepath.Join(resultVersion, chunksName) - os.MkdirAll(resultChunks, 0775) - reader1, writer1 := io.Pipe() - reader2, writer2 := io.Pipe() - chunks1 := make(chan []byte, 16) - chunks2 := make(chan []byte, 16) - chunks3 := make(chan IdentifiedChunk, 16) - files := listFiles(dataDir) - go concatFiles(&files, writer1) - go concatFiles(&files, writer2) - go repo.chunkStream(reader1, chunks1) - go repo.chunkStream(reader2, chunks2) - storeChunks(resultChunks, chunks1) - versions := []string{resultVersion} - go repo.loadChunks(versions, chunks3) - - i := 0 - for c2 := range chunks2 { - c3 := <-chunks3 - buff, err := io.ReadAll(c3.Reader()) - if err != nil { - t.Errorf("Error reading from chunk %d: %s\n", c3, err) - } - if bytes.Compare(c2, buff) != 0 { - t.Errorf("Chunk %d does not match file content", i) - t.Log("Expected: ", c2[:10], "...") - t.Log("Actual:", buff) - } - i++ - } -} - -func prepareChunks(dataDir string, repo *Repo, streamFunc func(*[]File, io.WriteCloser)) { - resultVersion := filepath.Join(repo.path, "00000") - resultChunks := filepath.Join(resultVersion, chunksName) - os.MkdirAll(resultChunks, 0775) - reader := getDataStream(dataDir, streamFunc) - chunks := make(chan []byte, 16) - go repo.chunkStream(reader, chunks) - storeChunks(resultChunks, chunks) -} - -func getDataStream(dataDir string, streamFunc func(*[]File, io.WriteCloser)) io.Reader { - reader, writer := io.Pipe() - files := listFiles(dataDir) - go streamFunc(&files, writer) - return reader -} - -func TestBsdiff(t *testing.T) { - logger.SetLevel(3) - defer logger.SetLevel(4) - resultDir := t.TempDir() - repo := NewRepo(resultDir) - dataDir := filepath.Join("testdata", "logs") - addedFile1 := filepath.Join(dataDir, "2", "slogTest.log") - addedFile2 := filepath.Join(dataDir, "3", "slogTest.log") - // Store initial chunks - prepareChunks(dataDir, repo, concatFiles) - - // Modify data - ioutil.WriteFile(addedFile1, []byte("hello"), 0664) - defer os.Remove(addedFile1) - ioutil.WriteFile(addedFile2, make([]byte, 4000), 0664) - defer os.Remove(addedFile2) - - // configure repo - repo.patcher = Bsdiff{} - repo.differ = Bsdiff{} - repo.chunkReadWrapper = utils.NopReadWrapper - repo.chunkWriteWrapper = utils.NopWriteWrapper - - // Load previously stored chunks - oldChunks := make(chan IdentifiedChunk, 16) - versions := repo.loadVersions() - go repo.loadChunks(versions, oldChunks) - repo.hashChunks(oldChunks) - - // Read new data - newVersion := len(versions) - newPath := filepath.Join(repo.path, fmt.Sprintf(versionFmt, newVersion)) - os.MkdirAll(newPath, 0775) - reader := getDataStream(dataDir, concatFiles) - storeQueue := make(chan chunkData, 10) - storeEnd := make(chan bool) - go repo.storageWorker(newVersion, storeQueue, storeEnd) - recipe, _ := repo.matchStream(reader, storeQueue, newVersion, 0) - close(storeQueue) - <-storeEnd - newChunks := extractDeltaChunks(recipe) - testutils.AssertLen(t, 2, newChunks, "New delta chunks:") - for _, c := range newChunks { - logger.Info("Patch size:", len(c.Patch)) - if len(c.Patch) >= repo.chunkSize/10 { - t.Errorf("Bsdiff of chunk is too large: %d", len(c.Patch)) - } - } -} - -func TestCommit(t *testing.T) { - dest := t.TempDir() - source := filepath.Join("testdata", "logs") - expected := filepath.Join("testdata", "repo_8k") - repo := NewRepo(dest) - repo.patcher = Bsdiff{} - repo.differ = Bsdiff{} - repo.chunkReadWrapper = utils.NopReadWrapper - repo.chunkWriteWrapper = utils.NopWriteWrapper - - repo.Commit(source) - assertSameTree(t, assertCompatibleRepoFile, expected, dest, "Commit") -} - -func TestCommitZlib(t *testing.T) { - dest := t.TempDir() - source := filepath.Join("testdata", "logs") - expected := filepath.Join("testdata", "repo_8k_zlib") - repo := NewRepo(dest) - repo.patcher = Bsdiff{} - repo.differ = Bsdiff{} - repo.chunkReadWrapper = utils.ZlibReader - repo.chunkWriteWrapper = utils.ZlibWriter - - repo.Commit(source) - assertSameTree(t, assertCompatibleRepoFile, expected, dest, "Commit") -} - -func TestRestore(t *testing.T) { - logger.SetLevel(2) - defer logger.SetLevel(4) - dest := t.TempDir() - source := filepath.Join("testdata", "repo_8k") - expected := filepath.Join("testdata", "logs") - repo := NewRepo(source) - repo.patcher = Bsdiff{} - repo.differ = Bsdiff{} - repo.chunkReadWrapper = utils.NopReadWrapper - repo.chunkWriteWrapper = utils.NopWriteWrapper - - repo.Restore(dest) - assertSameTree(t, testutils.AssertSameFile, expected, dest, "Restore") -} - -func TestRestoreZlib(t *testing.T) { - logger.SetLevel(2) - defer logger.SetLevel(4) - dest := t.TempDir() - source := filepath.Join("testdata", "repo_8k_zlib") - expected := filepath.Join("testdata", "logs") - repo := NewRepo(source) - repo.patcher = Bsdiff{} - repo.differ = Bsdiff{} - repo.chunkReadWrapper = utils.ZlibReader - repo.chunkWriteWrapper = utils.ZlibWriter - - repo.Restore(dest) - assertSameTree(t, testutils.AssertSameFile, expected, dest, "Restore") -} - -func TestRoundtrip(t *testing.T) { - logger.SetLevel(2) - defer logger.SetLevel(4) - temp := t.TempDir() - dest := t.TempDir() - source := filepath.Join("testdata", "logs") - repo1 := NewRepo(temp) - repo2 := NewRepo(temp) - - repo1.Commit(source) - // Commit a second version, just to see if it does not destroy everything - // TODO: check that the second version is indeed empty - repo1.Commit(source) - repo2.Restore(dest) - - assertSameTree(t, assertCompatibleRepoFile, source, dest, "Commit") -} - -func TestHashes(t *testing.T) { - dest := t.TempDir() - source := filepath.Join("testdata", "repo_8k") - - chunks := make(chan IdentifiedChunk, 16) - storeQueue := make(chan chunkData, 16) - storeEnd := make(chan bool) - - repo1 := NewRepo(source) - repo1.chunkReadWrapper = utils.NopReadWrapper - repo1.chunkWriteWrapper = utils.NopWriteWrapper - go repo1.loadChunks([]string{filepath.Join(source, "00000")}, chunks) - for c := range chunks { - fp, sk := repo1.hashChunk(c.GetId(), c.Reader()) - content, err := io.ReadAll(c.Reader()) - if err != nil { - t.Error(err) - } - storeQueue <- chunkData{ - hashes: chunkHashes{fp, sk}, - content: content, - id: c.GetId(), - } - } - repo2 := NewRepo(dest) - repo2.chunkReadWrapper = utils.NopReadWrapper - repo2.chunkWriteWrapper = utils.NopWriteWrapper - os.MkdirAll(filepath.Join(dest, "00000", chunksName), 0775) - go repo2.storageWorker(0, storeQueue, storeEnd) - close(storeQueue) - <-storeEnd - testutils.AssertLen(t, 0, repo2.fingerprints, "Fingerprints") - testutils.AssertLen(t, 0, repo2.sketches, "Sketches") - repo2.loadHashes([]string{filepath.Join(dest, "00000")}) - testutils.AssertSame(t, repo1.fingerprints, repo2.fingerprints, "Fingerprint maps") - testutils.AssertSame(t, repo1.sketches, repo2.sketches, "Sketches maps") -} - -func assertSameTree(t *testing.T, apply func(t *testing.T, expected string, actual string, prefix string), expected string, actual string, prefix string) { - actualFiles := listFiles(actual) - expectedFiles := listFiles(expected) - efCount := len(expectedFiles) - if efCount <= 0 { - t.Fatalf("No expected files: %d", efCount) - } - afCount := len(actualFiles) - if efCount != afCount { - t.Fatalf("Incorrect number of files: %d, should be %d", afCount, efCount) - } - for i, ef := range expectedFiles { - af := actualFiles[i] - efRelPath := ef.Path[len(expected):] - afRelPath := af.Path[len(actual):] - if efRelPath != afRelPath { - t.Fatalf("File path '%s' does not match '%s'", afRelPath, efRelPath) - } - apply(t, ef.Path, af.Path, prefix) - } -} - -func assertCompatibleRepoFile(t *testing.T, expected string, actual string, prefix string) { - if filepath.Base(expected) == filesName { - // TODO: Check Filelist file - // eFiles := loadFileList(expected) - // aFiles := loadFileList(actual) - // testutils.AssertLen(t, len(eFiles), aFiles, prefix) - // for i, eFile := range eFiles { - // eFile.Path = filepath.FromSlash(eFile.Path) - // if eFile != aFiles[i] { - // t.Fatal(prefix, "file entry do not match:", aFiles[i], ", expected:", eFile) - // } - // } - } else if filepath.Base(expected) == recipeName { - // TODO: Check Recipe files - // eRecipe := loadRecipe(expected) - // aRecipe := loadRecipe(actual) - // testutils.AssertSame(t, eRecipe, aRecipe, prefix+"recipe") - } else if filepath.Base(expected) == hashesName { - // Hashes file is checked in TestHashes - } else { - // Chunk content file - testutils.AssertSameFile(t, expected, actual, prefix) - } -} - -func assertChunkContent(t *testing.T, expected []byte, c Chunk, prefix string) { - buf, err := io.ReadAll(c.Reader()) - if err != nil { - t.Fatal(err) - } - testutils.AssertSame(t, expected, buf, prefix+" Chunk content") -} diff --git a/tar.go b/tar.go index feb3b34..02b8957 100644 --- a/tar.go +++ b/tar.go @@ -6,9 +6,10 @@ import ( "os" "github.com/n-peugnet/dna-backup/logger" + "github.com/n-peugnet/dna-backup/repo" ) -func streamFilesTar(files []File, stream io.WriteCloser) { +func streamFilesTar(files []repo.File, stream io.WriteCloser) { tarStream := tar.NewWriter(stream) for _, f := range files { file, err := os.Open(f.Path) diff --git a/testdata/.gitattributes b/testdata/.gitattributes deleted file mode 100644 index 6313b56..0000000 --- a/testdata/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -* text=auto eol=lf diff --git a/testdata/logs/1/logTest.log b/testdata/logs/1/logTest.log deleted file mode 100644 index 6bf31c3..0000000 --- a/testdata/logs/1/logTest.log +++ /dev/null @@ -1,10 +0,0 @@ -2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) -2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) -2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T20:51 [INFO] testInfoString (LogTest.java:52) -2019-06-05T20:51 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T22:41 [INFO] testInfoString (LogTest.java:52) -2019-06-05T22:41 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T23:02 [INFO] testInfoString (LogTest.java:52) -2019-06-05T23:02 [INFO][YO] testInfoString (LogTest.java:57) diff --git a/testdata/logs/2/csvParserTest.log b/testdata/logs/2/csvParserTest.log deleted file mode 100644 index 328720a..0000000 --- a/testdata/logs/2/csvParserTest.log +++ /dev/null @@ -1,63 +0,0 @@ -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.52 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 160,128 (Loader.java:117) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 231,367 (Loader.java:138) -2019-06-05T20:13 [INFO][TIMER] Temps pris par le parsing: 0.898 s (LoaderTest.java:69) -2019-06-05T20:13 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:14 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.46 millis (Loader.java:116) -2019-06-05T20:14 [INFO][MEMORY] 267,348 (Loader.java:117) -2019-06-05T20:14 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:14 [INFO][MEMORY] 231,359 (Loader.java:138) -2019-06-05T20:14 [INFO][TIMER] Temps pris par le parsing: 0.839 s (LoaderTest.java:69) -2019-06-05T20:14 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.61 millis (Loader.java:116) -2019-06-05T20:51 [INFO][MEMORY] 189,561 (Loader.java:117) -2019-06-05T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:51 [INFO][MEMORY] 168,686 (Loader.java:138) -2019-06-05T20:51 [INFO][TIMER] Temps pris par le parsing: 0.930 s (LoaderTest.java:69) -2019-06-05T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T22:41 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.83 millis (Loader.java:116) -2019-06-05T22:41 [INFO][MEMORY] 189,494 (Loader.java:117) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 168,619 (Loader.java:138) -2019-06-05T22:41 [INFO][TIMER] Temps pris par le parsing: 0.882 s (LoaderTest.java:69) -2019-06-05T22:41 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) -2019-06-05T23:02 [INFO][MEMORY] 189,551 (Loader.java:117) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 168,676 (Loader.java:138) -2019-06-05T23:02 [INFO][TIMER] Temps pris par le parsing: 1.04 s (LoaderTest.java:69) -2019-06-05T23:02 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.82 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 82,958 (Loader.java:117) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 163,350 (Loader.java:138) -2019-06-10T20:34 [INFO][TIMER] Temps pris par le parsing: 0.938 s (LoaderTest.java:69) -2019-06-10T20:34 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) -2019-06-10T20:50 [INFO][MEMORY] 82,957 (Loader.java:117) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 163,349 (Loader.java:138) -2019-06-10T20:50 [INFO][TIMER] Temps pris par le parsing: 0.853 s (LoaderTest.java:69) -2019-06-10T20:50 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.51 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 189,538 (Loader.java:117) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 168,663 (Loader.java:138) -2019-06-10T20:51 [INFO][TIMER] Temps pris par le parsing: 1.27 s (LoaderTest.java:69) -2019-06-10T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.58 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 189,539 (Loader.java:117) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 168,664 (Loader.java:138) -2019-06-10T21:24 [INFO][TIMER] Temps pris par le parsing: 1.86 s (LoaderTest.java:69) -2019-06-10T21:24 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) diff --git a/testdata/logs/2/slipdb.log b/testdata/logs/2/slipdb.log deleted file mode 100644 index 1f0e3b4..0000000 --- a/testdata/logs/2/slipdb.log +++ /dev/null @@ -1,185 +0,0 @@ -2019-06-05T14:47 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T14:49 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T14:49 [INFO][MEMORY] 229,457 (ParallelLoader.java:114) -2019-06-05T14:49 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.522 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T14:49 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T14:49 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T14:49 [INFO] Parsing terminé !! temps écoulé = 2.78 s (ParallelLoader.java:217) -2019-06-05T14:49 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T14:53 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T14:53 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T14:53 [INFO][MEMORY] 227,095 (ParallelLoader.java:114) -2019-06-05T14:53 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.433 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T14:53 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T14:53 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T14:53 [INFO] Parsing terminé !! temps écoulé = 2.37 s (ParallelLoader.java:217) -2019-06-05T14:53 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T14:54 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) -2019-06-05T14:59 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) -2019-06-05T14:59 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) -2019-06-05T15:08 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:09 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:09 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) -2019-06-05T15:09 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:09 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) -2019-06-05T15:10 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:10 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) -2019-06-05T15:12 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:12 [INFO][MEMORY] 178,759 (ParallelLoader.java:114) -2019-06-05T15:12 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.220 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:12 [INFO] Parsing terminé !! temps écoulé = 2.25 s (ParallelLoader.java:217) -2019-06-05T15:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:13 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:14 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:328) -2019-06-05T15:14 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T15:14 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:328) -2019-06-05T15:14 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T15:14 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:14 [INFO][MEMORY] 308,168 (ParallelLoader.java:114) -2019-06-05T15:14 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.052 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:14 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:14 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:14 [INFO] Parsing terminé !! temps écoulé = 1.32 s (ParallelLoader.java:217) -2019-06-05T15:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:14 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:15 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) -2019-06-05T15:15 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:17 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:26 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:27 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:27 [INFO][MEMORY] 179,237 (ParallelLoader.java:114) -2019-06-05T15:27 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.098 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:27 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:27 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:27 [INFO] Parsing terminé !! temps écoulé = 2.22 s (ParallelLoader.java:217) -2019-06-05T15:27 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:31 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:31 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:31 [INFO][MEMORY] 178,254 (ParallelLoader.java:114) -2019-06-05T15:31 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.118 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:31 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:31 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:31 [INFO] Parsing terminé !! temps écoulé = 2.11 s (ParallelLoader.java:217) -2019-06-05T15:31 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:41 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:41 [INFO][MEMORY] 180,287 (ParallelLoader.java:114) -2019-06-05T15:41 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.195 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:41 [INFO] Parsing terminé !! temps écoulé = 1.92 s (ParallelLoader.java:217) -2019-06-05T15:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:42 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:43 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:43 [INFO][MEMORY] 178,731 (ParallelLoader.java:114) -2019-06-05T15:43 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.141 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:43 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:43 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:43 [INFO] Parsing terminé !! temps écoulé = 2.03 s (ParallelLoader.java:217) -2019-06-05T15:43 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:35 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T18:35 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:35 [INFO][MEMORY] 229,265 (ParallelLoader.java:114) -2019-06-05T18:35 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.641 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:35 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:35 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:35 [INFO] Parsing terminé !! temps écoulé = 3.04 s (ParallelLoader.java:217) -2019-06-05T18:35 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:36 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:36 [INFO][MEMORY] 326,934 (ParallelLoader.java:114) -2019-06-05T18:36 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.072 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:36 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:36 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:36 [INFO] Parsing terminé !! temps écoulé = 1.17 s (ParallelLoader.java:217) -2019-06-05T18:36 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:37 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:37 [INFO][MEMORY] 537,997 (ParallelLoader.java:114) -2019-06-05T18:37 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:37 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:37 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:37 [INFO] Parsing terminé !! temps écoulé = 1.23 s (ParallelLoader.java:217) -2019-06-05T18:37 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:38 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) -2019-06-05T18:38 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T18:38 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:38 [INFO][MEMORY] 498,154 (ParallelLoader.java:114) -2019-06-05T18:38 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.031 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:38 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:38 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:38 [INFO] Parsing terminé !! temps écoulé = 1.10 s (ParallelLoader.java:217) -2019-06-05T18:38 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:39 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) -2019-06-05T18:39 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T18:40 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:40 [INFO][MEMORY] 493,228 (ParallelLoader.java:114) -2019-06-05T18:40 [INFO] Loader : nombre de résultats (total) lus = 500000 temps écoulé = 0.032 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:40 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:40 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:40 [INFO] Parsing terminé !! temps écoulé = 1.02 s (ParallelLoader.java:217) -2019-06-05T18:40 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:14 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T20:14 [ERROR] null (SerialStructure.java:74) -2019-06-05T20:15 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) -2019-06-05T20:15 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T20:15 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:15 [INFO][MEMORY] 188,768 (ParallelLoader.java:114) -2019-06-05T20:15 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.301 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:15 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:15 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:15 [INFO] Parsing terminé !! temps écoulé = 2.42 s (ParallelLoader.java:217) -2019-06-05T20:15 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:15 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) -2019-06-05T20:15 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T20:16 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:16 [INFO][MEMORY] 457,918 (ParallelLoader.java:114) -2019-06-05T20:16 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.049 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:16 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:16 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:16 [INFO] Parsing terminé !! temps écoulé = 1.34 s (ParallelLoader.java:217) -2019-06-05T20:16 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:19 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:19 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:19 [INFO][MEMORY] 180,323 (ParallelLoader.java:114) -2019-06-05T20:19 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.178 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:19 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:19 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:19 [INFO] Parsing terminé !! temps écoulé = 1.92 s (ParallelLoader.java:217) -2019-06-05T20:19 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:24 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:25 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:43 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:43 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:43 [INFO][MEMORY] 179,894 (ParallelLoader.java:114) -2019-06-05T20:43 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.133 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:43 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:43 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:43 [INFO] Parsing terminé !! temps écoulé = 2.14 s (ParallelLoader.java:217) -2019-06-05T20:43 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:50 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:50 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T21:56 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T21:56 [ERROR] null -2019-06-05T22:41 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T22:41 [ERROR] null -2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T22:42 [ERROR] null -2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T22:42 [ERROR] null -2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T22:42 [ERROR] null -2019-06-05T23:03 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T23:03 [ERROR] null -2019-06-05T23:04 [INFO] SerialStructure.writeStructure : OK ! -2019-06-05T23:04 [INFO] totalLength = 0 -2019-06-05T23:04 [INFO] totalLength = 0 -2019-06-05T23:04 [INFO] PARSE : memusage init = -2019-06-05T23:04 [INFO][MEMORY] 111,311 -2019-06-05T23:04 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.256 sactiveThreadNb = 0 -2019-06-05T23:04 [WARNING] Fail to save entry number 0: incorrect data -2019-06-05T23:04 [INFO] Parsing terminé !! temps écoulé = 2.57 s -2019-06-05T23:04 [INFO] PARSE : Nombre de lignes (local) = 100000 -2019-06-05T23:04 [INFO] totalLength = 99999 -2019-06-05T23:04 [INFO] totalLength = 99999 diff --git a/testdata/logs/3/indexingTreeTest.log b/testdata/logs/3/indexingTreeTest.log deleted file mode 100644 index b115b45..0000000 --- a/testdata/logs/3/indexingTreeTest.log +++ /dev/null @@ -1,999 +0,0 @@ -2019-06-05T00:03 [ERROR] config.json (No such file or directory) (Database.java:33) -2019-06-05T00:03 [INFO][MEMORY] Mem usage début - 2,459 (ParallelLoaderTest.java:56) -2019-06-05T20:12 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T20:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:12 [INFO][MEMORY] 27,443 (ParallelLoader.java:114) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.214 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:12 [INFO] Parsing terminé !! temps écoulé = 1.44 s (ParallelLoader.java:217) -2019-06-05T20:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:12 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.45 s (IndexMemDicStorageTests.java:183) -2019-06-05T20:12 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.477 s (IndexMemDicStorageTests.java:83) -2019-06-05T20:12 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.054 s (IndexMemDicStorageTests.java:86) -2019-06-05T20:12 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-05T20:12 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-05T20:12 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-05T20:12 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-05T20:12 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T20:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:12 [INFO][MEMORY] 190,485 (ParallelLoader.java:114) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.064 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:12 [INFO] Parsing terminé !! temps écoulé = 0.908 s (ParallelLoader.java:217) -2019-06-05T20:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:12 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.917 s (NewLoaderAndIndexTest.java:189) -2019-06-05T20:12 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T20:12 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-05T20:12 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-05T20:12 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-05T20:12 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-05T20:12 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-05T20:12 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 66666 (NewLoaderAndIndexTest.java:109) -2019-06-05T20:12 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T20:12 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.57 millis (Loader.java:116) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) -2019-06-05T20:12 [INFO][MEMORY] 401,153 (Loader.java:117) -2019-06-05T20:12 [INFO][MEMORY] 401,153 (Loader.java:117) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.33 millis (Loader.java:116) -2019-06-05T20:12 [INFO][MEMORY] 273,252 (Loader.java:117) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.97 millis (Loader.java:116) -2019-06-05T20:12 [INFO][MEMORY] 273,252 (Loader.java:117) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 216,262 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 221,186 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 218,771 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 216,694 (Loader.java:138) -2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.57 s (IndexTreeMessyTest.java:314) -2019-06-05T20:13 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T20:13 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,432 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2bef09c0 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@62ce72ff (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@58a63629 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7de843ef (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2e1af49b (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3fd2322d (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@56ac5c80 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69e49a81 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1023150a (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ece1e79 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@ab20d7c (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@474fae39 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4f4c789f (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13f9ad9 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b7cae6f (Table.java:338) -2019-06-05T20:13 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T20:13 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T20:13 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T20:13 [ERROR] null (SerialStructure.java:74) -2019-06-05T20:13 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 48 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.051 s (IndexTreeMessyTest.java:408) -2019-06-05T20:13 [INFO][TIMER] Temps parcours des résultats: 1.72 millis (IndexTreeMessyTest.java:416) -2019-06-05T20:13 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 9 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.011 s (IndexTreeMessyTest.java:425) -2019-06-05T20:13 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.125 s (IndexTreeMessyTest.java:439) -2019-06-05T20:13 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.041 s (IndexTreeMessyTest.java:449) -2019-06-05T20:13 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T20:13 [INFO] 40 (HashMapTests.java:27) -2019-06-05T20:13 [INFO][MEMORY] Mem usage début - 157,206 (ParallelLoaderTest.java:56) -2019-06-05T20:13 [INFO] Parsing de csvName = E:/csv/yellow_tripdata_2015-01.csv (ParallelLoaderTest.java:64) -2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:13 [INFO][MEMORY] 28,086 (ParallelLoader.java:114) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.182 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:13 [INFO] Parsing terminé !! temps écoulé = 1.43 s (ParallelLoader.java:217) -2019-06-05T20:13 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.44 s (IndexMemDicStorageTests.java:183) -2019-06-05T20:13 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.356 s (IndexMemDicStorageTests.java:83) -2019-06-05T20:13 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.032 s (IndexMemDicStorageTests.java:86) -2019-06-05T20:13 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-05T20:13 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-05T20:13 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-05T20:13 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:13 [INFO][MEMORY] 184,068 (ParallelLoader.java:114) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.052 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:13 [INFO] Parsing terminé !! temps écoulé = 0.941 s (ParallelLoader.java:217) -2019-06-05T20:13 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.945 s (NewLoaderAndIndexTest.java:189) -2019-06-05T20:13 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T20:13 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-05T20:13 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-05T20:13 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-05T20:13 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-05T20:13 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-05T20:13 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 66666 (NewLoaderAndIndexTest.java:109) -2019-06-05T20:13 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T20:13 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.70 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 276,783 (Loader.java:117) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.69 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 341,077 (Loader.java:117) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.09 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 358,336 (Loader.java:117) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 5.74 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 358,336 (Loader.java:117) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 218,799 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 218,840 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 223,379 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 216,694 (Loader.java:138) -2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.18 s (IndexTreeMessyTest.java:314) -2019-06-05T20:13 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T20:13 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2881ad47 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2bef09c0 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@62ce72ff (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@58a63629 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7de843ef (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2e1af49b (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3fd2322d (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@56ac5c80 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69e49a81 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1023150a (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ece1e79 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@ab20d7c (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@474fae39 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4f4c789f (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13f9ad9 (Table.java:338) -2019-06-05T20:13 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T20:13 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T20:13 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T20:13 [ERROR] null (SerialStructure.java:74) -2019-06-05T20:13 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 58 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.061 s (IndexTreeMessyTest.java:408) -2019-06-05T20:13 [INFO][TIMER] Temps parcours des résultats: 1.25 millis (IndexTreeMessyTest.java:416) -2019-06-05T20:13 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 8.20 millis (IndexTreeMessyTest.java:425) -2019-06-05T20:13 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.072 s (IndexTreeMessyTest.java:439) -2019-06-05T20:13 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.040 s (IndexTreeMessyTest.java:449) -2019-06-05T20:13 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 3 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T20:13 [INFO] 40 (HashMapTests.java:27) -2019-06-05T20:13 [INFO][MEMORY] Mem usage début - 157,225 (ParallelLoaderTest.java:56) -2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:13 [INFO][MEMORY] 166,226 (ParallelLoader.java:114) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.041 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:14 [INFO] Parsing terminé !! temps écoulé = 0.638 s (ParallelLoader.java:217) -2019-06-05T20:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:14 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T20:14 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:14 [INFO][MEMORY] 422,898 (ParallelLoader.java:114) -2019-06-05T20:14 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.040 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:14 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:14 [INFO] Parsing terminé !! temps écoulé = 0.624 s (ParallelLoader.java:217) -2019-06-05T20:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:14 [INFO][MEMORY] Mem usage fin - 267,348 (ParallelLoaderTest.java:113) -2019-06-05T20:14 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.46 s (ParallelLoaderTest.java:114) -2019-06-05T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:50 [INFO][MEMORY] 12,904 (ParallelLoader.java:114) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.175 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:50 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:50 [INFO] Parsing terminé !! temps écoulé = 2.03 s (ParallelLoader.java:217) -2019-06-05T20:50 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 2.04 s (IndexMemDicStorageTests.java:183) -2019-06-05T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:50 [INFO][MEMORY] 157,253 (ParallelLoader.java:114) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.111 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:50 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:50 [INFO] Parsing terminé !! temps écoulé = 0.781 s (ParallelLoader.java:217) -2019-06-05T20:50 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.787 s (NewLoaderAndIndexTest.java:189) -2019-06-05T20:50 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T20:50 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T20:50 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 5.82 millis (Loader.java:116) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.66 millis (Loader.java:116) -2019-06-05T20:50 [INFO][MEMORY] 134,560 (Loader.java:117) -2019-06-05T20:50 [INFO][MEMORY] 134,560 (Loader.java:117) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.11 millis (Loader.java:116) -2019-06-05T20:50 [INFO][MEMORY] 152,063 (Loader.java:117) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.011 s (Loader.java:116) -2019-06-05T20:50 [INFO][MEMORY] 168,447 (Loader.java:117) -2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:50 [INFO][MEMORY] 155,685 (Loader.java:138) -2019-06-05T20:50 [INFO][MEMORY] 156,197 (Loader.java:138) -2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:50 [INFO][MEMORY] 155,691 (Loader.java:138) -2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:50 [INFO][MEMORY] 155,633 (Loader.java:138) -2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.14 s (IndexTreeMessyTest.java:314) -2019-06-05T20:50 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T20:50 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,657 (IndexTreeDic.java:517) -2019-06-05T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b0b0854 (Table.java:338) -2019-06-05T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,657 (IndexTreeDic.java:517) -2019-06-05T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@19bb07ed (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,033 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@10e41621 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,033 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@353d0772 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,082 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2667f029 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,594 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@67a20f67 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@57c758ac (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@a9cd3b1 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13e39c73 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,915 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@64cd705f (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@9225652 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@654f0d9c (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6a400542 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6580cfdd (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7e0b85f9 (Table.java:338) -2019-06-05T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T20:51 [ERROR] null (SerialStructure.java:74) -2019-06-05T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 31 millis (IndexTreeDic.java:1164) -2019-06-05T20:51 [INFO][TIMER] Temps total recherche: 0.037 s (IndexTreeMessyTest.java:408) -2019-06-05T20:51 [INFO][TIMER] Temps parcours des résultats: 1.96 millis (IndexTreeMessyTest.java:416) -2019-06-05T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) -2019-06-05T20:51 [INFO][TIMER] Temps total recherche: 7.48 millis (IndexTreeMessyTest.java:425) -2019-06-05T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.119 s (IndexTreeMessyTest.java:439) -2019-06-05T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.040 s (IndexTreeMessyTest.java:449) -2019-06-05T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 1 millis (IndexTreeDic.java:1164) -2019-06-05T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T20:51 [INFO] 40 (HashMapTests.java:27) -2019-06-05T20:51 [INFO][MEMORY] Mem usage début - 88,295 (ParallelLoaderTest.java:56) -2019-06-05T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:51 [INFO][MEMORY] 90,343 (ParallelLoader.java:114) -2019-06-05T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.064 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:51 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:51 [INFO] Parsing terminé !! temps écoulé = 0.862 s (ParallelLoader.java:217) -2019-06-05T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:51 [INFO][MEMORY] 227,179 (ParallelLoader.java:114) -2019-06-05T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.055 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:51 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:51 [INFO] Parsing terminé !! temps écoulé = 0.676 s (ParallelLoader.java:217) -2019-06-05T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:51 [INFO][MEMORY] Mem usage fin - 189,561 (ParallelLoaderTest.java:113) -2019-06-05T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.63 s (ParallelLoaderTest.java:114) -2019-06-05T21:56 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T21:56 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T21:56 [INFO][MEMORY] 13,925 (ParallelLoader.java:114) -2019-06-05T21:56 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.226 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T21:56 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T21:56 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T21:57 [INFO] Parsing terminé !! temps écoulé = 33.57 s (ParallelLoader.java:217) -2019-06-05T21:57 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T21:57 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 33.66 s (IndexMemDicStorageTests.java:183) -2019-06-05T21:57 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T21:57 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T21:57 [INFO][MEMORY] 58,123 (ParallelLoader.java:114) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.206 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T21:57 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T21:57 [INFO] Parsing terminé !! temps écoulé = 15.87 s (ParallelLoader.java:217) -2019-06-05T21:57 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T21:57 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 16.28 s (NewLoaderAndIndexTest.java:189) -2019-06-05T21:57 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T21:57 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T21:57 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 9.32 millis (Loader.java:116) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 7.80 millis (Loader.java:116) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 8.95 millis (Loader.java:116) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 8.32 millis (Loader.java:116) -2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) -2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) -2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) -2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 155,396 (Loader.java:138) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 155,945 (Loader.java:138) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 155,523 (Loader.java:138) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 155,565 (Loader.java:138) -2019-06-05T22:41 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 35.40 s (IndexTreeMessyTest.java:314) -2019-06-05T22:41 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T22:41 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,457 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@e056f20 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,457 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b0b0854 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,834 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@19bb07ed (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,834 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@10e41621 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,883 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@353d0772 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,067 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2667f029 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,067 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@67a20f67 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,579 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@57c758ac (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,579 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@a9cd3b1 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13e39c73 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@64cd705f (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@9225652 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@654f0d9c (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6a400542 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,139 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6580cfdd (Table.java:338) -2019-06-05T22:41 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T22:41 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T22:41 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T22:41 [ERROR] null (SerialStructure.java:74) -2019-06-05T22:41 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 50 millis (IndexTreeDic.java:1164) -2019-06-05T22:41 [INFO][TIMER] Temps total recherche: 0.058 s (IndexTreeMessyTest.java:408) -2019-06-05T22:41 [INFO][TIMER] Temps parcours des résultats: 1.04 millis (IndexTreeMessyTest.java:416) -2019-06-05T22:41 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 8 millis (IndexTreeDic.java:1164) -2019-06-05T22:41 [INFO][TIMER] Temps total recherche: 8.60 millis (IndexTreeMessyTest.java:425) -2019-06-05T22:41 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.122 s (IndexTreeMessyTest.java:439) -2019-06-05T22:41 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.072 s (IndexTreeMessyTest.java:449) -2019-06-05T22:41 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 22 millis (IndexTreeDic.java:1164) -2019-06-05T22:41 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T22:41 [INFO] 40 (HashMapTests.java:27) -2019-06-05T22:41 [INFO][MEMORY] Mem usage début - 88,230 (ParallelLoaderTest.java:56) -2019-06-05T22:41 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T22:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T22:41 [INFO][MEMORY] 90,316 (ParallelLoader.java:114) -2019-06-05T22:41 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.068 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T22:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T22:41 [INFO] Parsing terminé !! temps écoulé = 1.09 s (ParallelLoader.java:217) -2019-06-05T22:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T22:41 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T22:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T22:41 [INFO][MEMORY] 415,931 (ParallelLoader.java:114) -2019-06-05T22:41 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.070 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T22:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T22:41 [INFO] Parsing terminé !! temps écoulé = 0.968 s (ParallelLoader.java:217) -2019-06-05T22:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T22:41 [INFO][MEMORY] Mem usage fin - 189,494 (ParallelLoaderTest.java:113) -2019-06-05T22:41 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 2.17 s (ParallelLoaderTest.java:114) -2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T23:02 [INFO][MEMORY] 12,404 (ParallelLoader.java:114) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.154 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.61 s (ParallelLoader.java:217) -2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.62 s (IndexMemDicStorageTests.java:183) -2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-05T23:02 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.266 s (IndexMemDicStorageTests.java:83) -2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-05T23:02 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.188 s (IndexMemDicStorageTests.java:86) -2019-06-05T23:02 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-05T23:02 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-05T23:02 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-05T23:02 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T23:02 [INFO][MEMORY] 103,147 (ParallelLoader.java:114) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.174 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.01 s (ParallelLoader.java:217) -2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.02 s (NewLoaderAndIndexTest.java:189) -2019-06-05T23:02 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-05T23:02 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-05T23:02 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-05T23:02 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-05T23:02 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-05T23:02 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-05T23:02 [INFO] totalLength = 66666 (IndexMemDic.java:217) -2019-06-05T23:02 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) -2019-06-05T23:02 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T23:02 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.57 millis (Loader.java:116) -2019-06-05T23:02 [INFO][MEMORY] 223,892 (Loader.java:117) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.56 millis (Loader.java:116) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.00 millis (Loader.java:116) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.58 millis (Loader.java:116) -2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) -2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) -2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 155,525 (Loader.java:138) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 155,474 (Loader.java:138) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 156,042 (Loader.java:138) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 155,626 (Loader.java:138) -2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.03 s (IndexTreeMessyTest.java:314) -2019-06-05T23:02 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T23:02 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,650 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,038 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,414 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,414 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,464 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,674 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,674 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5ae50ce6 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6f96c77 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@be64738 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ba9ad43 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49d904ec (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@48e4374 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3d680b5a (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b5d6a01 (Table.java:338) -2019-06-05T23:02 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T23:02 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T23:02 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T23:02 [ERROR] null (SerialStructure.java:74) -2019-06-05T23:02 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 43 millis (IndexTreeDic.java:1164) -2019-06-05T23:02 [INFO][TIMER] Temps total recherche: 0.051 s (IndexTreeMessyTest.java:408) -2019-06-05T23:02 [INFO][TIMER] Temps parcours des résultats: 0.73 millis (IndexTreeMessyTest.java:416) -2019-06-05T23:02 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) -2019-06-05T23:02 [INFO][TIMER] Temps total recherche: 6.60 millis (IndexTreeMessyTest.java:425) -2019-06-05T23:02 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.096 s (IndexTreeMessyTest.java:439) -2019-06-05T23:02 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.069 s (IndexTreeMessyTest.java:449) -2019-06-05T23:02 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 10 millis (IndexTreeDic.java:1164) -2019-06-05T23:02 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T23:02 [INFO] 40 (HashMapTests.java:27) -2019-06-05T23:02 [INFO][MEMORY] Mem usage début - 88,284 (ParallelLoaderTest.java:56) -2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T23:02 [INFO][MEMORY] 90,332 (ParallelLoader.java:114) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.051 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.11 s (ParallelLoader.java:217) -2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T23:02 [INFO][MEMORY] 275,258 (ParallelLoader.java:114) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.076 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 0.696 s (ParallelLoader.java:217) -2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T23:02 [INFO][MEMORY] Mem usage fin - 189,551 (ParallelLoaderTest.java:114) -2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.87 s (ParallelLoaderTest.java:115) -2019-06-10T20:19 [ERROR] config.json (No such file or directory) (Database.java:33) -2019-06-10T20:19 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:19 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.37 millis (Loader.java:116) -2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.93 millis (Loader.java:116) -2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.37 millis (Loader.java:116) -2019-06-10T20:19 [INFO][MEMORY] 10,178 (Loader.java:117) -2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.010 s (Loader.java:116) -2019-06-10T20:19 [INFO][MEMORY] 10,178 (Loader.java:117) -2019-06-10T20:19 [INFO][MEMORY] 19,394 (Loader.java:117) -2019-06-10T20:19 [INFO][MEMORY] 19,394 (Loader.java:117) -2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:19 [INFO][MEMORY] 150,062 (Loader.java:138) -2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:19 [INFO][MEMORY] 150,767 (Loader.java:138) -2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:19 [INFO][MEMORY] 150,245 (Loader.java:138) -2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:19 [INFO][MEMORY] 150,333 (Loader.java:138) -2019-06-10T20:19 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.79 s (IndexTreeMessyTest.java:314) -2019-06-10T20:19 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:19 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,359 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b083826 (Table.java:338) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,871 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@105fece7 (Table.java:338) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,871 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ec300f1 (Table.java:338) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,381 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@482cd91f (Table.java:338) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,381 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@123f1134 (Table.java:338) -2019-06-10T20:34 [ERROR] config.json (No such file or directory) (Database.java:33) -2019-06-10T20:34 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:34 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.70 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 9,153 (Loader.java:117) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.74 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 14,616 (Loader.java:117) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.42 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 20,072 (Loader.java:117) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.10 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 32,705 (Loader.java:117) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 149,780 (Loader.java:138) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 150,170 (Loader.java:138) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 150,367 (Loader.java:138) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 150,299 (Loader.java:138) -2019-06-10T20:34 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.04 s (IndexTreeMessyTest.java:314) -2019-06-10T20:34 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:34 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,323 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7d68ef40 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,323 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5b0abc94 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,699 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@75c072cb (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,699 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1f1c7bf6 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,748 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25b485ba (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2b546384 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5d740a0f (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@214b199c (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@20d3d15a (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,349 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2893de87 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@55ca8de8 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2c34f934 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@12d3a4e9 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@240237d2 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25a65b77 (Table.java:338) -2019-06-10T20:34 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-10T20:34 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-10T20:34 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-10T20:34 [ERROR] null (SerialStructure.java:74) -2019-06-10T20:34 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 44 millis (IndexTreeDic.java:1164) -2019-06-10T20:34 [INFO][TIMER] Temps total recherche: 0.053 s (IndexTreeMessyTest.java:408) -2019-06-10T20:34 [INFO][TIMER] Temps parcours des résultats: 1.23 millis (IndexTreeMessyTest.java:416) -2019-06-10T20:34 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 14 millis (IndexTreeDic.java:1164) -2019-06-10T20:34 [INFO][TIMER] Temps total recherche: 0.015 s (IndexTreeMessyTest.java:425) -2019-06-10T20:34 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.119 s (IndexTreeMessyTest.java:439) -2019-06-10T20:34 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.045 s (IndexTreeMessyTest.java:449) -2019-06-10T20:34 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 2 millis (IndexTreeDic.java:1164) -2019-06-10T20:34 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-10T20:34 [INFO] 40 (HashMapTests.java:27) -2019-06-10T20:34 [INFO][MEMORY] Mem usage début - 82,958 (ParallelLoaderTest.java:56) -2019-06-10T20:49 [ERROR] config.json (No such file or directory) (Database.java:33) -2019-06-10T20:49 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:49 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.20 millis (Loader.java:116) -2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.69 millis (Loader.java:116) -2019-06-10T20:49 [INFO][MEMORY] 9,820 (Loader.java:117) -2019-06-10T20:49 [INFO][MEMORY] 9,820 (Loader.java:117) -2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.41 millis (Loader.java:116) -2019-06-10T20:49 [INFO][MEMORY] 19,381 (Loader.java:117) -2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.16 millis (Loader.java:116) -2019-06-10T20:49 [INFO][MEMORY] 24,844 (Loader.java:117) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 150,362 (Loader.java:138) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 150,428 (Loader.java:138) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 150,268 (Loader.java:138) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 150,648 (Loader.java:138) -2019-06-10T20:50 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.96 s (IndexTreeMessyTest.java:314) -2019-06-10T20:50 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:50 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,382 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b083826 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,382 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@105fece7 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,758 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ec300f1 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,758 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@482cd91f (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,807 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@123f1134 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7d68ef40 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5b0abc94 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@75c072cb (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1f1c7bf6 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,319 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25b485ba (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,406 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2b546384 (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,406 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5d740a0f (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@214b199c (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@20d3d15a (Table.java:338) -2019-06-10T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 153,926 (IndexTreeDic.java:517) -2019-06-10T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2893de87 (Table.java:338) -2019-06-10T20:50 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-10T20:50 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-10T20:50 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-10T20:50 [ERROR] null (SerialStructure.java:74) -2019-06-10T20:50 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 36 millis (IndexTreeDic.java:1164) -2019-06-10T20:50 [INFO][TIMER] Temps total recherche: 0.044 s (IndexTreeMessyTest.java:408) -2019-06-10T20:50 [INFO][TIMER] Temps parcours des résultats: 0.73 millis (IndexTreeMessyTest.java:416) -2019-06-10T20:50 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) -2019-06-10T20:50 [INFO][TIMER] Temps total recherche: 7.60 millis (IndexTreeMessyTest.java:425) -2019-06-10T20:50 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.070 s (IndexTreeMessyTest.java:439) -2019-06-10T20:50 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.043 s (IndexTreeMessyTest.java:449) -2019-06-10T20:50 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-10T20:50 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 3 millis (IndexTreeDic.java:1164) -2019-06-10T20:50 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-10T20:50 [INFO] 40 (HashMapTests.java:27) -2019-06-10T20:50 [INFO][MEMORY] Mem usage début - 82,957 (ParallelLoaderTest.java:56) -2019-06-10T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-10T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:50 [INFO][MEMORY] 11,889 (ParallelLoader.java:114) -2019-06-10T20:50 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.114 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 1.68 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.68 s (IndexMemDicStorageTests.java:183) -2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T20:51 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.208 s (IndexMemDicStorageTests.java:83) -2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T20:51 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.176 s (IndexMemDicStorageTests.java:86) -2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:51 [INFO][MEMORY] 108,834 (ParallelLoader.java:114) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.141 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.890 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.900 s (NewLoaderAndIndexTest.java:189) -2019-06-10T20:51 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-10T20:51 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-10T20:51 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-10T20:51 [INFO] totalLength = 66666 (IndexMemDic.java:217) -2019-06-10T20:51 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) -2019-06-10T20:51 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:51 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.67 millis (Loader.java:116) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.13 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) -2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.14 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 178,179 (Loader.java:117) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.71 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 199,412 (Loader.java:117) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,755 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,569 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 156,154 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,614 (Loader.java:138) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 4.79 s (IndexTreeMessyTest.java:314) -2019-06-10T20:51 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:51 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,063 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) -2019-06-10T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-10T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-10T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-10T20:51 [ERROR] null (SerialStructure.java:74) -2019-06-10T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 41 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 0.049 s (IndexTreeMessyTest.java:408) -2019-06-10T20:51 [INFO][TIMER] Temps parcours des résultats: 1.53 millis (IndexTreeMessyTest.java:416) -2019-06-10T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 7.95 millis (IndexTreeMessyTest.java:425) -2019-06-10T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.079 s (IndexTreeMessyTest.java:439) -2019-06-10T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) -2019-06-10T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-10T20:51 [INFO] 40 (HashMapTests.java:27) -2019-06-10T20:51 [INFO][MEMORY] Mem usage début - 88,271 (ParallelLoaderTest.java:56) -2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:51 [INFO][MEMORY] 90,319 (ParallelLoader.java:114) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.842 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:51 [INFO][MEMORY] 209,650 (ParallelLoader.java:114) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.056 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.637 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO][MEMORY] Mem usage fin - 189,538 (ParallelLoaderTest.java:114) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (ParallelLoaderTest.java:115) -2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T21:24 [INFO][MEMORY] 12,384 (ParallelLoader.java:114) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.122 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 1.54 s (ParallelLoader.java:217) -2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (IndexMemDicStorageTests.java:183) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.312 s (IndexMemDicStorageTests.java:83) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.172 s (IndexMemDicStorageTests.java:86) -2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T21:24 [INFO][MEMORY] 108,772 (ParallelLoader.java:114) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.168 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.889 s (ParallelLoader.java:217) -2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.904 s (NewLoaderAndIndexTest.java:189) -2019-06-10T21:24 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-10T21:24 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-10T21:24 [INFO] totalLength = 66666 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) -2019-06-10T21:24 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T21:24 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.99 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 234,123 (Loader.java:117) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 154,808 (Loader.java:117) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.013 s (Loader.java:116) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.63 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) -2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 154,918 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,713 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,525 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,614 (Loader.java:138) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 12.07 s (IndexTreeMessyTest.java:314) -2019-06-10T21:24 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T21:24 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,064 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,576 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,176 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) -2019-06-10T21:24 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-10T21:24 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-10T21:24 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-10T21:24 [ERROR] null (SerialStructure.java:74) -2019-06-10T21:24 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 37 millis (IndexTreeDic.java:1164) -2019-06-10T21:24 [INFO][TIMER] Temps total recherche: 0.044 s (IndexTreeMessyTest.java:408) -2019-06-10T21:24 [INFO][TIMER] Temps parcours des résultats: 0.59 millis (IndexTreeMessyTest.java:416) -2019-06-10T21:24 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) -2019-06-10T21:24 [INFO][TIMER] Temps total recherche: 7.06 millis (IndexTreeMessyTest.java:425) -2019-06-10T21:24 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.071 s (IndexTreeMessyTest.java:439) -2019-06-10T21:24 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) -2019-06-10T21:24 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-10T21:24 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 8 millis (IndexTreeDic.java:1164) -2019-06-10T21:24 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-10T21:24 [INFO] 40 (HashMapTests.java:27) -2019-06-10T21:24 [INFO][MEMORY] Mem usage début - 88,272 (ParallelLoaderTest.java:56) -2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T21:24 [INFO][MEMORY] 90,320 (ParallelLoader.java:114) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.046 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.898 s (ParallelLoader.java:217) -2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T21:24 [INFO][MEMORY] 232,213 (ParallelLoader.java:114) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.047 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.709 s (ParallelLoader.java:217) -2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T21:24 [INFO][MEMORY] Mem usage fin - 189,539 (ParallelLoaderTest.java:114) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.68 s (ParallelLoaderTest.java:115) diff --git a/testdata/repo_8k/00000/chunks/000000000000000 b/testdata/repo_8k/00000/chunks/000000000000000 deleted file mode 100644 index ac78620..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000000 +++ /dev/null @@ -1,85 +0,0 @@ -2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) -2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T20:13 [INFO] testInfoString (LogTest.java:52) -2019-06-05T20:13 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T20:51 [INFO] testInfoString (LogTest.java:52) -2019-06-05T20:51 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T22:41 [INFO] testInfoString (LogTest.java:52) -2019-06-05T22:41 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T23:02 [INFO] testInfoString (LogTest.java:52) -2019-06-05T23:02 [INFO][YO] testInfoString (LogTest.java:57) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.52 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 160,128 (Loader.java:117) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 231,367 (Loader.java:138) -2019-06-05T20:13 [INFO][TIMER] Temps pris par le parsing: 0.898 s (LoaderTest.java:69) -2019-06-05T20:13 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:14 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.46 millis (Loader.java:116) -2019-06-05T20:14 [INFO][MEMORY] 267,348 (Loader.java:117) -2019-06-05T20:14 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:14 [INFO][MEMORY] 231,359 (Loader.java:138) -2019-06-05T20:14 [INFO][TIMER] Temps pris par le parsing: 0.839 s (LoaderTest.java:69) -2019-06-05T20:14 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.61 millis (Loader.java:116) -2019-06-05T20:51 [INFO][MEMORY] 189,561 (Loader.java:117) -2019-06-05T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:51 [INFO][MEMORY] 168,686 (Loader.java:138) -2019-06-05T20:51 [INFO][TIMER] Temps pris par le parsing: 0.930 s (LoaderTest.java:69) -2019-06-05T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T22:41 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.83 millis (Loader.java:116) -2019-06-05T22:41 [INFO][MEMORY] 189,494 (Loader.java:117) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 168,619 (Loader.java:138) -2019-06-05T22:41 [INFO][TIMER] Temps pris par le parsing: 0.882 s (LoaderTest.java:69) -2019-06-05T22:41 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) -2019-06-05T23:02 [INFO][MEMORY] 189,551 (Loader.java:117) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 168,676 (Loader.java:138) -2019-06-05T23:02 [INFO][TIMER] Temps pris par le parsing: 1.04 s (LoaderTest.java:69) -2019-06-05T23:02 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.82 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 82,958 (Loader.java:117) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 163,350 (Loader.java:138) -2019-06-10T20:34 [INFO][TIMER] Temps pris par le parsing: 0.938 s (LoaderTest.java:69) -2019-06-10T20:34 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-10T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) -2019-06-10T20:50 [INFO][MEMORY] 82,957 (Loader.java:117) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:50 [INFO][MEMORY] 163,349 (Loader.java:138) -2019-06-10T20:50 [INFO][TIMER] Temps pris par le parsing: 0.853 s (LoaderTest.java:69) -2019-06-10T20:50 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.51 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 189,538 (Loader.java:117) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 168,663 (Loader.java:138) -2019-06-10T20:51 [INFO][TIMER] Temps pris par le parsing: 1.27 s (LoaderTest.java:69) -2019-06-10T20:51 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.58 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 189,539 (Loader.java:117) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 168,664 (Loader.java:138) -2019-06-10T21:24 [INFO][TIMER] Temps pris par le parsing: 1.86 s (LoaderTest.java:69) -2019-06-10T21:24 [DEBUG][line][70] [2, Thu Apr 09 19:29:33 CEST 2015, Thu Apr 09 19:37:09 CEST 2015, 1, 0.83, -73.98651885986328, 40.76189422607422, 1, N, -73.97399139404297, 40.760414123535156, 1, 6.5, 1.0, 0.5, 1.66, 0.0, 0.3, 9.96] (LoaderTest.java:97) -2019-06-05T14:47 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T14:49 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T14:49 [INFO][MEMORY] 229,457 (ParallelLoader.java:114) -2019-06-05T14:49 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.522 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T14:49 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T14:49 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T14:49 [INFO] Parsing terminé !! temps écoulé = 2.78 s (ParallelLoader.java:217) -2019-06-05T14:49 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T14:53 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T14:53 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T14:53 [INFO][MEMORY] 227,095 (ParallelLoader.java:114) -2019-06-05T14:53 [INFO] Loader : nombre de résulta \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000001 b/testdata/repo_8k/00000/chunks/000000000000001 deleted file mode 100644 index 3e728e2..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000001 +++ /dev/null @@ -1,89 +0,0 @@ -ts (total) lus = 100000 temps écoulé = 0.433 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T14:53 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T14:53 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T14:53 [INFO] Parsing terminé !! temps écoulé = 2.37 s (ParallelLoader.java:217) -2019-06-05T14:53 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T14:54 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) -2019-06-05T14:59 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) -2019-06-05T14:59 [ERROR] dicStartIndex trop grand (IndexMemDic.java:461) -2019-06-05T15:08 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:09 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:09 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) -2019-06-05T15:09 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:09 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) -2019-06-05T15:10 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:10 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) -2019-06-05T15:12 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:12 [INFO][MEMORY] 178,759 (ParallelLoader.java:114) -2019-06-05T15:12 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.220 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:12 [INFO] Parsing terminé !! temps écoulé = 2.25 s (ParallelLoader.java:217) -2019-06-05T15:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:13 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:14 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:328) -2019-06-05T15:14 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T15:14 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:328) -2019-06-05T15:14 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T15:14 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:14 [INFO][MEMORY] 308,168 (ParallelLoader.java:114) -2019-06-05T15:14 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.052 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:14 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:14 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:14 [INFO] Parsing terminé !! temps écoulé = 1.32 s (ParallelLoader.java:217) -2019-06-05T15:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:14 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:15 [ERROR] delta != 0 1 dicCurrentIndex=0 realLinePos=0 (IndexMemDic.java:477) -2019-06-05T15:15 [WARNING] value can't be parsed in selected column : vendor_id 1.0 (FilterTermEntity.java:35) -2019-06-05T15:17 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:26 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:27 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:27 [INFO][MEMORY] 179,237 (ParallelLoader.java:114) -2019-06-05T15:27 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.098 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:27 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:27 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:27 [INFO] Parsing terminé !! temps écoulé = 2.22 s (ParallelLoader.java:217) -2019-06-05T15:27 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:31 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:31 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:31 [INFO][MEMORY] 178,254 (ParallelLoader.java:114) -2019-06-05T15:31 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.118 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:31 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:31 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:31 [INFO] Parsing terminé !! temps écoulé = 2.11 s (ParallelLoader.java:217) -2019-06-05T15:31 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:41 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:41 [INFO][MEMORY] 180,287 (ParallelLoader.java:114) -2019-06-05T15:41 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.195 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:41 [INFO] Parsing terminé !! temps écoulé = 1.92 s (ParallelLoader.java:217) -2019-06-05T15:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T15:42 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T15:43 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T15:43 [INFO][MEMORY] 178,731 (ParallelLoader.java:114) -2019-06-05T15:43 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.141 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T15:43 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T15:43 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T15:43 [INFO] Parsing terminé !! temps écoulé = 2.03 s (ParallelLoader.java:217) -2019-06-05T15:43 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:35 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T18:35 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:35 [INFO][MEMORY] 229,265 (ParallelLoader.java:114) -2019-06-05T18:35 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.641 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:35 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:35 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:35 [INFO] Parsing terminé !! temps écoulé = 3.04 s (ParallelLoader.java:217) -2019-06-05T18:35 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:36 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:36 [INFO][MEMORY] 326,934 (ParallelLoader.java:114) -2019-06-05T18:36 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.072 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:36 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:36 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:36 [INFO] Parsing terminé !! temps écoulé = 1.17 s (ParallelLoader.java:217) -2019-06-05T18:36 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:37 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:37 [INFO][MEMORY] 537,997 (ParallelLoader.java:114) -2019-06-05T18:37 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadN \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000002 b/testdata/repo_8k/00000/chunks/000000000000002 deleted file mode 100644 index 079ca90..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000002 +++ /dev/null @@ -1,98 +0,0 @@ -b = 0 (ParallelLoader.java:150) -2019-06-05T18:37 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:37 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:37 [INFO] Parsing terminé !! temps écoulé = 1.23 s (ParallelLoader.java:217) -2019-06-05T18:37 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:38 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) -2019-06-05T18:38 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T18:38 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:38 [INFO][MEMORY] 498,154 (ParallelLoader.java:114) -2019-06-05T18:38 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.031 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:38 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:38 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:38 [INFO] Parsing terminé !! temps écoulé = 1.10 s (ParallelLoader.java:217) -2019-06-05T18:38 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T18:39 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) -2019-06-05T18:39 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T18:40 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T18:40 [INFO][MEMORY] 493,228 (ParallelLoader.java:114) -2019-06-05T18:40 [INFO] Loader : nombre de résultats (total) lus = 500000 temps écoulé = 0.032 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T18:40 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T18:40 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T18:40 [INFO] Parsing terminé !! temps écoulé = 1.02 s (ParallelLoader.java:217) -2019-06-05T18:40 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:14 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T20:14 [ERROR] null (SerialStructure.java:74) -2019-06-05T20:15 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) -2019-06-05T20:15 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T20:15 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:15 [INFO][MEMORY] 188,768 (ParallelLoader.java:114) -2019-06-05T20:15 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.301 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:15 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:15 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:15 [INFO] Parsing terminé !! temps écoulé = 2.42 s (ParallelLoader.java:217) -2019-06-05T20:15 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:15 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 0 (Table.java:334) -2019-06-05T20:15 [INFO] SerialStructure.writeStructure : OK ! (SerialStructure.java:54) -2019-06-05T20:16 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:16 [INFO][MEMORY] 457,918 (ParallelLoader.java:114) -2019-06-05T20:16 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.049 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:16 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:16 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:16 [INFO] Parsing terminé !! temps écoulé = 1.34 s (ParallelLoader.java:217) -2019-06-05T20:16 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:19 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:19 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:19 [INFO][MEMORY] 180,323 (ParallelLoader.java:114) -2019-06-05T20:19 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.178 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:19 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:19 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:19 [INFO] Parsing terminé !! temps écoulé = 1.92 s (ParallelLoader.java:217) -2019-06-05T20:19 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:24 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:25 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:43 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:43 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:43 [INFO][MEMORY] 179,894 (ParallelLoader.java:114) -2019-06-05T20:43 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.133 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:43 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:43 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:43 [INFO] Parsing terminé !! temps écoulé = 2.14 s (ParallelLoader.java:217) -2019-06-05T20:43 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:50 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T20:50 [INFO] SerialStructure.loadStructure : OK ! (SerialStructure.java:71) -2019-06-05T21:56 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T21:56 [ERROR] null -2019-06-05T22:41 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T22:41 [ERROR] null -2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T22:42 [ERROR] null -2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T22:42 [ERROR] null -2019-06-05T22:42 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T22:42 [ERROR] null -2019-06-05T23:03 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. -2019-06-05T23:03 [ERROR] null -2019-06-05T23:04 [INFO] SerialStructure.writeStructure : OK ! -2019-06-05T23:04 [INFO] totalLength = 0 -2019-06-05T23:04 [INFO] totalLength = 0 -2019-06-05T23:04 [INFO] PARSE : memusage init = -2019-06-05T23:04 [INFO][MEMORY] 111,311 -2019-06-05T23:04 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.256 sactiveThreadNb = 0 -2019-06-05T23:04 [WARNING] Fail to save entry number 0: incorrect data -2019-06-05T23:04 [INFO] Parsing terminé !! temps écoulé = 2.57 s -2019-06-05T23:04 [INFO] PARSE : Nombre de lignes (local) = 100000 -2019-06-05T23:04 [INFO] totalLength = 99999 -2019-06-05T23:04 [INFO] totalLength = 99999 -2019-06-05T00:03 [ERROR] config.json (No such file or directory) (Database.java:33) -2019-06-05T00:03 [INFO][MEMORY] Mem usage début - 2,459 (ParallelLoaderTest.java:56) -2019-06-05T20:12 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T20:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:12 [INFO][MEMORY] 27,443 (ParallelLoader.java:114) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.214 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:12 [INFO] Parsing terminé !! temps écoulé = 1.44 s (ParallelLoader.java:217) -2019-06-05T20:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:12 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.45 s (IndexMemDicStorageTests.java:183) -2019-06-05T20:12 [INFO][TIME \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000003 b/testdata/repo_8k/00000/chunks/000000000000003 deleted file mode 100644 index 8720f6b..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000003 +++ /dev/null @@ -1,85 +0,0 @@ -R] Temps pris pour classer la 1ère fois: 0.477 s (IndexMemDicStorageTests.java:83) -2019-06-05T20:12 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.054 s (IndexMemDicStorageTests.java:86) -2019-06-05T20:12 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-05T20:12 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-05T20:12 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-05T20:12 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-05T20:12 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T20:12 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:12 [INFO][MEMORY] 190,485 (ParallelLoader.java:114) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.064 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:12 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:12 [INFO] Parsing terminé !! temps écoulé = 0.908 s (ParallelLoader.java:217) -2019-06-05T20:12 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:12 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.917 s (NewLoaderAndIndexTest.java:189) -2019-06-05T20:12 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T20:12 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-05T20:12 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-05T20:12 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-05T20:12 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-05T20:12 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-05T20:12 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 66666 (NewLoaderAndIndexTest.java:109) -2019-06-05T20:12 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T20:12 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.57 millis (Loader.java:116) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.56 millis (Loader.java:116) -2019-06-05T20:12 [INFO][MEMORY] 401,153 (Loader.java:117) -2019-06-05T20:12 [INFO][MEMORY] 401,153 (Loader.java:117) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.33 millis (Loader.java:116) -2019-06-05T20:12 [INFO][MEMORY] 273,252 (Loader.java:117) -2019-06-05T20:12 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:12 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.97 millis (Loader.java:116) -2019-06-05T20:12 [INFO][MEMORY] 273,252 (Loader.java:117) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 216,262 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 221,186 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 218,771 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 216,694 (Loader.java:138) -2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.57 s (IndexTreeMessyTest.java:314) -2019-06-05T20:13 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T20:13 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,432 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2bef09c0 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@62ce72ff (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@58a63629 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7de843ef (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2e1af49b (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3fd2322d (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@56ac5c80 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69e49a81 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1023150a (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ece1e79 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@ab20d7c (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@474fae39 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4f4c789f (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13f9ad9 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,693 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b7cae6f (Table.java:338) -2019-06-05T20:13 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T20:13 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T20:13 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T20:13 [ERROR] null (SerialStructure.java:74) -2019-06-05T20:13 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 48 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.051 s (IndexTreeMessyTest.java:408) -2019-06-05T20:13 [INFO][TIMER] Temps parcours des résultats: 1.72 millis (IndexTreeMessyTest.ja \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000004 b/testdata/repo_8k/00000/chunks/000000000000004 deleted file mode 100644 index f05dda7..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000004 +++ /dev/null @@ -1,87 +0,0 @@ -va:416) -2019-06-05T20:13 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 9 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.011 s (IndexTreeMessyTest.java:425) -2019-06-05T20:13 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.125 s (IndexTreeMessyTest.java:439) -2019-06-05T20:13 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.041 s (IndexTreeMessyTest.java:449) -2019-06-05T20:13 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T20:13 [INFO] 40 (HashMapTests.java:27) -2019-06-05T20:13 [INFO][MEMORY] Mem usage début - 157,206 (ParallelLoaderTest.java:56) -2019-06-05T20:13 [INFO] Parsing de csvName = E:/csv/yellow_tripdata_2015-01.csv (ParallelLoaderTest.java:64) -2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:13 [INFO][MEMORY] 28,086 (ParallelLoader.java:114) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.182 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:13 [INFO] Parsing terminé !! temps écoulé = 1.43 s (ParallelLoader.java:217) -2019-06-05T20:13 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.44 s (IndexMemDicStorageTests.java:183) -2019-06-05T20:13 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.356 s (IndexMemDicStorageTests.java:83) -2019-06-05T20:13 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.032 s (IndexMemDicStorageTests.java:86) -2019-06-05T20:13 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-05T20:13 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-05T20:13 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-05T20:13 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:13 [INFO][MEMORY] 184,068 (ParallelLoader.java:114) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.052 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:13 [INFO] Parsing terminé !! temps écoulé = 0.941 s (ParallelLoader.java:217) -2019-06-05T20:13 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.945 s (NewLoaderAndIndexTest.java:189) -2019-06-05T20:13 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T20:13 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-05T20:13 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-05T20:13 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-05T20:13 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-05T20:13 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-05T20:13 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 66666 (NewLoaderAndIndexTest.java:109) -2019-06-05T20:13 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T20:13 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.70 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 276,783 (Loader.java:117) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.69 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 341,077 (Loader.java:117) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.09 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 358,336 (Loader.java:117) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 5.74 millis (Loader.java:116) -2019-06-05T20:13 [INFO][MEMORY] 358,336 (Loader.java:117) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 218,799 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 218,840 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 223,379 (Loader.java:138) -2019-06-05T20:13 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:13 [INFO][MEMORY] 216,694 (Loader.java:138) -2019-06-05T20:13 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.18 s (IndexTreeMessyTest.java:314) -2019-06-05T20:13 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T20:13 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2881ad47 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2bef09c0 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@62ce72ff (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@58a63629 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7de843ef (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2e1af49b (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3fd2322d (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@56ac5c80 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69e49a81 (Table.java:338) -2019-06-05T20:13 [INFO] \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000005 b/testdata/repo_8k/00000/chunks/000000000000005 deleted file mode 100644 index 0134101..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000005 +++ /dev/null @@ -1,87 +0,0 @@ -[MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1023150a (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ece1e79 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@ab20d7c (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@474fae39 (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4f4c789f (Table.java:338) -2019-06-05T20:13 [INFO][MEMORY] IndexTreeDic.flushOnDisk 222,326 (IndexTreeDic.java:517) -2019-06-05T20:13 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13f9ad9 (Table.java:338) -2019-06-05T20:13 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T20:13 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T20:13 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T20:13 [ERROR] null (SerialStructure.java:74) -2019-06-05T20:13 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 58 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 0.061 s (IndexTreeMessyTest.java:408) -2019-06-05T20:13 [INFO][TIMER] Temps parcours des résultats: 1.25 millis (IndexTreeMessyTest.java:416) -2019-06-05T20:13 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO][TIMER] Temps total recherche: 8.20 millis (IndexTreeMessyTest.java:425) -2019-06-05T20:13 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.072 s (IndexTreeMessyTest.java:439) -2019-06-05T20:13 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.040 s (IndexTreeMessyTest.java:449) -2019-06-05T20:13 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T20:13 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 3 millis (IndexTreeDic.java:1164) -2019-06-05T20:13 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T20:13 [INFO] 40 (HashMapTests.java:27) -2019-06-05T20:13 [INFO][MEMORY] Mem usage début - 157,225 (ParallelLoaderTest.java:56) -2019-06-05T20:13 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T20:13 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:13 [INFO][MEMORY] 166,226 (ParallelLoader.java:114) -2019-06-05T20:13 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.041 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:13 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:13 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:14 [INFO] Parsing terminé !! temps écoulé = 0.638 s (ParallelLoader.java:217) -2019-06-05T20:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:14 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T20:14 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:14 [INFO][MEMORY] 422,898 (ParallelLoader.java:114) -2019-06-05T20:14 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.040 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:14 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:14 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:14 [INFO] Parsing terminé !! temps écoulé = 0.624 s (ParallelLoader.java:217) -2019-06-05T20:14 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:14 [INFO][MEMORY] Mem usage fin - 267,348 (ParallelLoaderTest.java:113) -2019-06-05T20:14 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.46 s (ParallelLoaderTest.java:114) -2019-06-05T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:50 [INFO][MEMORY] 12,904 (ParallelLoader.java:114) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.175 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:50 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:50 [INFO] Parsing terminé !! temps écoulé = 2.03 s (ParallelLoader.java:217) -2019-06-05T20:50 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 2.04 s (IndexMemDicStorageTests.java:183) -2019-06-05T20:50 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T20:50 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:50 [INFO][MEMORY] 157,253 (ParallelLoader.java:114) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.111 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:50 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:50 [INFO] Parsing terminé !! temps écoulé = 0.781 s (ParallelLoader.java:217) -2019-06-05T20:50 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.787 s (NewLoaderAndIndexTest.java:189) -2019-06-05T20:50 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T20:50 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T20:50 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 5.82 millis (Loader.java:116) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.66 millis (Loader.java:116) -2019-06-05T20:50 [INFO][MEMORY] 134,560 (Loader.java:117) -2019-06-05T20:50 [INFO][MEMORY] 134,560 (Loader.java:117) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.11 millis (Loader.java:116) -2019-06-05T20:50 [INFO][MEMORY] 152,063 (Loader.java:117) -2019-06-05T20:50 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T20:50 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.011 s (Loader.java:116) -2019-06-05T20:50 [INFO][MEMORY] 168,447 (Loader.java:117) -2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:50 [INFO][MEMORY] 155,685 (Loader.java:138) -2019-06-05T20:50 [INFO][MEMORY] 156,197 (Loader.java:138) -2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:50 [INFO][MEMORY] 155,691 (Lo \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000006 b/testdata/repo_8k/00000/chunks/000000000000006 deleted file mode 100644 index 91c72a3..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000006 +++ /dev/null @@ -1,84 +0,0 @@ -ader.java:138) -2019-06-05T20:50 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T20:50 [INFO][MEMORY] 155,633 (Loader.java:138) -2019-06-05T20:50 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.14 s (IndexTreeMessyTest.java:314) -2019-06-05T20:50 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T20:50 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,657 (IndexTreeDic.java:517) -2019-06-05T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b0b0854 (Table.java:338) -2019-06-05T20:50 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,657 (IndexTreeDic.java:517) -2019-06-05T20:50 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@19bb07ed (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,033 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@10e41621 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,033 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@353d0772 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,082 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2667f029 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,594 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@67a20f67 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@57c758ac (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@a9cd3b1 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,681 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13e39c73 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,915 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@64cd705f (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@9225652 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@654f0d9c (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6a400542 (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6580cfdd (Table.java:338) -2019-06-05T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,427 (IndexTreeDic.java:517) -2019-06-05T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7e0b85f9 (Table.java:338) -2019-06-05T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T20:51 [ERROR] null (SerialStructure.java:74) -2019-06-05T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 31 millis (IndexTreeDic.java:1164) -2019-06-05T20:51 [INFO][TIMER] Temps total recherche: 0.037 s (IndexTreeMessyTest.java:408) -2019-06-05T20:51 [INFO][TIMER] Temps parcours des résultats: 1.96 millis (IndexTreeMessyTest.java:416) -2019-06-05T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) -2019-06-05T20:51 [INFO][TIMER] Temps total recherche: 7.48 millis (IndexTreeMessyTest.java:425) -2019-06-05T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.119 s (IndexTreeMessyTest.java:439) -2019-06-05T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.040 s (IndexTreeMessyTest.java:449) -2019-06-05T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 1 millis (IndexTreeDic.java:1164) -2019-06-05T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T20:51 [INFO] 40 (HashMapTests.java:27) -2019-06-05T20:51 [INFO][MEMORY] Mem usage début - 88,295 (ParallelLoaderTest.java:56) -2019-06-05T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:51 [INFO][MEMORY] 90,343 (ParallelLoader.java:114) -2019-06-05T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.064 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:51 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:51 [INFO] Parsing terminé !! temps écoulé = 0.862 s (ParallelLoader.java:217) -2019-06-05T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T20:51 [INFO][MEMORY] 227,179 (ParallelLoader.java:114) -2019-06-05T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.055 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T20:51 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T20:51 [INFO] Parsing terminé !! temps écoulé = 0.676 s (ParallelLoader.java:217) -2019-06-05T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T20:51 [INFO][MEMORY] Mem usage fin - 189,561 (ParallelLoaderTest.java:113) -2019-06-05T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.63 s (ParallelLoaderTest.java:114) -2019-06-05T21:56 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T21:56 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T21:56 [INFO][MEMORY] 13,925 (ParallelLoader.java:114) -2019-06-05T21:56 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.226 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T21:56 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T21:56 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T21:57 [INFO] Parsing terminé !! temps écoulé = 33.57 s (ParallelLoader.java:217) -2019-06-05T21:57 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T21:57 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 33.66 s (IndexMemDicStorageTests.java:183) -2019-06-05T21:57 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T21:57 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T21:57 [INFO][MEMOR \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000007 b/testdata/repo_8k/00000/chunks/000000000000007 deleted file mode 100644 index f9f54db..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000007 +++ /dev/null @@ -1,87 +0,0 @@ -Y] 58,123 (ParallelLoader.java:114) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.206 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T21:57 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T21:57 [INFO] Parsing terminé !! temps écoulé = 15.87 s (ParallelLoader.java:217) -2019-06-05T21:57 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T21:57 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 16.28 s (NewLoaderAndIndexTest.java:189) -2019-06-05T21:57 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T21:57 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T21:57 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T21:57 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 9.32 millis (Loader.java:116) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 7.80 millis (Loader.java:116) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 8.95 millis (Loader.java:116) -2019-06-05T21:57 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 8.32 millis (Loader.java:116) -2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) -2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) -2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) -2019-06-05T21:57 [INFO][MEMORY] 181,960 (Loader.java:117) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 155,396 (Loader.java:138) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 155,945 (Loader.java:138) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 155,523 (Loader.java:138) -2019-06-05T22:41 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T22:41 [INFO][MEMORY] 155,565 (Loader.java:138) -2019-06-05T22:41 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 35.40 s (IndexTreeMessyTest.java:314) -2019-06-05T22:41 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T22:41 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,457 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@e056f20 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,457 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b0b0854 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,834 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@19bb07ed (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,834 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@10e41621 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,883 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@353d0772 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,067 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2667f029 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,067 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@67a20f67 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,579 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@57c758ac (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,579 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@a9cd3b1 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@13e39c73 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@64cd705f (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@9225652 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@654f0d9c (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,627 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6a400542 (Table.java:338) -2019-06-05T22:41 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,139 (IndexTreeDic.java:517) -2019-06-05T22:41 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6580cfdd (Table.java:338) -2019-06-05T22:41 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T22:41 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T22:41 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T22:41 [ERROR] null (SerialStructure.java:74) -2019-06-05T22:41 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 50 millis (IndexTreeDic.java:1164) -2019-06-05T22:41 [INFO][TIMER] Temps total recherche: 0.058 s (IndexTreeMessyTest.java:408) -2019-06-05T22:41 [INFO][TIMER] Temps parcours des résultats: 1.04 millis (IndexTreeMessyTest.java:416) -2019-06-05T22:41 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 8 millis (IndexTreeDic.java:1164) -2019-06-05T22:41 [INFO][TIMER] Temps total recherche: 8.60 millis (IndexTreeMessyTest.java:425) -2019-06-05T22:41 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.122 s (IndexTreeMessyTest.java:439) -2019-06-05T22:41 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.072 s (IndexTreeMessyTest.java:449) -2019-06-05T22:41 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T22:41 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 22 millis (IndexTreeDic.java:1164) -2019-06-05T22:41 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T22:41 [INFO] 40 (HashMapTests.java:27) -2019-06-05T22:41 [INFO][MEMORY] Mem usage début - 88,230 (ParallelLoaderTest.java:56) -2019-06-05T22:41 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T22:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T22:41 [INFO][MEMORY] 90,316 (ParallelLoader.java:114) -2019-06-05T22:41 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.068 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T22:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T22:41 [WARNING] Fail \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000008 b/testdata/repo_8k/00000/chunks/000000000000008 deleted file mode 100644 index 5995b3c..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000008 +++ /dev/null @@ -1,87 +0,0 @@ -to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T22:41 [INFO] Parsing terminé !! temps écoulé = 1.09 s (ParallelLoader.java:217) -2019-06-05T22:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T22:41 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:64) -2019-06-05T22:41 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T22:41 [INFO][MEMORY] 415,931 (ParallelLoader.java:114) -2019-06-05T22:41 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.070 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T22:41 [ERROR] debugCount = 0 (ParallelLoader.java:203) -2019-06-05T22:41 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T22:41 [INFO] Parsing terminé !! temps écoulé = 0.968 s (ParallelLoader.java:217) -2019-06-05T22:41 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T22:41 [INFO][MEMORY] Mem usage fin - 189,494 (ParallelLoaderTest.java:113) -2019-06-05T22:41 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 2.17 s (ParallelLoaderTest.java:114) -2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T23:02 [INFO][MEMORY] 12,404 (ParallelLoader.java:114) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.154 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.61 s (ParallelLoader.java:217) -2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.62 s (IndexMemDicStorageTests.java:183) -2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-05T23:02 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.266 s (IndexMemDicStorageTests.java:83) -2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-05T23:02 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.188 s (IndexMemDicStorageTests.java:86) -2019-06-05T23:02 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-05T23:02 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-05T23:02 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-05T23:02 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T23:02 [INFO][MEMORY] 103,147 (ParallelLoader.java:114) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.174 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.01 s (ParallelLoader.java:217) -2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.02 s (NewLoaderAndIndexTest.java:189) -2019-06-05T23:02 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-05T23:02 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-05T23:02 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-05T23:02 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-05T23:02 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-05T23:02 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-05T23:02 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-05T23:02 [INFO] totalLength = 66666 (IndexMemDic.java:217) -2019-06-05T23:02 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) -2019-06-05T23:02 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-05T23:02 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.57 millis (Loader.java:116) -2019-06-05T23:02 [INFO][MEMORY] 223,892 (Loader.java:117) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.56 millis (Loader.java:116) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.00 millis (Loader.java:116) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.58 millis (Loader.java:116) -2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) -2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) -2019-06-05T23:02 [INFO][MEMORY] 200,186 (Loader.java:117) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 155,525 (Loader.java:138) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 155,474 (Loader.java:138) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 156,042 (Loader.java:138) -2019-06-05T23:02 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-05T23:02 [INFO][MEMORY] 155,626 (Loader.java:138) -2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.03 s (IndexTreeMessyTest.java:314) -2019-06-05T23:02 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-05T23:02 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,650 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,038 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,414 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,414 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,464 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,674 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,674 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTree \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000009 b/testdata/repo_8k/00000/chunks/000000000000009 deleted file mode 100644 index 8a0d0af..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000009 +++ /dev/null @@ -1,88 +0,0 @@ -Dic@5ae50ce6 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6f96c77 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@be64738 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ba9ad43 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49d904ec (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@48e4374 (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3d680b5a (Table.java:338) -2019-06-05T23:02 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,194 (IndexTreeDic.java:517) -2019-06-05T23:02 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4b5d6a01 (Table.java:338) -2019-06-05T23:02 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-05T23:02 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-05T23:02 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-05T23:02 [ERROR] null (SerialStructure.java:74) -2019-06-05T23:02 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 43 millis (IndexTreeDic.java:1164) -2019-06-05T23:02 [INFO][TIMER] Temps total recherche: 0.051 s (IndexTreeMessyTest.java:408) -2019-06-05T23:02 [INFO][TIMER] Temps parcours des résultats: 0.73 millis (IndexTreeMessyTest.java:416) -2019-06-05T23:02 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 6 millis (IndexTreeDic.java:1164) -2019-06-05T23:02 [INFO][TIMER] Temps total recherche: 6.60 millis (IndexTreeMessyTest.java:425) -2019-06-05T23:02 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.096 s (IndexTreeMessyTest.java:439) -2019-06-05T23:02 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.069 s (IndexTreeMessyTest.java:449) -2019-06-05T23:02 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-05T23:02 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 10 millis (IndexTreeDic.java:1164) -2019-06-05T23:02 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-05T23:02 [INFO] 40 (HashMapTests.java:27) -2019-06-05T23:02 [INFO][MEMORY] Mem usage début - 88,284 (ParallelLoaderTest.java:56) -2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T23:02 [INFO][MEMORY] 90,332 (ParallelLoader.java:114) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.051 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 1.11 s (ParallelLoader.java:217) -2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T23:02 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-05T23:02 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-05T23:02 [INFO][MEMORY] 275,258 (ParallelLoader.java:114) -2019-06-05T23:02 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.076 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-05T23:02 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-05T23:02 [INFO] Parsing terminé !! temps écoulé = 0.696 s (ParallelLoader.java:217) -2019-06-05T23:02 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-05T23:02 [INFO][MEMORY] Mem usage fin - 189,551 (ParallelLoaderTest.java:114) -2019-06-05T23:02 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.87 s (ParallelLoaderTest.java:115) -2019-06-10T20:19 [ERROR] config.json (No such file or directory) (Database.java:33) -2019-06-10T20:19 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:19 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.37 millis (Loader.java:116) -2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.93 millis (Loader.java:116) -2019-06-10T20:19 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.37 millis (Loader.java:116) -2019-06-10T20:19 [INFO][MEMORY] 10,178 (Loader.java:117) -2019-06-10T20:19 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.010 s (Loader.java:116) -2019-06-10T20:19 [INFO][MEMORY] 10,178 (Loader.java:117) -2019-06-10T20:19 [INFO][MEMORY] 19,394 (Loader.java:117) -2019-06-10T20:19 [INFO][MEMORY] 19,394 (Loader.java:117) -2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:19 [INFO][MEMORY] 150,062 (Loader.java:138) -2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:19 [INFO][MEMORY] 150,767 (Loader.java:138) -2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:19 [INFO][MEMORY] 150,245 (Loader.java:138) -2019-06-10T20:19 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:19 [INFO][MEMORY] 150,333 (Loader.java:138) -2019-06-10T20:19 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 5.79 s (IndexTreeMessyTest.java:314) -2019-06-10T20:19 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:19 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,359 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1b083826 (Table.java:338) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,871 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@105fece7 (Table.java:338) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,871 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@3ec300f1 (Table.java:338) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,381 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@482cd91f (Table.java:338) -2019-06-10T20:19 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,381 (IndexTreeDic.java:517) -2019-06-10T20:19 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@123f1134 (Table.java:338) -2019-06-10T20:34 [ERROR] config.json (No such file or directory) (Database.java:33) -2019-06-10T20:34 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:34 [INFO] Nombre de résultats/ \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000010 b/testdata/repo_8k/00000/chunks/000000000000010 deleted file mode 100644 index 97a481a..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000010 +++ /dev/null @@ -1,88 +0,0 @@ -entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.70 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 9,153 (Loader.java:117) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.74 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 14,616 (Loader.java:117) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.42 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 20,072 (Loader.java:117) -2019-06-10T20:34 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:34 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 4.10 millis (Loader.java:116) -2019-06-10T20:34 [INFO][MEMORY] 32,705 (Loader.java:117) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 149,780 (Loader.java:138) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 150,170 (Loader.java:138) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 150,367 (Loader.java:138) -2019-06-10T20:34 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:34 [INFO][MEMORY] 150,299 (Loader.java:138) -2019-06-10T20:34 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 6.04 s (IndexTreeMessyTest.java:314) -2019-06-10T20:34 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:34 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,323 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7d68ef40 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,323 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5b0abc94 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,699 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@75c072cb (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,699 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1f1c7bf6 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 151,748 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25b485ba (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2b546384 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5d740a0f (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@214b199c (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,260 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@20d3d15a (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,349 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2893de87 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@55ca8de8 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2c34f934 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@12d3a4e9 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@240237d2 (Table.java:338) -2019-06-10T20:34 [INFO][MEMORY] IndexTreeDic.flushOnDisk 152,861 (IndexTreeDic.java:517) -2019-06-10T20:34 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@25a65b77 (Table.java:338) -2019-06-10T20:34 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-10T20:34 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-10T20:34 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-10T20:34 [ERROR] null (SerialStructure.java:74) -2019-06-10T20:34 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 44 millis (IndexTreeDic.java:1164) -2019-06-10T20:34 [INFO][TIMER] Temps total recherche: 0.053 s (IndexTreeMessyTest.java:408) -2019-06-10T20:34 [INFO][TIMER] Temps parcours des résultats: 1.23 millis (IndexTreeMessyTest.java:416) -2019-06-10T20:34 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 14 millis (IndexTreeDic.java:1164) -2019-06-10T20:34 [INFO][TIMER] Temps total recherche: 0.015 s (IndexTreeMessyTest.java:425) -2019-06-10T20:34 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.119 s (IndexTreeMessyTest.java:439) -2019-06-10T20:34 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.045 s (IndexTreeMessyTest.java:449) -2019-06-10T20:34 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-10T20:34 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 2 millis (IndexTreeDic.java:1164) -2019-06-10T20:34 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-10T20:34 [INFO] 40 (HashMapTests.java:27) -2019-06-10T20:34 [INFO][MEMORY] Mem usage début - 82,958 (ParallelLoaderTest.java:56) -2019-06-10T20:49 [ERROR] config.json (No such file or directory) (Database.java:33) -2019-06-10T20:49 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:49 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.20 millis (Loader.java:116) -2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.69 millis (Loader.java:116) -2019-06-10T20:49 [INFO][MEMORY] 9,820 (Loader.java:117) -2019-06-10T20:49 [INFO][MEMORY] 9,820 (Loader.java:117) -2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.41 millis (Loader.java:116) -2019-06-10T20:49 [INFO][MEMORY] 19,381 (Loader.java:117) -2019-06-10T20:49 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:49 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 1.16 millis (Loader.java:116) -2019-06-10T20:49 [INFO][MEMORY] 24,844 (Loader.java:117) -2019-06-10T20:50 [INFO] PARSE : FINAL USAGE (Loader.jav \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000011 b/testdata/repo_8k/00000/chunks/000000000000011 deleted file mode 100644 index 5a30329..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000011 +++ /dev/null @@ -1,87 +0,0 @@ -va:217) -2019-06-10T20:51 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-10T20:51 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-10T20:51 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-10T20:51 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-10T20:51 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-10T20:51 [INFO] totalLength = 66666 (IndexMemDic.java:217) -2019-06-10T20:51 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) -2019-06-10T20:51 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T20:51 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.67 millis (Loader.java:116) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.13 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) -2019-06-10T20:51 [INFO][MEMORY] 226,959 (Loader.java:117) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.14 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 178,179 (Loader.java:117) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.71 millis (Loader.java:116) -2019-06-10T20:51 [INFO][MEMORY] 199,412 (Loader.java:117) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,755 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,569 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 156,154 (Loader.java:138) -2019-06-10T20:51 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T20:51 [INFO][MEMORY] 155,614 (Loader.java:138) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 4.79 s (IndexTreeMessyTest.java:314) -2019-06-10T20:51 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T20:51 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,063 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,575 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@659a969b (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,662 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@76908cc0 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@2473d930 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@35047d03 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@49b0b76 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@769f71a9 (Table.java:338) -2019-06-10T20:51 [INFO][MEMORY] IndexTreeDic.flushOnDisk 158,182 (IndexTreeDic.java:517) -2019-06-10T20:51 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@4c9f8c13 (Table.java:338) -2019-06-10T20:51 [ERROR] SerialStructure.writeStructure : impossible de sauvegarder la structure du disque. (SerialStructure.java:56) -2019-06-10T20:51 [ERROR] java.nio.HeapByteBuffer (SerialStructure.java:57) -2019-06-10T20:51 [ERROR] SerialStructure.loadStructure : impossible de charger la structure du disque. (SerialStructure.java:73) -2019-06-10T20:51 [ERROR] null (SerialStructure.java:74) -2019-06-10T20:51 [INFO] Lancé (IndexTreeMessyTest.java:366) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 41 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 0.049 s (IndexTreeMessyTest.java:408) -2019-06-10T20:51 [INFO][TIMER] Temps parcours des résultats: 1.53 millis (IndexTreeMessyTest.java:416) -2019-06-10T20:51 [INFO] Nombre de résultats = 3116 (IndexTreeMessyTest.java:417) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 7 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO][TIMER] Temps total recherche: 7.95 millis (IndexTreeMessyTest.java:425) -2019-06-10T20:51 [INFO][TIMER] 1Temps d'acquisition des résultats (chargement du disque de tous les champs): 0.079 s (IndexTreeMessyTest.java:439) -2019-06-10T20:51 [INFO][TIMER] 2Temps d'acquisition des résultats certains champs seulement: 0.049 s (IndexTreeMessyTest.java:449) -2019-06-10T20:51 [INFO] Nombre de résultats = 700 (IndexTreeMessyTest.java:450) -2019-06-10T20:51 [INFO][TIMER] tempsPrisPourRecherchesSurFichiers: 4 millis (IndexTreeDic.java:1164) -2019-06-10T20:51 [INFO] Nombre de résultats (pour 18 exact) = 116 (IndexTreeMessyTest.java:456) -2019-06-10T20:51 [INFO] 40 (HashMapTests.java:27) -2019-06-10T20:51 [INFO][MEMORY] Mem usage début - 88,271 (ParallelLoaderTest.java:56) -2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:51 [INFO][MEMORY] 90,319 (ParallelLoader.java:114) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 300000 temps écoulé = 0.042 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20 \ No newline at end of file diff --git a/testdata/repo_8k/00000/chunks/000000000000012 b/testdata/repo_8k/00000/chunks/000000000000012 deleted file mode 100644 index 612b2ee..0000000 --- a/testdata/repo_8k/00000/chunks/000000000000012 +++ /dev/null @@ -1,87 +0,0 @@ -:51 [INFO] Parsing terminé !! temps écoulé = 0.842 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (ParallelLoaderTest.java:65) -2019-06-10T20:51 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T20:51 [INFO][MEMORY] 209,650 (ParallelLoader.java:114) -2019-06-10T20:51 [INFO] Loader : nombre de résultats (total) lus = 400000 temps écoulé = 0.056 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T20:51 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T20:51 [INFO] Parsing terminé !! temps écoulé = 0.637 s (ParallelLoader.java:217) -2019-06-10T20:51 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T20:51 [INFO][MEMORY] Mem usage fin - 189,538 (ParallelLoaderTest.java:114) -2019-06-10T20:51 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (ParallelLoaderTest.java:115) -2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (IndexMemDicStorageTests.java:180) -2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T21:24 [INFO][MEMORY] 12,384 (ParallelLoader.java:114) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 100000 temps écoulé = 0.122 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 1.54 s (ParallelLoader.java:217) -2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 1.55 s (IndexMemDicStorageTests.java:183) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 1ère fois: 0.312 s (IndexMemDicStorageTests.java:83) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO][TIMER] Temps pris pour classer la 2ème fois: 0.172 s (IndexMemDicStorageTests.java:86) -2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (IndexMemDicStorageTests.java:108) -2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (IndexMemDicStorageTests.java:111) -2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (IndexMemDicStorageTests.java:114) -2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (IndexMemDicStorageTests.java:117) -2019-06-10T21:24 [INFO] Parsing de csvName = testdata/SMALL_100_000_yellow_tripdata_2015-04.csv (NewLoaderAndIndexTest.java:170) -2019-06-10T21:24 [INFO] PARSE : memusage init = (ParallelLoader.java:112) -2019-06-10T21:24 [INFO][MEMORY] 108,772 (ParallelLoader.java:114) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (total) lus = 200000 temps écoulé = 0.168 sactiveThreadNb = 0 (ParallelLoader.java:150) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (ParallelLoaderRunnable.java:287) -2019-06-10T21:24 [INFO] Parsing terminé !! temps écoulé = 0.889 s (ParallelLoader.java:217) -2019-06-10T21:24 [INFO] PARSE : Nombre de lignes (local) = 100000 (ParallelLoader.java:218) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL PRIS PAR TOUS LES PARSINGS: 0.904 s (NewLoaderAndIndexTest.java:189) -2019-06-10T21:24 [INFO] Bon nombre total de lignes ! nombre = 99999 (NewLoaderAndIndexTest.java:193) -2019-06-10T21:24 [INFO] totalLength = 99999 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO] Recherche A OK ! Nb résultats selon A : 263 (NewLoaderAndIndexTest.java:71) -2019-06-10T21:24 [INFO] Recherche B OK ! Nb résultats selon B : 263 (NewLoaderAndIndexTest.java:74) -2019-06-10T21:24 [INFO] Recherche C OK ! Nb résultats selon C : 263 (NewLoaderAndIndexTest.java:77) -2019-06-10T21:24 [INFO] resultsPositionsArray.length = 263 (NewLoaderAndIndexTest.java:80) -2019-06-10T21:24 [INFO] Recherche FLAG OK ! Nb résultats avec FLAG : 175 index len = 99999 (NewLoaderAndIndexTest.java:101) -2019-06-10T21:24 [INFO] totalLength = 66666 (IndexMemDic.java:217) -2019-06-10T21:24 [INFO] Recherche STRUCT OK ! Nb résultats avec restructuration : 175 index len = 99999 (NewLoaderAndIndexTest.java:109) -2019-06-10T21:24 [INFO] setUpBeforeAll (IndexTreeMessyTest.java:73) -2019-06-10T21:24 [INFO] Nombre de résultats/entrées parsés INIT : 0 (IndexTreeMessyTest.java:186) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 2.99 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 234,123 (Loader.java:117) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.54 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 154,808 (Loader.java:117) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [WARNING] Fail to save entry number 0: incorrect data (Loader.java:107) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 0.013 s (Loader.java:116) -2019-06-10T21:24 [INFO] Loader : nombre de résultats (local) parsés = 0 temps écoulé = 3.63 millis (Loader.java:116) -2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) -2019-06-10T21:24 [INFO][MEMORY] 202,936 (Loader.java:117) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 154,918 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,713 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,525 (Loader.java:138) -2019-06-10T21:24 [INFO] PARSE : FINAL USAGE (Loader.java:136) -2019-06-10T21:24 [INFO][MEMORY] 155,614 (Loader.java:138) -2019-06-10T21:24 [INFO][TIMER] TEMPS TOTAL pris par le parsing: 12.07 s (IndexTreeMessyTest.java:314) -2019-06-10T21:24 [INFO] setUpBeforeAll OK (IndexTreeMessyTest.java:322) -2019-06-10T21:24 [DEBUG] TableHandler.flushAllIndexOnDisk : size = 15 (Table.java:334) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@7ce3cb8e (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 156,638 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@78b66d36 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@5223e5ee (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,014 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@bef2d72 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,064 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@69b2283a (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,576 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@22a637e7 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@6fe7aac8 (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flushAllIndexOnDisk : flush de l'arbre !index.indexTree.IndexTreeDic@1d119efb (Table.java:338) -2019-06-10T21:24 [INFO][MEMORY] IndexTreeDic.flushOnDisk 157,634 (IndexTreeDic.java:517) -2019-06-10T21:24 [INFO] Table.flu \ No newline at end of file diff --git a/testdata/repo_8k/00000/files b/testdata/repo_8k/00000/files deleted file mode 100644 index 56431d4..0000000 Binary files a/testdata/repo_8k/00000/files and /dev/null differ diff --git a/testdata/repo_8k/00000/hashes b/testdata/repo_8k/00000/hashes deleted file mode 100644 index ec622b2..0000000 Binary files a/testdata/repo_8k/00000/hashes and /dev/null differ diff --git a/testdata/repo_8k/00000/recipe b/testdata/repo_8k/00000/recipe deleted file mode 100644 index 66bde3d..0000000 Binary files a/testdata/repo_8k/00000/recipe and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000000 b/testdata/repo_8k_zlib/00000/chunks/000000000000000 deleted file mode 100644 index 11c1cab..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000000 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000001 b/testdata/repo_8k_zlib/00000/chunks/000000000000001 deleted file mode 100644 index b212097..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000001 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000002 b/testdata/repo_8k_zlib/00000/chunks/000000000000002 deleted file mode 100644 index 15d2a66..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000002 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000003 b/testdata/repo_8k_zlib/00000/chunks/000000000000003 deleted file mode 100644 index d3c7997..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000003 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000004 b/testdata/repo_8k_zlib/00000/chunks/000000000000004 deleted file mode 100644 index be13110..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000004 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000005 b/testdata/repo_8k_zlib/00000/chunks/000000000000005 deleted file mode 100644 index a92e0a6..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000005 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000006 b/testdata/repo_8k_zlib/00000/chunks/000000000000006 deleted file mode 100644 index 47e1d74..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000006 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000007 b/testdata/repo_8k_zlib/00000/chunks/000000000000007 deleted file mode 100644 index 2428c10..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000007 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000008 b/testdata/repo_8k_zlib/00000/chunks/000000000000008 deleted file mode 100644 index 0524676..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000008 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000009 b/testdata/repo_8k_zlib/00000/chunks/000000000000009 deleted file mode 100644 index e89cb70..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000009 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000010 b/testdata/repo_8k_zlib/00000/chunks/000000000000010 deleted file mode 100644 index 140ca5f..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000010 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000011 b/testdata/repo_8k_zlib/00000/chunks/000000000000011 deleted file mode 100644 index b61efae..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000011 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/chunks/000000000000012 b/testdata/repo_8k_zlib/00000/chunks/000000000000012 deleted file mode 100644 index 677a3a8..0000000 Binary files a/testdata/repo_8k_zlib/00000/chunks/000000000000012 and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/files b/testdata/repo_8k_zlib/00000/files deleted file mode 100644 index 70ca423..0000000 Binary files a/testdata/repo_8k_zlib/00000/files and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/hashes b/testdata/repo_8k_zlib/00000/hashes deleted file mode 100644 index ec622b2..0000000 Binary files a/testdata/repo_8k_zlib/00000/hashes and /dev/null differ diff --git a/testdata/repo_8k_zlib/00000/recipe b/testdata/repo_8k_zlib/00000/recipe deleted file mode 100644 index d8c788b..0000000 Binary files a/testdata/repo_8k_zlib/00000/recipe and /dev/null differ -- cgit v1.2.3