1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
|
package main
import (
"bytes"
"os"
"path"
"testing"
)
func prepareResult() {
result := path.Join("test", "result")
os.RemoveAll(result)
os.MkdirAll(result, 0775)
}
func chunkCompare(t *testing.T, dataDir string, testFiles []string, chunkCount int) {
files := make(chan File)
chunks := make(chan []byte)
go ListFiles(dataDir, files)
go ReadFiles(files, chunks)
offset := 0
buff := make([]byte, chunkSize*chunkCount)
for _, f := range testFiles {
content, err := os.ReadFile(path.Join(dataDir, f))
if err != nil {
t.Error("Error reading test data file")
}
for i := range content {
buff[offset+i] = content[i]
}
offset += len(content)
}
i := 0
for c := range chunks {
content := buff[i*chunkSize : (i+1)*chunkSize]
if len(c) != chunkSize {
t.Errorf("Chunk %d is not of chunkSize: %d", i, chunkSize)
}
if bytes.Compare(c, content) != 0 {
t.Errorf("Chunk %d does not match file content", i)
t.Log(c)
t.Log(content)
}
i++
}
if i != chunkCount {
t.Errorf("Incorrect number of chunks: %d, should be: %d", i, chunkCount)
}
}
func TestReadFiles1(t *testing.T) {
chunkCount := 1
dataDir := path.Join("test", "data", "logs.1")
files := []string{"logTest.log"}
chunkCompare(t, dataDir, files, chunkCount)
}
func TestReadFiles2(t *testing.T) {
chunkCount := 3
dataDir := path.Join("test", "data", "logs.2")
files := []string{"csvParserTest.log", "slipdb.log"}
chunkCompare(t, dataDir, files, chunkCount)
}
func TestReadFiles3(t *testing.T) {
chunkCount := 15
dataDir := path.Join("test", "data")
files := []string{
path.Join("logs.1", "logTest.log"),
path.Join("logs.2", "csvParserTest.log"),
path.Join("logs.2", "slipdb.log"),
path.Join("logs.3", "indexingTreeTest.log"),
}
chunkCompare(t, dataDir, files, chunkCount)
}
func TestLoadChunks(t *testing.T) {
prepareResult()
dataDir := path.Join("test", "data")
resultDir := path.Join("test", "result")
files1 := make(chan File)
files2 := make(chan File)
chunks1 := make(chan []byte)
chunks2 := make(chan []byte)
chunks3 := make(chan []byte)
go ListFiles(dataDir, files1)
go ListFiles(dataDir, files2)
go ReadFiles(files1, chunks1)
go ReadFiles(files2, chunks2)
StoreChunks(resultDir, chunks1)
go LoadChunks(resultDir, chunks3)
i := 0
for c2 := range chunks2 {
c3 := <-chunks3
if bytes.Compare(c2, c3) != 0 {
t.Errorf("Chunk %d does not match file content", i)
t.Log(c2)
t.Log(c3)
}
i++
}
}
|
