aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorn-peugnet <n.peugnet@free.fr>2021-08-20 17:22:13 +0200
committern-peugnet <n.peugnet@free.fr>2021-08-20 17:22:13 +0200
commit9f98d6ef4931f04b9023264f0e3408e4529c977d (patch)
treed05440a340287b88f2f037b9d9e2a49f68f55c69
parentda20d649b3775c4c061ff2aeefe1cea44bac1d19 (diff)
downloaddna-backup-9f98d6ef4931f04b9023264f0e3408e4529c977d.tar.gz
dna-backup-9f98d6ef4931f04b9023264f0e3408e4529c977d.zip
Store and Load chunks and
- also add a test for store and load - precise channel direction in function args
-rw-r--r--main.go2
-rw-r--r--repo.go39
-rw-r--r--repo_test.go30
3 files changed, 59 insertions, 12 deletions
diff --git a/main.go b/main.go
index 20289fd..f8a38ae 100644
--- a/main.go
+++ b/main.go
@@ -15,5 +15,5 @@ func main() {
chunks := make(chan []byte)
go ListFiles(path, files)
go ReadFiles(files, chunks)
- DumpChunks(".", chunks)
+ StoreChunks(".", chunks)
}
diff --git a/repo.go b/repo.go
index df1c8a2..17ad8f3 100644
--- a/repo.go
+++ b/repo.go
@@ -1,8 +1,6 @@
package main
import (
- "crypto/sha1"
- "encoding/hex"
"fmt"
"io"
"io/fs"
@@ -21,9 +19,9 @@ type File struct {
Size int64
}
-func ListFiles(path string, files chan File) {
+func ListFiles(path string, files chan<- File) {
err := filepath.Walk(path,
- func(path string, i fs.FileInfo, err error) error {
+ func(p string, i fs.FileInfo, err error) error {
if err != nil {
log.Println(err)
return err
@@ -31,7 +29,7 @@ func ListFiles(path string, files chan File) {
if i.IsDir() {
return nil
}
- var file = File{path, i.Size()}
+ var file = File{p, i.Size()}
files <- file
return nil
})
@@ -41,7 +39,7 @@ func ListFiles(path string, files chan File) {
close(files)
}
-func ReadFiles(files chan File, chunks chan []byte) {
+func ReadFiles(files <-chan File, chunks chan<- []byte) {
var buff []byte
var prev, read = chunkSize, 0
@@ -71,16 +69,37 @@ func ReadFiles(files chan File, chunks chan []byte) {
close(chunks)
}
-func PrintChunks(chunks chan []byte) {
+func PrintChunks(chunks <-chan []byte) {
for c := range chunks {
fmt.Println(c)
}
}
-func DumpChunks(dest string, chunks chan []byte) {
+func StoreChunks(dest string, chunks <-chan []byte) {
+ i := 0
for c := range chunks {
- sum := sha1.Sum(c)
- path := path.Join(dest, hex.EncodeToString(sum[:]))
+ path := path.Join(dest, fmt.Sprintf("%015d", i))
os.WriteFile(path, c, 0664)
+ i++
}
}
+
+func LoadChunks(repo string, chunks chan<- []byte) {
+ err := filepath.WalkDir(repo,
+ func(p string, e fs.DirEntry, err error) error {
+ if err != nil {
+ log.Println(err)
+ return err
+ }
+ if e.IsDir() {
+ return nil
+ }
+ buff, err := os.ReadFile(p)
+ chunks <- buff
+ return nil
+ })
+ if err != nil {
+ log.Println(err)
+ }
+ close(chunks)
+}
diff --git a/repo_test.go b/repo_test.go
index 27cdebf..956bd84 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -7,7 +7,7 @@ import (
"testing"
)
-func preparResult() {
+func prepareResult() {
result := path.Join("test", "result")
os.RemoveAll(result)
os.MkdirAll(result, 0775)
@@ -76,3 +76,31 @@ func TestReadFiles3(t *testing.T) {
}
chunkCompare(t, dataDir, files, chunkCount)
}
+
+func TestLoadChunks(t *testing.T) {
+ prepareResult()
+ dataDir := path.Join("test", "data")
+ resultDir := path.Join("test", "result")
+ files1 := make(chan File)
+ files2 := make(chan File)
+ chunks1 := make(chan []byte)
+ chunks2 := make(chan []byte)
+ chunks3 := make(chan []byte)
+ go ListFiles(dataDir, files1)
+ go ListFiles(dataDir, files2)
+ go ReadFiles(files1, chunks1)
+ go ReadFiles(files2, chunks2)
+ StoreChunks(resultDir, chunks1)
+ go LoadChunks(resultDir, chunks3)
+
+ i := 0
+ for c2 := range chunks2 {
+ c3 := <-chunks3
+ if bytes.Compare(c2, c3) != 0 {
+ t.Errorf("Chunk %d does not match file content", i)
+ t.Log(c2)
+ t.Log(c3)
+ }
+ i++
+ }
+}