diff options
author | n-peugnet <n.peugnet@free.fr> | 2021-08-20 17:22:13 +0200 |
---|---|---|
committer | n-peugnet <n.peugnet@free.fr> | 2021-08-20 17:22:13 +0200 |
commit | 9f98d6ef4931f04b9023264f0e3408e4529c977d (patch) | |
tree | d05440a340287b88f2f037b9d9e2a49f68f55c69 | |
parent | da20d649b3775c4c061ff2aeefe1cea44bac1d19 (diff) | |
download | dna-backup-9f98d6ef4931f04b9023264f0e3408e4529c977d.tar.gz dna-backup-9f98d6ef4931f04b9023264f0e3408e4529c977d.zip |
Store and Load chunks and
- also add a test for store and load
- precise channel direction in function args
-rw-r--r-- | main.go | 2 | ||||
-rw-r--r-- | repo.go | 39 | ||||
-rw-r--r-- | repo_test.go | 30 |
3 files changed, 59 insertions, 12 deletions
@@ -15,5 +15,5 @@ func main() { chunks := make(chan []byte) go ListFiles(path, files) go ReadFiles(files, chunks) - DumpChunks(".", chunks) + StoreChunks(".", chunks) } @@ -1,8 +1,6 @@ package main import ( - "crypto/sha1" - "encoding/hex" "fmt" "io" "io/fs" @@ -21,9 +19,9 @@ type File struct { Size int64 } -func ListFiles(path string, files chan File) { +func ListFiles(path string, files chan<- File) { err := filepath.Walk(path, - func(path string, i fs.FileInfo, err error) error { + func(p string, i fs.FileInfo, err error) error { if err != nil { log.Println(err) return err @@ -31,7 +29,7 @@ func ListFiles(path string, files chan File) { if i.IsDir() { return nil } - var file = File{path, i.Size()} + var file = File{p, i.Size()} files <- file return nil }) @@ -41,7 +39,7 @@ func ListFiles(path string, files chan File) { close(files) } -func ReadFiles(files chan File, chunks chan []byte) { +func ReadFiles(files <-chan File, chunks chan<- []byte) { var buff []byte var prev, read = chunkSize, 0 @@ -71,16 +69,37 @@ func ReadFiles(files chan File, chunks chan []byte) { close(chunks) } -func PrintChunks(chunks chan []byte) { +func PrintChunks(chunks <-chan []byte) { for c := range chunks { fmt.Println(c) } } -func DumpChunks(dest string, chunks chan []byte) { +func StoreChunks(dest string, chunks <-chan []byte) { + i := 0 for c := range chunks { - sum := sha1.Sum(c) - path := path.Join(dest, hex.EncodeToString(sum[:])) + path := path.Join(dest, fmt.Sprintf("%015d", i)) os.WriteFile(path, c, 0664) + i++ } } + +func LoadChunks(repo string, chunks chan<- []byte) { + err := filepath.WalkDir(repo, + func(p string, e fs.DirEntry, err error) error { + if err != nil { + log.Println(err) + return err + } + if e.IsDir() { + return nil + } + buff, err := os.ReadFile(p) + chunks <- buff + return nil + }) + if err != nil { + log.Println(err) + } + close(chunks) +} diff --git a/repo_test.go b/repo_test.go index 27cdebf..956bd84 100644 --- a/repo_test.go +++ b/repo_test.go @@ -7,7 +7,7 @@ import ( "testing" ) -func preparResult() { +func prepareResult() { result := path.Join("test", "result") os.RemoveAll(result) os.MkdirAll(result, 0775) @@ -76,3 +76,31 @@ func TestReadFiles3(t *testing.T) { } chunkCompare(t, dataDir, files, chunkCount) } + +func TestLoadChunks(t *testing.T) { + prepareResult() + dataDir := path.Join("test", "data") + resultDir := path.Join("test", "result") + files1 := make(chan File) + files2 := make(chan File) + chunks1 := make(chan []byte) + chunks2 := make(chan []byte) + chunks3 := make(chan []byte) + go ListFiles(dataDir, files1) + go ListFiles(dataDir, files2) + go ReadFiles(files1, chunks1) + go ReadFiles(files2, chunks2) + StoreChunks(resultDir, chunks1) + go LoadChunks(resultDir, chunks3) + + i := 0 + for c2 := range chunks2 { + c3 := <-chunks3 + if bytes.Compare(c2, c3) != 0 { + t.Errorf("Chunk %d does not match file content", i) + t.Log(c2) + t.Log(c3) + } + i++ + } +} |