From 9f98d6ef4931f04b9023264f0e3408e4529c977d Mon Sep 17 00:00:00 2001
From: n-peugnet <n.peugnet@free.fr>
Date: Fri, 20 Aug 2021 17:22:13 +0200
Subject: Store and Load chunks and

- also add a test for store and load
- precise channel direction in function args
---
 main.go      |  2 +-
 repo.go      | 39 +++++++++++++++++++++++++++++----------
 repo_test.go | 30 +++++++++++++++++++++++++++++-
 3 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/main.go b/main.go
index 20289fd..f8a38ae 100644
--- a/main.go
+++ b/main.go
@@ -15,5 +15,5 @@ func main() {
 	chunks := make(chan []byte)
 	go ListFiles(path, files)
 	go ReadFiles(files, chunks)
-	DumpChunks(".", chunks)
+	StoreChunks(".", chunks)
 }
diff --git a/repo.go b/repo.go
index df1c8a2..17ad8f3 100644
--- a/repo.go
+++ b/repo.go
@@ -1,8 +1,6 @@
 package main
 
 import (
-	"crypto/sha1"
-	"encoding/hex"
 	"fmt"
 	"io"
 	"io/fs"
@@ -21,9 +19,9 @@ type File struct {
 	Size int64
 }
 
-func ListFiles(path string, files chan File) {
+func ListFiles(path string, files chan<- File) {
 	err := filepath.Walk(path,
-		func(path string, i fs.FileInfo, err error) error {
+		func(p string, i fs.FileInfo, err error) error {
 			if err != nil {
 				log.Println(err)
 				return err
@@ -31,7 +29,7 @@ func ListFiles(path string, files chan File) {
 			if i.IsDir() {
 				return nil
 			}
-			var file = File{path, i.Size()}
+			var file = File{p, i.Size()}
 			files <- file
 			return nil
 		})
@@ -41,7 +39,7 @@ func ListFiles(path string, files chan File) {
 	close(files)
 }
 
-func ReadFiles(files chan File, chunks chan []byte) {
+func ReadFiles(files <-chan File, chunks chan<- []byte) {
 	var buff []byte
 	var prev, read = chunkSize, 0
 
@@ -71,16 +69,37 @@ func ReadFiles(files chan File, chunks chan []byte) {
 	close(chunks)
 }
 
-func PrintChunks(chunks chan []byte) {
+func PrintChunks(chunks <-chan []byte) {
 	for c := range chunks {
 		fmt.Println(c)
 	}
 }
 
-func DumpChunks(dest string, chunks chan []byte) {
+func StoreChunks(dest string, chunks <-chan []byte) {
+	i := 0
 	for c := range chunks {
-		sum := sha1.Sum(c)
-		path := path.Join(dest, hex.EncodeToString(sum[:]))
+		path := path.Join(dest, fmt.Sprintf("%015d", i))
 		os.WriteFile(path, c, 0664)
+		i++
 	}
 }
+
+func LoadChunks(repo string, chunks chan<- []byte) {
+	err := filepath.WalkDir(repo,
+		func(p string, e fs.DirEntry, err error) error {
+			if err != nil {
+				log.Println(err)
+				return err
+			}
+			if e.IsDir() {
+				return nil
+			}
+			buff, err := os.ReadFile(p)
+			chunks <- buff
+			return nil
+		})
+	if err != nil {
+		log.Println(err)
+	}
+	close(chunks)
+}
diff --git a/repo_test.go b/repo_test.go
index 27cdebf..956bd84 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -7,7 +7,7 @@ import (
 	"testing"
 )
 
-func preparResult() {
+func prepareResult() {
 	result := path.Join("test", "result")
 	os.RemoveAll(result)
 	os.MkdirAll(result, 0775)
@@ -76,3 +76,31 @@ func TestReadFiles3(t *testing.T) {
 	}
 	chunkCompare(t, dataDir, files, chunkCount)
 }
+
+func TestLoadChunks(t *testing.T) {
+	prepareResult()
+	dataDir := path.Join("test", "data")
+	resultDir := path.Join("test", "result")
+	files1 := make(chan File)
+	files2 := make(chan File)
+	chunks1 := make(chan []byte)
+	chunks2 := make(chan []byte)
+	chunks3 := make(chan []byte)
+	go ListFiles(dataDir, files1)
+	go ListFiles(dataDir, files2)
+	go ReadFiles(files1, chunks1)
+	go ReadFiles(files2, chunks2)
+	StoreChunks(resultDir, chunks1)
+	go LoadChunks(resultDir, chunks3)
+
+	i := 0
+	for c2 := range chunks2 {
+		c3 := <-chunks3
+		if bytes.Compare(c2, c3) != 0 {
+			t.Errorf("Chunk %d does not match file content", i)
+			t.Log(c2)
+			t.Log(c3)
+		}
+		i++
+	}
+}
-- 
cgit v1.2.3