initial commit

first working chunking logic

1 files changed, 86 insertions, 0 deletions

diff --git a/repo.go b/repo.go
new file mode 100644
index 0000000..df1c8a2
--- /dev/null
+++ b/repo.go
@@ -0,0 +1,86 @@
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+const (
+	chunkSize = 8 << 10
+)
+
+type File struct {
+	Path string
+	Size int64
+}
+
+func ListFiles(path string, files chan File) {
+	err := filepath.Walk(path,
+		func(path string, i fs.FileInfo, err error) error {
+			if err != nil {
+				log.Println(err)
+				return err
+			}
+			if i.IsDir() {
+				return nil
+			}
+			var file = File{path, i.Size()}
+			files <- file
+			return nil
+		})
+	if err != nil {
+		log.Println(err)
+	}
+	close(files)
+}
+
+func ReadFiles(files chan File, chunks chan []byte) {
+	var buff []byte
+	var prev, read = chunkSize, 0
+
+	for f := range files {
+		file, err := os.Open(f.Path)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+		for err != io.EOF {
+			if prev == chunkSize {
+				buff = make([]byte, chunkSize)
+				prev, err = file.Read(buff)
+			} else {
+				read, err = file.Read(buff[prev:])
+				prev += read
+			}
+			if err != nil && err != io.EOF {
+				log.Println(err)
+			}
+			if prev == chunkSize {
+				chunks <- buff
+			}
+		}
+	}
+	chunks <- buff
+	close(chunks)
+}
+
+func PrintChunks(chunks chan []byte) {
+	for c := range chunks {
+		fmt.Println(c)
+	}
+}
+
+func DumpChunks(dest string, chunks chan []byte) {
+	for c := range chunks {
+		sum := sha1.Sum(c)
+		path := path.Join(dest, hex.EncodeToString(sum[:]))
+		os.WriteFile(path, c, 0664)
+	}
+}