From bd1681cfe3554075b1b0827563b76c6b95bf8b94 Mon Sep 17 00:00:00 2001
From: n-peugnet <n.peugnet@free.fr>
Date: Mon, 18 Oct 2021 16:16:54 +0200
Subject: make the repo responsible of the export compression
---
dna/drive.go | 21 +++++++--------------
main.go | 3 +--
repo/export.go | 11 +++++++----
3 files changed, 15 insertions(+), 20 deletions(-)
diff --git a/dna/drive.go b/dna/drive.go
index 317d2f4..0fc5b03 100644
--- a/dna/drive.go
+++ b/dna/drive.go
@@ -27,7 +27,6 @@ import (
"github.com/n-peugnet/dna-backup/export"
"github.com/n-peugnet/dna-backup/logger"
- "github.com/n-peugnet/dna-backup/utils"
)
type Direction int
@@ -42,8 +41,6 @@ type DnaDrive struct {
trackSize int
tracksPerPool int
pools []Pool
- writeWrapper utils.WriteWrapper
- readWrapper utils.ReadWrapper
}
type Pool struct {
@@ -62,8 +59,6 @@ func New(
poolCount int,
trackSize int,
tracksPerPool int,
- writeWrapper utils.WriteWrapper,
- readWrapper utils.ReadWrapper,
) *DnaDrive {
pools := make([]Pool, poolCount)
os.MkdirAll(destination, 0755)
@@ -84,8 +79,6 @@ func New(
trackSize: trackSize,
tracksPerPool: tracksPerPool,
pools: pools,
- writeWrapper: writeWrapper,
- readWrapper: readWrapper,
}
}
@@ -112,7 +105,7 @@ func (d *DnaDrive) ExportVersion(end chan<- bool) export.Input {
func (d *DnaDrive) writeVersion(output export.Output, end chan<- bool) {
var err error
var recipe, files, version bytes.Buffer
- n := d.write(output.Chunks, d.pools[1:], Forward)
+ n := write(output.Chunks, d.pools[1:], d.trackSize, d.tracksPerPool, Forward)
_, err = io.Copy(&recipe, output.Recipe)
if err != nil {
logger.Error("dna export recipe ", err)
@@ -148,19 +141,19 @@ func (d *DnaDrive) writeVersion(output export.Output, end chan<- bool) {
} else if err != nil { // another error than EOF happened
logger.Error("dna export files: ", err)
} else { // files has not been fully written so we write what is left to pools
- d.write(&files, d.pools[1:], Backward)
+ write(&files, d.pools[1:], d.trackSize, d.tracksPerPool, Backward)
}
} else if err != nil { // another error than EOF happened
logger.Error("dna export recipe: ", err)
} else { // recipe has not been fully written so we concat with files and write what is left to pools
io.Copy(&recipe, &files)
- d.write(&recipe, d.pools[1:], Backward)
+ write(&recipe, d.pools[1:], d.trackSize, d.tracksPerPool, Backward)
}
- d.write(&version, d.pools[:1], Forward)
+ write(&version, d.pools[:1], d.trackSize, d.tracksPerPool, Forward)
end <- true
}
-func (d *DnaDrive) write(r io.Reader, pools []Pool, direction Direction) int64 {
+func write(r io.Reader, pools []Pool, trackSize int, tracksPerPool int, direction Direction) int64 {
var err error
var i, n int
var count int64
@@ -168,7 +161,7 @@ func (d *DnaDrive) write(r io.Reader, pools []Pool, direction Direction) int64 {
i = len(pools) - 1
}
for err != io.ErrUnexpectedEOF && err != io.EOF {
- if pools[i].TrackCount == d.tracksPerPool {
+ if pools[i].TrackCount == tracksPerPool {
if direction == Backward {
i--
} else {
@@ -179,7 +172,7 @@ func (d *DnaDrive) write(r io.Reader, pools []Pool, direction Direction) int64 {
}
continue
}
- buf := make([]byte, d.trackSize)
+ buf := make([]byte, trackSize)
n, err = io.ReadFull(r, buf)
if err == io.EOF {
break
diff --git a/main.go b/main.go
index ce1c4b3..fb0478e 100644
--- a/main.go
+++ b/main.go
@@ -25,7 +25,6 @@ import (
"github.com/n-peugnet/dna-backup/dna"
"github.com/n-peugnet/dna-backup/logger"
"github.com/n-peugnet/dna-backup/repo"
- "github.com/n-peugnet/dna-backup/utils"
)
type command struct {
@@ -139,7 +138,7 @@ func exportMain(args []string) error {
r := repo.NewRepo(source, chunkSize)
switch format {
case "dir":
- exporter := dna.New(dest, 96, trackSize, 10000, utils.ZlibWriter, utils.ZlibReader)
+ exporter := dna.New(dest, 96, trackSize, 10000)
r.Export(exporter)
case "csv":
fmt.Println("csv")
diff --git a/repo/export.go b/repo/export.go
index bcadd06..7597ce4 100644
--- a/repo/export.go
+++ b/repo/export.go
@@ -22,6 +22,7 @@ import (
"github.com/n-peugnet/dna-backup/export"
"github.com/n-peugnet/dna-backup/logger"
+ "github.com/n-peugnet/dna-backup/utils"
)
func (r *Repo) Export(exporter export.Exporter) {
@@ -32,22 +33,24 @@ func (r *Repo) Export(exporter export.Exporter) {
end := make(chan bool)
input := exporter.ExportVersion(end)
if len(chunks[i]) > 0 {
+ compressed := r.chunkWriteWrapper(input.Chunks)
for _, c := range chunks[i] {
- _, err := io.Copy(input.Chunks, c.Reader())
+ _, err := io.Copy(compressed, c.Reader())
if err != nil {
logger.Error(err)
}
}
- input.Chunks.Close()
+ compressed.Close()
}
- readDelta(r.versions[i], recipeName, r.chunkReadWrapper, func(rc io.ReadCloser) {
+ input.Chunks.Close()
+ readDelta(r.versions[i], recipeName, utils.NopReadWrapper, func(rc io.ReadCloser) {
_, err = io.Copy(input.Recipe, rc)
if err != nil {
logger.Error("load recipe ", err)
}
input.Recipe.Close()
})
- readDelta(r.versions[i], filesName, r.chunkReadWrapper, func(rc io.ReadCloser) {
+ readDelta(r.versions[i], filesName, utils.NopReadWrapper, func(rc io.ReadCloser) {
_, err = io.Copy(input.Files, rc)
if err != nil {
logger.Error("load files ", err)
--
cgit v1.2.3