From 5539220a8839519becd45b63be65ed86fa9286a4 Mon Sep 17 00:00:00 2001 From: n-peugnet Date: Mon, 18 Oct 2021 15:51:55 +0200 Subject: do the real export in a dir containing multiple pool files --- dna/drive.go | 181 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- dna/writer.go | 45 --------------- 2 files changed, 178 insertions(+), 48 deletions(-) delete mode 100644 dna/writer.go (limited to 'dna') diff --git a/dna/drive.go b/dna/drive.go index f99e7d4..013c4b4 100644 --- a/dna/drive.go +++ b/dna/drive.go @@ -17,14 +17,189 @@ package dna -import "io" +import ( + "bytes" + "encoding/gob" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/n-peugnet/dna-backup/logger" + "github.com/n-peugnet/dna-backup/utils" +) + +type Direction int + +const ( + Forward Direction = iota + Backward +) type DnaDrive struct { poolCount int trackSize int tracksPerPool int + pools []Pool + writeWrapper utils.WriteWrapper + readWrapper utils.ReadWrapper +} + +type Pool struct { + Data io.ReadWriteCloser + TrackCount int +} + +type Header struct { + Chunks uint64 + Recipe uint64 + Files uint64 +} + +type dnaVersion struct { + Input dnaInput + Output dnaOutput +} + +type dnaInput struct { + Chunks io.WriteCloser + Recipe io.WriteCloser + Files io.WriteCloser +} + +type dnaOutput struct { + Chunks io.ReadCloser + Recipe io.ReadCloser + Files io.ReadCloser +} + +func New( + destination string, + poolCount int, + trackSize int, + tracksPerPool int, + writeWrapper utils.WriteWrapper, + readWrapper utils.ReadWrapper, +) *DnaDrive { + pools := make([]Pool, poolCount) + os.MkdirAll(destination, 0755) + for i := range pools { + path := filepath.Join(destination, fmt.Sprintf("%02d", i)) + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + logger.Panic(err) + } + stat, err := file.Stat() + if err != nil { + logger.Panic(err) + } + pools[i] = Pool{file, int(stat.Size()) / trackSize} + } + return &DnaDrive{ + poolCount: poolCount, + trackSize: trackSize, + tracksPerPool: tracksPerPool, + pools: pools, + writeWrapper: writeWrapper, + readWrapper: readWrapper, + } +} + +func (d *DnaDrive) VersionInput() (dnaInput, <-chan bool) { + rChunks, wChunks := io.Pipe() + rRecipe, wRecipe := io.Pipe() + rFiles, wFiles := io.Pipe() + version := dnaVersion{ + Input: dnaInput{wChunks, wRecipe, wFiles}, + Output: dnaOutput{rChunks, rRecipe, rFiles}, + } + end := make(chan bool) + go d.writeVersion(version.Output, end) + return version.Input, end +} + +func (d *DnaDrive) writeVersion(output dnaOutput, end chan<- bool) { + var err error + var recipe, files, version bytes.Buffer + n := d.write(output.Chunks, d.pools[1:], Forward) + _, err = io.Copy(&recipe, output.Recipe) + if err != nil { + logger.Error("dna export recipe ", err) + } + _, err = io.Copy(&files, output.Files) + if err != nil { + logger.Error("dna export files ", err) + } + header := Header{ + uint64(n), + uint64(recipe.Len()), + uint64(files.Len()), + } + e := gob.NewEncoder(&version) + err = e.Encode(header) + if err != nil { + logger.Error("dna export version header: ", err) + } + logger.Debugf("version len %d", version.Len()) + rest := int64(d.trackSize - version.Len()) + logger.Debugf("version rest %d", rest) + n, err = io.CopyN(&version, &recipe, rest) + logger.Debugf("recipe copied in version %d", n) + rest -= n + logger.Debugf("version rest %d", rest) + if err == io.EOF && rest > 0 { // recipe is written to version but there is space left + n, err = io.CopyN(&version, &files, rest) + logger.Debugf("files copied in version %d", n) + rest -= n + logger.Debugf("version rest %d", rest) + if err == io.EOF && rest > 0 { // files is writtent to version but there is space left + // version.Write(make([]byte, rest)) + } else if err != nil { // another error than EOF happened + logger.Error("dna export files: ", err) + } else { // files has not been fully written so we write what is left to pools + d.write(&files, d.pools[1:], Backward) + } + } else if err != nil { // another error than EOF happened + logger.Error("dna export recipe: ", err) + } else { // recipe has not been fully written so we concat with files and write what is left to pools + io.Copy(&recipe, &files) + d.write(&recipe, d.pools[1:], Backward) + } + d.write(&version, d.pools[:1], Forward) + end <- true } -func (d *DnaDrive) Writer(w io.Writer) io.WriteCloser { - return NewWriter(w, d.trackSize) +func (d *DnaDrive) write(r io.Reader, pools []Pool, direction Direction) int64 { + var err error + var i, n int + var count int64 + if direction == Backward { + i = len(pools) - 1 + } + for err != io.ErrUnexpectedEOF && err != io.EOF { + if pools[i].TrackCount == d.tracksPerPool { + if direction == Backward { + i-- + } else { + i++ + } + if i < 0 || i >= len(pools) { + logger.Panic("dna export: no space left") + } + continue + } + buf := make([]byte, d.trackSize) + n, err = io.ReadFull(r, buf) + if err == io.EOF { + break + } + logger.Debug("written track:", n, err) + count += int64(n) + n, errw := pools[i].Data.Write(buf) + if errw != nil { + logger.Error("dna export: pool %d: %d/%d bytes written: %s", i, n, len(buf), errw) + } + pools[i].TrackCount++ + } + return count } diff --git a/dna/writer.go b/dna/writer.go deleted file mode 100644 index 6b232cd..0000000 --- a/dna/writer.go +++ /dev/null @@ -1,45 +0,0 @@ -/* Copyright (C) 2021 Nicolas Peugnet - - This file is part of dna-backup. - - dna-backup is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - dna-backup is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with dna-backup. If not, see . */ - -package dna - -import ( - "io" - - "github.com/n-peugnet/dna-backup/utils" -) - -type writer struct { - *utils.WriteCounter - trackSize int -} - -func NewWriter(w io.Writer, trackSize int) io.WriteCloser { - return &writer{ - WriteCounter: utils.NewWriteCounter(w), - trackSize: trackSize, - } -} - -func (d *writer) Close() (err error) { - // add padding for the last track - padding := make([]byte, d.trackSize-d.Count()%d.trackSize) - if _, err = d.Write(padding); err != nil { - return err - } - return nil -} -- cgit v1.2.3