support symlinks if it is internal to source

adapt tests

3 files changed, 73 insertions, 33 deletions

diff --git a/TODO.md b/TODO.md
index 96d8f1d..54de8f7 100644
--- a/TODO.md
+++ b/TODO.md
@@ -42,12 +42,16 @@ priority 2
     is implemented
 - [ ] keep hash workers so that they reuse the same hasher and reset it instead
     of creating a new one each time. This could save some processing time
-- [ ] support links (symbolic mainly and also hard)
-    - [ ] store this metadata somewhere, tar could be the solution, but this
+- [x] support symlinks
+    - [x] store this metadata somewhere, tar could be the solution, but this
         would bury the metadata down into the chunks, storing it into the files
         listing could be another solution but with this approach we would have
-        to think about what other metadata we want to store
-    - [ ] use a symlink aware Walk function (easy enough)
+        to think about what other metadata we want to store  
+        I stored it in the _files_ file so that we don't need to read the chunks
+        to get the content of a symlinked file and because it does not seem to
+        add a lot of weight to this file (after compression).
+- [ ] store and restore symlinks relatively if it was relative in source
+    directory.
 - [ ] add quick progress bar to CLI
 - [ ] `list` command to list versions
 - [ ] optional argument for `restore` to select the version to restore.
diff --git a/repo.go b/repo.go
index e633241..35edfbd 100644
--- a/repo.go
+++ b/repo.go
@@ -10,19 +10,17 @@ repo/
 │   │   ├── 000000000000000
 │   │   ├── 000000000000001
 │   │   ├── 000000000000002
-│   │   ├── 000000000000003
+│   │   └── 000000000000003
 │   ├── files
-│   ├── fingerprints
-│   ├── recipe
-│   └── sketches
+│   ├── hashes
+│   └── recipe
 └── 00001/
     ├── chunks/
     │   ├── 000000000000000
-    │   ├── 000000000000001
+    │   └── 000000000000001
     ├── files
-│   ├── fingerprints
-│   ├── recipe
-│   └── sketches
+    ├── hashes
+    └── recipe
 ```
 */
 
@@ -38,6 +36,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 
 	"github.com/chmduquesne/rollinghash/rabinkarp64"
 	"github.com/n-peugnet/dna-backup/cache"
@@ -100,6 +99,7 @@ type chunkData struct {
 type File struct {
 	Path string
 	Size int64
+	Link string
 }
 
 func NewRepo(path string) *Repo {
@@ -192,14 +192,21 @@ func (r *Repo) Restore(destination string) {
 	for _, file := range r.files {
 		filePath := filepath.Join(destination, file.Path)
 		dir := filepath.Dir(filePath)
-		os.MkdirAll(dir, 0775)      // TODO: handle errors
-		f, _ := os.Create(filePath) // TODO: handle errors
-		n, err := io.CopyN(f, bufReader, file.Size)
-		if err != nil {
-			logger.Errorf("storing file content for '%s', written %d/%d bytes: %s", filePath, n, file.Size, err)
-		}
-		if err := f.Close(); err != nil {
-			logger.Errorf("closing restored file '%s': %s", filePath, err)
+		os.MkdirAll(dir, 0775) // TODO: handle errors
+		if file.Link != "" {
+			err := os.Symlink(filepath.Join(destination, file.Link), filePath)
+			if err != nil {
+				logger.Errorf("restored symlink ", err)
+			}
+		} else {
+			f, _ := os.Create(filePath) // TODO: handle errors
+			n, err := io.CopyN(f, bufReader, file.Size)
+			if err != nil {
+				logger.Errorf("restored file, written %d/%d bytes: %s", filePath, n, file.Size, err)
+			}
+			if err := f.Close(); err != nil {
+				logger.Errorf("restored file ", err)
+			}
 		}
 	}
 }
@@ -226,28 +233,33 @@ func listFiles(path string) []File {
 			logger.Warning(err)
 			return nil
 		}
+		if i.IsDir() {
+			return nil
+		}
+		var link string
+		var size = i.Size()
 		if i.Mode()&fs.ModeSymlink != 0 {
 			target, err := filepath.EvalSymlinks(p)
 			if err != nil {
 				logger.Warning(err)
 				return nil
 			}
-			i, err = os.Stat(target)
+			if !strings.HasPrefix(target, path) {
+				logger.Warningf("skipping external symlink %s -> %s", p, target)
+				return nil
+			}
+			size = 0
+			link, err = filepath.Rel(path, target)
 			if err != nil {
 				logger.Warning(err)
 				return nil
 			}
-			if !i.IsDir() {
-				logger.Warningf("file symlink %s: content will be duplicated", p)
-			} else {
-				logger.Warningf("dir symlink %s: will not be followed", p)
+			if link == "" {
+				logger.Warningf("skipping empty symlink %s", p)
 				return nil
 			}
 		}
-		if i.IsDir() {
-			return nil
-		}
-		files = append(files, File{p, i.Size()})
+		files = append(files, File{p, size, link})
 		return nil
 	})
 	if err != nil {
@@ -276,6 +288,10 @@ func unprefixFiles(files []File, prefix string) (ret []File) {
 func concatFiles(files *[]File, stream io.WriteCloser) {
 	actual := make([]File, 0, len(*files))
 	for _, f := range *files {
+		if f.Link != "" {
+			actual = append(actual, f)
+			continue
+		}
 		file, err := os.Open(f.Path)
 		if err != nil {
 			logger.Warning(err)
diff --git a/repo_test.go b/repo_test.go
index 35b3101..b97ebae 100644
--- a/repo_test.go
+++ b/repo_test.go
@@ -178,12 +178,26 @@ func TestSymlinks(t *testing.T) {
 	defer logger.SetOutput(os.Stderr)
 	tmpDir := t.TempDir()
 	extDir := t.TempDir()
-	os.Symlink(extDir, filepath.Join(tmpDir, "linktodir"))
+	f, err := os.Create(filepath.Join(tmpDir, "existing"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	os.Symlink(extDir, filepath.Join(tmpDir, "linkexternal"))
 	os.Symlink("./notexisting", filepath.Join(tmpDir, "linknotexisting"))
+	os.Symlink("./existing", filepath.Join(tmpDir, "linkexisting"))
 	files := listFiles(tmpDir)
-	testutils.AssertLen(t, 0, files, "Files")
-	if !strings.Contains(output.String(), "linktodir") {
-		t.Errorf("log should contain a warning for linktodir, actual %q", &output)
+	testutils.AssertLen(t, 2, files, "Files")
+	if files[0].Link != "" {
+		t.Error("linkexisting should not be a link, actual:", files[0].Link)
+	}
+	if files[1].Link != "existing" {
+		t.Error("linkexisting should point to 'existing', actual:", files[1].Link)
+	}
+	if !strings.Contains(output.String(), "linkexternal") {
+		t.Errorf("log should contain a warning for linkexternal, actual %q", &output)
 	}
 	if !strings.Contains(output.String(), "notexisting") {
 		t.Errorf("log should contain a warning for notexisting, actual %q", &output)
@@ -325,6 +339,8 @@ func TestCommitZlib(t *testing.T) {
 }
 
 func TestRestore(t *testing.T) {
+	logger.SetLevel(2)
+	defer logger.SetLevel(4)
 	dest := t.TempDir()
 	source := filepath.Join("testdata", "repo_8k")
 	expected := filepath.Join("testdata", "logs")
@@ -339,6 +355,8 @@ func TestRestore(t *testing.T) {
 }
 
 func TestRestoreZlib(t *testing.T) {
+	logger.SetLevel(2)
+	defer logger.SetLevel(4)
 	dest := t.TempDir()
 	source := filepath.Join("testdata", "repo_8k_zlib")
 	expected := filepath.Join("testdata", "logs")
@@ -353,6 +371,8 @@ func TestRestoreZlib(t *testing.T) {
 }
 
 func TestRoundtrip(t *testing.T) {
+	logger.SetLevel(2)
+	defer logger.SetLevel(4)
 	temp := t.TempDir()
 	dest := t.TempDir()
 	source := filepath.Join("testdata", "logs")