From e79080490330a0c7d36bd85935c04200c63aa185 Mon Sep 17 00:00:00 2001 From: n-peugnet Date: Mon, 18 Oct 2021 16:48:06 +0200 Subject: make exp conditionnal and do not run Borg by default --- exp/Makefile | 2 +- exp/exp.sh | 170 +++++++++++++++++++++++++++++++++++------------------------ 2 files changed, 102 insertions(+), 70 deletions(-) (limited to 'exp') diff --git a/exp/Makefile b/exp/Makefile index 5437785..734c40b 100644 --- a/exp/Makefile +++ b/exp/Makefile @@ -38,7 +38,7 @@ DNA_4K ?= dna_4k DNA_8K ?= dna_8k NOPACK ?= nopack export GIT_NOPACK ?= $(abspath $(NOPACK)) -export BORG ?= borg +# export BORG ?= borg export TARGZ ?= targz export REAL ?= real export DIFFS ?= diffs diff --git a/exp/exp.sh b/exp/exp.sh index 5e2d9be..e8f6bd7 100755 --- a/exp/exp.sh +++ b/exp/exp.sh @@ -42,14 +42,20 @@ set-git-dir() { GITC="git -C $REPO_PATH" OUT=/tmp/dna-backup-exp-out -# Init git nopack dir -rm $REPO_PATH/.git -$GITC init --separate-git-dir=$GIT_NOPACK -$GITC --git-dir=$GIT_NOPACK config gc.auto 0 -set-git-dir $GIT_PATH +if [ -n "$GIT_NOPACK" ] +then + # Init git nopack dir + rm $REPO_PATH/.git + $GITC init --separate-git-dir=$GIT_NOPACK + $GITC --git-dir=$GIT_NOPACK config gc.auto 0 + set-git-dir $GIT_PATH +fi -# Init borg dir -borg init -e none $BORG +if [ -n "$BORG" ] +then + # Init borg dir + borg init -e none $BORG +fi # "empty tree" commit prev="4b825dc642cb6eb9a060e54bf8d69288fbee4904" @@ -66,78 +72,104 @@ do $GITC checkout $hash 2> $OUT \ || (log "error checking out"; cat $OUT; exit 1) - # Save real size for this version - log "save real size for this version" - du -b --summarize $REPO_PATH > $(printf "%s.versions/%05d" $REAL $i) - - # Create tar.gz for this version - log "create targ.gz for this version" - tar -czf $(printf "%s/%05d.tar.gz" $TARGZ $i) $REPO_PATH - - # Create git diff for this version - log "create git diff for this version" - diff=$(printf "%s/%05d.diff.gz" $DIFFS $i) - $GITC diff --minimal --binary --unified=0 -l0 $prev \ - | gzip \ - > $diff - - # Create git nopack for this version - log "create git nopack for this version" - set-git-dir $GIT_NOPACK - $GITC add . - $GITC commit -m $hash &> $OUT \ - || (log "error commiting to nopack"; cat $OUT; exit 1) - ls $GIT_NOPACK/objects/pack - find $GIT_NOPACK -type f -exec du -ba {} + \ - > $(printf "%s.versions/%05d" $GIT_NOPACK $i) - set-git-dir $GIT_PATH + if [ -n "$REAL" ] + then + # Save real size for this version + log "save real size for this version" + du -b --summarize $REPO_PATH > $(printf "%s.versions/%05d" $REAL $i) + fi + + if [ -n "$TARGZ" ] + then + # Create tar.gz for this version + log "create targ.gz for this version" + tar -czf $(printf "%s/%05d.tar.gz" $TARGZ $i) $REPO_PATH + fi - # Create borg backup for this versions - log "create borg backup for this versions" - borg create $BORG::$i $REPO_PATH - find $BORG/data -type f -exec du -ba {} + \ - | cut -f1 \ - | paste -sd+ \ - | bc \ - > $(printf "%s.versions/%05d" $BORG $i) - - # Create dna backups for this version - cat $DNA_PARAMS | while read name flags - do - log "create $name backup for this version" - $DNA_BACKUP commit -v 2 $flags $REPO_PATH $name - done + if [ -n "$DIFFS" ] + then + # Create git diff for this version + log "create git diff for this version" + diff=$(printf "%s/%05d.diff.gz" $DIFFS $i) + $GITC diff --minimal --binary --unified=0 -l0 $prev \ + | gzip \ + > $diff + fi - if [[ $(( $i % $SKIP_CHECK )) == 0 ]] + if [ -n "$GIT_NOPACK" ] then - # Check restore from git diffs - log "restore from git diffs" - TEMP=$(mktemp -d) - for n in $(seq 0 $i) - do - diff=$(printf "%s/%05d.diff.gz" $DIFFS $n) - cat $diff \ - | gzip --decompress \ - | git -C $TEMP apply --binary --unidiff-zero --whitespace=nowarn - - done - cp $REPO_PATH/.git $TEMP/ - log "check restore from diffs" - diff --brief --recursive $REPO_PATH $TEMP \ - || log "git patchs restore do not match source" - rm -rf $TEMP + # Create git nopack for this version + log "create git nopack for this version" + set-git-dir $GIT_NOPACK + $GITC add . + $GITC commit -m $hash &> $OUT \ + || (log "error commiting to nopack"; cat $OUT; exit 1) + ls $GIT_NOPACK/objects/pack + find $GIT_NOPACK -type f -exec du -ba {} + \ + > $(printf "%s.versions/%05d" $GIT_NOPACK $i) + set-git-dir $GIT_PATH + fi + + if [ -n "$BORG" ] + then + # Create borg backup for this versions + log "create borg backup for this versions" + borg create $BORG::$i $REPO_PATH + find $BORG/data -type f -exec du -ba {} + \ + | cut -f1 \ + | paste -sd+ \ + | bc \ + > $(printf "%s.versions/%05d" $BORG $i) + fi - # Check restore from dna backups + if [ -n "$DNA_PARAMS" ] + then + # Create dna backups for this version cat $DNA_PARAMS | while read name flags do - log "restore from $name backup" + log "create $name backup for this version" + $DNA_BACKUP commit -v 2 $flags $REPO_PATH $name + done + fi + + if [[ $(( $i % $SKIP_CHECK )) == 0 ]] + then + + if [ -n "$DIFFS" ] + then + # Check restore from git diffs + log "restore from git diffs" TEMP=$(mktemp -d) - $DNA_BACKUP restore -v 2 $flags $name $TEMP - log "check restore from backup" + for n in $(seq 0 $i) + do + diff=$(printf "%s/%05d.diff.gz" $DIFFS $n) + cat $diff \ + | gzip --decompress \ + | git -C $TEMP apply --binary --unidiff-zero --whitespace=nowarn - + done + cp $REPO_PATH/.git $TEMP/ + log "check restore from diffs" diff --brief --recursive $REPO_PATH $TEMP \ - || log "dna backup restore do not match source" + || log "git patchs restore do not match source" rm -rf $TEMP - done + fi + + + if [ -n "$DNA_PARAMS" ] + then + # Check restore from dna backups + cat $DNA_PARAMS | while read name flags + do + log "restore from $name backup" + TEMP=$(mktemp -d) + $DNA_BACKUP restore -v 2 $flags $name $TEMP + log "check restore from backup" + diff --brief --recursive $REPO_PATH $TEMP \ + || log "dna backup restore do not match source" + rm -rf $TEMP + done + fi fi prev=$hash -- cgit v1.2.3