Bump Restic to 0.13.1

backend/azure/azure.go

@@ -2,7 +2,9 @@ package azure
 
 import (
 	"context"
+	"crypto/md5"
 	"encoding/base64"
+	"hash"
 	"io"
 	"net/http"
 	"os"
@@ -112,6 +114,11 @@ func (be *Backend) Location() string {
 	return be.Join(be.container.Name, be.prefix)
 }
 
+// Hasher may return a hash function for calculating a content hash for the backend
+func (be *Backend) Hasher() hash.Hash {
+	return md5.New()
+}
+
 // Path returns the path in the bucket that is used for this backend.
 func (be *Backend) Path() string {
 	return be.prefix
@@ -148,7 +155,9 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 		dataReader := azureAdapter{rd}
 
 		// if it's smaller than 256miB, then just create the file directly from the reader
-		err = be.container.GetBlobReference(objName).CreateBlockBlobFromReader(dataReader, nil)
+		ref := be.container.GetBlobReference(objName)
+		ref.Properties.ContentMD5 = base64.StdEncoding.EncodeToString(rd.Hash())
+		err = ref.CreateBlockBlobFromReader(dataReader, nil)
 	} else {
 		// otherwise use the more complicated method
 		err = be.saveLarge(ctx, objName, rd)
@@ -192,10 +201,10 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
 		uploadedBytes += n
 
 		// upload it as a new "block", use the base64 hash for the ID
-		h := restic.Hash(buf)
+		h := md5.Sum(buf)
 		id := base64.StdEncoding.EncodeToString(h[:])
 		debug.Log("PutBlock %v with %d bytes", id, len(buf))
-		err = file.PutBlock(id, buf, nil)
+		err = file.PutBlock(id, buf, &storage.PutBlockOptions{ContentMD5: id})
 		if err != nil {
 			return errors.Wrap(err, "PutBlock")
 		}

backend/azure/azure_test.go

@@ -172,7 +172,7 @@ func TestUploadLargeFile(t *testing.T) {
 
 	t.Logf("hash of %d bytes: %v", len(data), id)
 
-	err = be.Save(ctx, h, restic.NewByteReader(data))
+	err = be.Save(ctx, h, restic.NewByteReader(data, be.Hasher()))
 	if err != nil {
 		t.Fatal(err)
 	}

backend/b2/b2.go

@@ -2,6 +2,7 @@ package b2
 
 import (
 	"context"
+	"hash"
 	"io"
 	"net/http"
 	"path"
@@ -137,6 +138,11 @@ func (be *b2Backend) Location() string {
 	return be.cfg.Bucket
 }
 
+// Hasher may return a hash function for calculating a content hash for the backend
+func (be *b2Backend) Hasher() hash.Hash {
+	return nil
+}
+
 // IsNotExist returns true if the error is caused by a non-existing file.
 func (be *b2Backend) IsNotExist(err error) bool {
 	return b2.IsNotExist(errors.Cause(err))
@@ -200,6 +206,7 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind
 	debug.Log("Save %v, name %v", h, name)
 	obj := be.bucket.Object(name)
 
+	// b2 always requires sha1 checksums for uploaded file parts
 	w := obj.NewWriter(ctx)
 	n, err := io.Copy(w, rd)
 	debug.Log("  saved %d bytes, err %v", n, err)
@@ -258,7 +265,13 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
 	defer be.sem.ReleaseToken()
 
 	obj := be.bucket.Object(be.Filename(h))
-	return errors.Wrap(obj.Delete(ctx), "Delete")
+	err := obj.Delete(ctx)
+	// consider a file as removed if b2 informs us that it does not exist
+	if b2.IsNotExist(err) {
+		return nil
+	}
+
+	return errors.Wrap(err, "Delete")
 }
 
 type semLocker struct {

backend/backend_retry_test.go

@@ -36,7 +36,7 @@ func TestBackendSaveRetry(t *testing.T) {
 	retryBackend := NewRetryBackend(be, 10, nil)
 
 	data := test.Random(23, 5*1024*1024+11241)
-	err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data))
+	err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -256,7 +256,7 @@ func TestBackendCanceledContext(t *testing.T) {
 	_, err = retryBackend.Stat(ctx, h)
 	assertIsCanceled(t, err)
 
-	err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}))
+	err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
 	assertIsCanceled(t, err)
 	err = retryBackend.Remove(ctx, h)
 	assertIsCanceled(t, err)

backend/dryrun/dry_backend.go

@@ -0,0 +1,84 @@
+package dryrun
+
+import (
+	"context"
+	"hash"
+	"io"
+
+	"github.com/rubiojr/rapi/internal/debug"
+	"github.com/rubiojr/rapi/restic"
+)
+
+// Backend passes reads through to an underlying layer and accepts writes, but
+// doesn't do anything. Also removes are ignored.
+// So in fact, this backend silently ignores all operations that would modify
+// the repo and does normal operations else.
+// This is used for `backup --dry-run`.
+type Backend struct {
+	b restic.Backend
+}
+
+// statically ensure that RetryBackend implements restic.Backend.
+var _ restic.Backend = &Backend{}
+
+// New returns a new backend that saves all data in a map in memory.
+func New(be restic.Backend) *Backend {
+	b := &Backend{b: be}
+	debug.Log("created new dry backend")
+	return b
+}
+
+// Save adds new Data to the backend.
+func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	debug.Log("faked saving %v bytes at %v", rd.Length(), h)
+
+	// don't save anything, just return ok
+	return nil
+}
+
+// Remove deletes a file from the backend.
+func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
+	return nil
+}
+
+// Location returns the location of the backend.
+func (be *Backend) Location() string {
+	return "DRY:" + be.b.Location()
+}
+
+// Delete removes all data in the backend.
+func (be *Backend) Delete(ctx context.Context) error {
+	return nil
+}
+
+func (be *Backend) Close() error {
+	return be.b.Close()
+}
+
+func (be *Backend) Hasher() hash.Hash {
+	return be.b.Hasher()
+}
+
+func (be *Backend) IsNotExist(err error) bool {
+	return be.b.IsNotExist(err)
+}
+
+func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
+	return be.b.List(ctx, t, fn)
+}
+
+func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
+	return be.b.Load(ctx, h, length, offset, fn)
+}
+
+func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+	return be.b.Stat(ctx, h)
+}
+
+func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	return be.b.Test(ctx, h)
+}

backend/dryrun/dry_backend_test.go

@@ -0,0 +1,137 @@
+package dryrun_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/rubiojr/rapi/restic"
+
+	"github.com/rubiojr/rapi/backend/dryrun"
+	"github.com/rubiojr/rapi/backend/mem"
+)
+
+// make sure that Backend implements backend.Backend
+var _ restic.Backend = &dryrun.Backend{}
+
+func newBackends() (*dryrun.Backend, restic.Backend) {
+	m := mem.New()
+	return dryrun.New(m), m
+}
+
+func TestDry(t *testing.T) {
+	ctx := context.TODO()
+
+	d, m := newBackends()
+	// Since the dry backend is a mostly write-only overlay, the standard backend test suite
+	// won't pass. Instead, perform a series of operations over the backend, testing the state
+	// at each step.
+	steps := []struct {
+		be      restic.Backend
+		op      string
+		fname   string
+		content string
+		wantErr string
+	}{
+		{d, "loc", "", "DRY:RAM", ""},
+		{d, "delete", "", "", ""},
+		{d, "stat", "a", "", "not found"},
+		{d, "list", "", "", ""},
+		{d, "save", "", "", "invalid"},
+		{d, "test", "a", "", ""},
+		{m, "save", "a", "baz", ""},  // save a directly to the mem backend
+		{d, "save", "b", "foob", ""}, // b is not saved
+		{d, "save", "b", "xxx", ""},  // no error as b is not saved
+		{d, "test", "a", "1", ""},
+		{d, "test", "b", "", ""},
+		{d, "stat", "", "", "invalid"},
+		{d, "stat", "a", "a 3", ""},
+		{d, "load", "a", "baz", ""},
+		{d, "load", "b", "", "not found"},
+		{d, "list", "", "a", ""},
+		{d, "remove", "c", "", ""},
+		{d, "stat", "b", "", "not found"},
+		{d, "list", "", "a", ""},
+		{d, "remove", "a", "", ""}, // a is in fact not removed
+		{d, "list", "", "a", ""},
+		{m, "remove", "a", "", ""}, // remove a from the mem backend
+		{d, "list", "", "", ""},
+		{d, "close", "", "", ""},
+		{d, "close", "", "", ""},
+	}
+
+	for i, step := range steps {
+		var err error
+		var boolRes bool
+
+		handle := restic.Handle{Type: restic.PackFile, Name: step.fname}
+		switch step.op {
+		case "save":
+			err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher()))
+		case "test":
+			boolRes, err = step.be.Test(ctx, handle)
+			if boolRes != (step.content != "") {
+				t.Errorf("%d. Test(%q) = %v, want %v", i, step.fname, boolRes, step.content != "")
+			}
+		case "list":
+			fileList := []string{}
+			err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
+				fileList = append(fileList, fi.Name)
+				return nil
+			})
+			sort.Strings(fileList)
+			files := strings.Join(fileList, " ")
+			if files != step.content {
+				t.Errorf("%d. List = %q, want %q", i, files, step.content)
+			}
+		case "loc":
+			loc := step.be.Location()
+			if loc != step.content {
+				t.Errorf("%d. Location = %q, want %q", i, loc, step.content)
+			}
+		case "delete":
+			err = step.be.Delete(ctx)
+		case "remove":
+			err = step.be.Remove(ctx, handle)
+		case "stat":
+			var fi restic.FileInfo
+			fi, err = step.be.Stat(ctx, handle)
+			if err == nil {
+				fis := fmt.Sprintf("%s %d", fi.Name, fi.Size)
+				if fis != step.content {
+					t.Errorf("%d. Stat = %q, want %q", i, fis, step.content)
+				}
+			}
+		case "load":
+			data := ""
+			err = step.be.Load(ctx, handle, 100, 0, func(rd io.Reader) error {
+				buf, err := ioutil.ReadAll(rd)
+				data = string(buf)
+				return err
+			})
+			if data != step.content {
+				t.Errorf("%d. Load = %q, want %q", i, data, step.content)
+			}
+		case "close":
+			err = step.be.Close()
+		default:
+			t.Fatalf("%d. unknown step operation %q", i, step.op)
+		}
+		if step.wantErr != "" {
+			if err == nil {
+				t.Errorf("%d. %s error = nil, want %q", i, step.op, step.wantErr)
+			} else if !strings.Contains(err.Error(), step.wantErr) {
+				t.Errorf("%d. %s error = %q, doesn't contain %q", i, step.op, err, step.wantErr)
+			} else if step.wantErr == "not found" && !step.be.IsNotExist(err) {
+				t.Errorf("%d. IsNotExist(%s error) = false, want true", i, step.op)
+			}
+
+		} else if err != nil {
+			t.Errorf("%d. %s error = %q, want nil", i, step.op, err)
+		}
+	}
+}

backend/foreground_windows.go

@@ -2,12 +2,16 @@ package backend
 
 import (
 	"os/exec"
+	"syscall"
 
 	"github.com/rubiojr/rapi/internal/errors"
+	"golang.org/x/sys/windows"
 )
 
 func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
 	// just start the process and hope for the best
+	cmd.SysProcAttr = &syscall.SysProcAttr{}
+	cmd.SysProcAttr.CreationFlags = windows.CREATE_NEW_PROCESS_GROUP
 	err = cmd.Start()
 	if err != nil {
 		return nil, errors.Wrap(err, "cmd.Start")

backend/gs/gs.go

@@ -3,6 +3,8 @@ package gs
 
 import (
 	"context"
+	"crypto/md5"
+	"hash"
 	"io"
 	"net/http"
 	"os"
@@ -188,6 +190,11 @@ func (be *Backend) Location() string {
 	return be.Join(be.bucketName, be.prefix)
 }
 
+// Hasher may return a hash function for calculating a content hash for the backend
+func (be *Backend) Hasher() hash.Hash {
+	return md5.New()
+}
+
 // Path returns the path in the bucket that is used for this backend.
 func (be *Backend) Path() string {
 	return be.prefix
@@ -234,6 +241,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 	// uploads are not providing significant benefit anyways.
 	w := be.bucket.Object(objName).NewWriter(ctx)
 	w.ChunkSize = 0
+	w.MD5 = rd.Hash()
 	wbytes, err := io.Copy(w, rd)
 	cerr := w.Close()
 	if err == nil {