Bump restic

Upstream ref: 8b84c96d9da9351edb7defb76ee851f1d528ae15

backend/backend_retry.go

@@ -33,6 +33,16 @@ func NewRetryBackend(be restic.Backend, maxTries int, report func(string, error,
 }
 
 func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) error {
+	// Don't do anything when called with an already cancelled context. There would be
+	// no retries in that case either, so be consistent and abort always.
+	// This enforces a strict contract for backend methods: Using a cancelled context
+	// will prevent any backup repository modifications. This simplifies ensuring that
+	// a backup repository is not modified any further after a context was cancelled.
+	// The 'local' backend for example does not provide this guarantee on its own.
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
 	err := backoff.RetryNotify(f,
 		backoff.WithContext(backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)), ctx),
 		func(err error, d time.Duration) {

backend/backend_retry_test.go

@@ -236,3 +236,38 @@ func TestBackendLoadRetry(t *testing.T) {
 	test.Equals(t, data, buf)
 	test.Equals(t, 2, attempt)
 }
+
+func assertIsCanceled(t *testing.T, err error) {
+	test.Assert(t, err == context.Canceled, "got unexpected err %v", err)
+}
+
+func TestBackendCanceledContext(t *testing.T) {
+	// unimplemented mock backend functions return an error by default
+	// check that we received the expected context canceled error instead
+	retryBackend := NewRetryBackend(mock.NewBackend(), 2, nil)
+	h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()}
+
+	// create an already canceled context
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	_, err := retryBackend.Test(ctx, h)
+	assertIsCanceled(t, err)
+	_, err = retryBackend.Stat(ctx, h)
+	assertIsCanceled(t, err)
+
+	err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}))
+	assertIsCanceled(t, err)
+	err = retryBackend.Remove(ctx, h)
+	assertIsCanceled(t, err)
+	err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
+		return nil
+	})
+	assertIsCanceled(t, err)
+	err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error {
+		return nil
+	})
+	assertIsCanceled(t, err)
+
+	// don't test "Delete" as it is not used by normal code
+}

backend/foreground_sysv.go

@@ -0,0 +1,27 @@
+// +build aix solaris
+
+package backend
+
+import (
+	"os/exec"
+	"syscall"
+
+	"github.com/rubiojr/rapi/internal/errors"
+)
+
+func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
+	// run the command in it's own process group so that SIGINT
+	// is not sent to it.
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+
+	// start the process
+	err = cmd.Start()
+	if err != nil {
+		return nil, errors.Wrap(err, "cmd.Start")
+	}
+
+	bg = func() error { return nil }
+	return bg, nil
+}

backend/foreground_unix.go

@@ -1,5 +1,4 @@
-// +build !solaris
-// +build !windows
+// +build !aix,!solaris,!windows
 
 package backend
 

backend/local/local.go

@@ -91,9 +91,8 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade
 	filename := b.Filename(h)
 
 	defer func() {
-		// Mark non-retriable errors as such (currently only
-		// "no space left on device").
-		if errors.Is(err, syscall.ENOSPC) {
+		// Mark non-retriable errors as such
+		if errors.Is(err, syscall.ENOSPC) || os.IsPermission(err) {
 			err = backoff.Permanent(err)
 		}
 	}()

internal/archiver/archiver.go

@@ -178,6 +178,10 @@ func (arch *Archiver) saveTree(ctx context.Context, t *restic.Tree) (restic.ID,
 		s.TreeBlobs++
 		s.TreeSize += uint64(len(buf))
 	}
+	// The context was canceled in the meantime, res.ID() might be invalid
+	if ctx.Err() != nil {
+		return restic.ID{}, s, ctx.Err()
+	}
 	return res.ID(), s, nil
 }
 
@@ -780,32 +784,31 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
 
 	var t tomb.Tomb
 	wctx := t.Context(ctx)
-
-	arch.runWorkers(wctx, &t)
-
 	start := time.Now()
 
-	debug.Log("starting snapshot")
-	rootTreeID, stats, err := func() (restic.ID, ItemStats, error) {
+	var rootTreeID restic.ID
+	var stats ItemStats
+	t.Go(func() error {
+		arch.runWorkers(wctx, &t)
+
+		debug.Log("starting snapshot")
 		tree, err := arch.SaveTree(wctx, "/", atree, arch.loadParentTree(wctx, opts.ParentSnapshot))
 		if err != nil {
-			return restic.ID{}, ItemStats{}, err
+			return err
 		}
 
 		if len(tree.Nodes) == 0 {
-			return restic.ID{}, ItemStats{}, errors.New("snapshot is empty")
+			return errors.New("snapshot is empty")
 		}
 
-		return arch.saveTree(wctx, tree)
-	}()
-	debug.Log("saved tree, error: %v", err)
+		rootTreeID, stats, err = arch.saveTree(wctx, tree)
+		// trigger shutdown but don't set an error
+		t.Kill(nil)
+		return err
+	})
 
-	t.Kill(nil)
-	werr := t.Wait()
-	debug.Log("err is %v, werr is %v", err, werr)
-	if err == nil || errors.Cause(err) == context.Canceled {
-		err = werr
-	}
+	err = t.Wait()
+	debug.Log("err is %v", err)
 
 	if err != nil {
 		debug.Log("error while saving tree: %v", err)

internal/archiver/archiver_test.go

@@ -3,6 +3,7 @@ package archiver
 import (
 	"bytes"
 	"context"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -15,6 +16,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/rubiojr/rapi/backend/mem"
 	"github.com/rubiojr/rapi/internal/checker"
 	"github.com/rubiojr/rapi/internal/errors"
 	"github.com/rubiojr/rapi/internal/fs"
@@ -1814,6 +1816,69 @@ func TestArchiverErrorReporting(t *testing.T) {
 	}
 }
 
+type noCancelBackend struct {
+	restic.Backend
+}
+
+func (c *noCancelBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	return c.Backend.Test(context.Background(), h)
+}
+
+func (c *noCancelBackend) Remove(ctx context.Context, h restic.Handle) error {
+	return c.Backend.Remove(context.Background(), h)
+}
+
+func (c *noCancelBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+	return c.Backend.Save(context.Background(), h, rd)
+}
+
+func (c *noCancelBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+	return c.Backend.Load(context.Background(), h, length, offset, fn)
+}
+
+func (c *noCancelBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+	return c.Backend.Stat(context.Background(), h)
+}
+
+func (c *noCancelBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
+	return c.Backend.List(context.Background(), t, fn)
+}
+
+func (c *noCancelBackend) Delete(ctx context.Context) error {
+	return c.Backend.Delete(context.Background())
+}
+
+func TestArchiverContextCanceled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	tempdir, removeTempdir := restictest.TempDir(t)
+	TestCreateFiles(t, tempdir, TestDir{
+		"targetfile": TestFile{Content: "foobar"},
+	})
+	defer removeTempdir()
+
+	// Ensure that the archiver itself reports the canceled context and not just the backend
+	repo, _ := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()})
+
+	back := restictest.Chdir(t, tempdir)
+	defer back()
+
+	arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
+
+	_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+
+	if err != nil {
+		t.Logf("found expected error (%v)", err)
+		return
+	}
+	if snapshotID.IsNull() {
+		t.Fatalf("no error returned but found null id")
+	}
+
+	t.Fatalf("expected error not returned by archiver")
+}
+
 // TrackFS keeps track which files are opened. For some files, an error is injected.
 type TrackFS struct {
 	fs.FS

internal/checker/checker.go

@@ -567,18 +567,21 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
 				errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q has nil blob list", node.Name)})
 			}
 
-			var size uint64
 			for b, blobID := range node.Content {
 				if blobID.IsNull() {
 					errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q blob %d has null ID", node.Name, b)})
 					continue
 				}
-				blobSize, found := c.repo.LookupBlobSize(blobID, restic.DataBlob)
+				// Note that we do not use the blob size. The "obvious" check
+				// whether the sum of the blob sizes matches the file size
+				// unfortunately fails in some cases that are not resolveable
+				// by users, so we omit this check, see #1887
+
+				_, found := c.repo.LookupBlobSize(blobID, restic.DataBlob)
 				if !found {
 					debug.Log("tree %v references blob %v which isn't contained in index", id, blobID)
 					errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q blob %v not found in index", node.Name, blobID)})
 				}
-				size += uint64(blobSize)
 			}
 
 			if c.trackUnused {

internal/options/options_test.go

@@ -125,6 +125,7 @@ type Target struct {
 	Name    string        `option:"name"`
 	ID      int           `option:"id"`
 	Timeout time.Duration `option:"timeout"`
+	Switch  bool          `option:"switch"`
 	Other   string
 }
 
@@ -155,7 +156,15 @@ var setTests = []struct {
 			"timeout": "10m3s",
 		},
 		Target{
-			Timeout: time.Duration(10*time.Minute + 3*time.Second),
+			Timeout: 10*time.Minute + 3*time.Second,
+		},
+	},
+	{
+		Options{
+			"switch": "true",
+		},
+		Target{
+			Switch: true,
 		},
 	},
 }
@@ -202,6 +211,13 @@ var invalidSetTests = []struct {
 		"ns",
 		`time: missing unit in duration "?2134"?`,
 	},
+	{
+		Options{
+			"switch": "yes",
+		},
+		"ns",
+		`strconv.ParseBool: parsing "yes": invalid syntax`,
+	},
 }
 
 func TestOptionsApplyInvalid(t *testing.T) {
@@ -213,9 +229,9 @@ func TestOptionsApplyInvalid(t *testing.T) {
 				t.Fatalf("expected error %v not found", test.err)
 			}
 
-			matched, err := regexp.MatchString(test.err, err.Error())
-			if err != nil {
-				t.Fatal(err)
+			matched, e := regexp.MatchString(test.err, err.Error())
+			if e != nil {
+				t.Fatal(e)
 			}
 
 			if !matched {
@@ -226,11 +242,11 @@ func TestOptionsApplyInvalid(t *testing.T) {
 }
 
 func TestListOptions(t *testing.T) {
-	var teststruct = struct {
+	teststruct := struct {
 		Foo string `option:"foo" help:"bar text help"`
 	}{}
 
-	var tests = []struct {
+	tests := []struct {
 		cfg  interface{}
 		opts []Help
 	}{
@@ -281,7 +297,7 @@ func TestListOptions(t *testing.T) {
 }
 
 func TestAppendAllOptions(t *testing.T) {
-	var tests = []struct {
+	tests := []struct {
 		cfgs map[string]interface{}
 		opts []Help
 	}{

internal/ui/progress/signals_sysv.go

@@ -1,4 +1,4 @@
-// +build linux solaris
+// +build aix linux solaris
 
 package progress
 

restic/node_aix.go

@@ -0,0 +1,39 @@
+// +build aix
+
+package restic
+
+import "syscall"
+
+func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
+	return nil
+}
+
+func (node Node) device() int {
+	return int(node.Device)
+}
+
+// AIX has a funny timespec type in syscall, with 32-bit nanoseconds.
+// golang.org/x/sys/unix handles this cleanly, but we're stuck with syscall
+// because os.Stat returns a syscall type in its os.FileInfo.Sys().
+func toTimespec(t syscall.StTimespec_t) syscall.Timespec {
+	return syscall.Timespec{Sec: t.Sec, Nsec: int64(t.Nsec)}
+}
+
+func (s statT) atim() syscall.Timespec { return toTimespec(s.Atim) }
+func (s statT) mtim() syscall.Timespec { return toTimespec(s.Mtim) }
+func (s statT) ctim() syscall.Timespec { return toTimespec(s.Ctim) }
+
+// Getxattr is a no-op on AIX.
+func Getxattr(path, name string) ([]byte, error) {
+	return nil, nil
+}
+
+// Listxattr is a no-op on AIX.
+func Listxattr(path string) ([]string, error) {
+	return nil, nil
+}
+
+// Setxattr is a no-op on AIX.
+func Setxattr(path, name string, data []byte) error {
+	return nil
+}