Bump restic

Upstream ref: 6ac032be64d8d7047bb30ce3dcc1e6914d801421

backend/azure/azure.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/base64"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
@@ -118,6 +117,16 @@ func (be *Backend) Path() string {
 	return be.prefix
 }
 
+type azureAdapter struct {
+	restic.RewindReader
+}
+
+func (azureAdapter) Close() error { return nil }
+
+func (a azureAdapter) Len() int {
+	return int(a.Length())
+}
+
 // Save stores data in the backend at the handle.
 func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
 	if err := h.Valid(); err != nil {
@@ -135,7 +144,8 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 	var err error
 	if rd.Length() < 256*1024*1024 {
 		// wrap the reader so that net/http client cannot close the reader
-		dataReader := ioutil.NopCloser(rd)
+		// CreateBlockBlobFromReader reads length from `Len()``
+		dataReader := azureAdapter{rd}
 
 		// if it's smaller than 256miB, then just create the file directly from the reader
 		err = be.container.GetBlobReference(objName).CreateBlockBlobFromReader(dataReader, nil)
@@ -162,6 +172,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
 	// read the data, in 100 MiB chunks
 	buf := make([]byte, 100*1024*1024)
 	var blocks []storage.Block
+	uploadedBytes := 0
 
 	for {
 		n, err := io.ReadFull(rd, buf)
@@ -178,6 +189,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
 		}
 
 		buf = buf[:n]
+		uploadedBytes += n
 
 		// upload it as a new "block", use the base64 hash for the ID
 		h := restic.Hash(buf)
@@ -194,6 +206,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
 		})
 	}
 
+	// sanity check
+	if uploadedBytes != int(rd.Length()) {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", uploadedBytes, rd.Length())
+	}
+
 	debug.Log("uploaded %d parts: %v", len(blocks), blocks)
 	err = file.PutBlockList(blocks, nil)
 	debug.Log("PutBlockList returned %v", err)

backend/b2/b2.go

@@ -209,6 +209,10 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind
 		return errors.Wrap(err, "Copy")
 	}
 
+	// sanity check
+	if n != rd.Length() {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
+	}
 	return errors.Wrap(w.Close(), "Close")
 }
 

backend/backend_retry_test.go

@@ -62,11 +62,11 @@ func TestBackendListRetry(t *testing.T) {
 			// fail during first retry, succeed during second
 			retry++
 			if retry == 1 {
-				fn(restic.FileInfo{Name: ID1})
+				_ = fn(restic.FileInfo{Name: ID1})
 				return errors.New("test list error")
 			}
-			fn(restic.FileInfo{Name: ID1})
-			fn(restic.FileInfo{Name: ID2})
+			_ = fn(restic.FileInfo{Name: ID1})
+			_ = fn(restic.FileInfo{Name: ID2})
 			return nil
 		},
 	}

backend/foreground_test.go

@@ -23,7 +23,9 @@ func TestForeground(t *testing.T) {
 
 	bg, err := backend.StartForeground(cmd)
 	rtest.OK(t, err)
-	defer cmd.Wait()
+	defer func() {
+		rtest.OK(t, cmd.Wait())
+	}()
 
 	err = bg()
 	rtest.OK(t, err)

backend/gs/gs.go

@@ -235,7 +235,10 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 	w := be.bucket.Object(objName).NewWriter(ctx)
 	w.ChunkSize = 0
 	wbytes, err := io.Copy(w, rd)
-	w.Close()
+	cerr := w.Close()
+	if err == nil {
+		err = cerr
+	}
 
 	be.sem.ReleaseToken()
 
@@ -245,6 +248,10 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 	}
 
 	debug.Log("%v -> %v bytes", objName, wbytes)
+	// sanity check
+	if wbytes != rd.Length() {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
+	}
 	return nil
 }
 

backend/local/local.go

@@ -118,11 +118,16 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade
 	}
 
 	// save data, then sync
-	_, err = io.Copy(f, rd)
+	wbytes, err := io.Copy(f, rd)
 	if err != nil {
 		_ = f.Close()
 		return errors.Wrap(err, "Write")
 	}
+	// sanity check
+	if wbytes != rd.Length() {
+		_ = f.Close()
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
+	}
 
 	if err = f.Sync(); err != nil {
 		pathErr, ok := err.(*os.PathError)
@@ -260,9 +265,15 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error)
 	if err != nil {
 		return err
 	}
-	defer d.Close()
 
 	sub, err := d.Readdirnames(-1)
+	if err != nil {
+		// ignore subsequent errors
+		_ = d.Close()
+		return err
+	}
+
+	err = d.Close()
 	if err != nil {
 		return err
 	}
@@ -281,9 +292,15 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error)
 	if err != nil {
 		return err
 	}
-	defer d.Close()
 
 	sub, err := d.Readdir(-1)
+	if err != nil {
+		// ignore subsequent errors
+		_ = d.Close()
+		return err
+	}
+
+	err = d.Close()
 	if err != nil {
 		return err
 	}

backend/local/local_internal_test.go

@@ -30,7 +30,9 @@ func TestNoSpacePermanent(t *testing.T) {
 
 	be, err := Open(context.Background(), Config{Path: dir})
 	rtest.OK(t, err)
-	defer be.Close()
+	defer func() {
+		rtest.OK(t, be.Close())
+	}()
 
 	h := restic.Handle{Type: restic.ConfigFile}
 	err = be.Save(context.Background(), h, nil)

backend/mem/mem_backend.go

@@ -81,6 +81,11 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
 		return err
 	}
 
+	// sanity check
+	if int64(len(buf)) != rd.Length() {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", len(buf), rd.Length())
+	}
+
 	be.data[h] = buf
 	debug.Log("saved %v bytes at %v", len(buf), h)
 

backend/rclone/backend.go

@@ -63,9 +63,9 @@ func run(command string, args ...string) (*StdioConn, *exec.Cmd, *sync.WaitGroup
 
 	stdout, w, err := os.Pipe()
 	if err != nil {
-		// close first pipe
-		r.Close()
-		stdin.Close()
+		// close first pipe and ignore subsequent errors
+		_ = r.Close()
+		_ = stdin.Close()
 		return nil, nil, nil, nil, err
 	}
 
@@ -197,8 +197,8 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
 		err := cmd.Wait()
 		debug.Log("Wait returned %v", err)
 		be.waitResult = err
-		// close our side of the pipes to rclone
-		stdioConn.CloseAll()
+		// close our side of the pipes to rclone, ignore errors
+		_ = stdioConn.CloseAll()
 		close(waitCh)
 	}()
 
@@ -228,22 +228,25 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
 	// rclone is able to accept HTTP requests.
 	url := fmt.Sprintf("http://localhost/file-%d", rand.Uint64())
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Set("Accept", rest.ContentTypeV2)
-	req.Cancel = ctx.Done()
 
 	res, err := ctxhttp.Do(ctx, client, req)
 	if err != nil {
-		bg()
+		// ignore subsequent errors
+		_ = bg()
 		_ = cmd.Process.Kill()
 		return nil, errors.Errorf("error talking HTTP to rclone: %v", err)
 	}
 
 	debug.Log("HTTP status %q returned, moving instance to background", res.Status)
-	bg()
+	err = bg()
+	if err != nil {
+		return nil, fmt.Errorf("error moving process to background: %w", err)
+	}
 
 	return be, nil
 }

backend/rclone/internal_test.go

@@ -23,7 +23,10 @@ func TestRcloneExit(t *testing.T) {
 		return
 	}
 	rtest.OK(t, err)
-	defer be.Close()
+	defer func() {
+		// ignore the error as the test will kill rclone (see below)
+		_ = be.Close()
+	}()
 
 	err = be.cmd.Process.Kill()
 	rtest.OK(t, err)

backend/rest/rest.go

@@ -132,15 +132,10 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
 	resp, err := ctxhttp.Do(ctx, b.client, req)
 	b.sem.ReleaseToken()
 
+	var cerr error
 	if resp != nil {
-		defer func() {
-			_, _ = io.Copy(ioutil.Discard, resp.Body)
-			e := resp.Body.Close()
-
-			if err == nil {
-				err = errors.Wrap(e, "Close")
-			}
-		}()
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		cerr = resp.Body.Close()
 	}
 
 	if err != nil {
@@ -151,7 +146,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
 		return errors.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode)
 	}
 
-	return nil
+	return errors.Wrap(cerr, "Close")
 }
 
 // ErrIsNotExist is returned whenever the requested file does not exist on the

backend/s3/s3.go

@@ -272,9 +272,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 	opts.ContentType = "application/octet-stream"
 
 	debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length())
-	n, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, ioutil.NopCloser(rd), int64(rd.Length()), opts)
+	info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, ioutil.NopCloser(rd), int64(rd.Length()), opts)
 
-	debug.Log("%v -> %v bytes, err %#v: %v", objName, n, err, err)
+	debug.Log("%v -> %v bytes, err %#v: %v", objName, info.Size, err, err)
+
+	// sanity check
+	if err != nil && info.Size != rd.Length() {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", info.Size, rd.Length())
+	}
 
 	return errors.Wrap(err, "client.PutObject")
 }

backend/sftp/sftp.go

@@ -287,13 +287,19 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
 		return errors.Wrap(err, "OpenFile")
 	}
 
-	// save data
-	_, err = io.Copy(f, rd)
+	// save data, make sure to use the optimized sftp upload method
+	wbytes, err := f.ReadFrom(rd)
 	if err != nil {
 		_ = f.Close()
 		return errors.Wrap(err, "Write")
 	}
 
+	// sanity check
+	if wbytes != rd.Length() {
+		_ = f.Close()
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
+	}
+
 	err = f.Close()
 	if err != nil {
 		return errors.Wrap(err, "Close")
@@ -332,6 +338,8 @@ func (r *SFTP) openReader(ctx context.Context, h restic.Handle, length int, offs
 	}
 
 	if length > 0 {
+		// unlimited reads usually use io.Copy which needs WriteTo support at the underlying reader
+		// limited reads are usually combined with io.ReadFull which reads all required bytes into a buffer in one go
 		return backend.LimitReadCloser(f, int64(length)), nil
 	}
 

backend/swift/swift.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"net/http"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
@@ -176,7 +177,9 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 	encoding := "binary/octet-stream"
 
 	debug.Log("PutObject(%v, %v, %v)", be.container, objName, encoding)
-	_, err := be.conn.ObjectPut(be.container, objName, rd, true, "", encoding, nil)
+	hdr := swift.Headers{"Content-Length": strconv.FormatInt(rd.Length(), 10)}
+	_, err := be.conn.ObjectPut(be.container, objName, rd, true, "", encoding, hdr)
+	// swift does not return the upload length
 	debug.Log("%v, err %#v", objName, err)
 
 	return errors.Wrap(err, "client.PutObject")

backend/test/tests.go

@@ -557,6 +557,40 @@ func (s *Suite) TestSave(t *testing.T) {
 	}
 }
 
+type incompleteByteReader struct {
+	restic.ByteReader
+}
+
+func (r *incompleteByteReader) Length() int64 {
+	return r.ByteReader.Length() + 42
+}
+
+// TestSaveError tests saving data in the backend.
+func (s *Suite) TestSaveError(t *testing.T) {
+	seedRand(t)
+
+	b := s.open(t)
+	defer func() {
+		// rclone will report an error when closing the backend. We have to ignore it
+		// otherwise this test will always fail
+		_ = b.Close()
+	}()
+
+	length := rand.Intn(1<<23) + 200000
+	data := test.Random(24, length)
+	var id restic.ID
+	copy(id[:], data)
+
+	// test that incomplete uploads fail
+	h := restic.Handle{Type: restic.PackFile, Name: id.String()}
+	err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *restic.NewByteReader(data)})
+	// try to delete possible leftovers
+	_ = s.delayedRemove(t, b, h)
+	if err == nil {
+		t.Fatal("incomplete upload did not fail")
+	}
+}
+
 var filenameTests = []struct {
 	name string
 	data string