Skip to content

Commit

Permalink
Bump restic
Browse files Browse the repository at this point in the history
Upstream ref: dc7a8aab2426e7d78d0a41b1608e75edf56a74d0 (v0.12.1)
  • Loading branch information
rubiojr committed Nov 5, 2021
1 parent 0aa7988 commit 57cd364
Show file tree
Hide file tree
Showing 43 changed files with 2,859 additions and 165 deletions.
2 changes: 1 addition & 1 deletion backend/azure/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ type Config struct {
Container string
Prefix string

Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"`
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
}

// NewConfig returns a new Config with the default values filled in.
Expand Down
2 changes: 1 addition & 1 deletion backend/gs/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ type Config struct {
Bucket string
Prefix string

Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"`
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
}

// NewConfig returns a new Config with the default values filled in.
Expand Down
35 changes: 21 additions & 14 deletions backend/local/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) {
for _, d := range be.Paths() {
err := fs.MkdirAll(d, backend.Modes.Dir)
if err != nil {
return nil, errors.Wrap(err, "MkdirAll")
return nil, errors.WithStack(err)
}
}

Expand All @@ -78,7 +78,7 @@ func (b *Local) Location() string {

// IsNotExist returns true if the error is caused by a non existing file.
func (b *Local) IsNotExist(err error) bool {
return os.IsNotExist(errors.Cause(err))
return errors.Is(err, os.ErrNotExist)
}

// Save stores data in the backend at the handle.
Expand Down Expand Up @@ -114,14 +114,14 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade
}

if err != nil {
return errors.Wrap(err, "OpenFile")
return errors.WithStack(err)
}

// save data, then sync
wbytes, err := io.Copy(f, rd)
if err != nil {
_ = f.Close()
return errors.Wrap(err, "Write")
return errors.WithStack(err)
}
// sanity check
if wbytes != rd.Length() {
Expand All @@ -135,21 +135,21 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade
// ignore error if filesystem does not support the sync operation
if !isNotSupported {
_ = f.Close()
return errors.Wrap(err, "Sync")
return errors.WithStack(err)
}
}

err = f.Close()
if err != nil {
return errors.Wrap(err, "Close")
return errors.WithStack(err)
}

// try to mark file as read-only to avoid accidential modifications
// ignore if the operation fails as some filesystems don't allow the chmod call
// e.g. exfat and network file systems with certain mount options
err = setFileReadonly(filename, backend.Modes.File)
if err != nil && !os.IsPermission(err) {
return errors.Wrap(err, "Chmod")
return errors.WithStack(err)
}

return nil
Expand Down Expand Up @@ -202,7 +202,7 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err

fi, err := fs.Stat(b.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat")
return restic.FileInfo{}, errors.WithStack(err)
}

return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
Expand All @@ -213,10 +213,10 @@ func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) {
debug.Log("Test %v", h)
_, err := fs.Stat(b.Filename(h))
if err != nil {
if os.IsNotExist(errors.Cause(err)) {
if b.IsNotExist(err) {
return false, nil
}
return false, errors.Wrap(err, "Stat")
return false, errors.WithStack(err)
}

return true, nil
Expand All @@ -230,7 +230,7 @@ func (b *Local) Remove(ctx context.Context, h restic.Handle) error {
// reset read-only flag
err := fs.Chmod(fn, 0666)
if err != nil && !os.IsPermission(err) {
return errors.Wrap(err, "Chmod")
return errors.WithStack(err)
}

return fs.Remove(fn)
Expand All @@ -245,7 +245,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File
if subdirs {
err = visitDirs(ctx, basedir, fn)
} else {
err = visitFiles(ctx, basedir, fn)
err = visitFiles(ctx, basedir, fn, false)
}

if b.IsNotExist(err) {
Expand Down Expand Up @@ -279,20 +279,27 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error)
}

for _, f := range sub {
err = visitFiles(ctx, filepath.Join(dir, f), fn)
err = visitFiles(ctx, filepath.Join(dir, f), fn, true)
if err != nil {
return err
}
}
return ctx.Err()
}

func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error) error {
func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error {
d, err := fs.Open(dir)
if err != nil {
return err
}

if ignoreNotADirectory {
fi, err := d.Stat()
if err != nil || !fi.IsDir() {
return err
}
}

sub, err := d.Readdir(-1)
if err != nil {
// ignore subsequent errors
Expand Down
48 changes: 48 additions & 0 deletions backend/rest/rest.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ import (
"io"
"io/ioutil"
"net/http"
"net/textproto"
"net/url"
"path"
"strconv"
"strings"

"golang.org/x/net/context/ctxhttp"
Expand Down Expand Up @@ -197,6 +199,44 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return err
}

// checkContentLength returns an error if the server returned a value in the
// Content-Length header in an HTTP2 connection, but closed the connection
// before any data was sent.
//
// This is a workaround for https://github.com/golang/go/issues/46071
//
// See also https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
func checkContentLength(resp *http.Response) error {
// the following code is based on
// https://github.com/golang/go/blob/b7a85e0003cedb1b48a1fd3ae5b746ec6330102e/src/net/http/h2_bundle.go#L8646

if resp.ContentLength != 0 {
return nil
}

if resp.ProtoMajor != 2 && resp.ProtoMinor != 0 {
return nil
}

if len(resp.Header[textproto.CanonicalMIMEHeaderKey("Content-Length")]) != 1 {
return nil
}

// make sure that if the server returned a content length and we can
// parse it, it is really zero, otherwise return an error
contentLength := resp.Header.Get("Content-Length")
cl, err := strconv.ParseUint(contentLength, 10, 63)
if err != nil {
return fmt.Errorf("unable to parse Content-Length %q: %w", contentLength, err)
}

if cl != 0 {
return errors.Errorf("unexpected EOF: got 0 instead of %v bytes", cl)
}

return nil
}

func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Load %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
Expand Down Expand Up @@ -246,6 +286,14 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
}

// workaround https://github.com/golang/go/issues/46071
// see also https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
err = checkContentLength(resp)
if err != nil {
_ = resp.Body.Close()
return nil, err
}

return resp.Body, nil
}

Expand Down
16 changes: 8 additions & 8 deletions backend/s3/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,12 +186,12 @@ type fileInfo struct {
isDir bool
}

func (fi fileInfo) Name() string { return fi.name } // base name of the file
func (fi fileInfo) Size() int64 { return fi.size } // length in bytes for regular files; system-dependent for others
func (fi fileInfo) Mode() os.FileMode { return fi.mode } // file mode bits
func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for Mode().IsDir()
func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil)
func (fi *fileInfo) Name() string { return fi.name } // base name of the file
func (fi *fileInfo) Size() int64 { return fi.size } // length in bytes for regular files; system-dependent for others
func (fi *fileInfo) Mode() os.FileMode { return fi.mode } // file mode bits
func (fi *fileInfo) ModTime() time.Time { return fi.modTime } // modification time
func (fi *fileInfo) IsDir() bool { return fi.isDir } // abbreviation for Mode().IsDir()
func (fi *fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil)

// ReadDir returns the entries for a directory.
func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, err error) {
Expand Down Expand Up @@ -225,7 +225,7 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo,
if name == "" {
continue
}
entry := fileInfo{
entry := &fileInfo{
name: name,
size: obj.Size,
modTime: obj.LastModified,
Expand Down Expand Up @@ -277,7 +277,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
debug.Log("%v -> %v bytes, err %#v: %v", objName, info.Size, err, err)

// sanity check
if err != nil && info.Size != rd.Length() {
if err == nil && info.Size != rd.Length() {
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", info.Size, rd.Length())
}

Expand Down
60 changes: 46 additions & 14 deletions backend/sftp/sftp.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,17 +165,7 @@ func (r *SFTP) ReadDir(ctx context.Context, dir string) ([]os.FileInfo, error) {
// IsNotExist returns true if the error is caused by a not existing file.
func (r *SFTP) IsNotExist(err error) bool {
err = errors.Cause(err)

if os.IsNotExist(err) {
return true
}

statusError, ok := err.(*sftp.StatusError)
if !ok {
return false
}

return statusError.Error() == `sftp: "No such file" (SSH_FX_NO_SUCH_FILE)`
return errors.Is(err, os.ErrNotExist)
}

func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
Expand Down Expand Up @@ -268,6 +258,7 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
}

filename := r.Filename(h)
dirname := r.Dirname(h)

// create new file
f, err := r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
Expand All @@ -283,10 +274,30 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
}
}

// pkg/sftp doesn't allow creating with a mode.
// Chmod while the file is still empty.
if err == nil {
err = f.Chmod(backend.Modes.File)
}
if err != nil {
return errors.Wrap(err, "OpenFile")
}

defer func() {
if err == nil {
return
}

// Try not to leave a partial file behind.
rmErr := r.c.Remove(f.Name())
if rmErr != nil {
debug.Log("sftp: failed to remove broken file %v: %v",
filename, rmErr)
}

err = r.checkNoSpace(dirname, rd.Length(), err)
}()

// save data, make sure to use the optimized sftp upload method
wbytes, err := f.ReadFrom(rd)
if err != nil {
Expand All @@ -301,11 +312,32 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
}

err = f.Close()
if err != nil {
return errors.Wrap(err, "Close")
return errors.Wrap(err, "Close")
}

// checkNoSpace checks if err was likely caused by lack of available space
// on the remote, and if so, makes it permanent.
func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error {
// The SFTP protocol has a message for ENOSPC,
// but pkg/sftp doesn't export it and OpenSSH's sftp-server
// sends FX_FAILURE instead.

e, ok := origErr.(*sftp.StatusError)
_, hasExt := r.c.HasExtension("[email protected]")
if !ok || e.FxCode() != sftp.ErrSSHFxFailure || !hasExt {
return origErr
}

return errors.Wrap(r.c.Chmod(filename, backend.Modes.File), "Chmod")
fsinfo, err := r.c.StatVFS(dir)
if err != nil {
debug.Log("sftp: StatVFS returned %v", err)
return origErr
}
if fsinfo.Favail == 0 || fsinfo.FreeSpace() < uint64(size) {
err := errors.New("sftp: no space left on device")
return backoff.Permanent(err)
}
return origErr
}

// Load runs fn with a reader that yields the contents of the file at h at the
Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ require (
github.com/ncw/swift v1.0.52
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/pkg/errors v0.9.1
github.com/pkg/sftp v1.12.0
github.com/pkg/sftp v1.13.2
github.com/pkg/xattr v0.4.1
github.com/prometheus/procfs v0.2.0
github.com/restic/chunker v0.4.0
Expand All @@ -39,11 +39,11 @@ require (
github.com/smartystreets/assertions v1.2.0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/urfave/cli/v2 v2.2.0
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
golang.org/x/net v0.0.0-20200904194848-62affa334b73
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
golang.org/x/sys v0.0.0-20200918174421-af09f7315aff
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7
golang.org/x/text v0.3.3
google.golang.org/api v0.33.0
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
Expand Down
Loading

0 comments on commit 57cd364

Please sign in to comment.