Feature: 上传时默认忽略隐藏的文件 (#92)

* Feature: 上传时默认忽略隐藏的文件

- 用户上传目录时，目录中包含的 . 开头的文件默认会被忽略
- 用户可以使用 --all 参数 强制上传所以文件

* 忽略windows下的隐藏文件

- 增加对windows系统下隐藏文件忽略的支持

* 上传隐藏目录触发提示

如果上传的是目录，并且是隐藏的目录，没有使用 `-all` 参数则触发提示

* chore: 增加忽略文件的测试用例

Author: arrebole

README.md

@@ -363,7 +363,7 @@ upx get -c /baima_text_auditer.tar
 | options | 说明                          |
 |---------|-----------------------------|
 | -w | 多线程下载 (1-10) (default: 5) |
-
+| -all |  上传包含目录下隐藏的文件和文件夹 |
 #### 语法
 ```bash
 upx put <local-file> [remote-file]
@@ -391,6 +391,7 @@ upx put ./video /myfiles
 | options | 说明                          |
 |---------|-----------------------------|
 | -w | 多线程下载 (1-10) (default: 5) |
+| -all |  上传包含目录下隐藏的文件和文件夹 |
 | --remote | 远程路径 |
 
 #### 语法

commands.go

@@ -347,11 +347,13 @@ func NewPutCommand() cli.Command {
 				localPath,
 				upPath,
 				c.Int("w"),
+				c.Bool("all"),
 			)
 			return nil
 		},
 		Flags: []cli.Flag{
 			cli.IntFlag{Name: "w", Usage: "max concurrent threads", Value: 5},
+			cli.BoolFlag{Name: "all", Usage: "upload all files including hidden files"},
 		},
 	}
 }
@@ -370,10 +372,12 @@ func NewUploadCommand() cli.Command {
 				c.Args(),
 				c.String("remote"),
 				c.Int("w"),
+				c.Bool("all"),
 			)
 			return nil
 		},
 		Flags: []cli.Flag{
+			cli.BoolFlag{Name: "all", Usage: "upload all files including hidden files"},
 			cli.IntFlag{Name: "w", Usage: "max concurrent threads", Value: 5},
 			cli.StringFlag{Name: "remote", Usage: "remote path", Value: "./"},
 		},

fsutil/ignore.go

@@ -0,0 +1,10 @@
+package fsutil
+
+import (
+	"strings"
+)
+
+// 判断文件是否是 . 开头的
+func hasDotPrefix(filename string) bool {
+	return strings.HasPrefix(filename, ".")
+}

fsutil/ignore_unix.go

@@ -0,0 +1,13 @@
+//go:build linux || darwin
+
+package fsutil
+
+import (
+	"io/fs"
+	"path/filepath"
+)
+
+// 判断文件是否是需要忽略的文件
+func IsIgnoreFile(path string, fileInfo fs.FileInfo) bool {
+	return hasDotPrefix(filepath.Base(path))
+}

fsutil/ignore_windows.go

@@ -0,0 +1,23 @@
+//go:build windows
+
+package fsutil
+
+import (
+	"io/fs"
+	"path/filepath"
+	"syscall"
+)
+
+// 判断文件是否是需要忽略的文件
+func IsIgnoreFile(path string, fileInfo fs.FileInfo) bool {
+	for hasDotPrefix(filepath.Base(path)) {
+		return true
+	}
+
+	underlyingData := fileInfo.Sys().(*syscall.Win32FileAttributeData)
+	if underlyingData != nil {
+		return underlyingData.FileAttributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0
+	}
+
+	return false
+}

putiginore_test.go

@@ -0,0 +1,99 @@
+package upx
+
+import (
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Ls(up string) ([]string, error) {
+	b, err := Upx("ls", up)
+	if err != nil {
+		return nil, err
+	}
+
+	var ups = make([]string, 0)
+	output := strings.TrimRight(string(b), "\n")
+	for _, line := range strings.Split(output, "\n") {
+		items := strings.Split(line, " ")
+		ups = append(ups, items[len(items)-1])
+	}
+	return ups, nil
+}
+
+func TestPutIgnore(t *testing.T) {
+	SetUp()
+	defer TearDown()
+
+	upRootPath := path.Join(ROOT, "iginore")
+	Upx("mkdir", upRootPath)
+
+	localRootPath, err := ioutil.TempDir("", "test")
+	assert.NoError(t, err)
+	localRootName := filepath.Base(localRootPath)
+
+	CreateFile(path.Join(localRootPath, "FILE1"))
+	CreateFile(path.Join(localRootPath, "FILE2"))
+	CreateFile(path.Join(localRootPath, ".FILE3"))
+	CreateFile(path.Join(localRootPath, ".FILES/FILE"))
+
+	// 上传文件夹
+	// 不包含隐藏的文件，所以只有FILE1和FILE2
+	Upx("put", localRootPath, upRootPath)
+	files, err := Ls(path.Join(upRootPath, localRootName))
+
+	assert.NoError(t, err)
+	assert.Len(t, files, 2)
+	assert.ElementsMatch(
+		t,
+		files,
+		[]string{"FILE1", "FILE2"},
+	)
+
+	// 上传隐藏的文件夹, 无all，上传失效
+	Upx(
+		"put",
+		path.Join(localRootPath, ".FILES"),
+		path.Join(upRootPath, localRootName, ".FILES"),
+	)
+	files, err = Ls(path.Join(upRootPath, localRootName))
+	assert.NoError(t, err)
+
+	assert.Len(t, files, 2)
+	assert.ElementsMatch(
+		t,
+		files,
+		[]string{"FILE1", "FILE2"},
+	)
+
+	// 上传隐藏的文件夹, 有all，上传成功
+	Upx(
+		"put",
+		"-all",
+		path.Join(localRootPath, ".FILES"),
+		path.Join(upRootPath, localRootName, ".FILES"),
+	)
+	files, err = Ls(path.Join(upRootPath, localRootName))
+	assert.NoError(t, err)
+	assert.Len(t, files, 3)
+	assert.ElementsMatch(
+		t,
+		files,
+		[]string{"FILE1", "FILE2", ".FILES"},
+	)
+
+	// 上传所有文件
+	Upx("put", "-all", localRootPath, upRootPath)
+	files, err = Ls(path.Join(upRootPath, localRootName))
+	assert.NoError(t, err)
+	assert.Len(t, files, 4)
+	assert.ElementsMatch(
+		t,
+		files,
+		[]string{"FILE1", "FILE2", ".FILE3", ".FILES"},
+	)
+}

session.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"io/ioutil"
 	"log"
 	"math/rand"
@@ -20,6 +21,7 @@ import (
 	"github.com/arrebole/progressbar"
 	"github.com/fatih/color"
 	"github.com/upyun/go-sdk/v3/upyun"
+	"github.com/upyun/upx/fsutil"
 	"github.com/upyun/upx/partial"
 )
 
@@ -496,7 +498,6 @@ func (sess *Session) putFileWithProgress(localPath, upPath string, localInfo os.
 		return err
 	}
 	defer fd.Close()
-
 	cfg := &upyun.PutObjectConfig{
 		Path: upPath,
 		Headers: map[string]string{
@@ -601,20 +602,32 @@ func (sess *Session) putFilesWitchProgress(localFiles []*UploadedFile, workers i
 	wg.Wait()
 }
 
-func (sess *Session) putDir(localPath, upPath string, workers int) {
+func (sess *Session) putDir(localPath, upPath string, workers int, withIgnore bool) {
+	localAbsPath, err := filepath.Abs(localPath)
+	if err != nil {
+		PrintErrorAndExit(err.Error())
+	}
+	// 如果上传的是目录，并且是隐藏的目录，则触发提示
+	rootDirInfo, err := os.Stat(localAbsPath)
+	if err != nil {
+		PrintErrorAndExit(err.Error())
+	}
+	if !withIgnore && fsutil.IsIgnoreFile(localAbsPath, rootDirInfo) {
+		PrintErrorAndExit("%s is a ignore dir, use `-all` to force put all files", localAbsPath)
+	}
+
 	type FileInfo struct {
 		fpath string
 		fInfo os.FileInfo
 	}
 	localFiles := make(chan *FileInfo, workers*2)
 	var wg sync.WaitGroup
-	var err error
 	wg.Add(workers)
 	for w := 0; w < workers; w++ {
 		go func() {
 			defer wg.Done()
 			for info := range localFiles {
-				rel, _ := filepath.Rel(localPath, info.fpath)
+				rel, _ := filepath.Rel(localAbsPath, info.fpath)
 				desPath := path.Join(upPath, filepath.ToSlash(rel))
 				if fInfo, err := os.Stat(info.fpath); err == nil && fInfo.IsDir() {
 					err = sess.updriver.Mkdir(desPath)
@@ -628,21 +641,29 @@ func (sess *Session) putDir(localPath, upPath string, workers int) {
 		}()
 	}
 
-	walk(localPath, func(fpath string, fInfo os.FileInfo, err error) {
-		if err == nil {
+	filepath.Walk(localAbsPath, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !withIgnore && fsutil.IsIgnoreFile(path, info) {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+		} else {
 			localFiles <- &FileInfo{
-				fpath: fpath,
-				fInfo: fInfo,
+				fpath: path,
+				fInfo: info,
 			}
 		}
+		return nil
 	})
 
 	close(localFiles)
 	wg.Wait()
 }
 
 // / Put 上传单文件或单目录
-func (sess *Session) Put(localPath, upPath string, workers int) {
+func (sess *Session) Put(localPath, upPath string, workers int, withIgnore bool) {
 	upPath = sess.AbsPath(upPath)
 
 	exist, isDir := false, false
@@ -693,7 +714,7 @@ func (sess *Session) Put(localPath, upPath string, workers int) {
 				upPath = path.Join(upPath, filepath.Base(localPath))
 			}
 		}
-		sess.putDir(localPath, upPath, workers)
+		sess.putDir(localPath, upPath, workers, withIgnore)
 	} else {
 		if isDir {
 			upPath = path.Join(upPath, filepath.Base(localPath))
@@ -703,7 +724,7 @@ func (sess *Session) Put(localPath, upPath string, workers int) {
 }
 
 // put 的升级版命令, 支持多文件上传
-func (sess *Session) Upload(filenames []string, upPath string, workers int) {
+func (sess *Session) Upload(filenames []string, upPath string, workers int, withIgnore bool) {
 	upPath = sess.AbsPath(upPath)
 
 	// 检测云端的目的地目录
@@ -741,7 +762,12 @@ func (sess *Session) Upload(filenames []string, upPath string, workers int) {
 
 	// 上传目录
 	for _, localPath := range dirs {
-		sess.putDir(localPath, path.Join(upPath, filepath.Base(localPath)), workers)
+		sess.putDir(
+			localPath,
+			path.Join(upPath, filepath.Base(localPath)),
+			workers,
+			withIgnore,
+		)
 	}
 
 	// 上传文件

utils.go

@@ -4,9 +4,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -134,19 +132,6 @@ func md5File(fpath string) (string, error) {
 	return fmt.Sprintf("%x", hash.Sum(nil)), nil
 }
 
-func walk(root string, f func(string, os.FileInfo, error)) {
-	fi, err := os.Stat(root)
-	if err == nil && fi != nil && fi.IsDir() {
-		fInfos, err := ioutil.ReadDir(root)
-		f(root, fi, err)
-		for _, fInfo := range fInfos {
-			walk(filepath.Join(root, fInfo.Name()), f)
-		}
-	} else {
-		f(root, fi, err)
-	}
-}
-
 func contains(slice []string, item string) bool {
 	for _, s := range slice {
 		if s == item {