Support file matching for folders

This allows go-getter to not download everything in a folder

Our usecase is that our S3 bucket contains large files and metadata and we want to only get metadata
With this, we can set a filename regex on the extension
bcfc71

I just cherry picked the changes that Julien did 4 years ago to make ensure that we don't break our fork of this repo but ensuring that we are up to date since there are vulnerabilities in the version we used.

We will have to check if the changes are still required and maybe reopen the PR that was orignially opened by Julien (hashicorp#289)

Author: jocgir

client.go

@@ -83,6 +83,10 @@ type Client struct {
 	// Disable symlinks
 	DisableSymlinks bool
 
+	// FileMatches allows to filter files that should be fetched from a source directory
+	// By default, all files are matched
+	FileMatches FileMatcher
+
 	Options []ClientOption
 }
 

client_option.go

@@ -17,6 +17,11 @@ func (c *Client) Configure(opts ...ClientOption) error {
 		c.Ctx = context.Background()
 	}
 
+	c.FileMatches = func(string, string) bool {
+		return true
+	}
+
+
 	// Store the options used to configure this client.
 	c.Options = opts
 

client_option_matching_files.go

@@ -0,0 +1,29 @@
+package getter
+
+import (
+	"regexp"
+	"strings"
+)
+
+// FileMatcher allows to filter files based on their source and name
+type FileMatcher = func(source, fullFilename string) bool
+
+// WithFileMatcher adds a FileMatcher to the client
+// Any kind of function that returns a boolean from file properties can be used
+func WithFileMatcher(fm FileMatcher) func(*Client) error {
+	return func(c *Client) error {
+		c.FileMatches = fm
+		return nil
+	}
+}
+
+// WithFileMatcher adds a regex FileMatcher to the client
+func WithRegexFileMatcher(regex string) func(*Client) error {
+	return func(c *Client) error {
+		expression, err := regexp.Compile(regex)
+		c.FileMatches = func(source, fullFilename string) bool {
+			return expression.MatchString(fullFilename) || expression.MatchString((strings.TrimPrefix(fullFilename, source)))
+		}
+		return err
+	}
+}

get_s3.go

@@ -138,6 +138,9 @@ func (g *S3Getter) Get(dst string, u *url.URL) error {
 		for _, object := range resp.Contents {
 			lastMarker = aws.StringValue(object.Key)
 			objPath := aws.StringValue(object.Key)
+			if !g.client.FileMatches(path, objPath) {
+				continue
+			}
 
 			// If the key ends with a backslash assume it is a directory and ignore
 			if strings.HasSuffix(objPath, "/") {