Add test suite; rewrite comment processor

The comment processor is now even more cursed. Surely no one will find any regressions this time.

Author: reteps

cmd/version.go

@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print the version number of dockerfmt",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("dockerfmt 0.3.0")
+		fmt.Println("dockerfmt 0.3.1")
 	},
 }

dockerfmt_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/reteps/dockerfmt/lib"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFormatter(t *testing.T) {
+	// assert equality
+	assert.Equal(t, 123, 123, "they should be equal")
+	matchingFiles, err := filepath.Glob("tests/in/*.dockerfile")
+	if err != nil {
+		t.Fatalf("Failed to find test files: %v", err)
+	}
+	c := &lib.Config{
+		IndentSize:      4,
+		TrailingNewline: true,
+		SpaceRedirects:  false,
+	}
+	for _, fileName := range matchingFiles {
+		t.Run(fileName, func(t *testing.T) {
+			outFile := strings.Replace(fileName, "in", "out", 1)
+			originalLines, err := lib.GetFileLines(fileName)
+			if err != nil {
+				t.Fatalf("Failed to read file %s: %v", fileName, err)
+			}
+			fmt.Printf("Comparing file %s with %s\n", fileName, outFile)
+			formattedLines := lib.FormatFileLines(originalLines, c)
+
+			// Write outFile to directory
+			err = os.WriteFile(outFile, []byte(formattedLines), 0644)
+			if err != nil {
+				t.Fatalf("Failed to write to file %s: %v", outFile, err)
+			}
+
+			// Read outFile
+			outLines, err := lib.GetFileLines(outFile)
+			if err != nil {
+				t.Fatalf("Failed to read file %s: %v", outFile, err)
+			}
+			// Compare outLines with formattedLines
+			assert.Equal(t, strings.Join(outLines, ""), formattedLines, "Files should be equal")
+
+		})
+	}
+}

go.mod

@@ -6,15 +6,19 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/moby/buildkit v0.20.2
 	github.com/spf13/cobra v1.9.1
+	github.com/stretchr/testify v1.10.0
 	mvdan.cc/sh/v3 v3.11.0
 )
 
 require (
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	google.golang.org/protobuf v1.35.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -65,6 +65,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
 google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

js/format.wasm

@@ -1,6 +1,6 @@
 {
   "name": "@reteps/dockerfmt",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "type": "module",
   "description": "",
   "repository": "git+https://github.com/reteps/dockerfmt/tree/main/js",

js/package-lock.json

@@ -229,11 +229,65 @@ func formatShell(content string, hereDoc bool, c *Config) string {
 	}
 
 	if !hereDoc {
+		// Here lies some cursed magic. Be careful.
+
 		// Replace comments with a subshell evaluation -- they won't be run so we can do this.
 		content = StripWhitespace(content, true)
+		lineComment := regexp.MustCompile(`(\n\s*)(#.*)`)
+		content = lineComment.ReplaceAllString(content, "$1`$2#`\\")
+
+		/*
+			```
+			foo \
+			`#comment#`\
+			&& bar
+			```
+
+			```
+			foo && \
+			`#comment#` \
+			bar
+			```
+		*/
+
+		commentContinuation := regexp.MustCompile(`(\\(?:\s*` + "`#.*#`" + `\\){1,}\s*)&&`)
+		content = commentContinuation.ReplaceAllString(content, "&&$1")
+
 		// log.Printf("Content0: %s\n", content)
-		re := regexp.MustCompile(`(\\\n\s+)((?:\s*#.*){1,})`)
-		content = re.ReplaceAllString(content, `'dummynode';$1$( $2`+"\n"+`)\`)
+		lines := strings.SplitAfter(content, "\n")
+		/**
+		if the next line is not a comment, and we didn't start with a continuation, don't add the `&&`.
+		*/
+		inContinutation := false
+		for i := range lines {
+			lineTrim := strings.Trim(lines[i], " \t\\\n")
+			// fmt.Printf("LineTrim: %s\n", lineTrim)
+			nextLine := ""
+			isComment := false
+			nextLineIsComment := false
+			if i+1 < len(lines) {
+				nextLine = strings.Trim(lines[i+1], " \t\\\n")
+			}
+			if len(nextLine) >= 2 && nextLine[:2] == "`#" {
+				nextLineIsComment = true
+			}
+			if len(lineTrim) >= 2 && lineTrim[:2] == "`#" {
+				isComment = true
+			}
+
+			// fmt.Printf("isComment: %v, nextLineIsComment: %v, inContinutation: %v\n", isComment, nextLineIsComment, inContinutation)
+			if isComment && (inContinutation || nextLineIsComment) {
+				lines[i] = strings.Replace(lines[i], "#`\\", "#`&&\\", 1)
+			}
+
+			if len(lineTrim) >= 2 && !isComment && lineTrim[len(lineTrim)-2:] == "&&" {
+				inContinutation = true
+			} else if !isComment {
+				inContinutation = false
+			}
+		}
+
+		content = strings.Join(lines, "")
 	}
 
 	// Now that we have a valid bash-style command, we can format it with shfmt
@@ -242,25 +296,28 @@ func formatShell(content string, hereDoc bool, c *Config) string {
 	// log.Printf("Content2: %s\n", content)
 
 	if !hereDoc {
-		content = regexp.MustCompile(`\$\(\s+(#[\w\W]*?)\s+\) \\`).ReplaceAllString(content, "$1")
-		// log.Printf("Content3: %s\n", content)
-		content = strings.ReplaceAll(content, "'dummynode' ", "")
-		content = strings.ReplaceAll(content, "'dummynode'", "")
-		content = regexp.MustCompile(`(\s*#.*)`).ReplaceAllString(content, "$1")
-		// log.Printf("Content4: %s\n", content)
-		content = regexp.MustCompile("(?m)^ *(#.*)").ReplaceAllString(content, strings.Repeat(" ", int(c.IndentSize))+"$1")
-
-		// Add backslashes if needed
+		reBacktickComment := regexp.MustCompile(`([ \t]*)(?:&& )?` + "`(#.*)#` " + `\\`)
+		content = reBacktickComment.ReplaceAllString(content, "$1$2")
+
+		// Fixup the comment indentation
 		lines := strings.SplitAfter(content, "\n")
-		for i := 0; i < len(lines); i++ {
-			line := lines[i]
-			if len(line) > 0 && line[len(line)-2] != '\\' && line[len(line)-1] == '\n' {
-				// Check if the next line is empty and if the current line is not a comment
-				if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" && strings.TrimSpace(line)[0] != '#' {
-					line = strings.TrimRight(line, " \n") + " \\\n"
+		prevIsComment := false
+		prevCommentSpacing := ""
+		for i := range lines {
+			lineTrim := strings.TrimLeft(lines[i], " \t")
+			// fmt.Printf("LineTrim: %s, %v\n", lineTrim, prevIsComment)
+			if len(lineTrim) >= 1 && lineTrim[0] == '#' {
+				lineParts := strings.SplitN(lines[i], "#", 2)
+
+				if prevIsComment {
+					lines[i] = prevCommentSpacing + "#" + lineParts[1]
+				} else {
+					prevCommentSpacing = lineParts[0]
 				}
+				prevIsComment = true
+			} else {
+				prevIsComment = false
 			}
-			lines[i] = line
 		}
 		content = strings.Join(lines, "")
 

js/package.json

@@ -27,4 +27,6 @@ RUN if [[ x$LATEST_NPM = xtrue ]]; then yarn global add npm@latest; fi
 
 FROM ubuntu
 
-RUN (cd out && ls)
+RUN (cd out && ls)
+
+RUN ls ; ls

lib/format.go

@@ -0,0 +1,45 @@
+RUN apt-get update \
+    # Comment here
+    && apt-get install -y --no-install-recommends \
+        # Another comment here
+        man-db unminimize \
+                    # Multiline comment here
+        # Here
+        gosu curl git htop less nano unzip vim wget zip && \
+    yes | unminimize && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* \
+    && find /tmp -not -path /tmp -delete
+
+RUN apt-get update && \
+    # Run 'unminimize' to add docs
+    apt-get install -y --no-install-recommends man-db unminimize \
+    && yes | unminimize \
+    && apt-get install -y --no-install-recommends \
+    # Reverse proxy workaround for PrairieLearn:
+    nginx \
+    gettext \
+    gosu \
+    fonts-dejavu \
+    # Utilities for convenience debugging this container:
+    less htop vim nano silversearcher-ag zip unzip git cmake curl wget sqlite3 && \
+    # Test:
+    gosu nobody true && \
+    # Cleanup:
+    apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && find /tmp -not -path /tmp -delete
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends man-db unminimize \
+    && yes | unminimize \
+    && apt-get install -y --no-install-recommends \
+    nginx \
+    gettext \
+    gosu \
+    fonts-dejavu \
+    less htop vim nano silversearcher-ag zip unzip git cmake curl wget sqlite3 && \
+    gosu nobody true && \
+    apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && find /tmp -not -path /tmp -delete

tests/comment.dockerfile renamed to tests/in/comment.dockerfile

@@ -0,0 +1,13 @@
+RUN apt-get update \
+    # Comment here
+    && apt-get install -y --no-install-recommends \
+        # Another comment here
+        man-db unminimize \
+                    # Multiline comment here
+        # Here
+        # And here
+        gosu curl git htop less nano unzip vim wget zip && \
+    yes | unminimize && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* \
+    && find /tmp -not -path /tmp -delete

tests/env.dockerfile renamed to tests/in/env.dockerfile

@@ -0,0 +1,43 @@
+# https://github.com/jessfraz/dockfmt/issues/2
+# set up PrairieLearn and run migrations to initialize the DB
+RUN chmod +x /PrairieLearn/scripts/init.sh \
+    && mkdir /course{,{2..9}} \
+    && mkdir -p /workspace_{main,host}_zips \
+    && mkdir -p /jobs \
+    # Here is a comment in the middle of my command -- `docker run -it --rm`
+    && /PrairieLearn/scripts/start_postgres.sh \
+                && cd /PrairieLearn \               
+    && make build \
+    # Here is a multiline comment in my command
+    # The parser has to handle this case, and strip out the
+    # $() chars surrounding
+    && node apps/prairielearn/dist/server.js --migrate-and-exit \            
+    && su postgres -c "createuser -s root" \
+    && /PrairieLearn/scripts/start_postgres.sh stop \
+    && /PrairieLearn/scripts/gen_ssl.sh \
+    && git config --global user.email "[email protected]" \
+    && git config --global user.name "Dev User" \
+    && git config --global safe.directory '*'
+
+RUN chmod +x /PrairieLearn/scripts/init.sh \
+    && mkdir /course{,{2..9}} \
+    && mkdir -p /workspace_{main,host}_zips \
+    && mkdir -p /jobs && \
+        # Here is a comment in the middle of my command -- `docker run -it --rm`
+    /PrairieLearn/scripts/start_postgres.sh \
+    && cd /PrairieLearn && \
+    make build && \
+        # Here is a multiline comment in my command
+        # The parser has to handle this case, and strip out the
+        # $() chars surrounding
+    node apps/prairielearn/dist/server.js --migrate-and-exit \
+    && su postgres -c "createuser -s root" \
+    && /PrairieLearn/scripts/start_postgres.sh stop \
+    && /PrairieLearn/scripts/gen_ssl.sh \
+    && git config --global user.email "[email protected]" \
+    && git config --global user.name "Dev User" \
+    && git config --global safe.directory '*'
+
+healthcheck --interval=5m --timeout=3s \
+  CMD curl -f http://localhost/ || exit 1
+CMD /PrairieLearn/scripts/init.sh

tests/flags.dockerfile renamed to tests/in/flags.dockerfile

@@ -11,4 +11,12 @@ RUN yarn           --frozen-lockfile &&            \
 # 最终的应用
 FROM          abiosoft/caddy
 COPY --from=builder          /app/packages/ufc-host-app/build /srv
-EXPOSE      2015
+EXPOSE      2015
+
+FROM foobar   
+RUN ls
+LABEL foo=bar   
+HEALTHCHECK NONE   
+CMD ls
+COPY . .   
+ADD . .   

tests/formatted.dockerfile renamed to tests/in/formatted.dockerfile

@@ -0,0 +1,3 @@
+# https://github.com/jessfraz/dockfmt/issues/12
+FROM scratch
+# a comment

tests/heredoc.dockerfile renamed to tests/in/heredoc.dockerfile

@@ -0,0 +1,13 @@
+# https://github.com/un-ts/prettier/issues/398
+ENV a=1 \
+    b=2 \
+    # comment
+    c=3 \
+    d=4 \
+    # comment
+    e=5
+
+ENV MY_VAR=my-value
+ENV MY_VAR=my-value2 \
+    c=4
+ENV MY_VAR=my-value3

tests/label.dockerfile renamed to tests/in/label.dockerfile

@@ -0,0 +1,8 @@
+# https://github.com/jessfraz/dockfmt/issues/23
+FROM --platform=linux/arm64 debian
+
+RUN --network=host apt-get install vim
+RUN --security echo "test"
+COPY --chown=my-user:my-group --chmod=644 ./config.conf /data/config.conf
+COPY --link ./another-file /data/linked-file
+ADD --keep-git-dir ./ /data/src