add a simple spanner sql parser for migration

Author: yad-unext

.golangci.yml

@@ -28,3 +28,6 @@ issues:
   exclude:
     # gosec: Duplicated errcheck checks
     - G104
+    - bad syntax for struct tag pair
+    - struct literal uses unkeyed fields
+    - bad syntax for struct tag key

database/spanner/ddl/.got/snapshots/TestBasic/ddl-string.json

@@ -0,0 +1 @@
+"A B;\nA;\nA (B C);\nA \"B\";\nA <B , C>;\nA B , A (B (C D)) E;\n"

database/spanner/ddl/.got/snapshots/TestBasic/ddl.json

@@ -0,0 +1,164 @@
+{
+  "Statements": [
+    {
+      "Expression": {
+        "Values": [
+          {
+            "Component": [
+              "A",
+              "B"
+            ],
+            "Braced": null,
+            "Bracketed": null
+          }
+        ]
+      },
+      "End": ";"
+    },
+    {
+      "Expression": {
+        "Values": [
+          {
+            "Component": [
+              "A"
+            ],
+            "Braced": null,
+            "Bracketed": null
+          }
+        ]
+      },
+      "End": ";"
+    },
+    {
+      "Expression": {
+        "Values": [
+          {
+            "Component": [
+              "A"
+            ],
+            "Braced": null,
+            "Bracketed": null
+          },
+          {
+            "Component": null,
+            "Braced": {
+              "Values": [
+                {
+                  "Component": [
+                    "B",
+                    "C"
+                  ],
+                  "Braced": null,
+                  "Bracketed": null
+                }
+              ]
+            },
+            "Bracketed": null
+          }
+        ]
+      },
+      "End": ";"
+    },
+    {
+      "Expression": {
+        "Values": [
+          {
+            "Component": [
+              "A",
+              "\"B\""
+            ],
+            "Braced": null,
+            "Bracketed": null
+          }
+        ]
+      },
+      "End": ";"
+    },
+    {
+      "Expression": {
+        "Values": [
+          {
+            "Component": [
+              "A"
+            ],
+            "Braced": null,
+            "Bracketed": null
+          },
+          {
+            "Component": null,
+            "Braced": null,
+            "Bracketed": {
+              "Values": [
+                {
+                  "Component": [
+                    "B",
+                    ",",
+                    "C"
+                  ],
+                  "Braced": null,
+                  "Bracketed": null
+                }
+              ]
+            }
+          }
+        ]
+      },
+      "End": ";"
+    },
+    {
+      "Expression": {
+        "Values": [
+          {
+            "Component": [
+              "A",
+              "B",
+              ",",
+              "A"
+            ],
+            "Braced": null,
+            "Bracketed": null
+          },
+          {
+            "Component": null,
+            "Braced": {
+              "Values": [
+                {
+                  "Component": [
+                    "B"
+                  ],
+                  "Braced": null,
+                  "Bracketed": null
+                },
+                {
+                  "Component": null,
+                  "Braced": {
+                    "Values": [
+                      {
+                        "Component": [
+                          "C",
+                          "D"
+                        ],
+                        "Braced": null,
+                        "Bracketed": null
+                      }
+                    ]
+                  },
+                  "Bracketed": null
+                }
+              ]
+            },
+            "Bracketed": null
+          },
+          {
+            "Component": [
+              "E"
+            ],
+            "Braced": null,
+            "Bracketed": null
+          }
+        ]
+      },
+      "End": ";"
+    }
+  ]
+}

database/spanner/ddl/ddl_test.go

@@ -0,0 +1,32 @@
+package ddl
+
+import (
+	"testing"
+
+	"github.com/ysmood/got"
+)
+
+func TestBasic(t *testing.T) {
+	g := got.T(t)
+
+	ddl, err := parser.ParseString("", `
+		-- Comment
+		A B;
+		A;
+		A (B C); /* 
+			Multi-line
+			comment
+		*/
+		A "B";
+		A <B, C>;
+		A -- Comment
+		B,
+		-- Comment
+		A (B (C D)) E;
+	`)
+	g.E(err)
+
+	g.Snapshot("ddl", ddl)
+
+	g.Snapshot("ddl-string", ddl.String())
+}

database/spanner/ddl/format.go

@@ -0,0 +1,53 @@
+package ddl
+
+import (
+	"fmt"
+	"strings"
+)
+
+func ToMigrationStatements(path, ddl string) ([]string, error) {
+	_, err := parser.ParseString(path, ddl)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse DDL: %w", err)
+	}
+
+	return nil, nil
+}
+
+func (d *SpannerDDL) String() string {
+	out := ""
+
+	for _, s := range d.Statements {
+		out += s.String() + "\n"
+	}
+
+	return out
+}
+
+func (s *Statement) String() string {
+	return s.Expression.String() + s.End
+}
+
+func (e *Expression) String() string {
+	list := []string{}
+
+	for _, v := range e.Values {
+		list = append(list, v.String())
+	}
+
+	return strings.Join(list, " ")
+}
+
+func (v *Value) String() string {
+	out := ""
+
+	if v.Braced != nil {
+		out += "(" + v.Braced.String() + ")"
+	} else if v.Bracketed != nil {
+		out += "<" + v.Bracketed.String() + ">"
+	} else {
+		out += strings.Join(v.Component, " ")
+	}
+
+	return out
+}

database/spanner/ddl/parser.go

@@ -0,0 +1,46 @@
+package ddl
+
+import (
+	"github.com/alecthomas/participle/v2"
+	"github.com/alecthomas/participle/v2/lexer"
+)
+
+// SpannerDDL only parses the components that are relevant for the migration tool.
+// Main objectives:
+//   - Remove comments: spanner does allow comments in DDL.
+//   - Split DDL into statements: spanner requires the migration request to be a list of statements, not a single DDL file.
+//     Also each request should not exceed 10 statements.
+type SpannerDDL struct {
+	Statements []*Statement `@@*`
+}
+
+type Statement struct {
+	Expression *Expression `@@`
+	End        string      `@";"`
+}
+
+type Expression struct {
+	Values []*Value `@@+`
+}
+
+type Value struct {
+	Component []string    `  @(Ident | String | Comma)+`
+	Braced    *Expression `| "(" @@ ")"`
+	Bracketed *Expression `| "<" @@ ">"`
+}
+
+var parser = participle.MustBuild[SpannerDDL](
+	participle.Lexer(lexer.MustSimple([]lexer.SimpleRule{
+		{"SingleLineComment", `--[^\n]*`},
+		{"MultiLineComment", `(?s)/\*.*?\*/`},
+		{"Whitespace", `\s+`},
+
+		{"StatementEnd", `;`},
+		{"Group", `[()<>]`},
+
+		{"Ident", `[a-zA-Z\d_]+`},
+		{"String", `"(?:\\.|[^"])*"`},
+		{"Comma", `,`},
+	})),
+	participle.Elide("SingleLineComment", "MultiLineComment", "Whitespace"),
+)

go.mod

@@ -7,6 +7,7 @@ require (
 	cloud.google.com/go/storage v1.38.0
 	github.com/Azure/go-autorest/autorest/adal v0.9.16
 	github.com/ClickHouse/clickhouse-go v1.4.3
+	github.com/alecthomas/participle/v2 v2.1.4
 	github.com/aws/aws-sdk-go v1.49.6
 	github.com/cenkalti/backoff/v4 v4.1.2
 	github.com/cockroachdb/cockroach-go/v2 v2.1.1
@@ -33,6 +34,7 @@ require (
 	github.com/snowflakedb/gosnowflake v1.6.19
 	github.com/stretchr/testify v1.9.0
 	github.com/xanzy/go-gitlab v0.15.0
+	github.com/ysmood/got v0.41.0
 	go.mongodb.org/mongo-driver v1.7.5
 	go.uber.org/atomic v1.7.0
 	golang.org/x/oauth2 v0.18.0
@@ -56,6 +58,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/ysmood/gop v0.2.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
 	go.opentelemetry.io/otel v1.29.0 // indirect

go.sum

@@ -69,6 +69,12 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
+github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U=
+github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI=
@@ -321,6 +327,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -581,6 +589,10 @@ github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCO
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
+github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
+github.com/ysmood/got v0.41.0 h1:XiFH311ltTSGyxjeKcNvy7dzbJjjTzn6DBgK313JHBs=
+github.com/ysmood/got v0.41.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=