Merge pull request #181 from openconfig/fix-adjacent-string-concat

Fix lexer to identify single-quote string that immediately occurs after a concatenation (plus sign).

Author: wenovus

pkg/yang/lex.go

@@ -386,8 +386,18 @@ func lexGround(l *lexer) stateFn {
 			l.next()
 			l.next()
 			return lexGround
+		default:
+			return lexIdentifier
+		}
+	case '+':
+		l.next()
+		switch l.peek() {
+		case '"', '\'':
+			l.emit(tIdentifier)
+			return lexGround
+		default:
+			return lexIdentifier
 		}
-		fallthrough
 	default:
 		return lexIdentifier
 	}

pkg/yang/lex_test.go

@@ -122,6 +122,17 @@ Tests:
 			T(tIdentifier, "fred"),
 		}},
 		{line(), `
+pattern '[a-zA-Z0-9!#$%&'+"'"+'*+/=?^_` + "`" + `{|}~-]+';
+`, []*token{
+			T(tIdentifier, "pattern"),
+			T(tString, "[a-zA-Z0-9!#$%&"),
+			T(tIdentifier, "+"),
+			T(tString, "'"),
+			T(tIdentifier, "+"),
+			T(tString, "*+/=?^_`{|}~-]+"),
+			T(';', ";"),
+		}},
+		{line(), `
 // tab indent both lines
 	"Broken
 	line"
@@ -182,7 +193,7 @@ Tests:
 				continue Tests
 			}
 			if len(tt.tokens) > i && !token.Equal(tt.tokens[i]) {
-				t.Errorf("%d: got %v want %v", tt.line, token, tt.tokens[i])
+				t.Errorf("%d, %d: got (%v, %q) want (%v, %q)", tt.line, i, token.code, token.Text, tt.tokens[i].code, tt.tokens[i].Text)
 			}
 		}
 	}

pkg/yang/parse_test.go

@@ -217,6 +217,27 @@ foo1 {
 			},
 		},
 		{line: line(), in: `
+foo1 {
+    key value1;
+    foo2 {
+      pattern '[a-zA-Z0-9!#$%&'+"'"+'*+/=?^_` + "`" + `{|}~-]+'
+            + '(\.[a-zA-Z0-9!#$%&'+"'"+'*+/=?^_` + "`" + `{|}~-]+)*'
+            + '@'
+            + '[a-zA-Z0-9!#$%&'+"'"+'*+/=?^_` + "`" + `{|}~-]+'
+            + '(\.[a-zA-Z0-9!#$%&'+"'"+'*+/=?^_` + "`" + `{|}~-]+)*';
+    }
+}
+`,
+			out: []*Statement{
+				S("foo1",
+					SA("key", "value1"),
+					S("foo2",
+						SA("pattern", "[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*"),
+					),
+				),
+			},
+		},
+		{line: line(), in: `
  }
 `,
 			err: `test.yang:2:2: unexpected }`,