diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..374ab16 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,26 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master, master-intersight ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... + diff --git a/errors.go b/errors.go index 07bd302..c9b8ddb 100644 --- a/errors.go +++ b/errors.go @@ -1,38 +1,66 @@ package godata +import "fmt" + type GoDataError struct { ResponseCode int Message string + Cause error } func (err *GoDataError) Error() string { return err.Message } +func (err *GoDataError) Unwrap() error { + return err.Cause +} + +func (err *GoDataError) SetCause(e error) *GoDataError { + err.Cause = e + return err +} + func BadRequestError(message string) *GoDataError { - return &GoDataError{400, message} + return &GoDataError{400, message, nil} } func NotFoundError(message string) *GoDataError { - return &GoDataError{404, message} + return &GoDataError{404, message, nil} } func MethodNotAllowedError(message string) *GoDataError { - return &GoDataError{405, message} + return &GoDataError{405, message, nil} } func GoneError(message string) *GoDataError { - return &GoDataError{410, message} + return &GoDataError{410, message, nil} } func PreconditionFailedError(message string) *GoDataError { - return &GoDataError{412, message} + return &GoDataError{412, message, nil} } func InternalServerError(message string) *GoDataError { - return &GoDataError{500, message} + return &GoDataError{500, message, nil} } func NotImplementedError(message string) *GoDataError { - return &GoDataError{501, message} + return &GoDataError{501, message, nil} +} + +type UnsupportedQueryParameterError struct { + Parameter string +} + +func (err *UnsupportedQueryParameterError) Error() string { + return fmt.Sprintf("Query parameter '%s' is not supported", err.Parameter) +} + +type DuplicateQueryParameterError struct { + Parameter string +} + +func (err *DuplicateQueryParameterError) Error() string { + return fmt.Sprintf("Query parameter '%s' cannot be specified more than once", err.Parameter) } diff --git a/example/main.go b/example/main.go index 7ce43d2..ff494d7 100644 --- a/example/main.go +++ b/example/main.go @@ -1,8 +1,10 @@ package example +/* import ( . "godata" ) +*/ func HelloWorld() { @@ -17,58 +19,60 @@ func AuthorizationMiddleware() { } func main() { - provider := &MySQLGoDataProvider{ - Hostname: "localhost", - Port: "3306", - Database: "Coffee", - Username: "dev", - Password: "dev", - } - - roaster := provider.ExposeEntity("roaster", "Roaster") - roaster.ExposeKey("id", "RoasterID", GoDataInt32) - roaster.ExposePrimitive("name", "Name", GoDataString) - roaster.ExposePrimitive("location", "Location", GoDataString) - roaster.ExposePrimitive("website", "Website", GoDataString) - roasterSet := provider.ExposeEntitySet(roaster, "Roasters") - - variety := provider.ExposeEntity("variety", "Variety") - variety.ExposeKey("id", "VarietyID", GoDataInt32) - variety.ExposePrimitive("name", "Name", GoDataString) - varietySet := provider.ExposeEntitySet(variety, "Varieties") - - roastLevel := provider.ExposeEntity("roast_level", "RoastLevel") - roastLevel.ExposeKey("id", "RoastLevelID", GoDataInt32) - roastLevel.ExposePrimitive("order", "Order", GoDataInt32) - roastLevel.ExposePrimitive("name", "Name", GoDataString) - roastLevel.ExposePrimitive("qualifier", "Qualifier", GoDataString) - roastLevelSet := provider.ExposeEntitySet(roastLevel, "RoastLevels") - - process := provider.ExposeEntity("process", "Process") - process.ExposeKey("id", "ProcessID", GoDataInt32) - process.ExposePrimitive("name", "Name", GoDataString) - processSet := provider.ExposeEntitySet(process, "Processes") - - bean := provider.ExposeEntity("bean", "Bean") - bean.ExposeKey("id", "BeanID", GoDataInt32) - bean.ExposePrimitive("country", "Country", GoDataString) - bean.ExposePrimitive("region", "Region", GoDataString) - bean.ExposePrimitive("min_elevation", "MinElevation", GoDataInt32) - bean.ExposePrimitive("max_elevation", "MaxElevation", GoDataInt32) - beanSet := provider.ExposeEntitySet(bean, "Beans") - - // map many beans to one roaster - provider.ExposeManyToOne(bean, roaster, "roaster_id", "Roaster", "Beans") - provider.ExposeManyToOne(bean, roastLevel, "roaster_level_id", "RoastLevel", "Beans") - provider.ExposeManyToOne(bean, process, "process_id", "Process", "Beans") - provider.ExposeManyToMany(bean, variety, "bean_variety_map", "Varieties", "Beans") - provider.BindProperty(beanSet, roasterSet, "Roaster", "Roaster", "Beans", "Beans") - provider.BindProperty(beanSet, roastLevelSet, "RoastLevel", "RoastLevel", "Beans", "Beans") - provider.BindProperty(beanSet, processSet, "Process", "Process", "Beans", "Beans") - provider.BindProperty(beanSet, varietySet, "Varieties", "Varieties", "Beans", "Beans") - - service := BuildService(provider) - service.ListenAndServe(":8080", "http://localhost") + /* + provider := &MySQLGoDataProvider{ + Hostname: "localhost", + Port: "3306", + Database: "Coffee", + Username: "dev", + Password: "dev", + } + + roaster := provider.ExposeEntity("roaster", "Roaster") + roaster.ExposeKey("id", "RoasterID", GoDataInt32) + roaster.ExposePrimitive("name", "Name", GoDataString) + roaster.ExposePrimitive("location", "Location", GoDataString) + roaster.ExposePrimitive("website", "Website", GoDataString) + roasterSet := provider.ExposeEntitySet(roaster, "Roasters") + + variety := provider.ExposeEntity("variety", "Variety") + variety.ExposeKey("id", "VarietyID", GoDataInt32) + variety.ExposePrimitive("name", "Name", GoDataString) + varietySet := provider.ExposeEntitySet(variety, "Varieties") + + roastLevel := provider.ExposeEntity("roast_level", "RoastLevel") + roastLevel.ExposeKey("id", "RoastLevelID", GoDataInt32) + roastLevel.ExposePrimitive("order", "Order", GoDataInt32) + roastLevel.ExposePrimitive("name", "Name", GoDataString) + roastLevel.ExposePrimitive("qualifier", "Qualifier", GoDataString) + roastLevelSet := provider.ExposeEntitySet(roastLevel, "RoastLevels") + + process := provider.ExposeEntity("process", "Process") + process.ExposeKey("id", "ProcessID", GoDataInt32) + process.ExposePrimitive("name", "Name", GoDataString) + processSet := provider.ExposeEntitySet(process, "Processes") + + bean := provider.ExposeEntity("bean", "Bean") + bean.ExposeKey("id", "BeanID", GoDataInt32) + bean.ExposePrimitive("country", "Country", GoDataString) + bean.ExposePrimitive("region", "Region", GoDataString) + bean.ExposePrimitive("min_elevation", "MinElevation", GoDataInt32) + bean.ExposePrimitive("max_elevation", "MaxElevation", GoDataInt32) + beanSet := provider.ExposeEntitySet(bean, "Beans") + + // map many beans to one roaster + provider.ExposeManyToOne(bean, roaster, "roaster_id", "Roaster", "Beans") + provider.ExposeManyToOne(bean, roastLevel, "roaster_level_id", "RoastLevel", "Beans") + provider.ExposeManyToOne(bean, process, "process_id", "Process", "Beans") + provider.ExposeManyToMany(bean, variety, "bean_variety_map", "Varieties", "Beans") + provider.BindProperty(beanSet, roasterSet, "Roaster", "Roaster", "Beans", "Beans") + provider.BindProperty(beanSet, roastLevelSet, "RoastLevel", "RoastLevel", "Beans", "Beans") + provider.BindProperty(beanSet, processSet, "Process", "Process", "Beans", "Beans") + provider.BindProperty(beanSet, varietySet, "Varieties", "Varieties", "Beans", "Beans") + + service := BuildService(provider) + service.ListenAndServe(":8080", "http://localhost") + */ //service.AttachMiddleware(CacheMiddleware) //service.AttachMiddleware(AuthorizationMiddleware) diff --git a/expand_parser.go b/expand_parser.go index 7b95ccd..2d0f413 100644 --- a/expand_parser.go +++ b/expand_parser.go @@ -21,6 +21,7 @@ var GlobalExpandTokenizer = ExpandTokenizer() type ExpandItem struct { Path []*Token Filter *GoDataFilterQuery + At *GoDataFilterQuery Search *GoDataSearchQuery OrderBy *GoDataOrderByQuery Skip *GoDataSkipQuery @@ -161,7 +162,7 @@ func ParseExpandOption(queue *tokenQueue, item *ExpandItem) error { return BadRequestError("Invalid expand clause.") } queue.Dequeue() // drop the '=' from the front of the queue - body := queue.String() + body := queue.GetValue() if head == "$filter" { filter, err := ParseFilterString(body) @@ -172,6 +173,15 @@ func ParseExpandOption(queue *tokenQueue, item *ExpandItem) error { } } + if head == "at" { + at, err := ParseFilterString(body) + if err == nil { + item.At = at + } else { + return err + } + } + if head == "$search" { search, err := ParseSearchString(body) if err == nil { diff --git a/expand_parser_test.go b/expand_parser_test.go index 9062637..653348a 100644 --- a/expand_parser_test.go +++ b/expand_parser_test.go @@ -72,13 +72,13 @@ func TestExpandNestedCommas(t *testing.T) { if output.ExpandItems[0].Select.SelectItems[0].Segments[0].Value != "FirstName" { actual := output.ExpandItems[0].Select.SelectItems[0].Segments[0] - t.Error("First select segment is '" + actual.Value + "' not 'FirstName'") + t.Error("First select segment is '" + actual.Value + "', expected 'FirstName'") return } if output.ExpandItems[0].Select.SelectItems[1].Segments[0].Value != "LastName" { actual := output.ExpandItems[0].Select.SelectItems[1].Segments[0] - t.Error("First select segment is '" + actual.Value + "' not 'LastName'") + t.Error("First select segment is '" + actual.Value + "', expected 'LastName'") return } diff --git a/filter_parser.go b/filter_parser.go index cec25c5..a4b042b 100644 --- a/filter_parser.go +++ b/filter_parser.go @@ -4,30 +4,32 @@ const ( FilterTokenOpenParen int = iota FilterTokenCloseParen FilterTokenWhitespace - FilterTokenNav - FilterTokenColon // for 'any' and 'all' lambda operators - FilterTokenComma - FilterTokenLogical - FilterTokenOp + FilterTokenNav // Property navigation + FilterTokenColon // Function arg separator for 'any(v:boolExpr)' and 'all(v:boolExpr)' lambda operators + FilterTokenComma // 5 + FilterTokenLogical // eq|ne|gt|ge|lt|le|and|or|not|has|in + FilterTokenOp // add|sub|mul|divby|div|mod and "/" token when used in lambda expression, e.g. tags/any() FilterTokenFunc - FilterTokenLambda - FilterTokenNull + FilterTokenLambda // any(), all() lambda functions + FilterTokenNull // 10 FilterTokenIt FilterTokenRoot FilterTokenFloat FilterTokenInteger - FilterTokenString + FilterTokenString // 15 FilterTokenDate FilterTokenTime FilterTokenDateTime FilterTokenBoolean - FilterTokenLiteral + FilterTokenLiteral // 20 + FilterTokenDuration // duration = [ "duration" ] SQUOTE durationValue SQUOTE + FilterTokenGuid ) var GlobalFilterTokenizer = FilterTokenizer() var GlobalFilterParser = FilterParser() -// Convert an input string from the $filter part of the URL into a parse +// ParseFilterString converts an input string from the $filter part of the URL into a parse // tree that can be used by providers to create a response. func ParseFilterString(filter string) (*GoDataFilterQuery, error) { tokens, err := GlobalFilterTokenizer.Tokenize(filter) @@ -43,34 +45,86 @@ func ParseFilterString(filter string) (*GoDataFilterQuery, error) { if err != nil { return nil, err } - return &GoDataFilterQuery{tree}, nil + if tree == nil || tree.Token == nil || + (len(tree.Children) == 0 && tree.Token.Type != FilterTokenBoolean) { + return nil, BadRequestError("Value must be a boolean expression") + } + return &GoDataFilterQuery{tree, filter}, nil } -// Create a tokenizer capable of tokenizing filter statements +// FilterTokenDurationRe is a regex for a token of type duration. +// The token value is set to the ISO 8601 string inside the single quotes +// For example, if the input data is duration'PT2H', then the token value is set to PT2H without quotes. +const FilterTokenDurationRe = `^(duration)?'(?P-?P((([0-9]+Y([0-9]+M)?([0-9]+D)?|([0-9]+M)([0-9]+D)?|([0-9]+D))(T(([0-9]+H)([0-9]+M)?([0-9]+(\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\.[0-9]+)?S)?|([0-9]+(\.[0-9]+)?S)))?)|(T(([0-9]+H)([0-9]+M)?([0-9]+(\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\.[0-9]+)?S)?|([0-9]+(\.[0-9]+)?S)))))'` + +// FilterTokenizer creates a tokenizer capable of tokenizing filter statements +// 4.01 Services MUST support case-insensitive operator names. +// See https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part2-url-conventions.html#_Toc31360955 func FilterTokenizer() *Tokenizer { t := Tokenizer{} + // guidValue = 8HEXDIG "-" 4HEXDIG "-" 4HEXDIG "-" 4HEXDIG "-" 12HEXDIG + t.Add(`^[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}`, FilterTokenGuid) + // duration = [ "duration" ] SQUOTE durationValue SQUOTE + // durationValue = [ SIGN ] "P" [ 1*DIGIT "D" ] [ "T" [ 1*DIGIT "H" ] [ 1*DIGIT "M" ] [ 1*DIGIT [ "." 1*DIGIT ] "S" ] ] + // Duration literals in OData 4.0 required prefixing with “duration”. + // In OData 4.01, services MUST support duration and enumeration literals with or without the type prefix. + // OData clients that want to operate across OData 4.0 and OData 4.01 services should always include the prefix for duration and enumeration types. + t.Add(FilterTokenDurationRe, FilterTokenDuration) t.Add("^[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}T[0-9]{2,2}:[0-9]{2,2}(:[0-9]{2,2}(.[0-9]+)?)?(Z|[+-][0-9]{2,2}:[0-9]{2,2})", FilterTokenDateTime) t.Add("^-?[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}", FilterTokenDate) t.Add("^[0-9]{2,2}:[0-9]{2,2}(:[0-9]{2,2}(.[0-9]+)?)?", FilterTokenTime) t.Add("^\\(", FilterTokenOpenParen) t.Add("^\\)", FilterTokenCloseParen) - t.Add("^/", FilterTokenNav) - t.Add("^:", FilterTokenColon) - t.Add("^,", FilterTokenComma) - t.Add("^(eq|ne|gt|ge|lt|le|and|or|not|has)", FilterTokenLogical) - t.Add("^(add|sub|mul|div|mod)", FilterTokenOp) - t.Add("^(contains|endswith|startswith|length|indexof|substring|tolower|toupper|"+ + t.Add("^(?P/)(?i)(any|all)", FilterTokenOp) // '/' as a token between a collection expression and a lambda function any() or all() + t.Add("^/", FilterTokenNav) // '/' as a token for property navigation. + t.AddWithSubstituteFunc("^:", FilterTokenColon, func(in string) string { return "," }) // Function arg separator for lambda functions (any, all) + t.Add("^,", FilterTokenComma) // Default arg separator for functions + // Per ODATA ABNF grammar, functions must be followed by a open parenthesis. + // This implementation is a bit more lenient and allows space character between + // the function name and the open parenthesis. + // TODO: If we remove the optional space character, the function token will be + // mistakenly interpreted as a literal. + // E.g. ABNF for 'geo.distance': + // distanceMethodCallExpr = "geo.distance" OPEN BWS commonExpr BWS COMMA BWS commonExpr BWS CLOSE + t.Add("(?i)^(?P(geo.distance|geo.intersects|geo.length))[\\s(]", FilterTokenFunc) + // According to ODATA ABNF notation, functions must be followed by a open parenthesis with no space + // between the function name and the open parenthesis. + // However, we are leniently allowing space characters between the function and the open parenthesis. + // TODO make leniency configurable. + // E.g. ABNF for 'indexof': + // indexOfMethodCallExpr = "indexof" OPEN BWS commonExpr BWS COMMA BWS commonExpr BWS CLOSE + t.Add("(?i)^(?P(substringof|substring|length|indexof|exists))[\\s(]", FilterTokenFunc) + // Logical operators must be followed by a space character. + // However, in practice user have written requests such as not(City eq 'Seattle') + // We are leniently allowing space characters between the operator name and the open parenthesis. + // TODO make leniency configurable. + // Example: + // notExpr = "not" RWS boolCommonExpr + t.Add("(?i)^(?P(eq|ne|gt|ge|lt|le|and|or|not|has|in))[\\s(]", FilterTokenLogical) + // Arithmetic operators must be followed by a space character. + t.Add("(?i)^(?P(add|sub|mul|divby|div|mod))\\s", FilterTokenOp) + // According to ODATA ABNF notation, functions must be followed by a open parenthesis with no space + // between the function name and the open parenthesis. + // However, we are leniently allowing space characters between the function and the open parenthesis. + // TODO make leniency configurable. + // + // E.g. ABNF for 'contains': + // containsMethodCallExpr = "contains" OPEN BWS commonExpr BWS COMMA BWS commonExpr BWS CLOSE + t.Add("(?i)^(?P(contains|endswith|startswith|tolower|toupper|"+ "trim|concat|year|month|day|hour|minute|second|fractionalseconds|date|"+ "time|totaloffsetminutes|now|maxdatetime|mindatetime|totalseconds|round|"+ - "floor|ceiling|isof|cast|geo.distance|geo.intersects|geo.length)", FilterTokenFunc) - t.Add("^(any|all)", FilterTokenLambda) + "floor|ceiling|isof|cast))[\\s(]", FilterTokenFunc) + // anyExpr = "any" OPEN BWS [ lambdaVariableExpr BWS COLON BWS lambdaPredicateExpr ] BWS CLOSE + // allExpr = "all" OPEN BWS lambdaVariableExpr BWS COLON BWS lambdaPredicateExpr BWS CLOSE + t.Add("(?i)^(?P(any|all))[\\s(]", FilterTokenLambda) t.Add("^null", FilterTokenNull) t.Add("^\\$it", FilterTokenIt) t.Add("^\\$root", FilterTokenRoot) t.Add("^-?[0-9]+\\.[0-9]+", FilterTokenFloat) t.Add("^-?[0-9]+", FilterTokenInteger) t.Add("^'(''|[^'])*'", FilterTokenString) - t.Add("^[a-zA-Z][a-zA-Z0-9_.]*", FilterTokenLiteral) + t.Add("^(true|false)", FilterTokenBoolean) + t.Add("^@*[a-zA-Z][a-zA-Z0-9_.]*", FilterTokenLiteral) // The optional '@' character is used to identify parameter aliases t.Ignore("^ ", FilterTokenWhitespace) return &t @@ -78,13 +132,17 @@ func FilterTokenizer() *Tokenizer { func FilterParser() *Parser { parser := EmptyParser() - parser.DefineOperator("/", 2, OpAssociationLeft, 8) + parser.DefineOperator("/", 2, OpAssociationLeft, 8) // Note: '/' is used as a property navigator and between a collExpr and lambda function. parser.DefineOperator("has", 2, OpAssociationLeft, 8) + // 'in' operator takes a literal list. + // City in ('Seattle') needs to be interpreted as a list expression, not a paren expression. + parser.DefineOperator("in", 2, OpAssociationLeft, 8).SetPreferListExpr(true) parser.DefineOperator("-", 1, OpAssociationNone, 7) parser.DefineOperator("not", 1, OpAssociationLeft, 7) parser.DefineOperator("cast", 2, OpAssociationNone, 7) parser.DefineOperator("mul", 2, OpAssociationNone, 6) - parser.DefineOperator("div", 2, OpAssociationNone, 6) + parser.DefineOperator("div", 2, OpAssociationNone, 6) // Division + parser.DefineOperator("divby", 2, OpAssociationNone, 6) // Decimal Division parser.DefineOperator("mod", 2, OpAssociationNone, 6) parser.DefineOperator("add", 2, OpAssociationNone, 5) parser.DefineOperator("sub", 2, OpAssociationNone, 5) @@ -92,46 +150,46 @@ func FilterParser() *Parser { parser.DefineOperator("ge", 2, OpAssociationLeft, 4) parser.DefineOperator("lt", 2, OpAssociationLeft, 4) parser.DefineOperator("le", 2, OpAssociationLeft, 4) - parser.DefineOperator("isof", 2, OpAssociationLeft, 4) parser.DefineOperator("eq", 2, OpAssociationLeft, 3) parser.DefineOperator("ne", 2, OpAssociationLeft, 3) parser.DefineOperator("and", 2, OpAssociationLeft, 2) parser.DefineOperator("or", 2, OpAssociationLeft, 1) - parser.DefineOperator(":", 2, OpAssociationLeft, 1) - parser.DefineFunction("contains", 2) - parser.DefineFunction("endswith", 2) - parser.DefineFunction("startswith", 2) - parser.DefineFunction("length", 1) - parser.DefineFunction("indexof", 2) - parser.DefineFunction("substring", 2) - parser.DefineFunction("tolower", 1) - parser.DefineFunction("toupper", 1) - parser.DefineFunction("trim", 1) - parser.DefineFunction("concat", 2) - parser.DefineFunction("year", 1) - parser.DefineFunction("month", 1) - parser.DefineFunction("day", 1) - parser.DefineFunction("hour", 1) - parser.DefineFunction("minute", 1) - parser.DefineFunction("second", 1) - parser.DefineFunction("fractionalseconds", 1) - parser.DefineFunction("date", 1) - parser.DefineFunction("time", 1) - parser.DefineFunction("totaloffsetminutes", 1) - parser.DefineFunction("now", 0) - parser.DefineFunction("maxdatetime", 0) - parser.DefineFunction("mindatetime", 0) - parser.DefineFunction("totalseconds", 1) - parser.DefineFunction("round", 1) - parser.DefineFunction("floor", 1) - parser.DefineFunction("ceiling", 1) - parser.DefineFunction("isof", 2) - parser.DefineFunction("cast", 2) - parser.DefineFunction("geo.distance", 2) - parser.DefineFunction("geo.intesects", 2) - parser.DefineFunction("geo.length", 1) - parser.DefineFunction("any", 1) - parser.DefineFunction("all", 1) + parser.DefineFunction("contains", []int{2}) + parser.DefineFunction("endswith", []int{2}) + parser.DefineFunction("startswith", []int{2}) + parser.DefineFunction("exists", []int{2}) + parser.DefineFunction("length", []int{1}) + parser.DefineFunction("indexof", []int{2}) + parser.DefineFunction("substring", []int{2, 3}) + parser.DefineFunction("substringof", []int{2}) + parser.DefineFunction("tolower", []int{1}) + parser.DefineFunction("toupper", []int{1}) + parser.DefineFunction("trim", []int{1}) + parser.DefineFunction("concat", []int{2}) + parser.DefineFunction("year", []int{1}) + parser.DefineFunction("month", []int{1}) + parser.DefineFunction("day", []int{1}) + parser.DefineFunction("hour", []int{1}) + parser.DefineFunction("minute", []int{1}) + parser.DefineFunction("second", []int{1}) + parser.DefineFunction("fractionalseconds", []int{1}) + parser.DefineFunction("date", []int{1}) + parser.DefineFunction("time", []int{1}) + parser.DefineFunction("totaloffsetminutes", []int{1}) + parser.DefineFunction("now", []int{0}) + parser.DefineFunction("maxdatetime", []int{0}) + parser.DefineFunction("mindatetime", []int{0}) + parser.DefineFunction("totalseconds", []int{1}) + parser.DefineFunction("round", []int{1}) + parser.DefineFunction("floor", []int{1}) + parser.DefineFunction("ceiling", []int{1}) + parser.DefineFunction("isof", []int{1, 2}) // isof function can take one or two arguments. + parser.DefineFunction("cast", []int{2}) + parser.DefineFunction("geo.distance", []int{2}) + parser.DefineFunction("geo.intersects", []int{2}) + parser.DefineFunction("geo.length", []int{1}) + parser.DefineFunction("any", []int{0, 2}) // 'any' can take either zero or one argument. + parser.DefineFunction("all", []int{2}) return parser } diff --git a/filter_parser_test.go b/filter_parser_test.go index 04f7fdd..1339267 100644 --- a/filter_parser_test.go +++ b/filter_parser_test.go @@ -1,27 +1,33 @@ package godata import ( - "errors" "fmt" - "strconv" + "strings" "testing" ) func TestFilterDateTime(t *testing.T) { tokenizer := FilterTokenizer() tokens := map[string]int{ - "2011-08-29T21:58Z": FilterTokenDateTime, - "2011-08-29T21:58:33Z": FilterTokenDateTime, - "2011-08-29T21:58:33.123Z": FilterTokenDateTime, - "2011-08-29T21:58+11:23": FilterTokenDateTime, - "2011-08-29T21:58:33+11:23": FilterTokenDateTime, + "2011-08-29T21:58Z": FilterTokenDateTime, + "2011-08-29T21:58:33Z": FilterTokenDateTime, + "2011-08-29T21:58:33.123Z": FilterTokenDateTime, + "2011-08-29T21:58+11:23": FilterTokenDateTime, + "2011-08-29T21:58:33+11:23": FilterTokenDateTime, "2011-08-29T21:58:33.123+11:23": FilterTokenDateTime, - "2011-08-29T21:58:33-11:23": FilterTokenDateTime, - "2011-08-29": FilterTokenDate, - "21:58:33": FilterTokenTime, + "2011-08-29T21:58:33-11:23": FilterTokenDateTime, + "2011-08-29": FilterTokenDate, + "21:58:33": FilterTokenTime, } for tokenValue, tokenType := range tokens { - input := "CreateTime gt" + tokenValue + // Previously, the unit test had no space character after 'gt' + // E.g. 'CreateTime gt2011-08-29T21:58Z' was considered valid. + // However the ABNF notation for ODATA logical operators is: + // gtExpr = RWS "gt" RWS commonExpr + // RWS = 1*( SP / HTAB / "%20" / "%09" ) ; "required" whitespace + // + // See http://docs.oasis-open.org/odata/odata/v4.01/csprd03/abnf/odata-abnf-construction-rules.txt + input := "CreateTime gt " + tokenValue expect := []*Token{ &Token{Value: "CreateTime", Type: FilterTokenLiteral}, &Token{Value: "gt", Type: FilterTokenLogical}, @@ -29,26 +35,31 @@ func TestFilterDateTime(t *testing.T) { } output, err := tokenizer.Tokenize(input) if err != nil { - t.Error(err) + t.Errorf("Failed to tokenize input %s. Error: %v", input, err) } result, err := CompareTokens(expect, output) if !result { - t.Error(err) + var a []string + for _, t := range output { + a = append(a, t.Value) + } + + t.Errorf("Unexpected tokens for input '%s'. Tokens: %s Error: %v", input, strings.Join(a, ", "), err) } } } -func TestFilterAny(t *testing.T) { +func TestFilterAnyArrayOfObjects(t *testing.T) { tokenizer := FilterTokenizer() input := "Tags/any(d:d/Key eq 'Site' and d/Value lt 10)" expect := []*Token{ &Token{Value: "Tags", Type: FilterTokenLiteral}, - &Token{Value: "/", Type: FilterTokenNav}, + &Token{Value: "/", Type: FilterTokenOp}, &Token{Value: "any", Type: FilterTokenLambda}, &Token{Value: "(", Type: FilterTokenOpenParen}, &Token{Value: "d", Type: FilterTokenLiteral}, - &Token{Value: ":", Type: FilterTokenColon}, + &Token{Value: ",", Type: FilterTokenColon}, // ':' is replaced by ',' which is the function argument separator. &Token{Value: "d", Type: FilterTokenLiteral}, &Token{Value: "/", Type: FilterTokenNav}, &Token{Value: "Key", Type: FilterTokenLiteral}, @@ -73,207 +84,2139 @@ func TestFilterAny(t *testing.T) { } } -func TestFilterAll(t *testing.T) { +func TestFilterAnyArrayOfPrimitiveTypes(t *testing.T) { tokenizer := FilterTokenizer() - input := "Tags/all(d:d/Key eq 'Site')" + input := "Tags/any(d:d eq 'Site')" + { + expect := []*Token{ + &Token{Value: "Tags", Type: FilterTokenLiteral}, + &Token{Value: "/", Type: FilterTokenOp}, + &Token{Value: "any", Type: FilterTokenLambda}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "d", Type: FilterTokenLiteral}, + &Token{Value: ",", Type: FilterTokenColon}, + &Token{Value: "d", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'Site'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + q, err := ParseFilterString(input) + if err != nil { + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Tags", 1}, + {"any", 1}, + {"d", 2}, + {"eq", 2}, + {"d", 3}, + {"'Site'", 3}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + fmt.Printf("Got tree:\n%v\n", q.Tree.String()) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +// geographyPolygon = geographyPrefix SQUOTE fullPolygonLiteral SQUOTE +// geographyPrefix = "geography" +// fullPolygonLiteral = sridLiteral polygonLiteral +// sridLiteral = "SRID" EQ 1*5DIGIT SEMI +// polygonLiteral = "Polygon" polygonData +// polygonData = OPEN ringLiteral *( COMMA ringLiteral ) CLOSE +// positionLiteral = doubleValue SP doubleValue ; longitude, then latitude +/* +func TestFilterGeographyPolygon(t *testing.T) { + input := "geo.intersects(location, geography'SRID=0;Polygon(-122.031577 47.578581, -122.031577 47.678581, -122.131577 47.678581, -122.031577 47.578581)')" + q, err := ParseFilterString(input) + if err != nil { + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"geo.intersects", 0}, + {"location", 1}, + {"geography'SRID=0;Polygon(-122.031577 47.578581, -122.031577 47.678581, -122.131577 47.678581, -122.031577 47.578581)'", 1}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + fmt.Printf("Got tree:\n%v\n", q.Tree.String()) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} +*/ + +// TestFilterAnyGeography matches documents where any of the geo coordinates in the locations field is within the given polygon. +/* +func TestFilterAnyGeography(t *testing.T) { + input := "locations/any(loc: geo.intersects(loc, geography'Polygon((-122.031577 47.578581, -122.031577 47.678581, -122.131577 47.678581, -122.031577 47.578581))'))" + q, err := ParseFilterString(input) + if err != nil { + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Tags", 1}, + {"any", 1}, + {"d", 2}, + {"or", 2}, + {"or", 3}, + {"or", 4}, + {"eq", 5}, + {"d", 6}, + {"'Site'", 6}, + {"eq", 5}, + {"'Environment'", 6}, + {"/", 6}, + {"d", 7}, + {"Key", 7}, + {"eq", 4}, + {"/", 5}, + {"/", 6}, + {"d", 7}, + {"d", 7}, + {"d", 6}, + {"123456", 5}, + {"eq", 3}, + {"concat", 4}, + {"/", 5}, + {"d", 6}, + {"FirstName", 6}, + {"/", 5}, + {"d", 6}, + {"LastName", 6}, + {"/", 4}, + {"$it", 5}, + {"FullName", 5}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + fmt.Printf("Got tree:\n%v\n", q.Tree.String()) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} +*/ + +func TestFilterAnyMixedQuery(t *testing.T) { + /* + { + "Tags": [ + "Site", + { "Key": "Environment" }, + { "d" : { "d": 123456 }}, + { "FirstName" : "Bob", "LastName": "Smith"} + ], + "FullName": "BobSmith" + } + */ + // The argument of a lambda operator is a case-sensitive lambda variable name followed by a colon (:) and a Boolean expression that + // uses the lambda variable name to refer to properties of members of the collection identified by the navigation path. + // If the name chosen for the lambda variable matches a property name of the current resource referenced by the resource path, the lambda variable takes precedence. + // Clients can prefix properties of the current resource referenced by the resource path with $it. + // Other path expressions in the Boolean expression neither prefixed with the lambda variable nor $it are evaluated in the scope of + // the collection instances at the origin of the navigation path prepended to the lambda operator. + input := "Tags/any(d:d eq 'Site' or 'Environment' eq d/Key or d/d/d eq 123456 or concat(d/FirstName, d/LastName) eq $it/FullName)" + q, err := ParseFilterString(input) + if err != nil { + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Tags", 1}, + {"any", 1}, + {"d", 2}, + {"or", 2}, + {"or", 3}, + {"or", 4}, + {"eq", 5}, + {"d", 6}, + {"'Site'", 6}, + {"eq", 5}, + {"'Environment'", 6}, + {"/", 6}, + {"d", 7}, + {"Key", 7}, + {"eq", 4}, + {"/", 5}, + {"/", 6}, + {"d", 7}, + {"d", 7}, + {"d", 6}, + {"123456", 5}, + {"eq", 3}, + {"concat", 4}, + {"/", 5}, + {"d", 6}, + {"FirstName", 6}, + {"/", 5}, + {"d", 6}, + {"LastName", 6}, + {"/", 4}, + {"$it", 5}, + {"FullName", 5}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + fmt.Printf("Got tree:\n%v\n", q.Tree.String()) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestFilterGuid(t *testing.T) { + tokenizer := FilterTokenizer() + input := "GuidValue eq 01234567-89ab-cdef-0123-456789abcdef" + expect := []*Token{ - &Token{Value: "Tags", Type: FilterTokenLiteral}, - &Token{Value: "/", Type: FilterTokenNav}, - &Token{Value: "all", Type: FilterTokenLambda}, - &Token{Value: "(", Type: FilterTokenOpenParen}, - &Token{Value: "d", Type: FilterTokenLiteral}, - &Token{Value: ":", Type: FilterTokenColon}, - &Token{Value: "d", Type: FilterTokenLiteral}, - &Token{Value: "/", Type: FilterTokenNav}, - &Token{Value: "Key", Type: FilterTokenLiteral}, + &Token{Value: "GuidValue", Type: FilterTokenLiteral}, &Token{Value: "eq", Type: FilterTokenLogical}, - &Token{Value: "'Site'", Type: FilterTokenString}, - &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "01234567-89ab-cdef-0123-456789abcdef", Type: FilterTokenGuid}, } output, err := tokenizer.Tokenize(input) if err != nil { t.Error(err) } - result, err := CompareTokens(expect, output) if !result { t.Error(err) } } -func TestFilterTokenizer(t *testing.T) { - +func TestFilterDurationWithType(t *testing.T) { tokenizer := FilterTokenizer() - input := "Name eq 'Milk' and Price lt 2.55" + input := "Task eq duration'P12DT23H59M59.999999999999S'" + expect := []*Token{ - &Token{Value: "Name", Type: FilterTokenLiteral}, + &Token{Value: "Task", Type: FilterTokenLiteral}, &Token{Value: "eq", Type: FilterTokenLogical}, - &Token{Value: "'Milk'", Type: FilterTokenString}, - &Token{Value: "and", Type: FilterTokenLogical}, - &Token{Value: "Price", Type: FilterTokenLiteral}, - &Token{Value: "lt", Type: FilterTokenLogical}, - &Token{Value: "2.55", Type: FilterTokenFloat}, + // Note the duration token is extracted. + &Token{Value: "P12DT23H59M59.999999999999S", Type: FilterTokenDuration}, } output, err := tokenizer.Tokenize(input) if err != nil { t.Error(err) } - result, err := CompareTokens(expect, output) if !result { + printTokens(output) t.Error(err) } } -func TestFilterTokenizerFunc(t *testing.T) { - +func TestFilterDurationWithoutType(t *testing.T) { tokenizer := FilterTokenizer() - input := "not endswith(Name,'ilk')" + input := "Task eq 'P12DT23H59M59.999999999999S'" + expect := []*Token{ - &Token{Value: "not", Type: FilterTokenLogical}, - &Token{Value: "endswith", Type: FilterTokenFunc}, - &Token{Value: "(", Type: FilterTokenOpenParen}, - &Token{Value: "Name", Type: FilterTokenLiteral}, - &Token{Value: ",", Type: FilterTokenComma}, - &Token{Value: "'ilk'", Type: FilterTokenString}, - &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "Task", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "P12DT23H59M59.999999999999S", Type: FilterTokenDuration}, } output, err := tokenizer.Tokenize(input) if err != nil { t.Error(err) } - result, err := CompareTokens(expect, output) if !result { + printTokens(output) t.Error(err) } } -func BenchmarkFilterTokenizer(b *testing.B) { - t := FilterTokenizer() - for i := 0; i < b.N; i++ { - input := "Name eq 'Milk' and Price lt 2.55" - t.Tokenize(input) +func TestFilterAnyWithNoArgs(t *testing.T) { + tokenizer := FilterTokenizer() + input := "Tags/any()" + { + expect := []*Token{ + &Token{Value: "Tags", Type: FilterTokenLiteral}, + &Token{Value: "/", Type: FilterTokenOp}, + &Token{Value: "any", Type: FilterTokenLambda}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + q, err := ParseFilterString(input) + if err != nil { + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Tags", 1}, + {"any", 1}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + fmt.Printf("Got tree:\n%v\n", q.Tree.String()) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) } } +func TestFilterDivby(t *testing.T) { + { + tokenizer := FilterTokenizer() + input := "Price div 2 gt 3.5" + expect := []*Token{ + &Token{Value: "Price", Type: FilterTokenLiteral}, + &Token{Value: "div", Type: FilterTokenOp}, + &Token{Value: "2", Type: FilterTokenInteger}, + &Token{Value: "gt", Type: FilterTokenLogical}, + &Token{Value: "3.5", Type: FilterTokenFloat}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } -// Check if two slices of tokens are the same. -func CompareTokens(a, b []*Token) (bool, error) { - if len(a) != len(b) { - return false, errors.New("Different lengths") + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } } - for i, _ := range a { - if a[i].Value != b[i].Value || a[i].Type != b[i].Type { - return false, errors.New("Different at index " + strconv.Itoa(i) + " " + - a[i].Value + " != " + b[i].Value + " or types are different.") + { + tokenizer := FilterTokenizer() + input := "Price divby 2 gt 3.5" + expect := []*Token{ + &Token{Value: "Price", Type: FilterTokenLiteral}, + &Token{Value: "divby", Type: FilterTokenOp}, + &Token{Value: "2", Type: FilterTokenInteger}, + &Token{Value: "gt", Type: FilterTokenLogical}, + &Token{Value: "3.5", Type: FilterTokenFloat}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) } } - return true, nil } -func TestFilterParserTree(t *testing.T) { - - input := "not (A eq B)" - - tokens, err := GlobalFilterTokenizer.Tokenize(input) +func TestFilterNotBooleanProperty(t *testing.T) { + tokenizer := FilterTokenizer() + input := "not Enabled" + { + expect := []*Token{ + &Token{Value: "not", Type: FilterTokenLogical}, + &Token{Value: "Enabled", Type: FilterTokenLiteral}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + q, err := ParseFilterString(input) if err != nil { - t.Error(err) + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) return } - output, err := GlobalFilterParser.InfixToPostfix(tokens) + var expect []expectedParseNode = []expectedParseNode{ + {"not", 0}, + {"Enabled", 1}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + fmt.Printf("Got tree:\n%v\n", q.Tree.String()) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } + +} + +// Note: according to ODATA ABNF notation, there must be a space between not and open parenthesis. +// http://docs.oasis-open.org/odata/odata/v4.01/csprd03/abnf/odata-abnf-construction-rules.txt +func TestFilterNotWithNoSpace(t *testing.T) { + tokenizer := FilterTokenizer() + input := "not(City eq 'Seattle')" + { + expect := []*Token{ + &Token{Value: "not", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'Seattle'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + q, err := ParseFilterString(input) if err != nil { - t.Error(err) + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) return } + var expect []expectedParseNode = []expectedParseNode{ + {"not", 0}, + {"eq", 1}, + {"City", 2}, + {"'Seattle'", 2}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + fmt.Printf("Got tree:\n%v\n", q.Tree.String()) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} - tree, err := GlobalFilterParser.PostfixToTree(output) +// TestFilterInOperator tests the "IN" operator with a comma-separated list of values. +func TestFilterInOperator(t *testing.T) { + tokenizer := FilterTokenizer() + input := "City in ( 'Seattle', 'Atlanta', 'Paris' )" + expect := []*Token{ + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "in", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'Seattle'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'Atlanta'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'Paris'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + tokens, err := tokenizer.Tokenize(input) if err != nil { t.Error(err) - return } - - if tree.Token.Value != "not" { - t.Error("Root is '" + tree.Token.Value + "' not 'not'") + result, err := CompareTokens(expect, tokens) + if !result { + t.Error(err) } - if tree.Children[0].Token.Value != "eq" { - t.Error("First child is '" + tree.Children[1].Token.Value + "' not 'eq'") + var postfix *tokenQueue + postfix, err = GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + } + expect = []*Token{ + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "'Seattle'", Type: FilterTokenString}, + &Token{Value: "'Atlanta'", Type: FilterTokenString}, + &Token{Value: "'Paris'", Type: FilterTokenString}, + &Token{Value: "3", Type: TokenTypeArgCount}, + &Token{Value: TokenListExpr, Type: TokenTypeListExpr}, + &Token{Value: "in", Type: FilterTokenLogical}, + } + result, err = CompareQueue(expect, postfix) + if !result { + t.Error(err) } -} + tree, err := GlobalFilterParser.PostfixToTree(postfix) + if err != nil { + t.Error(err) + } -func printTree(n *ParseNode, level int) { - indent := "" - for i := 0; i < level; i++ { - indent += " " + var treeExpect []expectedParseNode = []expectedParseNode{ + {"in", 0}, + {"City", 1}, + {TokenListExpr, 1}, + {"'Seattle'", 2}, + {"'Atlanta'", 2}, + {"'Paris'", 2}, } - fmt.Printf("%s %-10s %-10d\n", indent, n.Token.Value, n.Token.Type) - for _, v := range n.Children { - printTree(v, level+1) + pos := 0 + err = CompareTree(tree, treeExpect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) } } -func TestNestedPath(t *testing.T) { - input := "Address/City eq 'Redmond'" - tokens, err := GlobalFilterTokenizer.Tokenize(input) +// TestFilterInOperatorSingleValue tests the "IN" operator with a list containing a single value. +func TestFilterInOperatorSingleValue(t *testing.T) { + tokenizer := FilterTokenizer() + input := "City in ( 'Seattle' )" + + expect := []*Token{ + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "in", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'Seattle'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + tokens, err := tokenizer.Tokenize(input) if err != nil { t.Error(err) - return } - output, err := GlobalFilterParser.InfixToPostfix(tokens) + result, err := CompareTokens(expect, tokens) + if !result { + t.Error(err) + } + var postfix *tokenQueue + postfix, err = GlobalFilterParser.InfixToPostfix(tokens) if err != nil { t.Error(err) - return + } + expect = []*Token{ + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "'Seattle'", Type: FilterTokenString}, + &Token{Value: "1", Type: TokenTypeArgCount}, + &Token{Value: TokenListExpr, Type: TokenTypeListExpr}, + &Token{Value: "in", Type: FilterTokenLogical}, + } + result, err = CompareQueue(expect, postfix) + if !result { + t.Error(err) } - tree, err := GlobalFilterParser.PostfixToTree(output) + tree, err := GlobalFilterParser.PostfixToTree(postfix) if err != nil { t.Error(err) - return - } - //printTree(tree, 0) - if tree.Token.Value != "eq" { - t.Error("Root is '" + tree.Token.Value + "' not 'eq'") } - if tree.Children[0].Token.Value != "/" { - t.Error("First child is \"" + tree.Children[0].Token.Value + "\", not '/'") + + var treeExpect []expectedParseNode = []expectedParseNode{ + {"in", 0}, + {"City", 1}, + {TokenListExpr, 1}, + {"'Seattle'", 2}, } - if tree.Children[1].Token.Value != "'Redmond'" { - t.Error("First child is \"" + tree.Children[1].Token.Value + "\", not 'Redmond'") + pos := 0 + err = CompareTree(tree, treeExpect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) } } -func TestLambda(t *testing.T) { - input := "Tags/any(var:var/Key eq 'Site' and var/Value eq 'London')" - tokens, err := GlobalFilterTokenizer.Tokenize(input) +// TestFilterInOperatorEmptyList tests the "IN" operator with a list containing no value. +func TestFilterInOperatorEmptyList(t *testing.T) { + tokenizer := FilterTokenizer() + input := "City in ( )" + + expect := []*Token{ + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "in", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + tokens, err := tokenizer.Tokenize(input) if err != nil { t.Error(err) - return } - output, err := GlobalFilterParser.InfixToPostfix(tokens) + result, err := CompareTokens(expect, tokens) + if !result { + t.Error(err) + } + var postfix *tokenQueue + postfix, err = GlobalFilterParser.InfixToPostfix(tokens) if err != nil { t.Error(err) - return + } + expect = []*Token{ + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "0", Type: TokenTypeArgCount}, + &Token{Value: TokenListExpr, Type: TokenTypeListExpr}, + &Token{Value: "in", Type: FilterTokenLogical}, + } + result, err = CompareQueue(expect, postfix) + if !result { + t.Error(err) } - tree, err := GlobalFilterParser.PostfixToTree(output) + tree, err := GlobalFilterParser.PostfixToTree(postfix) if err != nil { t.Error(err) - return } - //printTree(tree, 0) - if tree.Token.Value != "/" { - t.Error("Root is '" + tree.Token.Value + "' not '/'") + var treeExpect []expectedParseNode = []expectedParseNode{ + {"in", 0}, + {"City", 1}, + {TokenListExpr, 1}, } - if tree.Children[0].Token.Value != "Tags" { - t.Error("First child is '" + tree.Children[0].Token.Value + "' not 'Tags'") - } - if tree.Children[1].Token.Value != "any" { - t.Error("First child is '" + tree.Children[1].Token.Value + "' not 'any'") - } - if tree.Children[1].Children[0].Token.Value != ":" { - t.Error("First child is '" + tree.Children[1].Children[0].Token.Value + "' not ':'") + pos := 0 + err = CompareTree(tree, treeExpect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) } } + +// TestFilterInOperatorBothSides tests the "IN" operator. +// Use a listExpr on both sides of the IN operator. +// listExpr = OPEN BWS commonExpr BWS *( COMMA BWS commonExpr BWS ) CLOSE +// Validate if a list is within another list. +func TestFilterInOperatorBothSides(t *testing.T) { + tokenizer := FilterTokenizer() + input := "(1, 2) in ( ('ab', 'cd'), (1, 2), ('abc', 'def') )" + + expect := []*Token{ + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "1", Type: FilterTokenInteger}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "2", Type: FilterTokenInteger}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "in", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'ab'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'cd'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: ",", Type: FilterTokenComma}, + + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "1", Type: FilterTokenInteger}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "2", Type: FilterTokenInteger}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: ",", Type: FilterTokenComma}, + + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'abc'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'def'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + tokens, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + result, err := CompareTokens(expect, tokens) + if !result { + t.Error(err) + } + var postfix *tokenQueue + postfix, err = GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + } + expect = []*Token{ + &Token{Value: "1", Type: FilterTokenInteger}, + &Token{Value: "2", Type: FilterTokenInteger}, + &Token{Value: "2", Type: TokenTypeArgCount}, + &Token{Value: TokenListExpr, Type: TokenTypeListExpr}, + + &Token{Value: "'ab'", Type: FilterTokenString}, + &Token{Value: "'cd'", Type: FilterTokenString}, + &Token{Value: "2", Type: TokenTypeArgCount}, + &Token{Value: TokenListExpr, Type: TokenTypeListExpr}, + + &Token{Value: "1", Type: FilterTokenInteger}, + &Token{Value: "2", Type: FilterTokenInteger}, + &Token{Value: "2", Type: TokenTypeArgCount}, + &Token{Value: TokenListExpr, Type: TokenTypeListExpr}, + + &Token{Value: "'abc'", Type: FilterTokenString}, + &Token{Value: "'def'", Type: FilterTokenString}, + &Token{Value: "2", Type: TokenTypeArgCount}, + &Token{Value: TokenListExpr, Type: TokenTypeListExpr}, + + &Token{Value: "3", Type: TokenTypeArgCount}, + &Token{Value: TokenListExpr, Type: TokenTypeListExpr}, + + &Token{Value: "in", Type: FilterTokenLogical}, + } + result, err = CompareQueue(expect, postfix) + if !result { + fmt.Printf("postfix notation: %s\n", postfix.String()) + t.Error(err) + } + + tree, err := GlobalFilterParser.PostfixToTree(postfix) + if err != nil { + t.Error(err) + } + + var treeExpect []expectedParseNode = []expectedParseNode{ + {"in", 0}, + {TokenListExpr, 1}, + {"1", 2}, + {"2", 2}, + // ('ab', 'cd'), (1, 2), ('abc', 'def') + {TokenListExpr, 1}, + {TokenListExpr, 2}, + {"'ab'", 3}, + {"'cd'", 3}, + {TokenListExpr, 2}, + {"1", 3}, + {"2", 3}, + {TokenListExpr, 2}, + {"'abc'", 3}, + {"'def'", 3}, + } + pos := 0 + err = CompareTree(tree, treeExpect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +// TestFilterInOperatorWithFunc tests the "IN" operator with a comma-separated list +// of values, one of which is a function call which itself has a comma-separated list of values. +func TestFilterInOperatorWithFunc(t *testing.T) { + tokenizer := FilterTokenizer() + // 'Atlanta' is enclosed in a unecessary parenExpr to validate the expression is properly unwrapped. + input := "City in ( 'Seattle', concat('San', 'Francisco'), ('Atlanta') )" + + { + expect := []*Token{ + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "in", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'Seattle'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "concat", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'San'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'Francisco'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'Atlanta'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + q, err := ParseFilterString(input) + if err != nil { + t.Errorf("Error parsing filter: %s", err.Error()) + } + var expect []expectedParseNode = []expectedParseNode{ + {"in", 0}, + {"City", 1}, + {TokenListExpr, 1}, + {"'Seattle'", 2}, + {"concat", 2}, + {"'San'", 3}, + {"'Francisco'", 3}, + {"'Atlanta'", 2}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + printTree(q.Tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestFilterNotInListExpr(t *testing.T) { + tokenizer := FilterTokenizer() + input := "not ( City in ( 'Seattle', 'Atlanta' ) )" + + { + expect := []*Token{ + &Token{Value: "not", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "in", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'Seattle'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'Atlanta'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + { + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"not", 0}, + {"in", 1}, + {"City", 2}, + {TokenListExpr, 2}, + {"'Seattle'", 3}, + {"'Atlanta'", 3}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } + + } +} + +func TestFilterAll(t *testing.T) { + tokenizer := FilterTokenizer() + input := "Tags/all(d:d/Key eq 'Site')" + expect := []*Token{ + &Token{Value: "Tags", Type: FilterTokenLiteral}, + &Token{Value: "/", Type: FilterTokenOp}, + &Token{Value: "all", Type: FilterTokenLambda}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "d", Type: FilterTokenLiteral}, + &Token{Value: ",", Type: FilterTokenColon}, + &Token{Value: "d", Type: FilterTokenLiteral}, + &Token{Value: "/", Type: FilterTokenNav}, + &Token{Value: "Key", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'Site'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } +} + +func TestFilterTokenizer(t *testing.T) { + + tokenizer := FilterTokenizer() + input := "Name eq 'Milk' and Price lt 2.55" + expect := []*Token{ + &Token{Value: "Name", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'Milk'", Type: FilterTokenString}, + &Token{Value: "and", Type: FilterTokenLogical}, + &Token{Value: "Price", Type: FilterTokenLiteral}, + &Token{Value: "lt", Type: FilterTokenLogical}, + &Token{Value: "2.55", Type: FilterTokenFloat}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } +} + +func TestFunction(t *testing.T) { + tokenizer := FilterTokenizer() + // The syntax for ODATA functions follows the inline parameter syntax. The function name must be followed + // by an opening parenthesis, followed by a comma-separated list of parameters, followed by a closing parenthesis. + // For example: + // GET serviceRoot/Airports?$filter=contains(Location/Address, 'San Francisco') + input := "contains(LastName, 'Smith') and FirstName eq 'John' and City eq 'Houston'" + expect := []*Token{ + &Token{Value: "contains", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "LastName", Type: FilterTokenLiteral}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'Smith'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "and", Type: FilterTokenLogical}, + &Token{Value: "FirstName", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'John'", Type: FilterTokenString}, + &Token{Value: "and", Type: FilterTokenLogical}, + &Token{Value: "City", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'Houston'", Type: FilterTokenString}, + } + { + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + { + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + if tree.Token.Value != "and" { + t.Errorf("Root is '%v', not 'and'", tree.Token.Value) + } + if len(tree.Children) != 2 { + t.Errorf("Unexpected number of operators. Expected 2, got %d", len(tree.Children)) + } + if tree.Children[0].Token.Value != "and" { + t.Errorf("First child is '%v', not 'and'", tree.Children[0].Token.Value) + } + if len(tree.Children[0].Children) != 2 { + t.Errorf("Unexpected number of operators. Expected 2, got %d", len(tree.Children)) + } + if tree.Children[0].Children[0].Token.Value != "contains" { + t.Errorf("First child is '%v', not 'contains'", tree.Children[0].Children[0].Token.Value) + } + if tree.Children[1].Token.Value != "eq" { + t.Errorf("First child is '%v', not 'eq'", tree.Children[1].Token.Value) + } + } +} + +func TestNestedFunction(t *testing.T) { + tokenizer := FilterTokenizer() + // Test ODATA syntax with nested function calls + input := "contains(LastName, toupper('Smith')) or FirstName eq 'John'" + expect := []*Token{ + &Token{Value: "contains", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "LastName", Type: FilterTokenLiteral}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "toupper", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'Smith'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "or", Type: FilterTokenLogical}, + &Token{Value: "FirstName", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'John'", Type: FilterTokenString}, + } + { + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + { + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + if tree.Token.Value != "or" { + t.Errorf("Root is '%v', not 'or'", tree.Token.Value) + } + if len(tree.Children) != 2 { + t.Errorf("Unexpected number of operators. Expected 2, got %d", len(tree.Children)) + } + if tree.Children[0].Token.Value != "contains" { + t.Errorf("First child is '%v', not 'contains'", tree.Children[0].Token.Value) + } + if len(tree.Children[0].Children) != 2 { + t.Errorf("Unexpected number of nested children. Expected 2, got %d", len(tree.Children[0].Children)) + } + if tree.Children[0].Children[1].Token.Value != "toupper" { + t.Errorf("First child is '%v', not 'toupper'", tree.Children[0].Children[1].Token.Value) + } + if tree.Children[1].Token.Value != "eq" { + t.Errorf("First child is '%v', not 'eq'", tree.Children[1].Token.Value) + } + } +} + +func TestValidFilterSyntax(t *testing.T) { + queries := []string{ + "substring(CompanyName,1,2) eq 'lf'", // substring with 3 arguments. + // Bolean values + "true", + "false", + "(true)", + "((true))", + "((true)) or false", + "not true", + "not false", + "not (not true)", + //"not not true", // TODO: I think this should work. 'not not true' is true + // String functions + "contains(CompanyName,'freds')", + "endswith(CompanyName,'Futterkiste')", + "startswith(CompanyName,'Alfr')", + "length(CompanyName) eq 19", + "indexof(CompanyName,'lfreds') eq 1", + "substring(CompanyName,1) eq 'lfreds Futterkiste'", // substring() with 2 arguments. + "'lfreds Futterkiste' eq substring(CompanyName,1)", // Same as above, but order of operands is reversed. + "substring(CompanyName,1,2) eq 'lf'", // substring() with 3 arguments. + "'lf' eq substring(CompanyName,1,2) ", // Same as above, but order of operands is reversed. + "substringof('Alfreds', CompanyName) eq true", + "tolower(CompanyName) eq 'alfreds futterkiste'", + "toupper(CompanyName) eq 'ALFREDS FUTTERKISTE'", + "trim(CompanyName) eq 'Alfreds Futterkiste'", + "concat(concat(City,', '), Country) eq 'Berlin, Germany'", + // GUID + "GuidValue eq 01234567-89ab-cdef-0123-456789abcdef", // TODO According to ODATA ABNF notation, GUID values do not have quotes. + // Date and Time functions + "StartDate eq 2012-12-03", + "DateTimeOffsetValue eq 2012-12-03T07:16:23Z", + // duration = [ "duration" ] SQUOTE durationValue SQUOTE + // "DurationValue eq duration'P12DT23H59M59.999999999999S'", // TODO See ODATA ABNF notation + "TimeOfDayValue eq 07:59:59.999", + "year(BirthDate) eq 0", + "month(BirthDate) eq 12", + "day(StartTime) eq 8", + "hour(StartTime) eq 1", + "hour (StartTime) eq 12", // function followed by space characters + "hour ( StartTime ) eq 15", // function followed by space characters + "minute(StartTime) eq 0", + "totaloffsetminutes(StartTime) eq 0", + "second(StartTime) eq 0", + "fractionalsecond(StartTime) lt 0.123456", // The fractionalseconds function returns the fractional seconds component of the + // DateTimeOffset or TimeOfDay parameter value as a non-negative decimal value less than 1. + "date(StartTime) ne date(EndTime)", + "totaloffsetminutes(StartTime) eq 60", + "StartTime eq mindatetime()", + // "totalseconds(EndTime sub StartTime) lt duration'PT23H59'", // TODO The totalseconds function returns the duration of the value in total seconds, including fractional seconds. + "EndTime eq maxdatetime()", + "time(StartTime) le StartOfDay", + "time('2015-10-14T23:30:00.104+02:00') lt now()", + "time(2015-10-14T23:30:00.104+02:00) lt now()", + // Math functions + "round(Freight) eq 32", + "floor(Freight) eq 32", + "ceiling(Freight) eq 33", + "Rating mod 5 eq 0", + "Price div 2 eq 3", + // Type functions + "isof(ShipCountry,Edm.String)", + "isof(NorthwindModel.BigOrder)", + "cast(ShipCountry,Edm.String)", + // Parameter aliases + // See http://docs.oasis-open.org/odata/odata/v4.0/errata03/os/complete/part1-protocol/odata-v4.0-errata03-os-part1-protocol-complete.html#_Toc453752288 + "Region eq @p1", // Aliases start with @ + // Geo functions + "geo.distance(CurrentPosition,TargetPosition)", + "geo.length(DirectRoute)", + "geo.intersects(Position,TargetArea)", + "GEO.INTERSECTS(Position,TargetArea)", // functions are case insensitive in ODATA 4.0.1 + // Logical operators + "'Milk' eq 'Milk'", // Compare two literals + "'Water' ne 'Milk'", // Compare two literals + "Name eq 'Milk'", + "Name EQ 'Milk'", // operators are case insensitive in ODATA 4.0.1 + "Name ne 'Milk'", + "Name NE 'Milk'", + "Name gt 'Milk'", + "Name ge 'Milk'", + "Name lt 'Milk'", + "Name le 'Milk'", + "Name eq Name", // parameter equals to itself + "Name eq 'Milk' and Price lt 2.55", + "not endswith(Name,'ilk')", + "Name eq 'Milk' or Price lt 2.55", + "City eq 'Dallas' or City eq 'Houston'", + // Nested properties + "Product/Name eq 'Milk'", + "Region/Product/Name eq 'Milk'", + "Country/Region/Product/Name eq 'Milk'", + //"style has Sales.Pattern'Yellow'", // TODO + // Arithmetic operators + "Price add 2.45 eq 5.00", + "2.46 add Price eq 5.00", + "Price add (2.47) eq 5.00", + "(Price add (2.48)) eq 5.00", + "Price ADD 2.49 eq 5.00", // 4.01 Services MUST support case-insensitive operator names. + "Price sub 0.55 eq 2.00", + "Price SUB 0.56 EQ 2.00", // 4.01 Services MUST support case-insensitive operator names. + "Price mul 2.0 eq 5.10", + "Price div 2.55 eq 1", + "Rating div 2 eq 2", + "Rating mod 5 eq 0", + // Grouping + "(4 add 5) mod (4 sub 1) eq 0", + "not (City eq 'Dallas') or Name in ('a', 'b', 'c') and not (State eq 'California')", + // Nested functions + "length(trim(CompanyName)) eq length(CompanyName)", + "concat(concat(City, ', '), Country) eq 'Berlin, Germany'", + // Various parenthesis combinations + "City eq 'Dallas'", + "City eq ('Dallas')", + "'Dallas' eq City", + "not (City eq 'Dallas')", + "City in ('Dallas')", + "(City in ('Dallas'))", + "(City in ('Dallas', 'Houston'))", + "not (City in ('Dallas'))", + "not (City in ('Dallas', 'Houston'))", + "not (((City eq 'Dallas')))", + "not(S1 eq 'foo')", + // Lambda operators + "Tags/any()", // The any operator without an argument returns true if the collection is not empty + "Tags/any(tag:tag eq 'London')", // 'Tags' is array of strings + "Tags/any(tag:tag eq 'London' or tag eq 'Berlin')", // 'Tags' is array of strings + "Tags/any(var:var/Key eq 'Site' and var/Value eq 'London')", // 'Tags' is array of {"Key": "abc", "Value": "def"} + "Tags/ANY(var:var/Key eq 'Site' AND var/Value eq 'London')", + "Tags/any(var:var/Key eq 'Site' and var/Value eq 'London') and not (City in ('Dallas'))", + "Tags/all(var:var/Key eq 'Site' and var/Value eq 'London')", + "Price/any(t:not (12345 eq t))", + // A long query. + "Tags/any(var:var/Key eq 'Site' and var/Value eq 'London') or " + + "Tags/any(var:var/Key eq 'Site' and var/Value eq 'Berlin') or " + + "Tags/any(var:var/Key eq 'Site' and var/Value eq 'Paris') or " + + "Tags/any(var:var/Key eq 'Site' and var/Value eq 'New York City') or " + + "Tags/any(var:var/Key eq 'Site' and var/Value eq 'San Francisco')", + } + for _, input := range queries { + q, err := ParseFilterString(input) + if err != nil { + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) + return + } else if q.Tree == nil { + t.Errorf("Error parsing query %s. Tree is nil", input) + } + if q.Tree.Token == nil { + t.Errorf("Error parsing query %s. Root token is nil", input) + } + if q.Tree.Token.Type == FilterTokenLiteral { + t.Errorf("Error parsing query %s. Unexpected root token type: %+v", input, q.Tree.Token) + } + //printTree(q.Tree) + } +} + +// The URLs below are not valid ODATA syntax, the parser should return an error. +func TestInvalidFilterSyntax(t *testing.T) { + queries := []string{ + "()", // It's not a boolean expression + "(TRUE)", + "(City)", + "(", + "((((", + ")", + "12345", // Number 12345 is not a boolean expression + "0", // Number 0 is not a boolean expression + "'123'", // String '123' is not a boolean expression + "TRUE", // Should be 'true' lowercase + "FALSE", // Should be 'false' lowercase + "yes", // yes is not a boolean expression + "no", // yes is not a boolean expression + "", // Empty string. + "eq", // Just a single logical operator + "and", // Just a single logical operator + "add", // Just a single arithmetic operator + "add ", // Just a single arithmetic operator + "add 2", // Missing operands + "add 2 3", // Missing operands + "City", // Just a single literal + "City City City City", // Sequence of literals + "City eq", // Missing operand + "City eq (", // Wrong operand + "City eq )", // Wrong operand + "City equals 'Dallas'", // Unknown operator that starts with the same letters as a known operator + "City near 'Dallas'", // Unknown operator that starts with the same letters as a known operator + "City isNot 'Dallas'", // Unknown operator + "not [City eq 'Dallas']", // Wrong delimiter + "not (City eq )", // Missing operand + "not ((City eq 'Dallas'", // Missing closing parenthesis + "not (City eq 'Dallas'", // Missing closing parenthesis + "not (City eq 'Dallas'))", // Extraneous closing parenthesis + "not City eq 'Dallas')", // Missing open parenthesis + "City eq 'Dallas' orCity eq 'Houston'", // missing space between or and City + // TODO: the query below should fail. + //"Tags/any(var:var/Key eq 'Site') orTags/any(var:var/Key eq 'Site')", + "not (City eq 'Dallas') and Name eq 'Houston')", + "Tags/all()", // The all operator cannot be used without an argument expression. + "LastName contains 'Smith'", // Previously the godata library was not returning an error. + "contains", // Function with missing parenthesis and arguments + "contains()", // Function with missing arguments + "contains LastName, 'Smith'", // Missing parenthesis + "contains(LastName)", // Insufficent number of function arguments + "contains(LastName, 'Smith'))", // Extraneous closing parenthesis + "contains(LastName, 'Smith'", // Missing closing parenthesis + "contains LastName, 'Smith')", // Missing open parenthesis + "City eq 'Dallas' 'Houston'", // extraneous string value + //"contains(Name, 'a', 'b', 'c', 'd')", // Too many function arguments + } + for _, input := range queries { + q, err := ParseFilterString(input) + if err == nil { + // The parser has incorrectly determined the syntax is valid. + printTree(q.Tree) + t.Errorf("The query '$filter=%s' is not valid ODATA syntax. The ODATA parser should return an error", input) + return + } + } +} + +// See http://docs.oasis-open.org/odata/odata/v4.01/csprd02/part1-protocol/odata-v4.01-csprd02-part1-protocol.html#_Toc486263411 +// Test 'in', which is the 'Is a member of' operator. +func TestFilterIn(t *testing.T) { + tokenizer := FilterTokenizer() + input := "contains(LastName, 'Smith') and Site in ('London', 'Paris', 'San Francisco', 'Dallas') and FirstName eq 'John'" + expect := []*Token{ + &Token{Value: "contains", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "LastName", Type: FilterTokenLiteral}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'Smith'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "and", Type: FilterTokenLogical}, + &Token{Value: "Site", Type: FilterTokenLiteral}, + &Token{Value: "in", Type: FilterTokenLogical}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'London'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'Paris'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'San Francisco'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'Dallas'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "and", Type: FilterTokenLogical}, + &Token{Value: "FirstName", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'John'", Type: FilterTokenString}, + } + { + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } + } + { + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + + /* + The expected tree is: + and 6 + and 6 + contains 8 + LastName 20 + 'Smith' 15 + in 6 + Site 20 + ( 0 + 'London' 15 + 'Paris' 15 + 'San Francisco' 15 + 'Dallas' 15 + eq 6 + FirstName 20 + 'John' 15 + + */ + if tree.Token.Value != "and" { + t.Errorf("Root is '%v', not 'and'", tree.Token.Value) + } + if len(tree.Children) != 2 { + t.Errorf("Unexpected number of operators. Expected 2, got %d", len(tree.Children)) + } + if tree.Children[0].Token.Value != "and" { + t.Errorf("First child is '%v', not 'and'", tree.Children[0].Token.Value) + } + if len(tree.Children[0].Children) != 2 { + t.Errorf("Unexpected number of operators. Expected 2, got %d", len(tree.Children)) + } + if tree.Children[0].Children[0].Token.Value != "contains" { + t.Errorf("First child is '%v', not 'contains'", tree.Children[0].Children[0].Token.Value) + } + if tree.Children[0].Children[1].Token.Value != "in" { + t.Errorf("First child is '%v', not 'in'", tree.Children[0].Children[1].Token.Value) + } + if len(tree.Children[0].Children[1].Children) != 2 { + t.Errorf("Unexpected number of operands for the 'in' operator. Expected 2, got %d", + len(tree.Children[0].Children[1].Children)) + } + if tree.Children[0].Children[1].Children[0].Token.Value != "Site" { + t.Errorf("Unexpected operand for the 'in' operator. Expected 'Site', got %s", + tree.Children[0].Children[1].Children[0].Token.Value) + } + if tree.Children[0].Children[1].Children[1].Token.Value != TokenListExpr { + t.Errorf("Unexpected operand for the 'in' operator. Expected 'list', got %s", + tree.Children[0].Children[1].Children[1].Token.Value) + } + if len(tree.Children[0].Children[1].Children[1].Children) != 4 { + t.Errorf("Unexpected number of operands for the 'in' operator. Expected 4, got %d", + len(tree.Children[0].Children[1].Children[1].Token.Value)) + } + if tree.Children[1].Token.Value != "eq" { + t.Errorf("First child is '%v', not 'eq'", tree.Children[1].Token.Value) + } + } +} + +func TestFilterTokenizerFunc(t *testing.T) { + + tokenizer := FilterTokenizer() + input := "not endswith(Name,'ilk')" + expect := []*Token{ + &Token{Value: "not", Type: FilterTokenLogical}, + &Token{Value: "endswith", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "Name", Type: FilterTokenLiteral}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "'ilk'", Type: FilterTokenString}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } +} + +func BenchmarkFilterTokenizer(b *testing.B) { + t := FilterTokenizer() + for i := 0; i < b.N; i++ { + input := "Name eq 'Milk' and Price lt 2.55" + t.Tokenize(input) + } +} + +// Check if two slices of tokens are the same. +func CompareTokens(expected, actual []*Token) (bool, error) { + if len(expected) != len(actual) { + return false, fmt.Errorf("Different lengths. Expected %d, Got %d", len(expected), len(actual)) + } + for i := range expected { + if expected[i].Type != actual[i].Type { + return false, fmt.Errorf("Different token types at index %d. Expected %v, Got %v. Value: %v", + i, expected[i].Type, actual[i].Type, expected[i].Value) + } + if expected[i].Value != actual[i].Value { + return false, fmt.Errorf("Different token values at index %d. Expected %v, Got %v", + i, expected[i].Value, actual[i].Value) + } + } + return true, nil +} + +func CompareQueue(expect []*Token, b *tokenQueue) (bool, error) { + bl := func() int { + if b.Empty() { + return 0 + } + l := 1 + for node := b.Head; node != b.Tail; node = node.Next { + l++ + } + return l + }() + if len(expect) != bl { + return false, fmt.Errorf("Different lengths. Got %d, expected %d", bl, len(expect)) + } + node := b.Head + for i := range expect { + if expect[i].Type != node.Token.Type { + return false, fmt.Errorf("Different token types at index %d. Got: %v, expected: %v. Expected value: %v", + i, node.Token.Type, expect[i].Type, expect[i].Value) + } + if expect[i].Value != node.Token.Value { + return false, fmt.Errorf("Different token values at index %d. Got: %v, expected: %v", + i, node.Token.Value, expect[i].Value) + } + node = node.Next + } + return true, nil +} + +func TestFilterParserTree(t *testing.T) { + + input := "not (A eq B)" + + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + + if err != nil { + t.Error(err) + return + } + + tree, err := GlobalFilterParser.PostfixToTree(output) + + if err != nil { + t.Error(err) + return + } + + if tree.Token.Value != "not" { + t.Error("Root is '" + tree.Token.Value + "' not 'not'") + } + if tree.Children[0].Token.Value != "eq" { + t.Error("First child is '" + tree.Children[1].Token.Value + "' not 'eq'") + } + +} + +func printTree(n *ParseNode) { + fmt.Printf("Tree:\n%s\n", n.String()) +} + +func printTokens(tokens []*Token) { + s := make([]string, len(tokens)) + for i := range tokens { + s[i] = tokens[i].Value + } + fmt.Printf("TOKENS: %s\n", strings.Join(s, " ")) +} + +func TestNestedPath(t *testing.T) { + input := "Address/City eq 'Redmond'" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + + var expect []expectedParseNode = []expectedParseNode{ + {"eq", 0}, + {"/", 1}, + {"Address", 2}, + {"City", 2}, + {"'Redmond'", 1}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestMultipleNestedPath(t *testing.T) { + input := "Product/Address/City eq 'Redmond'" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + + var expect []expectedParseNode = []expectedParseNode{ + {"eq", 0}, + {"/", 1}, + {"/", 2}, + {"Product", 3}, + {"Address", 3}, + {"City", 2}, + {"'Redmond'", 1}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestSubstringFunction(t *testing.T) { + // substring can take 2 or 3 arguments. + { + input := "substring(CompanyName,1) eq 'Foo'" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"eq", 0}, + {"substring", 1}, + {"CompanyName", 2}, + {"1", 2}, + {"'Foo'", 1}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } + } + { + input := "substring(CompanyName,1,2) eq 'lf'" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"eq", 0}, + {"substring", 1}, + {"CompanyName", 2}, + {"1", 2}, + {"2", 2}, + {"'lf'", 1}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } + } +} + +func TestSubstringofFunction(t *testing.T) { + // Previously, the parser was incorrectly interpreting the 'substringof' function as the 'sub' operator. + input := "substringof('Alfreds', CompanyName) eq true" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + { + expect := []*Token{ + &Token{Value: "substringof", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'Alfreds'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "CompanyName", Type: FilterTokenLiteral}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "true", Type: FilterTokenBoolean}, + } + result, err := CompareTokens(expect, tokens) + if !result { + t.Error(err) + } + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + { + expect := []*Token{ + &Token{Value: "'Alfreds'", Type: FilterTokenString}, + &Token{Value: "CompanyName", Type: FilterTokenLiteral}, + &Token{Value: "2", Type: TokenTypeArgCount}, // The number of function arguments. + &Token{Value: "substringof", Type: FilterTokenFunc}, + &Token{Value: "true", Type: FilterTokenBoolean}, + &Token{Value: "eq", Type: FilterTokenLogical}, + } + result, err := CompareQueue(expect, output) + if !result { + t.Error(err) + } + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"eq", 0}, + {"substringof", 1}, + {"'Alfreds'", 2}, + {"CompanyName", 2}, + {"true", 1}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +// TestSubstringNestedFunction tests the substring function with a nested call +// to substring, with the use of 2-argument and 3-argument substring. +func TestSubstringNestedFunction(t *testing.T) { + // Previously, the parser was incorrectly interpreting the 'substringof' function as the 'sub' operator. + input := "substring(substring('Francisco', 1), 3, 2) eq 'ci'" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + { + expect := []*Token{ + &Token{Value: "substring", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "substring", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "'Francisco'", Type: FilterTokenString}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "1", Type: FilterTokenInteger}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "3", Type: FilterTokenInteger}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "2", Type: FilterTokenInteger}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "'ci'", Type: FilterTokenString}, + } + result, err := CompareTokens(expect, tokens) + if !result { + t.Error(err) + } + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + { + expect := []*Token{ + &Token{Value: "'Francisco'", Type: FilterTokenString}, + &Token{Value: "1", Type: FilterTokenInteger}, + &Token{Value: "2", Type: TokenTypeArgCount}, // The number of function arguments. + &Token{Value: "substring", Type: FilterTokenFunc}, + &Token{Value: "3", Type: FilterTokenInteger}, + &Token{Value: "2", Type: FilterTokenInteger}, + &Token{Value: "3", Type: TokenTypeArgCount}, // The number of function arguments. + &Token{Value: "substring", Type: FilterTokenFunc}, + &Token{Value: "'ci'", Type: FilterTokenString}, + &Token{Value: "eq", Type: FilterTokenLogical}, + } + result, err := CompareQueue(expect, output) + if !result { + t.Error(err) + } + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"eq", 0}, + {"substring", 1}, + {"substring", 2}, + {"'Francisco'", 3}, + {"1", 3}, + {"3", 2}, + {"2", 2}, + {"'ci'", 1}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} +func TestGeoFunctions(t *testing.T) { + // Previously, the parser was incorrectly interpreting the 'geo.xxx' functions as the 'ge' operator. + input := "geo.distance(CurrentPosition,TargetPosition)" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"geo.distance", 0}, + {"CurrentPosition", 1}, + {"TargetPosition", 1}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestLambdaAny(t *testing.T) { + input := "Tags/any(var:var/Key eq 'Site')" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Tags", 1}, + {"any", 1}, + {"var", 2}, + {"eq", 2}, + {"/", 3}, + {"var", 4}, + {"Key", 4}, + {"'Site'", 3}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestLambdaAnyNot(t *testing.T) { + input := "Price/any(t:not (12345 eq t ))" + + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + { + expect := []*Token{ + &Token{Value: "Price", Type: FilterTokenLiteral}, + &Token{Value: "t", Type: FilterTokenLiteral}, + &Token{Value: "12345", Type: FilterTokenInteger}, + &Token{Value: "t", Type: FilterTokenLiteral}, + &Token{Value: "eq", Type: FilterTokenLogical}, + &Token{Value: "not", Type: FilterTokenLogical}, + &Token{Value: "2", Type: TokenTypeArgCount}, + &Token{Value: "any", Type: FilterTokenLambda}, + &Token{Value: "/", Type: FilterTokenOp}, + } + var result bool + result, err = CompareQueue(expect, output) + if !result { + t.Error(err) + } + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Price", 1}, + {"any", 1}, + {"t", 2}, + {"not", 2}, + {"eq", 3}, + {"12345", 4}, + {"t", 4}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestLambdaAnyAnd(t *testing.T) { + input := "Tags/any(var:var/Key eq 'Site' and var/Value eq 'London')" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Tags", 1}, + {"any", 1}, + {"var", 2}, + {"and", 2}, + {"eq", 3}, + {"/", 4}, + {"var", 5}, + {"Key", 5}, + {"'Site'", 4}, + {"eq", 3}, + {"/", 4}, + {"var", 5}, + {"Value", 5}, + {"'London'", 4}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestLambdaNestedAny(t *testing.T) { + input := "Enabled/any(t:t/Value eq Config/any(c:c/AdminState eq 'TRUE'))" + q, err := ParseFilterString(input) + if err != nil { + t.Errorf("Error parsing query %s. Error: %s", input, err.Error()) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Enabled", 1}, + {"any", 1}, + {"t", 2}, + {"eq", 2}, + {"/", 3}, + {"t", 4}, + {"Value", 4}, + {"/", 3}, + {"Config", 4}, + {"any", 4}, + {"c", 5}, + {"eq", 5}, + {"/", 6}, + {"c", 7}, + {"AdminState", 7}, + {"'TRUE'", 6}, + } + pos := 0 + err = CompareTree(q.Tree, expect, &pos, 0) + if err != nil { + printTree(q.Tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +// TestLambdaAnyNested validates the any() lambda function with multiple nested properties. +func TestLambdaAnyNestedProperties(t *testing.T) { + input := "Config/any(var:var/Config/Priority eq 123)" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Config", 1}, + {"any", 1}, + {"var", 2}, + {"eq", 2}, + {"/", 3}, + {"/", 4}, + {"var", 5}, + {"Config", 5}, + {"Priority", 4}, + {"123", 3}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestLambda2(t *testing.T) { + input := "Tags/any(var:var/Key eq 'Site' and var/Value eq 'London' or Price gt 1.0)" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Tags", 1}, + {"any", 1}, + {"var", 2}, + {"or", 2}, + {"and", 3}, + {"eq", 4}, + {"/", 5}, + {"var", 6}, + {"Key", 6}, + {"'Site'", 5}, + {"eq", 4}, + {"/", 5}, + {"var", 6}, + {"Value", 6}, + {"'London'", 5}, + {"gt", 3}, + {"Price", 4}, + {"1.0", 4}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestLambda3(t *testing.T) { + input := "Tags/any(var:var/Key eq 'Site' and var/Value eq 'London' or Price gt 1.0 or contains(var/Value, 'Smith'))" + tokens, err := GlobalFilterTokenizer.Tokenize(input) + if err != nil { + t.Error(err) + return + } + output, err := GlobalFilterParser.InfixToPostfix(tokens) + if err != nil { + t.Error(err) + return + } + + tree, err := GlobalFilterParser.PostfixToTree(output) + if err != nil { + t.Error(err) + return + } + var expect []expectedParseNode = []expectedParseNode{ + {"/", 0}, + {"Tags", 1}, + {"any", 1}, + {"var", 2}, + {"or", 2}, + {"or", 3}, + {"and", 4}, + {"eq", 5}, + {"/", 6}, + {"var", 7}, + {"Key", 7}, + {"'Site'", 6}, + {"eq", 5}, + {"/", 6}, + {"var", 7}, + {"Value", 7}, + {"'London'", 6}, + {"gt", 4}, + {"Price", 5}, + {"1.0", 5}, + {"contains", 3}, + {"/", 4}, + {"var", 5}, + {"Value", 5}, + {"'Smith'", 4}, + } + pos := 0 + err = CompareTree(tree, expect, &pos, 0) + if err != nil { + printTree(tree) + t.Errorf("Tree representation does not match expected value. error: %s", err.Error()) + } +} + +func TestFilterTokenizerExists(t *testing.T) { + + tokenizer := FilterTokenizer() + input := "exists(Name,false)" + expect := []*Token{ + &Token{Value: "exists", Type: FilterTokenFunc}, + &Token{Value: "(", Type: FilterTokenOpenParen}, + &Token{Value: "Name", Type: FilterTokenLiteral}, + &Token{Value: ",", Type: FilterTokenComma}, + &Token{Value: "false", Type: FilterTokenBoolean}, + &Token{Value: ")", Type: FilterTokenCloseParen}, + } + output, err := tokenizer.Tokenize(input) + if err != nil { + t.Error(err) + } + + result, err := CompareTokens(expect, output) + if !result { + t.Error(err) + } +} + +// CompareTree compares a tree representing a ODATA filter with the expected results. +// The expected values are a slice of nodes in breadth-first traversal. +func CompareTree(node *ParseNode, expect []expectedParseNode, pos *int, level int) error { + if *pos >= len(expect) { + return fmt.Errorf("Unexpected token. Got %s, expected no value", node.Token.Value) + } + if node.Token.Value != expect[*pos].Value { + return fmt.Errorf("Unexpected token. Got %s -> %d, expected: %s -> %d", node.Token.Value, level, expect[*pos].Value, expect[*pos].Level) + } + if level != expect[*pos].Level { + return fmt.Errorf("Unexpected level. Got %s -> %d, expected: %s -> %d", node.Token.Value, level, expect[*pos].Value, expect[*pos].Level) + } + for _, v := range node.Children { + *pos++ + if err := CompareTree(v, expect, pos, level+1); err != nil { + return err + } + } + if level == 0 && *pos+1 != len(expect) { + return fmt.Errorf("Expected number of tokens: %d, got %d", len(expect), *pos+1) + } + return nil +} + +type expectedParseNode struct { + Value string + Level int +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6783181 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module godata + +go 1.16 diff --git a/metadata_model.go b/metadata_model.go index 3e4c647..5a34bc8 100644 --- a/metadata_model.go +++ b/metadata_model.go @@ -34,6 +34,10 @@ func (t *GoDataMetadata) Bytes() ([]byte, error) { return append([]byte(xml.Header), output...), nil } +func (t *GoDataMetadata) String() string { + return "" +} + type GoDataReference struct { XMLName xml.Name `xml:"edmx:Reference"` Uri string `xml:"Uri,attr"` diff --git a/orderby_parser.go b/orderby_parser.go index eb041df..c87696f 100644 --- a/orderby_parser.go +++ b/orderby_parser.go @@ -35,7 +35,7 @@ func ParseOrderByString(orderby string) (*GoDataOrderByQuery, error) { result = append(result, &OrderByItem{field, order}) } - return &GoDataOrderByQuery{result}, nil + return &GoDataOrderByQuery{result, orderby}, nil } func SemanticizeOrderByQuery(orderby *GoDataOrderByQuery, service *GoDataService, entity *GoDataEntityType) error { diff --git a/parser.go b/parser.go index 072ea3d..8c9852d 100644 --- a/parser.go +++ b/parser.go @@ -1,7 +1,11 @@ package godata import ( + "fmt" "regexp" + "sort" + "strconv" + "strings" ) const ( @@ -10,10 +14,24 @@ const ( OpAssociationNone ) +// TokenTypeArgCount is used to specify the number of arguments of a function or listExpr +// This is used to handle variadic functions and listExpr. +const TokenTypeArgCount int = -1 + +// TokenTypeListExpr represents a parent node for a variadic listExpr. +// "list" +// "item1" +// "item2" +// ... +const TokenTypeListExpr int = -2 + +const TokenListExpr = "list" + +// TokenComma is the default separator for function arguments. const ( - NodeTypeLiteral int = iota - NodeTypeOp - NodeTypeFunc + TokenComma = "," + TokenOpenParen = "(" + TokenCloseParen = ")" ) type Tokenizer struct { @@ -22,9 +40,11 @@ type Tokenizer struct { } type TokenMatcher struct { - Pattern string - Re *regexp.Regexp - Token int + Pattern string // The regular expression matching a ODATA query token, such as literal value, operator or function + Re *regexp.Regexp // The compiled regex + Token int // The token identifier + CaseInsensitive bool // Regex is case-insensitive + Subst func(in string) string // A function that substitutes the raw input token with another representation. By default it is the identity. } type Token struct { @@ -32,19 +52,35 @@ type Token struct { Type int // Holds information about the semantic meaning of this token taken from the // context of the GoDataService. - SemanticType int + SemanticType SemanticType SemanticReference interface{} } func (t *Tokenizer) Add(pattern string, token int) { + t.AddWithSubstituteFunc(pattern, token, func(in string) string { return in }) +} + +func (t *Tokenizer) AddWithSubstituteFunc(pattern string, token int, subst func(string) string) { rxp := regexp.MustCompile(pattern) - matcher := &TokenMatcher{pattern, rxp, token} + matcher := &TokenMatcher{ + Pattern: pattern, + Re: rxp, + Token: token, + CaseInsensitive: strings.Contains(pattern, "(?i)"), + Subst: subst, + } t.TokenMatchers = append(t.TokenMatchers, matcher) } func (t *Tokenizer) Ignore(pattern string, token int) { rxp := regexp.MustCompile(pattern) - matcher := &TokenMatcher{pattern, rxp, token} + matcher := &TokenMatcher{ + Pattern: pattern, + Re: rxp, + Token: token, + CaseInsensitive: strings.Contains(pattern, "(?i)"), + Subst: func(in string) string { return in }, + } t.IgnoreMatchers = append(t.IgnoreMatchers, matcher) } @@ -53,22 +89,64 @@ func (t *Tokenizer) TokenizeBytes(target []byte) ([]*Token, error) { match := true // false when no match is found for len(target) > 0 && match { match = false - for _, m := range t.TokenMatchers { - token := m.Re.Find(target) - if len(token) > 0 { - parsed := Token{Value: string(token), Type: m.Token} - result = append(result, &parsed) - target = target[len(token):] // remove the token from the input + ignore := false + var tokens [][]byte + var m *TokenMatcher + for _, m = range t.TokenMatchers { + tokens = m.Re.FindSubmatch(target) + if len(tokens) > 0 { match = true break } } - for _, m := range t.IgnoreMatchers { - token := m.Re.Find(target) - if len(token) > 0 { - match = true - target = target[len(token):] // remove the token from the input - break + if len(tokens) == 0 { + for _, m = range t.IgnoreMatchers { + tokens = m.Re.FindSubmatch(target) + if len(tokens) > 0 { + ignore = true + break + } + } + } + if len(tokens) > 0 { + match = true + var parsed Token + var token []byte + // If the regex includes a named group and the name of that group is "token" + // then the value of the token is set to the subgroup. Other characters are + // not consumed by the tokenization process. + // For example, the regex: + // ^(?P(eq|ne|gt|ge|lt|le|and|or|not|has|in))\\s + // has a group named 'token' and the group is followed by a mandatory space character. + // If the input data is `Name eq 'Bob'`, the token is correctly set to 'eq' and + // the space after eq is not consumed, because the space character itself is supposed + // to be the next token. + // + // If Token.Value needs to be a sub-regex but the entire token needs to be consumed, + // use 'subtoken' + // ^(duration)?'(?P[0-9])' + l := 0 + if idx := m.Re.SubexpIndex("token"); idx > 0 { + token = tokens[idx] + l = len(token) + } else if idx := m.Re.SubexpIndex("subtoken"); idx > 0 { + token = tokens[idx] + l = len(tokens[0]) + } else { + token = tokens[0] + l = len(token) + } + target = target[l:] // remove the token from the input + if !ignore { + var v string + if m.CaseInsensitive { + // In ODATA 4.0.1, operators and functions are case insensitive. + v = strings.ToLower(string(token)) + } else { + v = string(token) + } + parsed = Token{Value: m.Subst(v), Type: m.Token} + result = append(result, &parsed) } } } @@ -76,7 +154,9 @@ func (t *Tokenizer) TokenizeBytes(target []byte) ([]*Token, error) { if len(target) > 0 && !match { return result, BadRequestError("No matching token for " + string(target)) } - + if len(result) < 1 { + return result, BadRequestError("Empty query parameter") + } return result, nil } @@ -99,12 +179,24 @@ type Operator struct { Operands int // Rank of precedence Precedence int + // Determine if the operands should be interpreted as a ListExpr or parenExpr according + // to ODATA ABNF grammar. + // This is only used when a listExpr has zero or one items. + // When a listExpr has 2 or more items, there is no ambiguity between listExpr and parenExpr. + // For example: + // 2 + (3) ==> the right operand is a parenExpr + // City IN ('Seattle', 'Atlanta') ==> the right operand is unambiguously a listExpr + // City IN ('Seattle') ==> the right operand should be a listExpr + PreferListExpr bool +} + +func (o *Operator) SetPreferListExpr(v bool) { + o.PreferListExpr = v } type Function struct { - Token string - // The number of parameters this function accepts - Params int + Token string // The function token + Params []int // The number of parameters this function accepts } type ParseNode struct { @@ -113,45 +205,82 @@ type ParseNode struct { Children []*ParseNode } +func (p *ParseNode) String() string { + var sb strings.Builder + var treePrinter func(n *ParseNode, sb *strings.Builder, level int) + + treePrinter = func(n *ParseNode, s *strings.Builder, level int) { + if n == nil || n.Token == nil { + s.WriteRune('\n') + return + } + s.WriteString(fmt.Sprintf("[%2d]", n.Token.Type)) + s.WriteString(strings.Repeat(" ", level)) + s.WriteString(n.Token.Value) + s.WriteRune('\n') + for _, v := range n.Children { + treePrinter(v, s, level+1) + } + } + treePrinter(p, &sb, 0) + return sb.String() +} + func EmptyParser() *Parser { - return &Parser{make(map[string]*Operator, 0), make(map[string]*Function)} + return &Parser{ + Operators: make(map[string]*Operator, 0), + Functions: make(map[string]*Function), + } } -// Add an operator to the language. Provide the token, a precedence, and -// whether the operator is left, right, or not associative. -func (p *Parser) DefineOperator(token string, operands, assoc, precedence int) { - p.Operators[token] = &Operator{token, assoc, operands, precedence} +// DefineOperator adds an operator to the language. Provide the token, the expected number of arguments, +// whether the operator is left, right, or not associative, and a precedence. +func (p *Parser) DefineOperator(token string, operands, assoc, precedence int) *Operator { + op := &Operator{ + Token: token, + Association: assoc, + Operands: operands, + Precedence: precedence, + } + p.Operators[token] = op + return op } -// Add a function to the language -func (p *Parser) DefineFunction(token string, params int) { +// DefineFunction adds a function to the language +// params is the number of parameters this function accepts +func (p *Parser) DefineFunction(token string, params []int) { + sort.Sort(sort.Reverse(sort.IntSlice(params))) p.Functions[token] = &Function{token, params} } -// Parse the input string of tokens using the given definitions of operators -// and functions. (Everything else is assumed to be a literal.) Uses the -// Shunting-Yard algorithm. +// InfixToPostfix parses the input string of tokens using the given definitions of operators +// and functions. +// Everything else is assumed to be a literal. +// Uses the Shunting-Yard algorithm. +// +// Infix notation for variadic functions and operators: f ( a, b, c, d ) +// Postfix notation with wall notation: | a b c d f +// Postfix notation with count notation: a b c d 4 f +// func (p *Parser) InfixToPostfix(tokens []*Token) (*tokenQueue, error) { - queue := tokenQueue{} - stack := tokenStack{} + queue := tokenQueue{} // output queue in postfix + stack := tokenStack{} // Operator stack + isCurrentTokenLiteral := false for len(tokens) > 0 { token := tokens[0] tokens = tokens[1:] - if _, ok := p.Functions[token.Value]; ok { + isCurrentTokenLiteral = false + if len(tokens) == 0 || tokens[0].Value != TokenOpenParen { + // A function token must be followed by open parenthesis token. + return nil, BadRequestError(fmt.Sprintf("Function '%s' must be followed by '('", token.Value)) + } + stack.incrementListArgCount() // push functions onto the stack stack.Push(token) - } else if token.Value == "," { - // function parameter separator, pop off stack until we see a "(" - for !stack.Empty() && stack.Peek().Value != "(" { - queue.Enqueue(stack.Pop()) - } - // there was an error parsing - if stack.Empty() { - return nil, BadRequestError("Parse error") - } } else if o1, ok := p.Operators[token.Value]; ok { + isCurrentTokenLiteral = false // push operators onto stack according to precedence if !stack.Empty() { for o2, ok := p.Operators[stack.Peek().Value]; ok && @@ -165,45 +294,151 @@ func (p *Parser) InfixToPostfix(tokens []*Token) (*tokenQueue, error) { o2, ok = p.Operators[stack.Peek().Value] } } - stack.Push(token) - } else if token.Value == "(" { - // push open parens onto the stack - stack.Push(token) - } else if token.Value == ")" { - // if we find a close paren, pop things off the stack - for !stack.Empty() && stack.Peek().Value != "(" { - queue.Enqueue(stack.Pop()) - } - // there was an error parsing - if stack.Empty() { - return nil, BadRequestError("Parse error. Mismatched parenthesis.") + if o1.Operands == 1 { // not, - + stack.incrementListArgCount() } - // pop off open paren - stack.Pop() - // if next token is a function, move it to the queue - if !stack.Empty() { - if _, ok := p.Functions[stack.Peek().Value]; ok { + stack.Push(token) + } else { + switch token.Value { + case TokenOpenParen: + isCurrentTokenLiteral = false + // In OData, the parenthesis tokens can be used: + // - As a parenExpr to set explicit precedence order, such as "(a + 2) x b" + // These precedence tokens are removed while parsing the OData query. + // - As a listExpr for multi-value sets, such as "City in ('San Jose', 'Chicago', 'Dallas')" + // The list tokens are retained while parsing the OData query. + // ABNF grammar: + // listExpr = OPEN BWS commonExpr BWS *( COMMA BWS commonExpr BWS ) CLOSE + stack.incrementListArgCount() + // Push open parens onto the stack + stack.Push(token) + case TokenCloseParen: + isCurrentTokenLiteral = false + // if we find a close paren, pop things off the stack + for !stack.Empty() { + if stack.Peek().Value == TokenOpenParen { + break + } else { + queue.Enqueue(stack.Pop()) + } + } + if stack.Empty() { + // there was an error parsing + return nil, BadRequestError("Parse error. Mismatched parenthesis.") + } + // Get the argument count associated with the open paren. + // Example: (a, b, c) is a listExpr with three arguments. + argCount := stack.getArgCount() + // pop off open paren + stack.Pop() + + // Determine if the parenthesis delimiters are: + // - A listExpr, possibly an empty list or single element. + // Note a listExpr may be on the left-side or right-side of operators, + // or it may be a list of function arguments. + // - A parenExpr, which is used as a precedence delimiter. + // + // (1, 2, 3) is a listExpr, there is no ambiguity. + // (1) matches both listExpr or parenExpr. + // parenExpr takes precedence over listExpr. + // + // For example: + // 1 IN (1, 2) ==> parenthesis is used to create a list of two elements. + // (1) + (2) ==> parenthesis is a precedence delimiter, i.e. parenExpr. + var isFunc, isListExpr bool + if !stack.Empty() { + _, isFunc = p.Functions[stack.Peek().Value] + } + switch { + case isFunc: + isListExpr = true + case argCount <= 1: + // When a listExpr contains a single item, it is ambiguous whether it is a listExpr or parenExpr. + if !stack.Empty() { + if o1, ok := p.Operators[stack.Peek().Value]; ok { + if o1.PreferListExpr { + // The expression is the right operand of an operator that has a preference for listExpr vs parenExpr. + isListExpr = true + } + } + } + if !isListExpr && len(tokens) > 0 { + if o1, ok := p.Operators[tokens[0].Value]; ok { + // The expression is the left operand of an operator that has a preference for listExpr vs parenExpr. + if o1.PreferListExpr { + isListExpr = true + } + } + } + case argCount > 1: + isListExpr = true + } + if isListExpr { + // The open parenthesis was a delimiter for a listExpr. + // Add a token indicating the number of arguments in the list. + queue.Enqueue(&Token{ + Value: strconv.Itoa(argCount), + Type: TokenTypeArgCount, + }) + // Enqueue a 'list' token if we are processing a ListExpr. + if !isFunc { + queue.Enqueue(&Token{ + Value: TokenListExpr, + Type: TokenTypeListExpr, + }) + } + } + // if next token is a function or multi-value operator, move it to the queue + if isFunc { queue.Enqueue(stack.Pop()) } + case TokenComma: + // Function argument separator (",") + isCurrentTokenLiteral = false + // Comma may be used as: + // 1. Separator of function parameters, + // 2. Separator for listExpr such as "City IN ('Seattle', 'San Francisco')" + // + // Pop off stack until we see a TokenOpenParen + for !stack.Empty() && stack.Peek().Value != TokenOpenParen { + // This happens when the previous function argument is an expression composed + // of multiple tokens, as opposed to a single token. For example: + // max(sin( 5 mul pi ) add 3, sin( 5 )) + queue.Enqueue(stack.Pop()) + } + if stack.Empty() { + // there was an error parsing. The top of the stack must be open parenthesis + return nil, BadRequestError("Parse error") + } + if stack.Peek().Value != TokenOpenParen { + panic("unexpected token") + } + default: + if isCurrentTokenLiteral { + // In most cases, it is invalid to have two consecutive literal values. + // TODO: The exception is for a positionLiteral: + // positionLiteral = doubleValue SP doubleValue ; longitude, then latitude + return nil, BadRequestError("Request cannot include two consecutive literal values") + } + isCurrentTokenLiteral = true + // Token is a literal -- put it in the queue + stack.incrementListArgCount() + queue.Enqueue(token) } - } else { - // Token is a literal -- put it in the queue - queue.Enqueue(token) } } // pop off the remaining operators onto the queue for !stack.Empty() { - if stack.Peek().Value == "(" || stack.Peek().Value == ")" { + if stack.Peek().Value == TokenOpenParen || stack.Peek().Value == TokenCloseParen { return nil, BadRequestError("parse error. Mismatched parenthesis.") } queue.Enqueue(stack.Pop()) } - return &queue, nil } -// Convert a Postfix token queue to a parse tree +// PostfixToTree converts a Postfix token queue to a parse tree func (p *Parser) PostfixToTree(queue *tokenQueue) (*ParseNode, error) { stack := &nodeStack{} currNode := &ParseNode{} @@ -212,20 +447,58 @@ func (p *Parser) PostfixToTree(queue *tokenQueue) (*ParseNode, error) { for t != nil { t = t.Next } - + processVariadicArgs := func(parent *ParseNode) (int, error) { + if stack.Empty() { + return 0, fmt.Errorf("No argCount token found. '%s'", parent.Token.Value) + } + n := stack.Pop() + if n.Token.Type != TokenTypeArgCount { + return 0, fmt.Errorf("No argCount token found. '%s'", parent.Token.Value) + } + argCount, err := strconv.Atoi(n.Token.Value) + if err != nil { + return 0, err + } + for i := 0; i < argCount; i++ { + if stack.Empty() { + return 0, fmt.Errorf("Missing argument found. '%s'", parent.Token.Value) + } + c := stack.Pop() + // Attach the operand to its parent node which represents the function/operator + c.Parent = parent + // prepend children so they get added in the right order + parent.Children = append([]*ParseNode{c}, parent.Children...) + } + return argCount, nil + } for !queue.Empty() { // push the token onto the stack as a tree node - currNode = &ParseNode{queue.Dequeue(), nil, make([]*ParseNode, 0)} + currToken := queue.Dequeue() + currNode = &ParseNode{currToken, nil, make([]*ParseNode, 0)} stack.Push(currNode) if _, ok := p.Functions[stack.Peek().Token.Value]; ok { // if the top of the stack is a function node := stack.Pop() f := p.Functions[node.Token.Value] - // pop off function parameters - for i := 0; i < f.Params; i++ { - // prepend children so they get added in the right order - node.Children = append([]*ParseNode{stack.Pop()}, node.Children...) + // Pop off function parameters + // The first token is the number of function arguments, which is useful + // when parsing variadic functions. + // Some functions, e.g. substring, can take a variable number of arguments. + if argCount, err := processVariadicArgs(node); err != nil { + return nil, err + } else { + foundMatch := false + for _, expectedArgCount := range f.Params { + if argCount == expectedArgCount { + foundMatch = true + break + } + } + if !foundMatch { + return nil, fmt.Errorf("Invalid number of arguments for function '%s'. Got %d argument", + node.Token.Value, argCount) + } } stack.Push(node) } else if _, ok := p.Operators[stack.Peek().Token.Value]; ok { @@ -234,13 +507,25 @@ func (p *Parser) PostfixToTree(queue *tokenQueue) (*ParseNode, error) { o := p.Operators[node.Token.Value] // pop off operands for i := 0; i < o.Operands; i++ { + if stack.Empty() { + return nil, fmt.Errorf("Insufficient number of operands for operator '%s'", node.Token.Value) + } // prepend children so they get added in the right order - node.Children = append([]*ParseNode{stack.Pop()}, node.Children...) + c := stack.Pop() + c.Parent = node + node.Children = append([]*ParseNode{c}, node.Children...) } stack.Push(node) + } else if stack.Peek().Token.Type == TokenTypeListExpr { + // ListExpr: List of items + node := stack.Pop() + if _, err := processVariadicArgs(node); err != nil { + return nil, err + } + stack.Push(node) + } } - return currNode, nil } @@ -250,12 +535,13 @@ type tokenStack struct { } type tokenStackNode struct { - Token *Token - Prev *tokenStackNode + Token *Token // The token value. + Prev *tokenStackNode // The previous node in the stack. + tokenCount int // The number of arguments in a listExpr. } func (s *tokenStack) Push(t *Token) { - node := tokenStackNode{t, s.Head} + node := tokenStackNode{Token: t, Prev: s.Head} //fmt.Println("Pushed:", t.Value, "->", s.String()) s.Head = &node s.Size++ @@ -277,6 +563,16 @@ func (s *tokenStack) Empty() bool { return s.Head == nil } +func (s *tokenStack) incrementListArgCount() { + if !s.Empty() && s.Head.Token.Value == TokenOpenParen { + s.Head.tokenCount++ + } +} + +func (s *tokenStack) getArgCount() int { + return s.Head.tokenCount +} + func (s *tokenStack) String() string { output := "" currNode := s.Head @@ -298,6 +594,7 @@ type tokenQueueNode struct { Next *tokenQueueNode } +// Enqueue adds the specified token at the tail of the queue. func (q *tokenQueue) Enqueue(t *Token) { node := tokenQueueNode{t, q.Tail, nil} //fmt.Println(t.Value) @@ -311,6 +608,7 @@ func (q *tokenQueue) Enqueue(t *Token) { q.Tail = &node } +// Dequeue removes the token at the head of the queue and returns the token. func (q *tokenQueue) Dequeue() *Token { node := q.Head if node.Next != nil { @@ -328,13 +626,26 @@ func (q *tokenQueue) Empty() bool { } func (q *tokenQueue) String() string { - result := "" + var sb strings.Builder + node := q.Head + for node != nil { + sb.WriteString(node.Token.Value) + node = node.Next + if node != nil { + sb.WriteRune(' ') + } + } + return sb.String() +} + +func (q *tokenQueue) GetValue() string { + var sb strings.Builder node := q.Head for node != nil { - result += node.Token.Value + sb.WriteString(node.Token.Value) node = node.Next } - return result + return sb.String() } type nodeStack struct { @@ -364,3 +675,14 @@ func (s *nodeStack) Peek() *ParseNode { func (s *nodeStack) Empty() bool { return s.Head == nil } + +func (s *nodeStack) String() string { + var sb strings.Builder + currNode := s.Head + for currNode != nil { + sb.WriteRune(' ') + sb.WriteString(currNode.ParseNode.Token.Value) + currNode = currNode.Prev + } + return sb.String() +} diff --git a/parser_test.go b/parser_test.go index 5beb576..4054ce3 100644 --- a/parser_test.go +++ b/parser_test.go @@ -7,8 +7,8 @@ import ( func TestPEMDAS(t *testing.T) { parser := EmptyParser() - parser.DefineFunction("sin", 1) - parser.DefineFunction("max", 2) + parser.DefineFunction("sin", []int{1}) + parser.DefineFunction("max", []int{2}) parser.DefineOperator("^", 2, OpAssociationRight, 5) parser.DefineOperator("*", 2, OpAssociationLeft, 5) parser.DefineOperator("/", 2, OpAssociationLeft, 5) @@ -61,8 +61,8 @@ func TestPEMDAS(t *testing.T) { func BenchmarkPEMDAS(b *testing.B) { parser := EmptyParser() - parser.DefineFunction("sin", 1) - parser.DefineFunction("max", 2) + parser.DefineFunction("sin", []int{1}) + parser.DefineFunction("max", []int{2}) parser.DefineOperator("^", 2, OpAssociationRight, 5) parser.DefineOperator("*", 2, OpAssociationLeft, 5) parser.DefineOperator("/", 2, OpAssociationLeft, 5) @@ -138,16 +138,16 @@ func TestBoolean(t *testing.T) { func TestFunc(t *testing.T) { parser := EmptyParser() - parser.DefineFunction("sin", 1) - parser.DefineFunction("max", 2) - parser.DefineFunction("volume", 3) + parser.DefineFunction("sin", []int{1}) + parser.DefineFunction("max", []int{2}) + parser.DefineFunction("volume", []int{3}) parser.DefineOperator("^", 2, OpAssociationRight, 5) parser.DefineOperator("*", 2, OpAssociationLeft, 5) parser.DefineOperator("/", 2, OpAssociationLeft, 5) parser.DefineOperator("+", 2, OpAssociationLeft, 4) parser.DefineOperator("-", 2, OpAssociationLeft, 4) - // max(sin(5*pi)+3, sin(5)+volume(3,2,4)/2) + // max(sin(5*pi)+3, sin(5)+volume(3,2,4)/[]int{3}) tokens := []*Token{ &Token{Value: "max"}, &Token{Value: "("}, @@ -180,8 +180,10 @@ func TestFunc(t *testing.T) { // 5 pi * sin 3 + 5 sin 3 2 4 volume 2 / + max expected := []string{ - "5", "pi", "*", "sin", "3", "+", "5", "sin", "3", "2", "4", "volume", "2", - "/", "+", "max"} + "5", "pi", "*", "1" /* arg count */, "sin", "3", "+", "5", "1" /* arg count */, "sin", + "3", "2", "4", "3" /* arg count */, "volume", "2", + "/", "+", + "2" /* arg count */, "max"} result, err := parser.InfixToPostfix(tokens) if err != nil { @@ -205,8 +207,8 @@ func TestFunc(t *testing.T) { func TestTree(t *testing.T) { parser := EmptyParser() - parser.DefineFunction("sin", 1) - parser.DefineFunction("max", 2) + parser.DefineFunction("sin", []int{1}) + parser.DefineFunction("max", []int{2}) parser.DefineOperator("^", 2, OpAssociationRight, 5) parser.DefineOperator("*", 2, OpAssociationLeft, 5) parser.DefineOperator("/", 2, OpAssociationLeft, 5) @@ -237,8 +239,11 @@ func TestTree(t *testing.T) { t.Error(err) } - root, _ := parser.PostfixToTree(result) - + root, err := parser.PostfixToTree(result) + if err != nil { + t.Error("Error parsing query") + return + } if root.Token.Value != "sin" { t.Error("Root node is not sin") } @@ -267,8 +272,8 @@ func TestTree(t *testing.T) { func BenchmarkBuildTree(b *testing.B) { parser := EmptyParser() - parser.DefineFunction("sin", 1) - parser.DefineFunction("max", 2) + parser.DefineFunction("sin", []int{1}) + parser.DefineFunction("max", []int{2}) parser.DefineOperator("^", 2, OpAssociationRight, 5) parser.DefineOperator("*", 2, OpAssociationLeft, 5) parser.DefineOperator("/", 2, OpAssociationLeft, 5) diff --git a/providers/mysql.go b/providers/mysql.go index 4846566..4a2a726 100644 --- a/providers/mysql.go +++ b/providers/mysql.go @@ -1,7 +1,6 @@ package mysql import ( - "fmt" . "godata" //"database/sql" //"errors" @@ -119,74 +118,78 @@ func BuildMySQLProvider(cxnParams *MySQLConnectionParams, namespace string) *MyS } func (p *MySQLGoDataProvider) BuildQuery(r *GoDataRequest) (string, error) { - setName := r.FirstSegment.Name - entitySet := p.EntitySets[setName] - tableName := entitySet.Entity.TableName - pKeyValue := r.FirstSegment.Identifier - - query := []byte{} - params := []string{} - - selectClause, selectParams, selectErr := p.BuildSelectClause(r) - if selectErr != nil { - return nil, selectErr - } - query = append(query, selectClause) - params = append(params, selectParams) - - fromClause, fromParams, fromErr := p.BuildFromClause(r) - if fromErr != nil { - return nil, fromErr - } - query = append(query, selectClause) - params = append(params, selectParams) + /* + setName := r.FirstSegment.Name + entitySet := p.EntitySets[setName] + tableName := entitySet.Entity.TableName + pKeyValue := r.FirstSegment.Identifier + + query := []byte{} + params := []string{} + + selectClause, selectParams, selectErr := p.BuildSelectClause(r) + if selectErr != nil { + return nil, selectErr + } + query = append(query, selectClause) + params = append(params, selectParams) + fromClause, fromParams, fromErr := p.BuildFromClause(r) + if fromErr != nil { + return nil, fromErr + } + query = append(query, selectClause) + params = append(params, selectParams) + */ + return "", NotImplementedError("not implemented") } // Build the select clause to begin the query, and also return the values to // send to a prepared statement. func (p *MySQLGoDataProvider) BuildSelectClause(r *GoDataRequest) ([]byte, []string, error) { - + return nil, nil, NotImplementedError("not implemented") } // Build the from clause in the query, and also return the values to send to // the prepared statement. func (p *MySQLGoDataProvider) BuildFromClause(r *GoDataRequest) ([]byte, []string, error) { - + return nil, nil, NotImplementedError("not implemented") } // Build a where clause that can be appended to an SQL query, and also return // the values to send to a prepared statement. func (p *MySQLGoDataProvider) BuildWhereClause(r *GoDataRequest) ([]byte, []string, error) { + /* + // Builds the WHERE clause recursively using DFS + recursiveBuildWhere := func(n *ParseNode) ([]byte, []string, error) { + if n.Token.Type == FilterTokenLiteral { + // TODO: map to columns + return []byte("?"), []byte(n.Token.Value), nil + } - // Builds the WHERE clause recursively using DFS - recursiveBuildWhere := func(n *ParseNode) ([]byte, []string, error) { - if n.Token.Type == FilterTokenLiteral { - // TODO: map to columns - return []byte("?"), []byte(n.Token.Value), nil - } - - if v, ok := MySQLNodeMap[n.Token.Value]; ok { - params := string - children := []byte{} - // build each child first using DFS - for _, child := range n.Children { - q, o, err := recursiveBuildWhere(child) - if err != nil { - return nil, nil, err + if v, ok := MySQLNodeMap[n.Token.Value]; ok { + params := string + children := []byte{} + // build each child first using DFS + for _, child := range n.Children { + q, o, err := recursiveBuildWhere(child) + if err != nil { + return nil, nil, err + } + children := append(children, q) + // make the assumption that all params appear LTR and are never added + // out of order + params := append(params, o) } - children := append(children, q) - // make the assumption that all params appear LTR and are never added - // out of order - params := append(params, o) + // merge together the children and the current node + result := fmt.Sprintf(v, children...) + return []byte(result), params, nil + } else { + return nil, nil, NotImplementedError(n.Token.Value + " is not implemented.") } - // merge together the children and the current node - result := fmt.Sprintf(v, children...) - return []byte(result), params, nil - } else { - return nil, nil, NotImplementedError(n.Token.Value + " is not implemented.") } - } + */ + return nil, nil, NotImplementedError("not implemented") } // Respond to a GoDataRequest using the MySQL provider. diff --git a/providers/mysql_test.go b/providers/mysql_test.go index 1468c20..1ac6f43 100644 --- a/providers/mysql_test.go +++ b/providers/mysql_test.go @@ -1,8 +1,9 @@ package mysql +/* + import ( . "godata" - "testing" ) func TestCoffeeDatabase(t *testing.T) { @@ -214,3 +215,4 @@ func BenchmarkCoffeeDatabase(b *testing.B) { provider.BuildMetadata() } } +*/ diff --git a/request_model.go b/request_model.go index 73b5511..09f58d9 100644 --- a/request_model.go +++ b/request_model.go @@ -15,8 +15,10 @@ const ( RequestKindCount ) +type SemanticType int + const ( - SemanticTypeUnknown int = iota + SemanticTypeUnknown SemanticType = iota SemanticTypeEntity SemanticTypeEntitySet SemanticTypeDerivedEntity @@ -43,7 +45,7 @@ type GoDataSegment struct { RawValue string // The kind of resource being pointed at by this segment - SemanticType int + SemanticType SemanticType // A pointer to the metadata type this object represents, as defined by a // particular service @@ -65,6 +67,7 @@ type GoDataSegment struct { type GoDataQuery struct { Filter *GoDataFilterQuery + At *GoDataFilterQuery Apply *GoDataApplyQuery Expand *GoDataExpandQuery Select *GoDataSelectQuery @@ -82,6 +85,8 @@ type GoDataQuery struct { // is stored as a parse tree that can be traversed. type GoDataFilterQuery struct { Tree *ParseNode + // The raw filter string + RawValue string } type GoDataApplyQuery string @@ -92,10 +97,14 @@ type GoDataExpandQuery struct { type GoDataSelectQuery struct { SelectItems []*SelectItem + // The raw select string + RawValue string } type GoDataOrderByQuery struct { OrderByItems []*OrderByItem + // The raw orderby string + RawValue string } type GoDataTopQuery int @@ -108,6 +117,8 @@ type GoDataInlineCountQuery string type GoDataSearchQuery struct { Tree *ParseNode + // The raw search string + RawValue string } type GoDataFormatQuery struct { diff --git a/search_parser.go b/search_parser.go index f935c10..71cb76c 100644 --- a/search_parser.go +++ b/search_parser.go @@ -26,7 +26,7 @@ func ParseSearchString(filter string) (*GoDataSearchQuery, error) { if err != nil { return nil, err } - return &GoDataSearchQuery{tree}, nil + return &GoDataSearchQuery{tree, filter}, nil } // Create a tokenizer capable of tokenizing filter statements diff --git a/select_parser.go b/select_parser.go index aef4bb9..4888656 100644 --- a/select_parser.go +++ b/select_parser.go @@ -22,7 +22,7 @@ func ParseSelectString(sel string) (*GoDataSelectQuery, error) { result = append(result, &SelectItem{segments}) } - return &GoDataSelectQuery{result}, nil + return &GoDataSelectQuery{result, sel}, nil } func SemanticizeSelectQuery(sel *GoDataSelectQuery, service *GoDataService, entity *GoDataEntityType) error { diff --git a/service.go b/service.go index f8b9e0e..a03247d 100644 --- a/service.go +++ b/service.go @@ -143,7 +143,7 @@ func BuildService(provider GoDataProvider, serviceUrl string) (*GoDataService, e // to a GoData provider, and then building a response. func (service *GoDataService) GoDataHTTPHandler(w http.ResponseWriter, r *http.Request) { - request, err := ParseRequest(r.URL.Path, r.URL.Query()) + request, err := ParseRequest(r.URL.Path, r.URL.Query(), false) if err != nil { panic(err) // TODO: return proper error diff --git a/service_test.go b/service_test.go index 7256540..b478dc2 100644 --- a/service_test.go +++ b/service_test.go @@ -112,7 +112,7 @@ func TestSemanticizeRequest(t *testing.T) { return } - req, err := ParseRequest(url.Path, url.Query()) + req, err := ParseRequest(url.Path, url.Query(), false) if err != nil { t.Error(err) @@ -180,7 +180,7 @@ func TestSemanticizeRequestWildcard(t *testing.T) { return } - req, err := ParseRequest(url.Path, url.Query()) + req, err := ParseRequest(url.Path, url.Query(), false) if err != nil { t.Error(err) @@ -197,7 +197,7 @@ func TestSemanticizeRequestWildcard(t *testing.T) { err = SemanticizeRequest(req, service) if err != nil { - t.Error(err) + t.Errorf("Failed to semanticize request. Error: %v", err) return } @@ -263,7 +263,7 @@ func BenchmarkTypicalParseSemanticizeRequest(b *testing.B) { for n := 0; n < b.N; n++ { - req, err := ParseRequest(url.Path, url.Query()) + req, err := ParseRequest(url.Path, url.Query(), false) if err != nil { b.Error(err) @@ -300,7 +300,7 @@ func BenchmarkWildcardParseSemanticizeRequest(b *testing.B) { for n := 0; n < b.N; n++ { - req, err := ParseRequest(url.Path, url.Query()) + req, err := ParseRequest(url.Path, url.Query(), false) if err != nil { b.Error(err) diff --git a/url_parser.go b/url_parser.go index b2092f4..7fc53dd 100644 --- a/url_parser.go +++ b/url_parser.go @@ -1,19 +1,20 @@ package godata import ( + "fmt" "net/url" "strings" ) // Parse a request from the HTTP server and format it into a GoDaataRequest type // to be passed to a provider to produce a result. -func ParseRequest(path string, query url.Values) (*GoDataRequest, error) { +func ParseRequest(path string, query url.Values, lenient bool) (*GoDataRequest, error) { firstSegment, lastSegment, err := ParseUrlPath(path) if err != nil { return nil, err } - parsedQuery, err := ParseUrlQuery(query) + parsedQuery, err := ParseUrlQuery(query, lenient) if err != nil { return nil, err } @@ -204,8 +205,38 @@ func SemanticizePathSegment(segment *GoDataSegment, service *GoDataService) erro return BadRequestError("Invalid segment " + segment.RawValue) } -func ParseUrlQuery(query url.Values) (*GoDataQuery, error) { +var supportedOdataKeywords = map[string]bool{ + "$filter": true, + "$apply": true, + "$expand": true, + "$select": true, + "$orderby": true, + "$top": true, + "$skip": true, + "$count": true, + "$inlinecount": true, + "$search": true, + "$format": true, + "at": true, + "tags": true, +} + +func ParseUrlQuery(query url.Values, lenient bool) (*GoDataQuery, error) { + if !lenient { + // Validate each query parameter is a valid ODATA keyword. + for key, val := range query { + if _, ok := supportedOdataKeywords[key]; !ok { + return nil, BadRequestError(fmt.Sprintf("Query parameter '%s' is not supported", key)). + SetCause(&UnsupportedQueryParameterError{key}) + } + if len(val) > 1 { + return nil, BadRequestError(fmt.Sprintf("Query parameter '%s' cannot be specified more than once", key)). + SetCause(&DuplicateQueryParameterError{key}) + } + } + } filter := query.Get("$filter") + at := query.Get("at") apply := query.Get("$apply") expand := query.Get("$expand") sel := query.Get("$select") @@ -226,6 +257,18 @@ func ParseUrlQuery(query url.Values) (*GoDataQuery, error) { if err != nil { return nil, err } + if at != "" { + result.At, err = ParseFilterString(at) + } + if err != nil { + return nil, err + } + if at != "" { + result.At, err = ParseFilterString(at) + } + if err != nil { + return nil, err + } if apply != "" { result.Apply, err = ParseApplyString(apply) } diff --git a/url_parser_test.go b/url_parser_test.go index e8d2ee5..1088cdc 100644 --- a/url_parser_test.go +++ b/url_parser_test.go @@ -14,7 +14,7 @@ func TestUrlParser(t *testing.T) { return } - request, err := ParseRequest(parsedUrl.Path, parsedUrl.Query()) + request, err := ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) if err != nil { t.Error(err) @@ -34,3 +34,196 @@ func TestUrlParser(t *testing.T) { return } } + +func TestUrlParserStrictValidation(t *testing.T) { + testUrl := "Employees(1)/Sales.Manager?$expand=DirectReports%28$select%3DFirstName%2CLastName%3B$levels%3D4%29" + parsedUrl, err := url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) + if err != nil { + t.Error(err) + return + } + + testUrl = "Employees(1)/Sales.Manager?$filter=FirstName eq 'Bob'" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) + if err != nil { + t.Error(err) + return + } + + // Wrong filter with an extraneous single quote + testUrl = "Employees(1)/Sales.Manager?$filter=FirstName eq' 'Bob'" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) + if err == nil { + t.Errorf("Parser should have returned invalid filter error: %s", testUrl) + return + } + + // Valid query with two parameters: + // $filter=FirstName eq 'Bob' + // at=Version eq '123' + testUrl = "Employees(1)/Sales.Manager?$filter=FirstName eq 'Bob'&at=Version eq '123'" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) + if err != nil { + t.Error(err) + return + } + + // Invalid query: + // $filter=FirstName eq' 'Bob' has extraneous single quote. + // at=Version eq '123' is valid + testUrl = "Employees(1)/Sales.Manager?$filter=FirstName eq' 'Bob'&at=Version eq '123'" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) + if err == nil { + t.Errorf("Parser should have returned invalid filter error: %s", testUrl) + return + } + + testUrl = "Employees(1)/Sales.Manager?$select=3DFirstName" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) + if err != nil { + t.Error(err) + return + } + + testUrl = "Employees(1)/Sales.Manager?$filter=Name in ('Bob','Alice')&$select=Name,Address%3B$expand=Address($select=City)" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false /*strict*/) + if err != nil { + t.Errorf("Unexpected parsing error: %v", err) + return + } + + // A $select option cannot be wrapped with parenthesis. This is not legal ODATA. + + /* + queryOptions = queryOption *( "&" queryOption ) + queryOption = systemQueryOption + / aliasAndValue + / nameAndValue + / customQueryOption + systemQueryOption = compute + / deltatoken + / expand + / filter + / format + / id + / inlinecount + / orderby + / schemaversion + / search + / select + / skip + / skiptoken + / top + / index + select = ( "$select" / "select" ) EQ selectItem *( COMMA selectItem ) + */ + testUrl = "Employees(1)/Sales.Manager?$filter=Name in ('Bob','Alice')&($select=Name,Address%3B$expand=Address($select=City))" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false /*strict*/) + if err == nil { + t.Errorf("Parser should have raised error") + return + } + + // Duplicate keyword: '$select' is present twice. + testUrl = "Employees(1)/Sales.Manager?$select=3DFirstName&$select=3DFirstName" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + // In lenient mode, do not return an error when there is a duplicate keyword. + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), true /*lenient*/) + if err != nil { + t.Error(err) + return + } + // In strict mode, return an error when there is a duplicate keyword. + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false /*strict*/) + if err == nil { + t.Error("Parser should have returned duplicate keyword error") + return + } + + // Unsupported keywords + testUrl = "Employees(1)/Sales.Manager?orderby=FirstName" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), true /*lenient*/) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false /*strict*/) + if err == nil { + t.Error("Parser should have returned unsupported keyword error") + return + } + + testUrl = "Employees(1)/Sales.Manager?$select=LastName&$expand=Address" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) + if err != nil { + t.Error(err) + return + } + + testUrl = "Employees(1)/Sales.Manager?$select=FirstName,LastName&$expand=Address" + parsedUrl, err = url.Parse(testUrl) + if err != nil { + t.Error(err) + return + } + _, err = ParseRequest(parsedUrl.Path, parsedUrl.Query(), false) + if err != nil { + t.Error(err) + return + } + +}