From f06cdc670dc5974df317f13ac9c9964c40ba04cf Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 18 Oct 2024 13:08:37 +0800 Subject: [PATCH 1/9] feat: fix Union type with dataclass ambiguous error Signed-off-by: mao3267 --- .run/single-binary.run.xml | 4 +- flyteadmin/go.mod | 6 +-- flyteadmin/go.sum | 3 ++ flytepropeller/go.mod | 7 +++ flytepropeller/go.sum | 16 +++++++ .../pkg/compiler/validators/typing.go | 47 +++++++++++++++++++ go.mod | 8 ++-- go.sum | 14 ++++-- 8 files changed, 93 insertions(+), 12 deletions(-) diff --git a/.run/single-binary.run.xml b/.run/single-binary.run.xml index 4b5de3705a..77de6fe10a 100644 --- a/.run/single-binary.run.xml +++ b/.run/single-binary.run.xml @@ -8,6 +8,8 @@ - + + \ No newline at end of file diff --git a/flyteadmin/go.mod b/flyteadmin/go.mod index 25340ec208..0f6ce014f5 100644 --- a/flyteadmin/go.mod +++ b/flyteadmin/go.mod @@ -47,7 +47,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - github.com/wI2L/jsondiff v0.5.0 + github.com/wI2L/jsondiff v0.6.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.24.0 golang.org/x/net v0.27.0 @@ -174,9 +174,9 @@ require ( github.com/spf13/viper v1.11.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/flyteadmin/go.sum b/flyteadmin/go.sum index bab1e3f1f3..3bc3b54615 100644 --- a/flyteadmin/go.sum +++ b/flyteadmin/go.sum @@ -1270,6 +1270,7 @@ github.com/tidwall/gjson v1.7.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -1279,6 +1280,7 @@ github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= @@ -1301,6 +1303,7 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= diff --git a/flytepropeller/go.mod b/flytepropeller/go.mod index a14f689a2b..73bbd27a69 100644 --- a/flytepropeller/go.mod +++ b/flytepropeller/go.mod @@ -22,11 +22,14 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 + github.com/santhosh-tekuri/jsonschema v1.2.4 github.com/shamaton/msgpack/v2 v2.2.2 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 + github.com/wI2L/jsondiff v0.6.0 + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 @@ -125,6 +128,10 @@ require ( github.com/spf13/viper v1.11.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect diff --git a/flytepropeller/go.sum b/flytepropeller/go.sum index 87d2b03633..70d0669dd6 100644 --- a/flytepropeller/go.sum +++ b/flytepropeller/go.sum @@ -374,6 +374,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/shamaton/msgpack/v2 v2.2.2 h1:GOIg0c9LV04VwzOOqZSrmsv/JzjNOOMxnS/HvOHGdgs= github.com/shamaton/msgpack/v2 v2.2.2/go.mod h1:6khjYnkx73f7VQU7wjcFS9DFjs+59naVWJv1TB7qdOI= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -411,11 +413,25 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/flytepropeller/pkg/compiler/validators/typing.go b/flytepropeller/pkg/compiler/validators/typing.go index 2bde60b47b..abd5010475 100644 --- a/flytepropeller/pkg/compiler/validators/typing.go +++ b/flytepropeller/pkg/compiler/validators/typing.go @@ -1,9 +1,14 @@ package validators import ( + "bytes" + "encoding/json" + "fmt" + jscmp "gitlab.com/yvesf/json-schema-compare" "strings" structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/santhosh-tekuri/jsonschema" flyte "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) @@ -16,6 +21,37 @@ type trivialChecker struct { literalType *flyte.LiteralType } +func isSameTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { + compiler := jsonschema.NewCompiler() + + addSchemaToCompiler := func(name string, data *structpb.Struct) error { + schemaByte, err := json.Marshal(data.Fields) + if err != nil { + return fmt.Errorf("failed to marshal %s: %w", name, err) + } + return compiler.AddResource(name, bytes.NewReader(schemaByte)) + } + + if err := addSchemaToCompiler("source", sourceMetaData); err != nil { + return false + } + if err := addSchemaToCompiler("target", targetMetaData); err != nil { + return false + } + + sourceSchema, err := compiler.Compile("source") + if err != nil { + return false + } + targetSchema, err := compiler.Compile("target") + if err != nil { + return false + } + + cmprErrList := jscmp.Compare(sourceSchema, targetSchema) + return len(cmprErrList) == 1 && strings.Contains(cmprErrList[0].Error(), "FIXME additionalProperties not implemented") +} + // CastsFrom is a trivial type checker merely checks if types match exactly. func (t trivialChecker) CastsFrom(upstreamType *flyte.LiteralType) bool { // If upstream is an enum, it can be consumed as a string downstream @@ -35,6 +71,17 @@ func (t trivialChecker) CastsFrom(upstreamType *flyte.LiteralType) bool { return false } + if upstreamType.GetSimple() == flyte.SimpleType_STRUCT && t.literalType.GetSimple() == flyte.SimpleType_STRUCT { + upstreamMetaData := upstreamType.GetMetadata() + downstreamMetaData := t.literalType.GetMetadata() + + if upstreamMetaData == nil || downstreamMetaData == nil { + return true + } + + return isSameTypeInJSON(upstreamMetaData, downstreamMetaData) + } + // Ignore metadata when comparing types. upstreamTypeCopy := *upstreamType downstreamTypeCopy := *t.literalType diff --git a/go.mod b/go.mod index 5b59c17763..085ae3b154 100644 --- a/go.mod +++ b/go.mod @@ -164,6 +164,7 @@ require ( github.com/ray-project/kuberay/ray-operator v1.1.0-rc.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/robfig/cron/v3 v3.0.0 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.10.0+incompatible // indirect github.com/shamaton/msgpack/v2 v2.2.2 // indirect @@ -175,11 +176,12 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/testify v1.9.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/wI2L/jsondiff v0.5.0 // indirect + github.com/wI2L/jsondiff v0.6.0 // indirect + gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/go.sum b/go.sum index cdb119abe5..fe4b3afb19 100644 --- a/go.sum +++ b/go.sum @@ -1192,6 +1192,7 @@ github.com/rubenv/sql-migrate v0.0.0-20190212093014-1007f53448d7/go.mod h1:WS0rl github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/santhosh-tekuri/jsonschema/v2 v2.1.0/go.mod h1:yzJzKUGV4RbWqWIBBP4wSOBqavX5saE02yirLS0OTyg= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -1304,8 +1305,8 @@ github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH github.com/tidwall/gjson v1.6.8/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/gjson v1.7.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -1313,8 +1314,9 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= @@ -1335,8 +1337,8 @@ github.com/unrolled/secure v0.0.0-20181005190816-ff9db2ff917f/go.mod h1:mnPT77IA github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= -github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= @@ -1352,6 +1354,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= +gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.elastic.co/apm v1.8.0/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= go.elastic.co/apm/module/apmhttp v1.8.0/go.mod h1:9LPFlEON51/lRbnWDfqAWErihIiAFDUMfMV27YjoWQ8= go.elastic.co/apm/module/apmot v1.8.0/go.mod h1:Q5Xzabte8G/fkvDjr1jlDuOSUt9hkVWNZEHh6ZNaTjI= From 47ccbd1e0ba8ae4cfefb489dc13c989ae4b94838 Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 1 Nov 2024 17:10:11 +0800 Subject: [PATCH 2/9] fix: direct json comparison for superset Signed-off-by: mao3267 --- flytepropeller/go.mod | 3 +- flytepropeller/go.sum | 6 +- .../pkg/compiler/validators/typing.go | 72 ++++++++++++------- go.mod | 2 - go.sum | 3 - 5 files changed, 49 insertions(+), 37 deletions(-) diff --git a/flytepropeller/go.mod b/flytepropeller/go.mod index c129312e44..33c2370781 100644 --- a/flytepropeller/go.mod +++ b/flytepropeller/go.mod @@ -20,16 +20,15 @@ require ( github.com/imdario/mergo v0.3.13 github.com/magiconair/properties v1.8.6 github.com/mitchellh/mapstructure v1.5.0 + github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 - github.com/santhosh-tekuri/jsonschema v1.2.4 github.com/shamaton/msgpack/v2 v2.2.2 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/wI2L/jsondiff v0.6.0 - gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 diff --git a/flytepropeller/go.sum b/flytepropeller/go.sum index 63c498dc77..bc37abe635 100644 --- a/flytepropeller/go.sum +++ b/flytepropeller/go.sum @@ -332,6 +332,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM= +github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -373,8 +375,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= -github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/shamaton/msgpack/v2 v2.2.2 h1:GOIg0c9LV04VwzOOqZSrmsv/JzjNOOMxnS/HvOHGdgs= github.com/shamaton/msgpack/v2 v2.2.2/go.mod h1:6khjYnkx73f7VQU7wjcFS9DFjs+59naVWJv1TB7qdOI= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -429,8 +429,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= -gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/flytepropeller/pkg/compiler/validators/typing.go b/flytepropeller/pkg/compiler/validators/typing.go index abd5010475..6ca244235e 100644 --- a/flytepropeller/pkg/compiler/validators/typing.go +++ b/flytepropeller/pkg/compiler/validators/typing.go @@ -1,16 +1,12 @@ package validators import ( - "bytes" "encoding/json" - "fmt" - jscmp "gitlab.com/yvesf/json-schema-compare" "strings" - structpb "github.com/golang/protobuf/ptypes/struct" - "github.com/santhosh-tekuri/jsonschema" - flyte "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" + structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/wI2L/jsondiff" ) type typeChecker interface { @@ -21,35 +17,59 @@ type trivialChecker struct { literalType *flyte.LiteralType } -func isSameTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { - compiler := jsonschema.NewCompiler() +func isSuperTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { + // Since there are lots of field differences between draft-07 and draft 2020-12, + // we only support json schema with 2020-12 draft, which is generated here: https://github.com/flyteorg/flytekit/blob/ff2d0da686c82266db4dbf764a009896cf062349/flytekit/core/type_engine.py#L630-L639 + _, upstreamIsDraft7 := sourceMetaData.Fields["$schema"] + _, downstreamIsDraft7 := targetMetaData.Fields["$schema"] - addSchemaToCompiler := func(name string, data *structpb.Struct) error { - schemaByte, err := json.Marshal(data.Fields) - if err != nil { - return fmt.Errorf("failed to marshal %s: %w", name, err) - } - return compiler.AddResource(name, bytes.NewReader(schemaByte)) + if upstreamIsDraft7 || downstreamIsDraft7 { + return false } - if err := addSchemaToCompiler("source", sourceMetaData); err != nil { - return false + copySourceSchema := make(map[string]interface{}) + copyTargetSchema := make(map[string]interface{}) + + for k, v := range sourceMetaData.Fields { + copySourceSchema[k] = v } - if err := addSchemaToCompiler("target", targetMetaData); err != nil { - return false + + for k, v := range targetMetaData.Fields { + copyTargetSchema[k] = v } - sourceSchema, err := compiler.Compile("source") - if err != nil { - return false + srcSchemaBytes, _ := json.Marshal(copySourceSchema) + tgtSchemaBytes, _ := json.Marshal(copyTargetSchema) + + patch, _ := jsondiff.CompareJSON(srcSchemaBytes, tgtSchemaBytes) + for _, p := range patch { + if p.Type != jsondiff.OperationAdd { + return false + } else if strings.Contains(p.Path, "required") { + return false + } } - targetSchema, err := compiler.Compile("target") - if err != nil { + return true +} + +func isSameTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { + // Since there are lots of field differences between draft-07 and draft 2020-12, + // we only support json schema with 2020-12 draft, which is generated here: https://github.com/flyteorg/flytekit/blob/ff2d0da686c82266db4dbf764a009896cf062349/flytekit/core/type_engine.py#L630-L639 + _, upstreamIsDraft7 := sourceMetaData.Fields["$schema"] + _, downstreamIsDraft7 := targetMetaData.Fields["$schema"] + + if upstreamIsDraft7 != downstreamIsDraft7 { return false } - cmprErrList := jscmp.Compare(sourceSchema, targetSchema) - return len(cmprErrList) == 1 && strings.Contains(cmprErrList[0].Error(), "FIXME additionalProperties not implemented") + sourceSchema := sourceMetaData.Fields + targetSchema := targetMetaData.Fields + + srcSchemaBytes, _ := json.Marshal(sourceSchema) + tgtSchemaBytes, _ := json.Marshal(targetSchema) + + patch, _ := jsondiff.CompareJSON(srcSchemaBytes, tgtSchemaBytes) + return len(patch) == 0 } // CastsFrom is a trivial type checker merely checks if types match exactly. @@ -79,7 +99,7 @@ func (t trivialChecker) CastsFrom(upstreamType *flyte.LiteralType) bool { return true } - return isSameTypeInJSON(upstreamMetaData, downstreamMetaData) + return isSameTypeInJSON(upstreamMetaData, downstreamMetaData) || isSuperTypeInJSON(upstreamMetaData, downstreamMetaData) } // Ignore metadata when comparing types. diff --git a/go.mod b/go.mod index afef96f916..3f82c1a968 100644 --- a/go.mod +++ b/go.mod @@ -164,7 +164,6 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/robfig/cron/v3 v3.0.0 // indirect github.com/samber/lo v1.47.0 // indirect - github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.10.0+incompatible // indirect github.com/shamaton/msgpack/v2 v2.2.2 // indirect @@ -181,7 +180,6 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/wI2L/jsondiff v0.6.0 // indirect - gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/go.sum b/go.sum index d244d3da7e..fe96eac22a 100644 --- a/go.sum +++ b/go.sum @@ -1198,7 +1198,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= -github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/santhosh-tekuri/jsonschema/v2 v2.1.0/go.mod h1:yzJzKUGV4RbWqWIBBP4wSOBqavX5saE02yirLS0OTyg= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -1360,8 +1359,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7 h1:BAkxmYRc1ZPl6Gap4HWqwPT8yLZMrgaAwx12Ft408sg= -gitlab.com/yvesf/json-schema-compare v0.0.0-20190604192943-a900c04201f7/go.mod h1:X40Z1OU8o1oiXWzBmkuYOaruzYGv60l0AxGiB0E9keI= go.elastic.co/apm v1.8.0/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= go.elastic.co/apm/module/apmhttp v1.8.0/go.mod h1:9LPFlEON51/lRbnWDfqAWErihIiAFDUMfMV27YjoWQ8= go.elastic.co/apm/module/apmot v1.8.0/go.mod h1:Q5Xzabte8G/fkvDjr1jlDuOSUt9hkVWNZEHh6ZNaTjI= From 85489dc859cd74a4489362474aaef696aa0c7d5b Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 1 Nov 2024 17:20:03 +0800 Subject: [PATCH 3/9] fix: go.mod missing entry for error Signed-off-by: mao3267 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3f82c1a968..5074ac4345 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( github.com/flyteorg/flyte/datacatalog v0.0.0-00010101000000-000000000000 - github.com/flyteorg/flyte/flyteadmin v0.0.0-00010101000000-000000000000 + github.com/flyteorg/flyte/flyteadmin v1.13.3 github.com/flyteorg/flyte/flytepropeller v0.0.0-00010101000000-000000000000 github.com/flyteorg/flyte/flytestdlib v0.0.0-00010101000000-000000000000 github.com/golang/glog v1.2.0 From cc685bb228b07acea5202cde3a8c88a541a66d1c Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 1 Nov 2024 17:56:47 +0800 Subject: [PATCH 4/9] fix: update go module and sum Signed-off-by: mao3267 --- flyteadmin/go.sum | 8 +++----- flytectl/go.mod | 6 +++++- flytectl/go.sum | 12 ++++++++++++ flytepropeller/go.mod | 1 - flytepropeller/go.sum | 2 -- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/flyteadmin/go.sum b/flyteadmin/go.sum index b54d149687..021117ae67 100644 --- a/flyteadmin/go.sum +++ b/flyteadmin/go.sum @@ -1271,8 +1271,7 @@ github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH github.com/tidwall/gjson v1.6.8/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/gjson v1.7.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= @@ -1281,8 +1280,8 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE= @@ -1304,8 +1303,7 @@ github.com/unrolled/secure v0.0.0-20181005190816-ff9db2ff917f/go.mod h1:mnPT77IA github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= -github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= diff --git a/flytectl/go.mod b/flytectl/go.mod index b657a02d4d..36cceeb16a 100644 --- a/flytectl/go.mod +++ b/flytectl/go.mod @@ -141,13 +141,17 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shamaton/msgpack/v2 v2.2.2 // indirect github.com/spf13/afero v1.9.2 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.11.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/wI2L/jsondiff v0.6.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/flytectl/go.sum b/flytectl/go.sum index f57ca65c0a..b314c7d268 100644 --- a/flytectl/go.sum +++ b/flytectl/go.sum @@ -457,6 +457,18 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/flytepropeller/go.mod b/flytepropeller/go.mod index 33c2370781..3e83107905 100644 --- a/flytepropeller/go.mod +++ b/flytepropeller/go.mod @@ -20,7 +20,6 @@ require ( github.com/imdario/mergo v0.3.13 github.com/magiconair/properties v1.8.6 github.com/mitchellh/mapstructure v1.5.0 - github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 github.com/shamaton/msgpack/v2 v2.2.2 diff --git a/flytepropeller/go.sum b/flytepropeller/go.sum index bc37abe635..3ab11fff27 100644 --- a/flytepropeller/go.sum +++ b/flytepropeller/go.sum @@ -332,8 +332,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM= -github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= From 3a629e1ce13069d08d0bec744dccaa14401fcb3b Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 1 Nov 2024 18:28:33 +0800 Subject: [PATCH 5/9] refactor: gci format Signed-off-by: mao3267 --- flytepropeller/pkg/compiler/validators/typing.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flytepropeller/pkg/compiler/validators/typing.go b/flytepropeller/pkg/compiler/validators/typing.go index 6ca244235e..4c0c3b4c55 100644 --- a/flytepropeller/pkg/compiler/validators/typing.go +++ b/flytepropeller/pkg/compiler/validators/typing.go @@ -4,9 +4,10 @@ import ( "encoding/json" "strings" - flyte "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" structpb "github.com/golang/protobuf/ptypes/struct" "github.com/wI2L/jsondiff" + + flyte "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) type typeChecker interface { From aa4d98e3a0ecfa8e5982cb4bed2cacfa7b6c9562 Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 1 Nov 2024 20:09:10 +0800 Subject: [PATCH 6/9] test: add dataset casting tests for same (one/two levels) and superset (one level) dataclass Signed-off-by: mao3267 --- .../pkg/compiler/validators/typing_test.go | 413 ++++++++++++++++++ 1 file changed, 413 insertions(+) diff --git a/flytepropeller/pkg/compiler/validators/typing_test.go b/flytepropeller/pkg/compiler/validators/typing_test.go index f2e407b986..e7e51b55a0 100644 --- a/flytepropeller/pkg/compiler/validators/typing_test.go +++ b/flytepropeller/pkg/compiler/validators/typing_test.go @@ -5,6 +5,7 @@ import ( structpb "github.com/golang/protobuf/ptypes/struct" "github.com/stretchr/testify/assert" + structpb2 "google.golang.org/protobuf/types/known/structpb" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) @@ -557,6 +558,418 @@ func TestMapCasting(t *testing.T) { assert.True(t, castable, "castable from Struct to struct") }) + t.Run("dataclass cast to same dataclass with draft 2020-12 (one level)", func(t *testing.T) { + /* + @dataclass + class A: + a: int + */ + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + ) + assert.True(t, castable, "dataclass castable with one level properties") + }) + + t.Run("dataclass cast to dataclass with draft 2020-12 (two level)", func(t *testing.T) { + /* + @dataclass + class A: + a: int + + @dataclass + class B: + b: A + */ + + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "b"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "B"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "b": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "required": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": { + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "b"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "B"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "b": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "required": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": { + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + ) + assert.True(t, castable, "dataclass castable with two level properties") + }) + + t.Run("dataclass cast to superset dataclass with draft 2020-12 (one level)", func(t *testing.T) { + /* + @dataclass + class A: + a: int + + @dataclass + class A: + a: int + b: Optional[str] + */ + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb2.Value{ + "required": &structpb.Value{ + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{{Kind: &structpb.Value_StringValue{StringValue: "a"}}}, + }, + }, + }, + "title": &structpb.Value{ + Kind: &structpb.Value_StringValue{StringValue: "A"}, + }, + "properties": &structpb.Value{ + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "integer"}, + }, + }, + }, + }, + }, + "b": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "default": { + Kind: &structpb.Value_NullValue{}, + }, + "anyOf": { + Kind: &structpb.Value_ListValue{ + ListValue: &structpb.ListValue{ + Values: []*structpb.Value{ + { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "string"}, + }, + }, + }, + }, + }, + { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": { + Kind: &structpb.Value_StringValue{StringValue: "null"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "type": { + Kind: &structpb.Value_StringValue{StringValue: "object"}, + }, + "AdditionalProperties": { + Kind: &structpb.Value_BoolValue{BoolValue: false}, + }, + }, + }, + }, + ) + assert.True(t, castable, "dataclass castable to superset with one level properties") + }) + t.Run("MismatchedMapNestLevels_Scalar", func(t *testing.T) { castable := AreTypesCastable( &core.LiteralType{ From 818afb7a28fb298ae700b7a20a244cf6be2036c6 Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 8 Nov 2024 16:21:35 +0800 Subject: [PATCH 7/9] fix: support Pydantic BaseModel comparison Signed-off-by: mao3267 --- .../pkg/compiler/validators/typing.go | 61 +++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/flytepropeller/pkg/compiler/validators/typing.go b/flytepropeller/pkg/compiler/validators/typing.go index 4c0c3b4c55..44502ce504 100644 --- a/flytepropeller/pkg/compiler/validators/typing.go +++ b/flytepropeller/pkg/compiler/validators/typing.go @@ -18,33 +18,58 @@ type trivialChecker struct { literalType *flyte.LiteralType } +func removeTitleFieldFromProperties(schema map[string]interface{}) { + properties, ok := schema["properties"].(*structpb.Value) + if !ok { + return + } + + for _, p := range properties.GetStructValue().Fields { + delete(p.GetStructValue().Fields, "title") + } +} + func isSuperTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { // Since there are lots of field differences between draft-07 and draft 2020-12, // we only support json schema with 2020-12 draft, which is generated here: https://github.com/flyteorg/flytekit/blob/ff2d0da686c82266db4dbf764a009896cf062349/flytekit/core/type_engine.py#L630-L639 _, upstreamIsDraft7 := sourceMetaData.Fields["$schema"] _, downstreamIsDraft7 := targetMetaData.Fields["$schema"] + // We only support super type check for draft 2020-12 if upstreamIsDraft7 || downstreamIsDraft7 { return false } - copySourceSchema := make(map[string]interface{}) - copyTargetSchema := make(map[string]interface{}) + copySrcSchema := make(map[string]interface{}) + copyTgtSchema := make(map[string]interface{}) for k, v := range sourceMetaData.Fields { - copySourceSchema[k] = v + copySrcSchema[k] = v } for k, v := range targetMetaData.Fields { - copyTargetSchema[k] = v + copyTgtSchema[k] = v } - srcSchemaBytes, _ := json.Marshal(copySourceSchema) - tgtSchemaBytes, _ := json.Marshal(copyTargetSchema) + // The JSON schema generated by Pydantic.BaseModel includes a title field in its properties, repeatedly recording the property name. + // Since this title field is absent in the JSON schema generated for dataclass, we need to remove the title field from the properties to ensure equivalence. + removeTitleFieldFromProperties(copySrcSchema) + removeTitleFieldFromProperties(copyTgtSchema) + + srcSchemaBytes, _ := json.Marshal(copySrcSchema) + tgtSchemaBytes, _ := json.Marshal(copyTgtSchema) patch, _ := jsondiff.CompareJSON(srcSchemaBytes, tgtSchemaBytes) for _, p := range patch { - if p.Type != jsondiff.OperationAdd { + // If additionalProperties is false, the field is not present in the schema from Pydantic.BaseModel. + // We handle this case by checking the relationships by ourselves. + if p.Type != jsondiff.OperationAdd && p.Path == "/additionalProperties" { + if p.Type == jsondiff.OperationRemove || p.Type == jsondiff.OperationReplace { + if p.OldValue != false { + return false + } + } + } else if p.Type != jsondiff.OperationAdd { return false } else if strings.Contains(p.Path, "required") { return false @@ -59,15 +84,29 @@ func isSameTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { _, upstreamIsDraft7 := sourceMetaData.Fields["$schema"] _, downstreamIsDraft7 := targetMetaData.Fields["$schema"] + // If the schema version is different, we can't compare them. if upstreamIsDraft7 != downstreamIsDraft7 { return false } - sourceSchema := sourceMetaData.Fields - targetSchema := targetMetaData.Fields + copySrcSchema := make(map[string]interface{}) + copyTgtSchema := make(map[string]interface{}) + + for k, v := range sourceMetaData.Fields { + copySrcSchema[k] = v + } + + for k, v := range targetMetaData.Fields { + copyTgtSchema[k] = v + } + + // The JSON schema generated by Pydantic.BaseModel includes a title field in its properties, repeatedly recording the property name. + // Since this title field is absent in the JSON schema generated for dataclass, we need to remove the title field from the properties to ensure equivalence. + removeTitleFieldFromProperties(copySrcSchema) + removeTitleFieldFromProperties(copyTgtSchema) - srcSchemaBytes, _ := json.Marshal(sourceSchema) - tgtSchemaBytes, _ := json.Marshal(targetSchema) + srcSchemaBytes, _ := json.Marshal(copySrcSchema) + tgtSchemaBytes, _ := json.Marshal(copyTgtSchema) patch, _ := jsondiff.CompareJSON(srcSchemaBytes, tgtSchemaBytes) return len(patch) == 0 From d6468b6dc6735dc47ed80da995bef30e29b7786d Mon Sep 17 00:00:00 2001 From: mao3267 Date: Fri, 8 Nov 2024 21:40:47 +0800 Subject: [PATCH 8/9] fix: handle nested pydantic basemodel Signed-off-by: mao3267 --- .../pkg/compiler/validators/typing.go | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/flytepropeller/pkg/compiler/validators/typing.go b/flytepropeller/pkg/compiler/validators/typing.go index 44502ce504..43fbb61760 100644 --- a/flytepropeller/pkg/compiler/validators/typing.go +++ b/flytepropeller/pkg/compiler/validators/typing.go @@ -18,17 +18,36 @@ type trivialChecker struct { literalType *flyte.LiteralType } -func removeTitleFieldFromProperties(schema map[string]interface{}) { - properties, ok := schema["properties"].(*structpb.Value) +func removeTitleFieldFromProperties(schema map[string]*structpb.Value) { + properties, ok := schema["properties"] if !ok { return } for _, p := range properties.GetStructValue().Fields { + if _, ok := p.GetStructValue().Fields["properties"]; ok { + removeTitleFieldFromProperties(p.GetStructValue().Fields) + } delete(p.GetStructValue().Fields, "title") } } +func resolveRef(schema, defs map[string]*structpb.Value) { + // Schema from Pydantic BaseModel includes a $def field, which is a reference to the actual schema. + // We need to resolve the reference to compare the schema with those from marshumaro. + // https://github.com/flyteorg/flytekit/blob/3475ddc41f2ba31d23dd072362be704d7c2470a0/flytekit/core/type_engine.py#L632-L641 + for _, p := range schema["properties"].GetStructValue().Fields { + if _, ok := p.GetStructValue().Fields["$ref"]; ok { + propName := strings.TrimPrefix(p.GetStructValue().Fields["$ref"].GetStringValue(), "#/$defs/") + p.GetStructValue().Fields = defs[propName].GetStructValue().Fields + resolveRef(p.GetStructValue().Fields, defs) + delete(p.GetStructValue().Fields, "$ref") + } + } + + delete(schema, "$defs") +} + func isSuperTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { // Since there are lots of field differences between draft-07 and draft 2020-12, // we only support json schema with 2020-12 draft, which is generated here: https://github.com/flyteorg/flytekit/blob/ff2d0da686c82266db4dbf764a009896cf062349/flytekit/core/type_engine.py#L630-L639 @@ -40,8 +59,8 @@ func isSuperTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { return false } - copySrcSchema := make(map[string]interface{}) - copyTgtSchema := make(map[string]interface{}) + copySrcSchema := make(map[string]*structpb.Value) + copyTgtSchema := make(map[string]*structpb.Value) for k, v := range sourceMetaData.Fields { copySrcSchema[k] = v @@ -51,6 +70,13 @@ func isSuperTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { copyTgtSchema[k] = v } + // For nested Pydantic BaseModel, we need to resolve the reference to compare the schema. + if _, ok := copySrcSchema["$defs"]; ok { + resolveRef(copySrcSchema, copySrcSchema["$defs"].GetStructValue().Fields) + } + if _, ok := copyTgtSchema["$defs"]; ok { + resolveRef(copyTgtSchema, copyTgtSchema["$defs"].GetStructValue().Fields) + } // The JSON schema generated by Pydantic.BaseModel includes a title field in its properties, repeatedly recording the property name. // Since this title field is absent in the JSON schema generated for dataclass, we need to remove the title field from the properties to ensure equivalence. removeTitleFieldFromProperties(copySrcSchema) @@ -63,7 +89,7 @@ func isSuperTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { for _, p := range patch { // If additionalProperties is false, the field is not present in the schema from Pydantic.BaseModel. // We handle this case by checking the relationships by ourselves. - if p.Type != jsondiff.OperationAdd && p.Path == "/additionalProperties" { + if p.Type != jsondiff.OperationAdd && strings.Contains(p.Path, "additionalProperties") { if p.Type == jsondiff.OperationRemove || p.Type == jsondiff.OperationReplace { if p.OldValue != false { return false @@ -89,8 +115,8 @@ func isSameTypeInJSON(sourceMetaData, targetMetaData *structpb.Struct) bool { return false } - copySrcSchema := make(map[string]interface{}) - copyTgtSchema := make(map[string]interface{}) + copySrcSchema := make(map[string]*structpb.Value) + copyTgtSchema := make(map[string]*structpb.Value) for k, v := range sourceMetaData.Fields { copySrcSchema[k] = v From ada05ed10febb83ff51fbd7166534f26fbf728e4 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Mon, 11 Nov 2024 22:28:26 +0800 Subject: [PATCH 9/9] Reviews from Eduardo Signed-off-by: Future-Outlier --- flytepropeller/pkg/compiler/validators/typing.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flytepropeller/pkg/compiler/validators/typing.go b/flytepropeller/pkg/compiler/validators/typing.go index 43fbb61760..1fc9c493ca 100644 --- a/flytepropeller/pkg/compiler/validators/typing.go +++ b/flytepropeller/pkg/compiler/validators/typing.go @@ -19,6 +19,8 @@ type trivialChecker struct { } func removeTitleFieldFromProperties(schema map[string]*structpb.Value) { + // TODO: Explain why we need this + // TODO: givse me example about dataclass vs. Pydantic BaseModel properties, ok := schema["properties"] if !ok { return