truncate in Convert function

Author: James Cor

enginetest/enginetests.go

@@ -3960,9 +3960,9 @@ func TestWindowRangeFrames(t *testing.T, harness Harness) {
 	TestQueryWithContext(t, ctx, e, harness, `SELECT sum(y) over (partition by z order by date range between unbounded preceding and interval '1' DAY following) FROM c order by x`, []sql.Row{{float64(1)}, {float64(1)}, {float64(1)}, {float64(1)}, {float64(5)}, {float64(5)}, {float64(10)}, {float64(10)}, {float64(10)}, {float64(10)}}, nil, nil, nil)
 	TestQueryWithContext(t, ctx, e, harness, `SELECT count(y) over (partition by z order by date range between interval '1' DAY following and interval '2' DAY following) FROM c order by x`, []sql.Row{{1}, {1}, {1}, {1}, {1}, {0}, {2}, {2}, {0}, {0}}, nil, nil, nil)
 	TestQueryWithContext(t, ctx, e, harness, `SELECT count(y) over (partition by z order by date range between interval '1' DAY preceding and interval '2' DAY following) FROM c order by x`, []sql.Row{{4}, {4}, {4}, {5}, {2}, {2}, {4}, {4}, {4}, {4}}, nil, nil, nil)
+	TestQueryWithContext(t, ctx, e, harness, "SELECT sum(y) over (partition by z order by date range interval 'e' DAY preceding) FROM c order by x", []sql.Row{{float64(0)}, {float64(0)}, {float64(0)}, {float64(1)}, {float64(1)}, {float64(3)}, {float64(1)}, {float64(1)}, {float64(4)}, {float64(4)}}, nil, nil, nil)
 
 	AssertErr(t, e, harness, "SELECT sum(y) over (partition by z range between unbounded preceding and interval '1' DAY following) FROM c order by x", nil, aggregation.ErrRangeInvalidOrderBy)
-	AssertErr(t, e, harness, "SELECT sum(y) over (partition by z order by date range interval 'e' DAY preceding) FROM c order by x", nil, sql.ErrInvalidValue)
 }
 
 func TestNamedWindows(t *testing.T, harness Harness) {

enginetest/queries/json_table_queries.go

@@ -571,7 +571,7 @@ var JSONTableScriptTests = []ScriptTest{
 			},
 			{
 				Query:          "SELECT * FROM JSON_TABLE('{\"c1\":\"abc\"}', '$' COLUMNS(c1 INT PATH '$.c1' DEFAULT 'def' ON ERROR)) as jt;",
-				ExpectedErrStr: "error: 'def' is not a valid value for 'int'",
+				ExpectedErrStr: "Invalid JSON text in argument 1 to function JSON_TABLE: \"Invalid value.\"",
 			},
 		},
 	},
@@ -612,7 +612,7 @@ var JSONTableScriptTests = []ScriptTest{
 			},
 			{
 				Query:          "SELECT * FROM JSON_TABLE('{\"c1\":\"abc\"}', '$' COLUMNS(c1 INT PATH '$.c1' ERROR ON ERROR)) as jt;",
-				ExpectedErrStr: "error: 'abc' is not a valid value for 'int'",
+				ExpectedErrStr: "Invalid JSON text in argument 1 to function JSON_TABLE: \"Invalid value.\"",
 			},
 		},
 	},

sql/core.go

@@ -317,7 +317,11 @@ func ConvertToBool(ctx *Context, v interface{}) (bool, error) {
 	case float64:
 		return b != 0, nil
 	case string:
-		bFloat, err := strconv.ParseFloat(TrimStringToNumberPrefix(ctx, b, false), 64)
+		truncStr, didTrunc := TruncateStringToInt(b)
+		if didTrunc {
+			ctx.Warn(mysql.ERTruncatedWrongValue, "%s", ErrTruncatedIncorrect.New("INTEGER", b))
+		}
+		bFloat, err := strconv.ParseFloat(truncStr, 64)
 		if err != nil {
 			return false, nil
 		}
@@ -382,6 +386,60 @@ func convertEmptyStringToZero(s string) string {
 	return s
 }
 
+// TruncateStringToInt trims any whitespace from s, then truncates the string to the left most characters that make
+// up a valid integer. Empty strings are converted "0". Additionally, returns a flag indicating if truncation occurred.
+func TruncateStringToInt(s string) (string, bool) {
+	s = strings.Trim(s, IntCutSet)
+	i, n := 0, len(s)
+	for ; i < n; i++ {
+		c := rune(s[i])
+		if unicode.IsDigit(c) {
+			continue
+		}
+		if i == 0 && (c == '-' || c == '+') {
+			continue
+		}
+		break
+	}
+	if i == 0 {
+		return "0", i != n
+	}
+	return s[:i], i != n
+}
+
+// TruncateStringToDouble trims any whitespace from s, then truncates the string to the left most characters that make
+// up a valid double. Empty strings are converted "0". Additionally, returns a flag indicating if truncation occurred.
+func TruncateStringToDouble(s string) (string, bool) {
+	var signIndex int
+	var seenDigit, seenDot, seenExp bool
+	s = strings.Trim(s, NumericCutSet)
+	i, n := 0, len(s)
+	for ; i < n; i++ {
+		char := rune(s[i])
+		if unicode.IsDigit(char) {
+			seenDigit = true
+			continue
+		}
+		if char == '.' && !seenDot {
+			seenDot = true
+			continue
+		}
+		if (char == 'e' || char == 'E') && !seenExp && seenDigit {
+			seenExp = true
+			signIndex = i + 1 // allow a sign following exponent
+			continue
+		}
+		if i == signIndex && (char == '-' || char == '+') {
+			continue
+		}
+		break
+	}
+	if i == 0 {
+		return "0", i != n
+	}
+	return s[:i], i != n
+}
+
 var ErrVectorInvalidBinaryLength = errors.NewKind("cannot convert BINARY(%d) to vector, byte length must be a multiple of 4 bytes")
 
 // DecodeVector decodes a byte slice that represents a vector. This is needed for distance functions.

sql/expression/convert.go

@@ -421,7 +421,7 @@ func convertValue(ctx *sql.Context, val interface{}, castTo string, originType s
 		}
 		return num, nil
 	case ConvertToYear:
-		value, err := convertHexBlobToDecimalForNumericContext(val, originType)
+		value, err := prepareForNumericContext(ctx, val, originType, true)
 		if err != nil {
 			return nil, err
 		}

sql/expression/interval.go

@@ -16,6 +16,7 @@ package expression
 
 import (
 	"fmt"
+	"github.com/dolthub/vitess/go/mysql"
 	"regexp"
 	"strconv"
 	"strings"
@@ -140,7 +141,10 @@ func (i *Interval) EvalDelta(ctx *sql.Context, row sql.Row) (*TimeDelta, error)
 	} else {
 		val, _, err = types.Int64.Convert(ctx, val)
 		if err != nil {
-			return nil, err
+			if !sql.ErrTruncatedIncorrect.Is(err) {
+				return nil, err
+			}
+			ctx.Warn(mysql.ERTruncatedWrongValue, "%s", err.Error())
 		}
 
 		num := val.(int64)

sql/expression/procedurereference.go

@@ -69,6 +69,9 @@ func (ppr *ProcedureReference) InitializeVariable(ctx *sql.Context, name string,
 	}
 	convertedVal, _, err := sqlType.Convert(ctx, val)
 	if err != nil {
+		if sql.ErrTruncatedIncorrect.Is(err) {
+			return sql.ErrInvalidValue.New(val, sqlType)
+		}
 		return err
 	}
 	lowerName := strings.ToLower(name)

sql/index_builder.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/shopspring/decimal"
 	"gopkg.in/src-d/go-errors.v1"
+
+	"github.com/dolthub/vitess/go/mysql"
 )
 
 var (
@@ -229,9 +231,14 @@ func (b *MySQLIndexBuilder) convertKey(ctx *Context, colType Type, keyType Type,
 	if et, ok := colType.(ExtendedType); ok {
 		return et.ConvertToType(ctx, keyType.(ExtendedType), key)
 	} else {
-		// TODO: would it make more sense for colType.Convert to handle the truncation or just do it here?
-		key, _, err := colType.Convert(ctx, key)
-		return key, err
+		k, _, err := colType.Convert(ctx, key)
+		if err != nil {
+			if !ErrTruncatedIncorrect.Is(err) {
+				return nil, err
+			}
+			ctx.Warn(mysql.ERTruncatedWrongValue, "%s", err.Error())
+		}
+		return k, nil
 	}
 }
 

sql/iters/rel_iters.go

@@ -290,11 +290,17 @@ func (c *JsonTableCol) Next(ctx *sql.Context, obj interface{}, pass bool, ord in
 	val, _, err = c.Opts.Typ.Convert(ctx, val)
 	if err != nil {
 		if c.Opts.ErrOnErr {
-			return nil, err
+			if sql.ErrTruncatedIncorrect.Is(err) {
+				return nil, sql.ErrInvalidJSONText.New(c.pos+1, "JSON_TABLE", "Invalid value.")
+			}
+			return nil, sql.ErrInvalidJSONText.New(c.pos+1, "JSON_TABLE", err.Error())
 		}
 		val, _, err = c.Opts.Typ.Convert(ctx, c.Opts.DefErrVal)
 		if err != nil {
-			return nil, err
+			if sql.ErrTruncatedIncorrect.Is(err) {
+				return nil, sql.ErrInvalidJSONText.New(c.pos+1, "JSON_TABLE", "Invalid value.")
+			}
+			return nil, sql.ErrInvalidJSONText.New(c.pos+1, "JSON_TABLE", err.Error())
 		}
 	}
 

sql/rowexec/insert.go

@@ -120,9 +120,14 @@ func (i *insertIter) Next(ctx *sql.Context) (returnRow sql.Row, returnErr error)
 			ctxWithValues := context.WithValue(ctx.Context, types.ColumnNameKey, col.Name)
 			ctxWithValues = context.WithValue(ctxWithValues, types.RowNumberKey, i.rowNumber)
 			ctxWithColumnInfo := ctx.WithContext(ctxWithValues)
-			converted, inRange, cErr := col.Type.Convert(ctxWithColumnInfo, row[idx])
+			val := row[idx]
+			// TODO: check mysql strict sql_mode
+			converted, inRange, cErr := col.Type.Convert(ctxWithColumnInfo, val)
 			if cErr == nil && !inRange {
-				cErr = sql.ErrValueOutOfRange.New(row[idx], col.Type)
+				cErr = sql.ErrValueOutOfRange.New(val, col.Type)
+			}
+			if sql.ErrTruncatedIncorrect.Is(cErr) {
+				cErr = sql.ErrInvalidValue.New(val, col.Type)
 			}
 			if cErr != nil {
 				// Ignore individual column errors when INSERT IGNORE, UPDATE IGNORE, etc. is specified.

sql/types/number.go

@@ -961,21 +961,24 @@ func convertToInt64(t NumberTypeImpl_, v interface{}) (int64, sql.ConvertInRange
 	case float32:
 		if v > float32(math.MaxInt64) {
 			return math.MaxInt64, sql.OutOfRange, nil
-		} else if v < float32(math.MinInt64) {
+		}
+		if v < float32(math.MinInt64) {
 			return math.MinInt64, sql.OutOfRange, nil
 		}
 		return int64(math.Round(float64(v))), sql.InRange, nil
 	case float64:
 		if v > float64(math.MaxInt64) {
 			return math.MaxInt64, sql.OutOfRange, nil
-		} else if v < float64(math.MinInt64) {
+		}
+		if v < float64(math.MinInt64) {
 			return math.MinInt64, sql.OutOfRange, nil
 		}
 		return int64(math.Round(v)), sql.InRange, nil
 	case decimal.Decimal:
 		if v.GreaterThan(dec_int64_max) {
 			return dec_int64_max.IntPart(), sql.OutOfRange, nil
-		} else if v.LessThan(dec_int64_min) {
+		}
+		if v.LessThan(dec_int64_min) {
 			return dec_int64_min.IntPart(), sql.OutOfRange, nil
 		}
 		return v.Round(0).IntPart(), sql.InRange, nil
@@ -986,23 +989,25 @@ func convertToInt64(t NumberTypeImpl_, v interface{}) (int64, sql.ConvertInRange
 		}
 		return i, sql.InRange, nil
 	case string:
-		v = strings.Trim(v, sql.IntCutSet)
-		if v == "" {
-			// StringType{}.Zero() returns empty string, but should represent "0" for number value
-			return 0, sql.InRange, nil
+		// TODO: this currently assumes we are always rounding to preserve behavior
+		//   but we should only be rounding on inserts
+		var err error
+		truncStr, didTrunc := sql.TruncateStringToDouble(v)
+		if didTrunc {
+			err = sql.ErrTruncatedIncorrect.New(t, v)
 		}
 		// Parse first an integer, which allows for more values than float64
-		i, err := strconv.ParseInt(v, 10, 64)
-		if err == nil {
-			return i, sql.InRange, nil
+		i, pErr := strconv.ParseInt(truncStr, 10, 64)
+		if pErr == nil {
+			return i, sql.InRange, err
 		}
-		// If that fails, try as a float and truncate it to integral
-		f, err := strconv.ParseFloat(v, 64)
-		if err != nil {
-			return 0, sql.OutOfRange, sql.ErrInvalidValue.New(v, t.String())
+		// If that fails, try as a float and round it to integral
+		f, pErr := strconv.ParseFloat(truncStr, 64)
+		if pErr == nil {
+			f = math.Round(f)
+			return int64(f), sql.InRange, err
 		}
-		f = math.Round(f)
-		return int64(f), sql.InRange, nil
+		return 0, sql.OutOfRange, sql.ErrInvalidValue.New(v, t.String())
 	case bool:
 		if v {
 			return 1, sql.InRange, nil