-
Notifications
You must be signed in to change notification settings - Fork 10
feat: support decimal #58
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 8 commits
dc15bfb
c186a73
6c3459f
f486bab
100bba6
5a3e29b
b2e697e
a1c3981
d2b6670
a7f4c07
b6e5de3
f3222af
8405773
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -188,6 +188,37 @@ will now also serialize ``float64`` (double-precision) columns as binary. | |
| You might see a performance uplift if this is a dominant data type in your | ||
| ingestion workload. | ||
|
|
||
| ## Decimal columns | ||
|
|
||
| QuestDB server version 9.2.0 and newer supports decimal columns with arbitrary precision and scale. | ||
| The Go client converts supported decimal values to QuestDB's text/binary wire format automatically, pass any of the following to `DecimalColumn`: | ||
|
|
||
| - `questdb.ScaledDecimal`, including helpers like `questdb.NewDecimalFromInt64` and `questdb.NewDecimal`. | ||
| - Types implementing `questdb.DecimalMarshaler`. | ||
| - `github.com/shopspring/decimal.Decimal` values or pointers. | ||
| - `nil` or `questdb.NullDecimal()` to send a `NULL`. | ||
|
||
|
|
||
| ```go | ||
| price := qdb.NewDecimalFromInt64(12345, 2) // 123.45 with scale 2 | ||
| commission := qdb.NewDecimal(big.NewInt(-750), 4) // -0.0750 with scale 4 | ||
|
|
||
| err = sender. | ||
| Table("trades"). | ||
| Symbol("symbol", "ETH-USD"). | ||
| DecimalColumn("price", price). | ||
| DecimalColumn("commission", commission). | ||
| AtNow(ctx) | ||
| ``` | ||
|
|
||
| To emit textual decimals, pass a validated string literal (without the trailing `d`—the client adds it): | ||
|
|
||
| ```go | ||
| err = sender. | ||
| Table("quotes"). | ||
| DecimalColumn("mid", "1.23456"). | ||
| AtNow(ctx) | ||
| ``` | ||
|
|
||
| ## Pooled Line Senders | ||
|
|
||
| **Warning: Experimental feature designed for use with HTTP senders ONLY** | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -573,6 +573,52 @@ func (b *buffer) Float64Column(name string, val float64) *buffer { | |
| return b | ||
| } | ||
|
|
||
| func (b *buffer) DecimalColumn(name string, val any) *buffer { | ||
|
||
| if !b.prepareForField() { | ||
| return b | ||
| } | ||
| b.lastErr = b.writeColumnName(name) | ||
| if b.lastErr != nil { | ||
| return b | ||
| } | ||
| if str, ok := val.(string); ok { | ||
| if err := validateDecimalText(str); err != nil { | ||
| b.lastErr = err | ||
| return b | ||
| } | ||
| b.WriteByte('=') | ||
| b.WriteString(str) | ||
| b.WriteByte('d') | ||
| b.hasFields = true | ||
| return b | ||
| } | ||
|
|
||
| dec, err := normalizeDecimalValue(val) | ||
| if err != nil { | ||
| b.lastErr = err | ||
| return b | ||
| } | ||
| scale, payload, err := dec.toBinary() | ||
| if err != nil { | ||
| b.lastErr = err | ||
| return b | ||
| } | ||
| if len(payload) == 0 { | ||
| // Don't write null decimals | ||
| return b | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. At this point, we already wrote column name to the buffer. ILP requires the value to be included. Can't we interpret zero length as null on the server side? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, we don't even need to send the value. |
||
| } | ||
| b.WriteByte('=') | ||
| b.WriteByte('=') | ||
| b.WriteByte(decimalBinaryTypeCode) | ||
| b.WriteByte(scale) | ||
| b.WriteByte(byte(len(payload))) | ||
| if len(payload) > 0 { | ||
| b.Write(payload) | ||
| } | ||
| b.hasFields = true | ||
| return b | ||
| } | ||
|
|
||
| func (b *buffer) Float64ColumnBinary(name string, val float64) *buffer { | ||
| if !b.prepareForField() { | ||
| return b | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,6 +39,19 @@ import ( | |
|
|
||
| type bufWriterFn func(b *qdb.Buffer) error | ||
|
|
||
| type fakeShopspringDecimal struct { | ||
| coeff *big.Int | ||
| exp int32 | ||
| } | ||
|
|
||
| func (f fakeShopspringDecimal) Coefficient() *big.Int { | ||
| return f.coeff | ||
| } | ||
|
|
||
| func (f fakeShopspringDecimal) Exponent() int32 { | ||
| return f.exp | ||
| } | ||
|
|
||
| func newTestBuffer() qdb.Buffer { | ||
| return qdb.NewBuffer(128*1024, 1024*1024, 127) | ||
| } | ||
|
|
@@ -481,6 +494,139 @@ func TestFloat64ColumnBinary(t *testing.T) { | |
| } | ||
| } | ||
|
|
||
| func TestDecimalColumnText(t *testing.T) { | ||
|
||
| prefix := []byte(testTable + " price==") | ||
| testCases := []struct { | ||
| name string | ||
| value any | ||
| expected []byte | ||
| }{ | ||
| { | ||
| name: "positive", | ||
| value: qdb.NewDecimalFromInt64(12345, 2), | ||
| expected: append(prefix, 0x17, 0x02, 0x02, 0x30, 0x39, 0x0A), | ||
| }, | ||
| { | ||
| name: "negative", | ||
| value: qdb.NewDecimal(big.NewInt(-12345), 3), | ||
| expected: append(prefix, 0x17, 0x03, 0x02, 0xCF, 0xC7, 0x0A), | ||
| }, | ||
| { | ||
| name: "zero with scale", | ||
| value: qdb.NewDecimalFromInt64(0, 4), | ||
| expected: append(prefix, 0x17, 0x04, 0x01, 0x0, 0x0A), | ||
| }, | ||
| { | ||
| name: "null decimal", | ||
| value: qdb.NullDecimal(), | ||
| expected: append(prefix, 0x17, 0x0, 0x0, 0x0A), | ||
| }, | ||
| { | ||
| name: "shopspring compatible", | ||
| value: fakeShopspringDecimal{coeff: big.NewInt(123456), exp: -4}, | ||
| expected: append(prefix, 0x17, 0x04, 0x03, 0x01, 0xE2, 0x40, 0x0A), | ||
| }, | ||
| { | ||
| name: "nil pointer treated as null", | ||
| value: (*fakeShopspringDecimal)(nil), | ||
| expected: append(prefix, 0x17, 0x0, 0x0, 0x0A), | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| buf := newTestBuffer() | ||
| err := buf.Table(testTable).DecimalColumn("price", tc.value).At(time.Time{}, false) | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, tc.expected, buf.Messages()) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestDecimalColumnStringValidation(t *testing.T) { | ||
| t.Run("valid strings", func(t *testing.T) { | ||
| testCases := []struct { | ||
| name string | ||
| value string | ||
| expected string | ||
| }{ | ||
| {"integer", "123", "123d"}, | ||
| {"decimal", "123.450", "123.450d"}, | ||
| {"negative", "-0.001", "-0.001d"}, | ||
| {"exponent positive", "1.2e3", "1.2e3d"}, | ||
| {"exponent negative", "-4.5E-2", "-4.5E-2d"}, | ||
| {"nan token", "NaN", "NaNd"}, | ||
| {"infinity token", "Infinity", "Infinityd"}, | ||
| } | ||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| buf := newTestBuffer() | ||
| err := buf.Table(testTable).DecimalColumn("price", tc.value).At(time.Time{}, false) | ||
| assert.NoError(t, err) | ||
| expected := []byte(testTable + " price=" + tc.expected + "\n") | ||
| assert.Equal(t, expected, buf.Messages()) | ||
| }) | ||
| } | ||
| }) | ||
|
|
||
| t.Run("invalid strings", func(t *testing.T) { | ||
| testCases := []struct { | ||
| name string | ||
| value string | ||
| }{ | ||
| {"empty", ""}, | ||
| {"sign only", "+"}, | ||
| {"double dot", "12.3.4"}, | ||
| {"invalid char", "12a3"}, | ||
| {"exponent missing mantissa", "e10"}, | ||
| {"exponent no digits", "1.2e"}, | ||
| {"exponent sign no digits", "1.2e+"}, | ||
| } | ||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| buf := newTestBuffer() | ||
| err := buf.Table(testTable).DecimalColumn("price", tc.value).At(time.Time{}, false) | ||
| assert.Error(t, err) | ||
| assert.Contains(t, err.Error(), "decimal") | ||
| assert.Empty(t, buf.Messages()) | ||
| }) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| func TestDecimalColumnErrors(t *testing.T) { | ||
| t.Run("invalid scale", func(t *testing.T) { | ||
| buf := newTestBuffer() | ||
| dec := qdb.NewDecimalFromInt64(1, 100) | ||
| err := buf.Table(testTable).DecimalColumn("price", dec).At(time.Time{}, false) | ||
| assert.ErrorContains(t, err, "decimal scale") | ||
| assert.Empty(t, buf.Messages()) | ||
| }) | ||
|
|
||
| t.Run("overflow", func(t *testing.T) { | ||
| buf := newTestBuffer() | ||
| bigVal := new(big.Int).Lsh(big.NewInt(1), 2100) | ||
| dec := qdb.NewDecimal(bigVal, 0) | ||
| err := buf.Table(testTable).DecimalColumn("price", dec).At(time.Time{}, false) | ||
| assert.ErrorContains(t, err, "exceeds 127-bytes limit") | ||
| assert.Empty(t, buf.Messages()) | ||
| }) | ||
|
|
||
| t.Run("unsupported type", func(t *testing.T) { | ||
| buf := newTestBuffer() | ||
| err := buf.Table(testTable).DecimalColumn("price", struct{}{}).At(time.Time{}, false) | ||
| assert.ErrorContains(t, err, "unsupported decimal column value type") | ||
| assert.Empty(t, buf.Messages()) | ||
| }) | ||
|
|
||
| t.Run("no column", func(t *testing.T) { | ||
| buf := newTestBuffer() | ||
| err := buf.Table(testTable).DecimalColumn("price", qdb.NullDecimal()).At(time.Time{}, false) | ||
| assert.ErrorContains(t, err, "no symbols or columns were provided: invalid message") | ||
| assert.Empty(t, buf.Messages()) | ||
| }) | ||
| } | ||
|
|
||
| func TestFloat64Array1DColumn(t *testing.T) { | ||
| testCases := []struct { | ||
| name string | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why do we want to support this 3rd-party library? I've checked it and the last release was 1.5 years ago while there are 100+ open GH issues, some of which look like straightforward bugs.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It seems to be the only popular decimal integration in the golang ecosystem.
Having initial support for it makes it easier for customers to connect to QDB.