diff --git a/conn_batch.go b/conn_batch.go index 10283ad33d..c30b81a2a6 100644 --- a/conn_batch.go +++ b/conn_batch.go @@ -226,18 +226,18 @@ func (b *batchColumn) Append(v interface{}) (err error) { } func (b *batchColumn) AppendRow(v interface{}) (err error) { - if b.batch.IsSent() { - return ErrBatchAlreadySent - } - if b.err != nil { - b.release(b.err) - return b.err - } - if err = b.column.AppendRow(v); err != nil { - b.release(err) - return err - } - return nil + if b.batch.IsSent() { + return ErrBatchAlreadySent + } + if b.err != nil { + b.release(b.err) + return b.err + } + if err = b.column.AppendRow(v); err != nil { + b.release(err) + return err + } + return nil } var ( diff --git a/contributors/contributors.go b/contributors/contributors.go index 7fcbae29fc..35c4047ae4 100644 --- a/contributors/contributors.go +++ b/contributors/contributors.go @@ -22,6 +22,7 @@ import ( "strings" ) +//go:generate bash -c "git log \"--pretty=%an <%ae>\" | sort -u > list" //go:embed list var source string diff --git a/internal/cmd/release_prep/main.go b/internal/cmd/release_prep/main.go new file mode 100644 index 0000000000..086b295235 --- /dev/null +++ b/internal/cmd/release_prep/main.go @@ -0,0 +1,271 @@ +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" +) + +var skipWorkingTreeIsDirtyCheck = flag.Bool("skip-working-tree-is-dirty-check", false, "Skip working tree is dirty check") + +func main() { + flag.Parse() + + if !(*skipWorkingTreeIsDirtyCheck) && gitRepositoryWorkingTreeIsDirty() { + log.Fatalln("Git working tree is dirty") + return + } + + releaseURL := getLatestDraftReleaseURL() + + log.Println("Latest draft release URL:") + log.Println(releaseURL) + + r := getRelease(releaseURL) + + log.Println("Release tag:") + log.Println(r.TagName) + log.Println("Release body:") + log.Println(r.Body) + + major, minor, patch, err := parseSemVer(r.TagName) + if err != nil { + log.Fatalln(err) + return + } + + if len(r.Body) == 0 { + log.Fatalln("Release body is empty") + return + } + + changelogPath := changelogFilePath() + prependReleaseToChangelog(changelogPath, r) + + if err := updateClientInfo(major, minor, patch); err != nil { + log.Fatalln(err) + return + } + + runGoGenerate() + runGoFmt() +} + +func runGoFmt() { + cmd := exec.Command("go", "fmt", "./...") + cmd.Dir = getRootPath() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalln(err) + return + } +} + +func runGoGenerate() { + cmd := exec.Command("go", "generate", "./...") + cmd.Dir = getRootPath() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalln(err) + return + } +} + +func gitRepositoryWorkingTreeIsDirty() bool { + cmd := exec.Command("git", "status", "--porcelain") + out, err := cmd.Output() + if err != nil { + log.Fatalln(err) + return false + } + return len(out) > 0 +} + +func parseSemVer(version string) (major, minor, patch int, err error) { + // Define a regular expression to match SemVer format + re := regexp.MustCompile(`^v?(\d+)\.(\d+)\.(\d+)$`) + + // Apply the regular expression to the version string + match := re.FindStringSubmatch(version) + + // Check if the version string matches the SemVer format + if len(match) != 4 { + err = fmt.Errorf("invalid SemVer format: %s", version) + return + } + + // Parse the major, minor, and patch components as integers + major, err = strconv.Atoi(match[1]) + if err != nil { + return + } + + minor, err = strconv.Atoi(match[2]) + if err != nil { + return + } + + patch, err = strconv.Atoi(match[3]) + if err != nil { + return + } + + return +} + +func prependReleaseToChangelog(changelogPath string, r release) { + f, err := os.OpenFile(changelogPath, os.O_RDWR, 0666) + if err != nil { + log.Fatalln(err) + } + defer f.Close() + + content, err := io.ReadAll(f) + if err != nil { + log.Fatalln(err) + return + } + + f.Seek(0, io.SeekStart) + f.WriteString(fmt.Sprintf("# %s, %s ", r.TagName, time.Now().Format("2006-01-02"))) + f.WriteString(r.Body) + f.WriteString("\n\n") + f.Write(content) +} + +func changelogFilePath() string { + rootPath := getRootPath() + changelogPath := rootPath + "/CHANGELOG.md" + return changelogPath +} + +func getRootPath() string { + wd, _ := os.Getwd() + rootPath := strings.Replace(wd, "internal/cmd/release_prep", "", 1) + return rootPath +} + +func getRelease(releaseURL string) release { + req, err := http.NewRequest(http.MethodGet, releaseURL, nil) + if err != nil { + log.Fatalln(err) + } + req.Header.Set("Authorization", "token "+os.Getenv("GITHUB_TOKEN")) + res, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatalln(err) + } + defer res.Body.Close() + + var release release + if err := json.NewDecoder(res.Body).Decode(&release); err != nil { + log.Fatalln(err) + } + + return release +} + +type release struct { + URL string `json:"url"` + Body string `json:"body"` + TagName string `json:"tag_name"` +} + +func getLatestDraftReleaseURL() string { + // Fetch the latest release from GitHub repository using GitHub API + req, err := http.NewRequest(http.MethodGet, "https://api.github.com/repos/clickhouse/clickhouse-go/releases?per_page=100", nil) + if err != nil { + log.Fatalln(err) + } + + req.Header.Set("Authorization", "token "+os.Getenv("GITHUB_TOKEN")) + res, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatalln(err) + } + defer res.Body.Close() + + var releases []struct { + URL string `json:"url"` + Draft bool `json:"draft"` + } + if err := json.NewDecoder(res.Body).Decode(&releases); err != nil { + log.Fatalln(err) + } + + // filter out releases that are not drafts + for i := 0; i < len(releases); { + if releases[i].Draft { + return releases[i].URL + } + } + + log.Fatalln("No draft releases found") + return "" +} + +func updateClientInfo(major, minor, patch int) error { + // Open the client_info.go file for reading and writing + file, err := os.OpenFile(getRootPath()+"/client_info.go", os.O_RDWR, 0644) + if err != nil { + return err + } + defer file.Close() + + // Read the contents of the file into memory + bytes, err := io.ReadAll(file) + if err != nil { + return err + } + + // Replace the ClientVersionMajor, ClientVersionMinor, and ClientVersionPatch lines + reMajor := regexp.MustCompile(`ClientVersionMajor\s+=\s+\d+`) + reMinor := regexp.MustCompile(`ClientVersionMinor\s+=\s+\d+`) + rePatch := regexp.MustCompile(`ClientVersionPatch\s+=\s+\d+`) + newLines := []string{ + fmt.Sprintf("ClientVersionMajor = %d", major), + fmt.Sprintf("ClientVersionMinor = %d", minor), + fmt.Sprintf("ClientVersionPatch = %d", patch), + } + scanner := bufio.NewScanner(strings.NewReader(string(bytes))) + var newContent string + for scanner.Scan() { + line := scanner.Text() + if reMajor.MatchString(line) { + line = newLines[0] + } else if reMinor.MatchString(line) { + line = newLines[1] + } else if rePatch.MatchString(line) { + line = newLines[2] + } + newContent += line + "\n" + } + if err := scanner.Err(); err != nil { + return err + } + + // Write the updated content back to the file + if _, err := file.Seek(0, 0); err != nil { + return err + } + if err := file.Truncate(0); err != nil { + return err + } + if _, err := file.Write([]byte(newContent)); err != nil { + return err + } + + return nil +} diff --git a/lib/cityhash102/doc.go b/lib/cityhash102/doc.go index e60469097b..859bf7d6a8 100644 --- a/lib/cityhash102/doc.go +++ b/lib/cityhash102/doc.go @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -/** COPY from https://github.com/zentures/cityhash/ +/* +* COPY from https://github.com/zentures/cityhash/ NOTE: The code is modified to be compatible with CityHash128 used in ClickHouse */ diff --git a/lib/driver/driver.go b/lib/driver/driver.go index d76c2fda1b..5300fc1b6d 100644 --- a/lib/driver/driver.go +++ b/lib/driver/driver.go @@ -87,7 +87,7 @@ type ( } BatchColumn interface { Append(interface{}) error - AppendRow(interface{}) error + AppendRow(interface{}) error } ColumnType interface { Name() string diff --git a/tests/columnar_batch_test.go b/tests/columnar_batch_test.go index 22badfb85e..32016d33d5 100644 --- a/tests/columnar_batch_test.go +++ b/tests/columnar_batch_test.go @@ -252,12 +252,12 @@ func TestNullableColumnarInterface(t *testing.T) { } func TestColumnarAppendRowInterface(t *testing.T) { - conn, err := GetNativeConnection(nil, nil, &clickhouse.Compression{ - Method: clickhouse.CompressionLZ4, - }) - ctx := context.Background() - require.NoError(t, err) - const ddl = ` + conn, err := GetNativeConnection(nil, nil, &clickhouse.Compression{ + Method: clickhouse.CompressionLZ4, + }) + ctx := context.Background() + require.NoError(t, err) + const ddl = ` CREATE TABLE test_column_interface ( Col1 UInt8 , Col2 String @@ -267,63 +267,63 @@ func TestColumnarAppendRowInterface(t *testing.T) { , Col6 Int64 ) Engine MergeTree() ORDER BY tuple() ` - defer func() { - conn.Exec(ctx, "DROP TABLE IF EXISTS test_column_interface") - }() - require.NoError(t, conn.Exec(ctx, ddl)) - batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_column_interface") - require.NoError(t, err) - var currentTime = time.Now().Truncate(time.Second) + defer func() { + conn.Exec(ctx, "DROP TABLE IF EXISTS test_column_interface") + }() + require.NoError(t, conn.Exec(ctx, ddl)) + batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_column_interface") + require.NoError(t, err) + var currentTime = time.Now().Truncate(time.Second) - for i := 0; i < 150; i++ { - require.NoError(t, batch.Column(0).AppendRow(uint8(i))) - require.NoError(t, batch.Column(1).AppendRow(fmt.Sprintf("value_%d", i))) - require.NoError(t, batch.Column(2).AppendRow(currentTime)) - require.NoError(t, batch.Column(3).AppendRow(sql.NullString{String: fmt.Sprintf("value_%d", i), Valid: true})) - require.NoError(t, batch.Column(4).AppendRow(sql.NullTime{Time: currentTime, Valid: true})) - require.NoError(t, batch.Column(5).AppendRow( sql.NullInt64{Int64: int64(i), Valid: true})) - } - require.NoError(t, batch.Send()) - var count uint64 - require.NoError(t, conn.QueryRow(ctx, "SELECT COUNT() FROM test_column_interface").Scan(&count)) - require.Equal(t, uint64(150), count) - rows, err := conn.Query(ctx, "SELECT * FROM test_column_interface WHERE Col1 >= $1 AND Col1 < $2", 10, 30) - require.NoError(t, err) - var ( - row uint8 = 10 - ) - iCount := 0 - for rows.Next() { - var ( - col1 uint8 - col2 string - col3 time.Time - col4 sql.NullString - col5 sql.NullTime - col6 sql.NullInt64 - ) - require.NoError(t, rows.Scan(&col1, &col2, &col3, &col4, &col5, &col6)) - assert.Equal(t, row, col1) - assert.Equal(t, fmt.Sprintf("value_%d", row), col2) - assert.Equal(t, currentTime.Unix(), col3.Unix()) - assert.Equal(t, fmt.Sprintf("value_%d", row), col4.String) - assert.Equal(t, currentTime.In(time.UTC), col5.Time) - assert.Equal(t, int64(row), col6.Int64) - row++ - iCount++ - } - rows.Close() - require.NoError(t, rows.Err()) - assert.Equal(t, 20, iCount) + for i := 0; i < 150; i++ { + require.NoError(t, batch.Column(0).AppendRow(uint8(i))) + require.NoError(t, batch.Column(1).AppendRow(fmt.Sprintf("value_%d", i))) + require.NoError(t, batch.Column(2).AppendRow(currentTime)) + require.NoError(t, batch.Column(3).AppendRow(sql.NullString{String: fmt.Sprintf("value_%d", i), Valid: true})) + require.NoError(t, batch.Column(4).AppendRow(sql.NullTime{Time: currentTime, Valid: true})) + require.NoError(t, batch.Column(5).AppendRow(sql.NullInt64{Int64: int64(i), Valid: true})) + } + require.NoError(t, batch.Send()) + var count uint64 + require.NoError(t, conn.QueryRow(ctx, "SELECT COUNT() FROM test_column_interface").Scan(&count)) + require.Equal(t, uint64(150), count) + rows, err := conn.Query(ctx, "SELECT * FROM test_column_interface WHERE Col1 >= $1 AND Col1 < $2", 10, 30) + require.NoError(t, err) + var ( + row uint8 = 10 + ) + iCount := 0 + for rows.Next() { + var ( + col1 uint8 + col2 string + col3 time.Time + col4 sql.NullString + col5 sql.NullTime + col6 sql.NullInt64 + ) + require.NoError(t, rows.Scan(&col1, &col2, &col3, &col4, &col5, &col6)) + assert.Equal(t, row, col1) + assert.Equal(t, fmt.Sprintf("value_%d", row), col2) + assert.Equal(t, currentTime.Unix(), col3.Unix()) + assert.Equal(t, fmt.Sprintf("value_%d", row), col4.String) + assert.Equal(t, currentTime.In(time.UTC), col5.Time) + assert.Equal(t, int64(row), col6.Int64) + row++ + iCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + assert.Equal(t, 20, iCount) } func TestNullableAppendRowColumnarInterface(t *testing.T) { - conn, err := GetNativeConnection(nil, nil, &clickhouse.Compression{ - Method: clickhouse.CompressionLZ4, - }) - ctx := context.Background() - require.NoError(t, err) - const ddl = ` + conn, err := GetNativeConnection(nil, nil, &clickhouse.Compression{ + Method: clickhouse.CompressionLZ4, + }) + ctx := context.Background() + require.NoError(t, err) + const ddl = ` CREATE TABLE test_column_interface ( Col1 Nullable(UInt8) , Col2 Nullable(String) @@ -331,120 +331,114 @@ func TestNullableAppendRowColumnarInterface(t *testing.T) { , Col4 Nullable(Decimal(10, 2)) ) Engine MergeTree() ORDER BY tuple() ` - defer func() { - conn.Exec(ctx, "DROP TABLE IF EXISTS test_column_interface") - }() - require.NoError(t, conn.Exec(ctx, ddl)) - batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_column_interface") - require.NoError(t, err) + defer func() { + conn.Exec(ctx, "DROP TABLE IF EXISTS test_column_interface") + }() + require.NoError(t, conn.Exec(ctx, ddl)) + batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_column_interface") + require.NoError(t, err) - var ( - currentTime = time.Now().Truncate(time.Second) - decimalVal = decimal.NewFromFloat(12.02) - ) + var ( + currentTime = time.Now().Truncate(time.Second) + decimalVal = decimal.NewFromFloat(12.02) + ) - for i := 0; i < 150; i++ { - a, b := uint8(i), fmt.Sprintf("value_%d", i) - { - require.NoError(t, batch.Column(0).AppendRow(&a)) - require.NoError(t, batch.Column(1).AppendRow(&b)) - require.NoError(t, batch.Column(2).AppendRow(¤tTime)) - require.NoError(t, batch.Column(3).AppendRow(&decimalVal)) - } - } - require.NoError(t, batch.Send()) - var count uint64 - require.NoError(t, conn.QueryRow(ctx, "SELECT COUNT() FROM test_column_interface").Scan(&count)) - require.Equal(t, uint64(150), count) - rows, err := conn.Query(ctx, "SELECT * FROM test_column_interface WHERE Col1 >= $1 AND Col1 < $2", 10, 30) - require.NoError(t, err) - var ( - row uint8 = 10 - ) - count = 0 - for rows.Next() { - var ( - col1 *uint8 - col2 *string - col3 *time.Time - col4 *decimal.Decimal - ) - if assert.NoError(t, rows.Scan(&col1, &col2, &col3, &col4)) { - assert.Equal(t, row, *col1) - assert.Equal(t, fmt.Sprintf("value_%d", row), *col2) - assert.Equal(t, currentTime.Unix(), col3.Unix()) - assert.Equal(t, decimalVal.String(), (*col4).String()) - } - row++ - count++ - } - rows.Close() - require.NoError(t, rows.Err()) - assert.Equal(t, uint64(20), count) - require.NoError(t, conn.Exec(ctx, "TRUNCATE TABLE test_column_interface")) - batch, err = conn.PrepareBatch(ctx, "INSERT INTO test_column_interface") - require.NoError(t, err) - { + for i := 0; i < 150; i++ { + a, b := uint8(i), fmt.Sprintf("value_%d", i) + { + require.NoError(t, batch.Column(0).AppendRow(&a)) + require.NoError(t, batch.Column(1).AppendRow(&b)) + require.NoError(t, batch.Column(2).AppendRow(¤tTime)) + require.NoError(t, batch.Column(3).AppendRow(&decimalVal)) + } + } + require.NoError(t, batch.Send()) + var count uint64 + require.NoError(t, conn.QueryRow(ctx, "SELECT COUNT() FROM test_column_interface").Scan(&count)) + require.Equal(t, uint64(150), count) + rows, err := conn.Query(ctx, "SELECT * FROM test_column_interface WHERE Col1 >= $1 AND Col1 < $2", 10, 30) + require.NoError(t, err) + var ( + row uint8 = 10 + ) + count = 0 + for rows.Next() { + var ( + col1 *uint8 + col2 *string + col3 *time.Time + col4 *decimal.Decimal + ) + if assert.NoError(t, rows.Scan(&col1, &col2, &col3, &col4)) { + assert.Equal(t, row, *col1) + assert.Equal(t, fmt.Sprintf("value_%d", row), *col2) + assert.Equal(t, currentTime.Unix(), col3.Unix()) + assert.Equal(t, decimalVal.String(), (*col4).String()) + } + row++ + count++ + } + rows.Close() + require.NoError(t, rows.Err()) + assert.Equal(t, uint64(20), count) + require.NoError(t, conn.Exec(ctx, "TRUNCATE TABLE test_column_interface")) + batch, err = conn.PrepareBatch(ctx, "INSERT INTO test_column_interface") + require.NoError(t, err) + { - currentTime = time.Now().Truncate(time.Second) + currentTime = time.Now().Truncate(time.Second) - for i := 0; i < 150; i++ { - a, b := uint8(i), fmt.Sprintf("value_%d", i) - require.NoError(t, batch.Column(0).AppendRow( &a )) + for i := 0; i < 150; i++ { + a, b := uint8(i), fmt.Sprintf("value_%d", i) + require.NoError(t, batch.Column(0).AppendRow(&a)) - switch { - case i%2 == 0: - require.NoError(t, batch.Column(1).AppendRow(&b)) - require.NoError(t, batch.Column(2).AppendRow(¤tTime)) - require.NoError(t, batch.Column(3).AppendRow(&decimalVal)) - default: - require.NoError(t, batch.Column(1).AppendRow(nil)) - require.NoError(t, batch.Column(2).AppendRow(nil)) - require.NoError(t, batch.Column(3).AppendRow(nil)) - } - } - require.NoError(t, batch.Send()) - var count uint64 - require.NoError(t, conn.QueryRow(ctx, "SELECT COUNT() FROM test_column_interface").Scan(&count)) - require.Equal(t, uint64(150), count) - rows, err := conn.Query(ctx, "SELECT * FROM test_column_interface WHERE Col1 >= $1 AND Col1 < $2", 10, 30) - require.NoError(t, err) - var ( - row uint8 = 10 - ) - count = 0 - for rows.Next() { - var ( - col1 *uint8 - col2 *string - col3 *time.Time - col4 *decimal.Decimal - ) - require.NoError(t, rows.Scan(&col1, &col2, &col3, &col4)) - switch { - case row%2 == 0: - assert.Equal(t, row, *col1) - assert.Equal(t, fmt.Sprintf("value_%d", row), *col2) - assert.Equal(t, currentTime.Unix(), col3.Unix()) - assert.Equal(t, decimalVal.String(), (*col4).String()) - default: - if assert.Equal(t, row, *col1) { - assert.Nil(t, col2) - assert.Nil(t, col3) - assert.Nil(t, col4) - } - } - row++ - count++ - } - rows.Close() - require.NoError(t, rows.Err()) - assert.Equal(t, uint64(20), count) - } + switch { + case i%2 == 0: + require.NoError(t, batch.Column(1).AppendRow(&b)) + require.NoError(t, batch.Column(2).AppendRow(¤tTime)) + require.NoError(t, batch.Column(3).AppendRow(&decimalVal)) + default: + require.NoError(t, batch.Column(1).AppendRow(nil)) + require.NoError(t, batch.Column(2).AppendRow(nil)) + require.NoError(t, batch.Column(3).AppendRow(nil)) + } + } + require.NoError(t, batch.Send()) + var count uint64 + require.NoError(t, conn.QueryRow(ctx, "SELECT COUNT() FROM test_column_interface").Scan(&count)) + require.Equal(t, uint64(150), count) + rows, err := conn.Query(ctx, "SELECT * FROM test_column_interface WHERE Col1 >= $1 AND Col1 < $2", 10, 30) + require.NoError(t, err) + var ( + row uint8 = 10 + ) + count = 0 + for rows.Next() { + var ( + col1 *uint8 + col2 *string + col3 *time.Time + col4 *decimal.Decimal + ) + require.NoError(t, rows.Scan(&col1, &col2, &col3, &col4)) + switch { + case row%2 == 0: + assert.Equal(t, row, *col1) + assert.Equal(t, fmt.Sprintf("value_%d", row), *col2) + assert.Equal(t, currentTime.Unix(), col3.Unix()) + assert.Equal(t, decimalVal.String(), (*col4).String()) + default: + if assert.Equal(t, row, *col1) { + assert.Nil(t, col2) + assert.Nil(t, col3) + assert.Nil(t, col4) + } + } + row++ + count++ + } + rows.Close() + require.NoError(t, rows.Err()) + assert.Equal(t, uint64(20), count) + } } - - - - - - diff --git a/tests/issues/485/main.go b/tests/issues/485/main.go index 0a68aad1dc..ba9aa249e4 100644 --- a/tests/issues/485/main.go +++ b/tests/issues/485/main.go @@ -6,7 +6,7 @@ // not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an diff --git a/tests/json_test.go b/tests/json_test.go index 1153a09e3c..5c865c5358 100644 --- a/tests/json_test.go +++ b/tests/json_test.go @@ -2456,13 +2456,13 @@ func TestMultipleJsonRowsWithNil(t *testing.T) { for k := range myMap { newMap[k] = myMap[k] } - + return newMap } type Login struct { - Username string `json:"username"` - Attachment map[string]interface{} + Username string `json:"username"` + Attachment map[string]interface{} } myAttachment := map[string]interface{}{