Skip to content

Commit 323adbe

Browse files
EnmkEnmk
authored
Merge pull request ClickHouse#173 from arthurpassos/array_of_lc_serialization_fix
Fix Array of low cardinality types serialization/deserialization
2 parents 277376e + b68d7f3 commit 323adbe

31 files changed

+327
-116
lines changed

‎clickhouse/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ SET ( clickhouse-cpp-lib-src
77
base/wire_format.cpp
88

99
columns/array.cpp
10+
columns/column.cpp
1011
columns/date.cpp
1112
columns/decimal.cpp
1213
columns/enum.cpp

‎clickhouse/columns/array.cpp

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,22 +54,34 @@ void ColumnArray::Append(ColumnRef column) {
5454
}
5555
}
5656

57-
bool ColumnArray::Load(InputStream* input, size_t rows) {
57+
bool ColumnArray::LoadPrefix(InputStream* input, size_t rows) {
5858
if (!rows) {
5959
return true;
6060
}
61-
if (!offsets_->Load(input, rows)) {
61+
62+
return data_->LoadPrefix(input, rows);
63+
}
64+
65+
bool ColumnArray::LoadBody(InputStream* input, size_t rows) {
66+
if (!rows) {
67+
return true;
68+
}
69+
if (!offsets_->LoadBody(input, rows)) {
6270
return false;
6371
}
64-
if (!data_->Load(input, (*offsets_)[rows - 1])) {
72+
if (!data_->LoadBody(input, (*offsets_)[rows - 1])) {
6573
return false;
6674
}
6775
return true;
6876
}
6977

70-
void ColumnArray::Save(OutputStream* output) {
71-
offsets_->Save(output);
72-
data_->Save(output);
78+
void ColumnArray::SavePrefix(OutputStream* output) {
79+
data_->SavePrefix(output);
80+
}
81+
82+
void ColumnArray::SaveBody(OutputStream* output) {
83+
offsets_->SaveBody(output);
84+
data_->SaveBody(output);
7385
}
7486

7587
void ColumnArray::Clear() {
@@ -92,6 +104,7 @@ void ColumnArray::OffsetsIncrease(size_t n) {
92104
}
93105

94106
size_t ColumnArray::GetOffset(size_t n) const {
107+
95108
return (n == 0) ? 0 : (*offsets_)[n - 1];
96109
}
97110

‎clickhouse/columns/array.h

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,17 @@ class ColumnArray : public Column {
2424
/// Appends content of given column to the end of current one.
2525
void Append(ColumnRef column) override;
2626

27+
/// Loads column prefix from input stream.
28+
bool LoadPrefix(InputStream* input, size_t rows) override;
29+
2730
/// Loads column data from input stream.
28-
bool Load(InputStream* input, size_t rows) override;
31+
bool LoadBody(InputStream* input, size_t rows) override;
32+
33+
/// Saves column prefix to output stream.
34+
void SavePrefix(OutputStream* output) override;
2935

3036
/// Saves column data to output stream.
31-
void Save(OutputStream* output) override;
37+
void SaveBody(OutputStream* output) override;
3238

3339
/// Clear column data .
3440
void Clear() override;

‎clickhouse/columns/column.cpp

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#include "column.h"
2+
3+
namespace clickhouse {
4+
5+
bool Column::LoadPrefix(InputStream*, size_t) {
6+
/// does nothing by default
7+
return true;
8+
}
9+
10+
bool Column::Load(InputStream* input, size_t rows) {
11+
return LoadPrefix(input, rows) && LoadBody(input, rows);
12+
}
13+
14+
void Column::SavePrefix(OutputStream*) {
15+
/// does nothing by default
16+
}
17+
18+
/// Saves column data to output stream.
19+
void Column::Save(OutputStream* output) {
20+
SavePrefix(output);
21+
SaveBody(output);
22+
}
23+
24+
}

‎clickhouse/columns/column.h

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,27 @@ class Column : public std::enable_shared_from_this<Column> {
4242
/// Appends content of given column to the end of current one.
4343
virtual void Append(ColumnRef column) = 0;
4444

45+
/// Template method to load column data from input stream. It'll call LoadPrefix and LoadBody.
46+
/// Should be called only once from the client. Derived classes should not call it.
47+
bool Load(InputStream* input, size_t rows);
48+
49+
/// Loads column prefix from input stream.
50+
virtual bool LoadPrefix(InputStream* input, size_t rows);
51+
4552
/// Loads column data from input stream.
46-
virtual bool Load(InputStream* input, size_t rows) = 0;
53+
virtual bool LoadBody(InputStream* input, size_t rows) = 0;
54+
55+
/// Saves column prefix to output stream. Column types with prefixes must implement it.
56+
virtual void SavePrefix(OutputStream* output);
57+
58+
/// Saves column body to output stream.
59+
virtual void SaveBody(OutputStream* output) = 0;
4760

48-
/// Saves column data to output stream.
49-
virtual void Save(OutputStream* output) = 0;
61+
/// Template method to save to output stream. It'll call SavePrefix and SaveBody respectively
62+
/// Should be called only once from the client. Derived classes should not call it.
63+
/// Save is split in Prefix and Body because some data types require prefixes and specific serialization order.
64+
/// For instance, Array(LowCardinality(X)) requires LowCardinality.key_version bytes to come before Array.offsets
65+
void Save(OutputStream* output);
5066

5167
/// Clear column data .
5268
virtual void Clear() = 0;

‎clickhouse/columns/date.cpp

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@ void ColumnDate::Append(ColumnRef column) {
2727
}
2828
}
2929

30-
bool ColumnDate::Load(InputStream* input, size_t rows) {
31-
return data_->Load(input, rows);
30+
bool ColumnDate::LoadBody(InputStream* input, size_t rows) {
31+
return data_->LoadBody(input, rows);
3232
}
3333

34-
void ColumnDate::Save(OutputStream* output) {
35-
data_->Save(output);
34+
void ColumnDate::SaveBody(OutputStream* output) {
35+
data_->SaveBody(output);
3636
}
3737

3838
size_t ColumnDate::Size() const {
@@ -89,12 +89,12 @@ void ColumnDateTime::Append(ColumnRef column) {
8989
}
9090
}
9191

92-
bool ColumnDateTime::Load(InputStream* input, size_t rows) {
93-
return data_->Load(input, rows);
92+
bool ColumnDateTime::LoadBody(InputStream* input, size_t rows) {
93+
return data_->LoadBody(input, rows);
9494
}
9595

96-
void ColumnDateTime::Save(OutputStream* output) {
97-
data_->Save(output);
96+
void ColumnDateTime::SaveBody(OutputStream* output) {
97+
data_->SaveBody(output);
9898
}
9999

100100
size_t ColumnDateTime::Size() const {
@@ -162,12 +162,12 @@ void ColumnDateTime64::Append(ColumnRef column) {
162162
}
163163
}
164164

165-
bool ColumnDateTime64::Load(InputStream* input, size_t rows) {
166-
return data_->Load(input, rows);
165+
bool ColumnDateTime64::LoadBody(InputStream* input, size_t rows) {
166+
return data_->LoadBody(input, rows);
167167
}
168168

169-
void ColumnDateTime64::Save(OutputStream* output) {
170-
data_->Save(output);
169+
void ColumnDateTime64::SaveBody(OutputStream* output) {
170+
data_->SaveBody(output);
171171
}
172172

173173
void ColumnDateTime64::Clear() {

‎clickhouse/columns/date.h

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ class ColumnDate : public Column {
2626
void Append(ColumnRef column) override;
2727

2828
/// Loads column data from input stream.
29-
bool Load(InputStream* input, size_t rows) override;
29+
bool LoadBody(InputStream* input, size_t rows) override;
3030

3131
/// Saves column data to output stream.
32-
void Save(OutputStream* output) override;
32+
void SaveBody(OutputStream* output) override;
3333

3434
/// Clear column data .
3535
void Clear() override;
@@ -70,13 +70,13 @@ class ColumnDateTime : public Column {
7070
void Append(ColumnRef column) override;
7171

7272
/// Loads column data from input stream.
73-
bool Load(InputStream* input, size_t rows) override;
73+
bool LoadBody(InputStream* input, size_t rows) override;
7474

7575
/// Clear column data .
7676
void Clear() override;
7777

7878
/// Saves column data to output stream.
79-
void Save(OutputStream* output) override;
79+
void SaveBody(OutputStream* output) override;
8080

8181
/// Returns count of rows in the column.
8282
size_t Size() const override;
@@ -118,13 +118,13 @@ class ColumnDateTime64 : public Column {
118118
void Append(ColumnRef column) override;
119119

120120
/// Loads column data from input stream.
121-
bool Load(InputStream* input, size_t rows) override;
121+
bool LoadBody(InputStream* input, size_t rows) override;
122122

123123
/// Clear column data .
124124
void Clear() override;
125125

126126
/// Saves column data to output stream.
127-
void Save(OutputStream* output) override;
127+
void SaveBody(OutputStream* output) override;
128128

129129
/// Returns count of rows in the column.
130130
size_t Size() const override;

‎clickhouse/columns/decimal.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,12 +197,12 @@ void ColumnDecimal::Append(ColumnRef column) {
197197
}
198198
}
199199

200-
bool ColumnDecimal::Load(InputStream * input, size_t rows) {
201-
return data_->Load(input, rows);
200+
bool ColumnDecimal::LoadBody(InputStream * input, size_t rows) {
201+
return data_->LoadBody(input, rows);
202202
}
203203

204-
void ColumnDecimal::Save(OutputStream* output) {
205-
data_->Save(output);
204+
void ColumnDecimal::SaveBody(OutputStream* output) {
205+
data_->SaveBody(output);
206206
}
207207

208208
void ColumnDecimal::Clear() {

‎clickhouse/columns/decimal.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ class ColumnDecimal : public Column {
2121

2222
public:
2323
void Append(ColumnRef column) override;
24-
bool Load(InputStream* input, size_t rows) override;
25-
void Save(OutputStream* output) override;
24+
bool LoadBody(InputStream* input, size_t rows) override;
25+
void SaveBody(OutputStream* output) override;
2626
void Clear() override;
2727
size_t Size() const override;
2828
ColumnRef Slice(size_t begin, size_t len) const override;

‎clickhouse/columns/enum.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,13 @@ void ColumnEnum<T>::Append(ColumnRef column) {
7474
}
7575

7676
template <typename T>
77-
bool ColumnEnum<T>::Load(InputStream* input, size_t rows) {
77+
bool ColumnEnum<T>::LoadBody(InputStream* input, size_t rows) {
7878
data_.resize(rows);
7979
return WireFormat::ReadBytes(*input, data_.data(), data_.size() * sizeof(T));
8080
}
8181

8282
template <typename T>
83-
void ColumnEnum<T>::Save(OutputStream* output) {
83+
void ColumnEnum<T>::SaveBody(OutputStream* output) {
8484
WireFormat::WriteBytes(*output, data_.data(), data_.size() * sizeof(T));
8585
}
8686

‎clickhouse/columns/enum.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,11 @@ class ColumnEnum : public Column {
3333
void Append(ColumnRef column) override;
3434

3535
/// Loads column data from input stream.
36-
bool Load(InputStream* input, size_t rows) override;
36+
bool LoadBody(InputStream* input, size_t rows) override;
3737

3838
/// Saves column data to output stream.
39-
void Save(OutputStream* output) override;
40-
39+
void SaveBody(OutputStream* output) override;
40+
4141
/// Clear column data .
4242
void Clear() override;
4343

‎clickhouse/columns/ip4.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,12 @@ void ColumnIPv4::Append(ColumnRef column) {
7171
}
7272
}
7373

74-
bool ColumnIPv4::Load(InputStream * input, size_t rows) {
75-
return data_->Load(input, rows);
74+
bool ColumnIPv4::LoadBody(InputStream * input, size_t rows) {
75+
return data_->LoadBody(input, rows);
7676
}
7777

78-
void ColumnIPv4::Save(OutputStream* output) {
79-
data_->Save(output);
78+
void ColumnIPv4::SaveBody(OutputStream* output) {
79+
data_->SaveBody(output);
8080
}
8181

8282
size_t ColumnIPv4::Size() const {

‎clickhouse/columns/ip4.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ class ColumnIPv4 : public Column {
4141
void Append(ColumnRef column) override;
4242

4343
/// Loads column data from input stream.
44-
bool Load(InputStream* input, size_t rows) override;
44+
bool LoadBody(InputStream* input, size_t rows) override;
4545

4646
/// Saves column data to output stream.
47-
void Save(OutputStream* output) override;
47+
void SaveBody(OutputStream* output) override;
4848

4949
/// Clear column data .
5050
void Clear() override;

‎clickhouse/columns/ip6.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,12 @@ void ColumnIPv6::Append(ColumnRef column) {
7171
}
7272
}
7373

74-
bool ColumnIPv6::Load(InputStream* input, size_t rows) {
75-
return data_->Load(input, rows);
74+
bool ColumnIPv6::LoadBody(InputStream* input, size_t rows) {
75+
return data_->LoadBody(input, rows);
7676
}
7777

78-
void ColumnIPv6::Save(OutputStream* output) {
79-
data_->Save(output);
78+
void ColumnIPv6::SaveBody(OutputStream* output) {
79+
data_->SaveBody(output);
8080
}
8181

8282
size_t ColumnIPv6::Size() const {

‎clickhouse/columns/ip6.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@ class ColumnIPv6 : public Column {
3939
void Append(ColumnRef column) override;
4040

4141
/// Loads column data from input stream.
42-
bool Load(InputStream* input, size_t rows) override;
42+
bool LoadBody(InputStream* input, size_t rows) override;
4343

4444
/// Saves column data to output stream.
45-
void Save(OutputStream* output) override;
45+
void SaveBody(OutputStream* output) override;
4646

4747
/// Clear column data .
4848
void Clear() override;

0 commit comments

Comments
 (0)