Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[env](compile)open compile check in some function #42452

Merged
merged 1 commit into from
Oct 28, 2024
Merged

[env](compile)open compile check in some function #42452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion be/src/olap/tablet_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ class TabletReader {

// used for compaction to record row ids
bool record_rowids = false;
RowIdConversion* rowid_conversion;
RowIdConversion* rowid_conversion = nullptr;
std::vector<int> topn_filter_source_node_ids;
int topn_filter_target_node_id = -1;
// used for special optimization for query : ORDER BY key LIMIT n
Expand Down
4 changes: 2 additions & 2 deletions be/src/util/hash_util.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ class HashUtil {
static const uint32_t MURMUR3_32_SEED = 104729;

// modify from https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
static uint32_t murmur_hash3_32(const void* key, int32_t len, uint32_t seed) {
static uint32_t murmur_hash3_32(const void* key, int64_t len, uint32_t seed) {
uint32_t out = 0;
murmur_hash3_x86_32(key, len, seed, &out);
return out;
Expand Down Expand Up @@ -227,7 +227,7 @@ class HashUtil {
// Our hash function is MurmurHash2, 64 bit version.
// It was modified in order to provide the same result in
// big and little endian archs (endian neutral).
static uint64_t murmur_hash64A(const void* key, int32_t len, unsigned int seed) {
static uint64_t murmur_hash64A(const void* key, int64_t len, unsigned int seed) {
const uint64_t m = MURMUR_PRIME;
const int r = 47;
uint64_t h = seed ^ (len * m);
Expand Down
4 changes: 2 additions & 2 deletions be/src/util/murmur_hash3.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ FORCE_INLINE uint64_t fmix64(uint64_t k) {

//-----------------------------------------------------------------------------

void murmur_hash3_x86_32(const void* key, int len, uint32_t seed, void* out) {
void murmur_hash3_x86_32(const void* key, int64_t len, uint32_t seed, void* out) {
const uint8_t* data = (const uint8_t*)key;
const int nblocks = len / 4;

Expand Down Expand Up @@ -435,7 +435,7 @@ void murmur_hash3_x64_128(const void* key, const int len, const uint32_t seed, v
((uint64_t*)out)[1] = h2;
}

void murmur_hash3_x64_64(const void* key, const int len, const uint64_t seed, void* out) {
void murmur_hash3_x64_64(const void* key, const int64_t len, const uint64_t seed, void* out) {
const uint8_t* data = (const uint8_t*)key;
const int nblocks = len / 8;
uint64_t h1 = seed;
Expand Down
4 changes: 2 additions & 2 deletions be/src/util/murmur_hash3.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@ typedef unsigned __int64 uint64_t;

//-----------------------------------------------------------------------------

void murmur_hash3_x86_32(const void* key, int len, uint32_t seed, void* out);
void murmur_hash3_x86_32(const void* key, int64_t len, uint32_t seed, void* out);

void murmur_hash3_x86_128(const void* key, int len, uint32_t seed, void* out);

void murmur_hash3_x64_128(const void* key, int len, uint32_t seed, void* out);

void murmur_hash3_x64_64(const void* key, int len, uint64_t seed, void* out);
void murmur_hash3_x64_64(const void* key, int64_t len, uint64_t seed, void* out);

//-----------------------------------------------------------------------------
6 changes: 3 additions & 3 deletions be/src/util/string_parser.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ class StringParser {

// Convert a string s representing a number in given base into a decimal number.
template <typename T>
static inline T string_to_int(const char* __restrict s, int len, int base,
static inline T string_to_int(const char* __restrict s, int64_t len, int base,
ParseResult* result) {
T ans = string_to_int_internal<T>(s, len, base, result);
if (LIKELY(*result == PARSE_SUCCESS)) {
Expand Down Expand Up @@ -207,7 +207,7 @@ class StringParser {
// Convert a string s representing a number in given base into a decimal number.
// Return PARSE_FAILURE on leading whitespace. Trailing whitespace is allowed.
template <typename T>
static inline T string_to_int_internal(const char* __restrict s, int len, int base,
static inline T string_to_int_internal(const char* __restrict s, int64_t len, int base,
ParseResult* result);

// Converts an ascii string to an integer of type T assuming it cannot overflow
Expand Down Expand Up @@ -385,7 +385,7 @@ T StringParser::string_to_unsigned_int_internal(const char* __restrict s, int le
}

template <typename T>
T StringParser::string_to_int_internal(const char* __restrict s, int len, int base,
T StringParser::string_to_int_internal(const char* __restrict s, int64_t len, int base,
ParseResult* result) {
typedef typename std::make_unsigned<T>::type UnsignedT;
UnsignedT val = 0;
Expand Down
5 changes: 4 additions & 1 deletion be/src/vec/functions/function_collection_in.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
#include "vec/functions/function.h"

namespace doris::vectorized {
#include "common/compile_check_begin.h"
struct ColumnRowRef {
ENABLE_FACTORY_CREATOR(ColumnRowRef);
ColumnPtr column;
Expand Down Expand Up @@ -128,7 +129,7 @@ class FunctionCollectionIn : public IFunction {
}
ColumnPtr column_ptr = std::move(args_column_ptr);
// make collection ref into set
int col_size = column_ptr->size();
auto col_size = column_ptr->size();
for (size_t i = 0; i < col_size; i++) {
state->args_set.insert({column_ptr, i});
}
Expand Down Expand Up @@ -191,3 +192,5 @@ class FunctionCollectionIn : public IFunction {
};

} // namespace doris::vectorized

#include "common/compile_check_end.h"
1 change: 1 addition & 0 deletions be/src/vec/functions/function_conv.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
#include "vec/functions/simple_function_factory.h"

namespace doris {
#include "common/compile_check_begin.h"
class FunctionContext;
} // namespace doris

Expand Down
5 changes: 4 additions & 1 deletion be/src/vec/functions/function_convert_tz.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
#include "vec/functions/function.h"
#include "vec/runtime/vdatetime_value.h"
namespace doris::vectorized {
#include "common/compile_check_begin.h"

struct ConvertTzState {
bool use_state = false;
Expand Down Expand Up @@ -215,7 +216,7 @@ class FunctionConvertTZ : public IFunction {
NullMap& result_null_map, size_t input_rows_count) {
cctz::time_zone& from_tz = convert_tz_state->from_tz;
cctz::time_zone& to_tz = convert_tz_state->to_tz;
auto push_null = [&](int row) {
auto push_null = [&](size_t row) {
result_null_map[row] = true;
result_column->insert_default();
};
Expand Down Expand Up @@ -310,3 +311,5 @@ class FunctionConvertTZ : public IFunction {
};

} // namespace doris::vectorized

#include "common/compile_check_end.h"
11 changes: 7 additions & 4 deletions be/src/vec/functions/function_date_or_datetime_computation.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include <type_traits>
#include <utility>

#include "common/cast_set.h"
#include "common/compiler_util.h"
#include "common/exception.h"
#include "common/logging.h"
Expand Down Expand Up @@ -645,7 +646,7 @@ struct DateTimeAddIntervalImpl {
col_to->get_data(), null_map->get_data(),
delta_vec_column->get_data());
} else {
Op::constant_vector(sources_const->template get_value<FromType2>(),
Op::constant_vector(sources_const->template get_value<FromType1>(),
col_to->get_data(), null_map->get_data(),
*not_nullable_column_ptr_arg1);
}
Expand Down Expand Up @@ -675,7 +676,7 @@ struct DateTimeAddIntervalImpl {
Op::constant_vector(sources_const->template get_value<FromType1>(),
col_to->get_data(), delta_vec_column->get_data());
} else {
Op::constant_vector(sources_const->template get_value<FromType2>(),
Op::constant_vector(sources_const->template get_value<FromType1>(),
col_to->get_data(),
*block.get_by_position(arguments[1]).column);
}
Expand Down Expand Up @@ -876,7 +877,7 @@ struct CurrentDateTimeImpl {
if constexpr (WithPrecision) {
if (const auto* const_column = check_and_get_column<ColumnConst>(
block.get_by_position(arguments[0]).column)) {
int scale = const_column->get_int(0);
int64_t scale = const_column->get_int(0);
dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->nano_seconds(),
context->state()->timezone_obj(), scale);
Expand Down Expand Up @@ -1002,6 +1003,7 @@ struct CurrentTimeImpl {
};

struct TimeToSecImpl {
// rethink the func should return int32
using ReturnType = DataTypeInt32;
static constexpr auto name = "time_to_sec";
static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
Expand All @@ -1012,7 +1014,8 @@ struct TimeToSecImpl {

auto& res_data = res_col->get_data();
for (int i = 0; i < input_rows_count; ++i) {
res_data[i] = static_cast<int64_t>(column_data.get_element(i)) / (1000 * 1000);
res_data[i] =
cast_set<int>(static_cast<int64_t>(column_data.get_element(i)) / (1000 * 1000));
}
block.replace_by_position(result, std::move(res_col));

Expand Down
6 changes: 5 additions & 1 deletion be/src/vec/functions/function_datetime_string_to_string.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <utility>
#include <variant>

#include "common/cast_set.h"
#include "common/status.h"
#include "vec/aggregate_functions/aggregate_function.h"
#include "vec/columns/column.h"
Expand All @@ -46,6 +47,7 @@
#include "vec/runtime/vdatetime_value.h"

namespace doris {
#include "common/compile_check_begin.h"
class FunctionContext;
} // namespace doris

Expand Down Expand Up @@ -189,7 +191,7 @@ class FunctionDateTimeStringToString : public IFunction {
for (int i = 0; i < len; ++i) {
null_map[i] = Transform::template execute<Impl>(
ts[i], format, res_data, offset, context->state()->timezone_obj());
res_offsets[i] = offset;
res_offsets[i] = cast_set<uint32_t>(offset);
}
res_data.resize(offset);
},
Expand All @@ -199,3 +201,5 @@ class FunctionDateTimeStringToString : public IFunction {
};

} // namespace doris::vectorized

#include "common/compile_check_end.h"
53 changes: 25 additions & 28 deletions be/src/vec/functions/function_encryption.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,25 @@
// specific language governing permissions and limitations
// under the License.

#include <stddef.h>
#include <stdint.h>

#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "common/cast_set.h"
#include "common/status.h"
#include "util/encryption_util.h"
#include "util/string_util.h"
#include "vec/aggregate_functions/aggregate_function.h"
#include "vec/columns/column.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_string.h"
#include "vec/columns/column_vector.h"
#include "vec/columns/columns_number.h"
#include "vec/common/assert_cast.h"
#include "vec/common/pod_array.h"
#include "vec/common/string_ref.h"
#include "vec/core/block.h"
#include "vec/core/column_numbers.h"
#include "vec/core/column_with_type_and_name.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_string.h"
Expand All @@ -50,6 +43,7 @@
#include "vec/utils/util.hpp"

namespace doris {
#include "common/compile_check_begin.h"
class FunctionContext;
} // namespace doris

Expand Down Expand Up @@ -136,9 +130,9 @@ void execute_result_vector(std::vector<const ColumnString::Offsets*>& offsets_li
template <typename Impl, bool is_encrypt>
void execute_result_const(const ColumnString::Offsets* offsets_column,
const ColumnString::Chars* chars_column, StringRef key_arg, size_t i,
EncryptionMode& encryption_mode, const char* iv_raw, int iv_length,
EncryptionMode& encryption_mode, const char* iv_raw, size_t iv_length,
ColumnString::Chars& result_data, ColumnString::Offsets& result_offset,
NullMap& null_map, const char* aad, int aad_length) {
NullMap& null_map, const char* aad, size_t aad_length) {
int src_size = (*offsets_column)[i] - (*offsets_column)[i - 1];
const auto* src_raw = reinterpret_cast<const char*>(&(*chars_column)[(*offsets_column)[i - 1]]);
execute_result<Impl, is_encrypt>(src_raw, src_size, key_arg.data, key_arg.size, i,
Expand All @@ -147,15 +141,15 @@ void execute_result_const(const ColumnString::Offsets* offsets_column,
}

template <typename Impl, bool is_encrypt>
void execute_result(const char* src_raw, int src_size, const char* key_raw, int key_size, size_t i,
EncryptionMode& encryption_mode, const char* iv_raw, int iv_length,
void execute_result(const char* src_raw, size_t src_size, const char* key_raw, size_t key_size,
size_t i, EncryptionMode& encryption_mode, const char* iv_raw, size_t iv_length,
ColumnString::Chars& result_data, ColumnString::Offsets& result_offset,
NullMap& null_map, const char* aad, int aad_length) {
NullMap& null_map, const char* aad, size_t aad_length) {
if (src_size == 0) {
StringOP::push_null_string(i, result_data, result_offset, null_map);
return;
}
int cipher_len = src_size;
auto cipher_len = src_size;
if constexpr (is_encrypt) {
cipher_len += 16;
// for output AEAD tag
Expand Down Expand Up @@ -438,22 +432,25 @@ struct EncryptionAndDecryptMultiImpl {
};

struct EncryptImpl {
static int execute_impl(EncryptionMode mode, const unsigned char* source,
uint32_t source_length, const unsigned char* key, uint32_t key_length,
const char* iv, int iv_length, bool padding, unsigned char* encrypt,
const unsigned char* aad, int aad_length) {
return EncryptionUtil::encrypt(mode, source, source_length, key, key_length, iv, iv_length,
true, encrypt, aad, aad_length);
static int execute_impl(EncryptionMode mode, const unsigned char* source, size_t source_length,
const unsigned char* key, size_t key_length, const char* iv,
size_t iv_length, bool padding, unsigned char* encrypt,
const unsigned char* aad, size_t aad_length) {
// now the openssl only support int, so here we need to cast size_t to uint32_t
return EncryptionUtil::encrypt(mode, source, cast_set<uint32_t>(source_length), key,
cast_set<uint32_t>(key_length), iv, cast_set<int>(iv_length),
true, encrypt, aad, cast_set<uint32_t>(aad_length));
}
};

struct DecryptImpl {
static int execute_impl(EncryptionMode mode, const unsigned char* source,
uint32_t source_length, const unsigned char* key, uint32_t key_length,
const char* iv, int iv_length, bool padding, unsigned char* encrypt,
const unsigned char* aad, int aad_length) {
return EncryptionUtil::decrypt(mode, source, source_length, key, key_length, iv, iv_length,
true, encrypt, aad, aad_length);
static int execute_impl(EncryptionMode mode, const unsigned char* source, size_t source_length,
const unsigned char* key, size_t key_length, const char* iv,
size_t iv_length, bool padding, unsigned char* encrypt,
const unsigned char* aad, size_t aad_length) {
return EncryptionUtil::decrypt(mode, source, cast_set<uint32_t>(source_length), key,
cast_set<uint32_t>(key_length), iv, cast_set<int>(iv_length),
true, encrypt, aad, cast_set<uint32_t>(aad_length));
}
};

Expand Down
1 change: 1 addition & 0 deletions be/src/vec/functions/function_hash.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
#include "vec/utils/template_helpers.hpp"

namespace doris::vectorized {
#include "common/compile_check_begin.h"
constexpr uint64_t emtpy_value = 0xe28dbde7fe22e41c;

template <typename ReturnType>
Expand Down
6 changes: 4 additions & 2 deletions be/src/vec/functions/function_hex.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <string_view>
#include <utility>

#include "common/cast_set.h"
#include "common/status.h"
#include "olap/hll.h"
#include "util/simd/vstring_function.h" //place this header file at last to compile
Expand All @@ -46,6 +47,7 @@
#include "vec/functions/simple_function_factory.h"

namespace doris {
#include "common/compile_check_begin.h"
class FunctionContext;
} // namespace doris

Expand Down Expand Up @@ -111,7 +113,7 @@ struct HexStringImpl {
auto source = reinterpret_cast<const unsigned char*>(&data[offsets[i - 1]]);
size_t srclen = offsets[i] - offsets[i - 1];
hex_encode(source, srclen, dst_data_ptr, offset);
dst_offsets[i] = offset;
dst_offsets[i] = cast_set<uint32_t>(offset);
}
return Status::OK();
}
Expand Down Expand Up @@ -184,7 +186,7 @@ struct HexHLLImpl {
dst_data_ptr = res_data.data() + offset;
hex_encode(reinterpret_cast<const unsigned char*>(hll_str.data()), hll_str.length(),
dst_data_ptr, offset);
res_offsets[i] = offset;
res_offsets[i] = cast_set<uint32_t>(offset);
hll_str.clear();
}
return Status::OK();
Expand Down
Loading
Loading