Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,9 @@ endif ()

option(BUILD_TEST "Build tests" ON)
message("cmake using: BUILD_TEST=${BUILD_TEST}")
if (BUILD_TEST)
add_definitions(-DENABLE_TEST)
endif ()

option(BUILD_TOOLS "Build the tsfile command-line tools" ON)
message("cmake using: BUILD_TOOLS=${BUILD_TOOLS}")
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/common/allocator/alloc_base.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ extern TSFILE_API const char* g_mod_names[__LAST_MOD_ID];
void* mem_alloc(uint32_t size, AllocModID mid);
void mem_free(void* ptr);
void* mem_realloc(void* ptr, uint32_t size);
#ifdef ENABLE_TEST
TSFILE_API void TEST_fail_next_mem_realloc();
#endif

class ModStat {
public:
Expand Down
18 changes: 17 additions & 1 deletion cpp/src/common/allocator/mem_alloc.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#endif
#include <string.h>

#include <atomic>
#include <iomanip>
#include <iostream>

Expand All @@ -33,6 +34,16 @@

namespace common {

#ifdef ENABLE_TEST
namespace {
std::atomic<bool> g_fail_next_mem_realloc(false);
}

void TEST_fail_next_mem_realloc() {
g_fail_next_mem_realloc.store(true, std::memory_order_release);
}
#endif

const char* g_mod_names[__LAST_MOD_ID] = {
/* 0 */ "DEFAULT",
/* 1 */ "TVLIST_DATA",
Expand Down Expand Up @@ -139,6 +150,11 @@ void mem_free(void* ptr) {
}

void* mem_realloc(void* ptr, uint32_t size) {
#ifdef ENABLE_TEST
if (g_fail_next_mem_realloc.exchange(false, std::memory_order_acq_rel)) {
return nullptr;
}
#endif
char* p = static_cast<char*>(ptr);
char* raw_ptr = p - ALIGNMENT;
const uint64_t header =
Expand Down Expand Up @@ -222,4 +238,4 @@ void ModStat::print_stat() {

BaseAllocator g_base_allocator;

} // end namespace common
} // end namespace common
10 changes: 6 additions & 4 deletions cpp/src/common/container/array.h
Original file line number Diff line number Diff line change
Expand Up @@ -136,12 +136,13 @@ class Array {
}

size_t new_capacity = (size_t)tmp;
array_ =
ValueType* new_array =
(ValueType*)mem_realloc(array_, new_capacity * sizeof(*(array_)));
if (UNLIKELY(nullptr == array_)) {
if (UNLIKELY(nullptr == new_array)) {
// log_err("realloc failed.");
return E_OOM;
}
array_ = new_array;
capacity_ = new_capacity;
return E_OK;
}
Expand All @@ -150,12 +151,13 @@ class Array {
size_t new_capacity = (size_t)(capacity_ / 2);
// if the size passed to realloc is smaller than before, OS will
// automatically release the rest memory
array_ =
ValueType* new_array =
(ValueType*)mem_realloc(array_, new_capacity * sizeof(*(array_)));
if (UNLIKELY(nullptr == array_)) {
if (UNLIKELY(nullptr == new_array)) {
// log_err("malloc failed.");
return E_OOM;
}
array_ = new_array;
capacity_ = new_capacity;
return E_OK;
}
Expand Down
23 changes: 17 additions & 6 deletions cpp/src/common/container/byte_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,18 @@ class ByteBuffer {

FORCE_INLINE void reset() { real_data_size_ = 0; }

FORCE_INLINE void extend_memory(uint32_t new_size) {
FORCE_INLINE int extend_memory(uint32_t new_size) {
ASSERT(new_size > reserved_size_);
data_ = static_cast<char*>(mem_realloc(data_, new_size));
char* new_data = static_cast<char*>(mem_realloc(data_, new_size));
if (UNLIKELY(new_data == nullptr)) {
return E_OOM;
}
data_ = new_data;
reserved_size_ = new_size;
return E_OK;
}

FORCE_INLINE void append_variable_value(const char* value, uint32_t len) {
FORCE_INLINE int append_variable_value(const char* value, uint32_t len) {
// dynamic growth
if (UNLIKELY((real_data_size_ + len + variable_type_len_) >
reserved_size_)) {
Expand All @@ -67,7 +72,9 @@ class ByteBuffer {
g_config_value_.tsblock_mem_inc_step_size_ > len
? g_config_value_.tsblock_mem_inc_step_size_
: (len + 1);
extend_memory(reserved_size_ + growth_size);
if (UNLIKELY(extend_memory(reserved_size_ + growth_size) != E_OK)) {
return E_OOM;
}
}

ASSERT(data_);
Expand All @@ -80,22 +87,26 @@ class ByteBuffer {
memcpy(&data_[real_data_size_], value, len);
real_data_size_ += len;
}
return E_OK;
}

FORCE_INLINE void append_fixed_value(const char* value, uint32_t len) {
FORCE_INLINE int append_fixed_value(const char* value, uint32_t len) {
// dynamic growth
if (UNLIKELY(real_data_size_ + len > reserved_size_)) {
// extreme scenarios, when encountering very long string
uint32_t growth_size =
g_config_value_.tsblock_mem_inc_step_size_ > len
? g_config_value_.tsblock_mem_inc_step_size_
: (len + 1);
extend_memory(reserved_size_ + growth_size);
if (UNLIKELY(extend_memory(reserved_size_ + growth_size) != E_OK)) {
return E_OOM;
}
}

ASSERT(data_);
memcpy(&data_[real_data_size_], value, len);
real_data_size_ += len;
return E_OK;
}

// for fixed len value
Expand Down
10 changes: 6 additions & 4 deletions cpp/src/common/container/sorted_array.h
Original file line number Diff line number Diff line change
Expand Up @@ -139,12 +139,13 @@ class SortedArray {
}

size_t new_capacity = (size_t)tmp;
array_ =
ValueType* new_array =
(ValueType*)mem_realloc(array_, new_capacity * sizeof(*(array_)));
if (UNLIKELY(nullptr == array_)) {
if (UNLIKELY(nullptr == new_array)) {
// log_err("realloc failed.");
return E_OOM;
}
array_ = new_array;
capacity_ = new_capacity;
return E_OK;
}
Expand All @@ -153,14 +154,15 @@ class SortedArray {
size_t new_capacity = (size_t)(capacity_ / 2);
// if the size passed to realloc is smaller than before, OS will
// automatically release the rest memory
array_ = (ValueType*)mem_realloc(
ValueType* new_array = (ValueType*)mem_realloc(
array_,
new_capacity *
sizeof(*(array_))); // TODO: user ourself's mem_alloc()
if (UNLIKELY(nullptr == array_)) {
if (UNLIKELY(nullptr == new_array)) {
// log_err("malloc failed.");
return E_OOM;
}
array_ = new_array;
capacity_ = new_capacity;
return E_OK;
}
Expand Down
19 changes: 12 additions & 7 deletions cpp/src/common/tablet.cc
Original file line number Diff line number Diff line change
Expand Up @@ -423,15 +423,19 @@ void* Tablet::get_value(int row_index, uint32_t schema_index,
}

template <>
void Tablet::process_val(uint32_t row_index, uint32_t schema_index,
common::String str) {
value_matrix_[schema_index].string_col->append(row_index, str.buf_,
str.len_);
int Tablet::process_val(uint32_t row_index, uint32_t schema_index,
common::String str) {
int ret = value_matrix_[schema_index].string_col->append(
row_index, str.buf_, str.len_);
if (ret != E_OK) {
return ret;
}
bitmaps_[schema_index].clear(row_index); /* mark as non-null */
return E_OK;
}

template <typename T>
void Tablet::process_val(uint32_t row_index, uint32_t schema_index, T val) {
int Tablet::process_val(uint32_t row_index, uint32_t schema_index, T val) {
switch (schema_vec_->at(schema_index).data_type_) {
case common::BOOLEAN:
(value_matrix_[schema_index].bool_data)[row_index] =
Expand Down Expand Up @@ -459,6 +463,7 @@ void Tablet::process_val(uint32_t row_index, uint32_t schema_index, T val) {
ASSERT(false);
}
bitmaps_[schema_index].clear(row_index); /* mark as non-null */
return E_OK;
}

template <typename T>
Expand All @@ -475,7 +480,7 @@ int Tablet::add_value(uint32_t row_index, uint32_t schema_index, T val) {
if (UNLIKELY(!TypeMatch<T>(schema.data_type_))) {
return E_TYPE_NOT_MATCH;
}
process_val(row_index, schema_index, val);
ret = process_val(row_index, schema_index, val);
}
return ret;
}
Expand All @@ -492,7 +497,7 @@ int Tablet::add_value(uint32_t row_index, uint32_t schema_index, std::tm val) {
}
int32_t date_int;
if (RET_SUCC(common::DateConverter::date_to_int(val, date_int))) {
process_val(row_index, schema_index, date_int);
ret = process_val(row_index, schema_index, date_int);
}
return ret;
}
Expand Down
20 changes: 16 additions & 4 deletions cpp/src/common/tablet.h
Original file line number Diff line number Diff line change
Expand Up @@ -80,16 +80,28 @@ class Tablet {
if (offsets) offsets[0] = 0;
}

void append(uint32_t row, const char* data, uint32_t len) {
int append(uint32_t row, const char* data, uint32_t len) {
// Grow buffer if needed
if (buf_used + len > buf_capacity) {
buf_capacity = buf_capacity * 2 + len;
buffer = (char*)common::mem_realloc(buffer, buf_capacity);
uint64_t new_capacity_64 =
static_cast<uint64_t>(buf_capacity) * 2 + len;
if (UNLIKELY(new_capacity_64 > UINT32_MAX)) {
return common::E_OVERFLOW;
}
uint32_t new_capacity = static_cast<uint32_t>(new_capacity_64);
char* new_buffer =
(char*)common::mem_realloc(buffer, new_capacity);
if (UNLIKELY(new_buffer == nullptr)) {
return common::E_OOM;
}
buffer = new_buffer;
buf_capacity = new_capacity;
}
memcpy(buffer + buf_used, data, len);
offsets[row] = static_cast<int32_t>(buf_used);
offsets[row + 1] = static_cast<int32_t>(buf_used + len);
buf_used += len;
return common::E_OK;
}

const char* get_str(uint32_t row) const {
Expand Down Expand Up @@ -393,7 +405,7 @@ class Tablet {

private:
template <typename T>
void process_val(uint32_t row_index, uint32_t schema_index, T val);
int process_val(uint32_t row_index, uint32_t schema_index, T val);
uint32_t max_row_num_;
uint32_t cur_row_size_;
std::string insert_target_name_;
Expand Down
Loading
Loading