Skip to content

Commit

Permalink
[Enhancement] No need to build_slice for get data from BinaryColumn (#…
Browse files Browse the repository at this point in the history 
…30702)

Signed-off-by: trueeyu <[email protected]>
(cherry picked from commit 2d86dd2)

# Conflicts:
#	be/src/util/arrow/starrocks_column_to_arrow.cpp
  • Loading branch information
@trueeyu @mergify
trueeyu authored and mergify[bot] committed Sep 11, 2023
1 parent dbbbefd commit 0ba85cb
Showing 1 changed file with 46 additions and 10 deletions.
56 changes: 46 additions & 10 deletions be/src/util/arrow/starrocks_column_to_arrow.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ struct ColumnToArrowConverter<PT, AT, is_nullable, ConvFloatAndIntegerGuard<PT,
using StarRocksCppType = RunTimeCppType<PT>;
using StarRocksColumnType = RunTimeColumnType<PT>;
using ArrowType = ArrowTypeIdToType<AT>;
using ArrowCppType = ArrowTypeIdToCppType<AT>;
using ArrowBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
static inline arrow::Status convert(const ColumnPtr& column, arrow::MemoryPool* pool,
std::shared_ptr<arrow::Array>& array) {
Expand Down Expand Up @@ -72,7 +71,6 @@ struct ColumnToArrowConverter<PT, AT, is_nullable, ConvDecimalGuard<PT, AT>> {
using StarRocksCppType = RunTimeCppType<PT>;
using StarRocksColumnType = RunTimeColumnType<PT>;
using ArrowType = ArrowTypeIdToType<AT>;
using ArrowCppType = ArrowTypeIdToCppType<AT>;
using ArrowBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;

static inline arrow::Decimal128 convert_datum(const StarRocksCppType& datum) {
Expand Down Expand Up @@ -149,7 +147,6 @@ struct ColumnToArrowConverter<PT, AT, is_nullable, ConvBinaryGuard<PT, AT>> {
using StarRocksCppType = RunTimeCppType<PT>;
using StarRocksColumnType = RunTimeColumnType<PT>;
using ArrowType = ArrowTypeIdToType<AT>;
using ArrowCppType = ArrowTypeIdToCppType<AT>;
using ArrowBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;

static inline std::string convert_datum(const StarRocksCppType& datum, [[maybe_unused]] int precision,
Expand Down Expand Up @@ -180,25 +177,46 @@ struct ColumnToArrowConverter<PT, AT, is_nullable, ConvBinaryGuard<PT, AT>> {
const auto* nullable_column = down_cast<NullableColumn*>(column.get());
const auto* data_column = down_cast<StarRocksColumnType*>(nullable_column->data_column().get());
const auto* null_column = down_cast<NullColumn*>(nullable_column->null_column().get());
<<<<<<< HEAD
const auto& data = data_column->get_data();
[[maybe_unused]] int precision = -1;
[[maybe_unused]] int scale = -1;
if constexpr (pt_is_decimal<PT>) {
precision = data_column->precision();
scale = data_column->scale();
}
=======
>>>>>>> 2d86dd2346 ([Enhancement] No need to build_slice for get data from BinaryColumn (#30702))
const auto num_rows = null_column->size();
for (auto i = 0; i < num_rows; ++i) {
if (nullable_column->is_null(i)) {
ARROW_RETURN_NOT_OK(builder->AppendNull());
} else {
ARROW_RETURN_NOT_OK(builder->Append(convert_datum(data[i], precision, scale)));
if constexpr (lt_is_string<LT>) {
const auto& data = data_column->get_proxy_data();
for (auto i = 0; i < num_rows; ++i) {
if (nullable_column->is_null(i)) {
ARROW_RETURN_NOT_OK(builder->AppendNull());
} else {
ARROW_RETURN_NOT_OK(builder->Append(convert_datum(data[i], -1, -1)));
}
}
} else {
const auto& data = data_column->get_data();
[[maybe_unused]] int precision = -1;
[[maybe_unused]] int scale = -1;
if constexpr (lt_is_decimal<LT>) {
precision = data_column->precision();
scale = data_column->scale();
}
for (auto i = 0; i < num_rows; ++i) {
if (nullable_column->is_null(i)) {
ARROW_RETURN_NOT_OK(builder->AppendNull());
} else {
ARROW_RETURN_NOT_OK(builder->Append(convert_datum(data[i], precision, scale)));
}
}
}
} else {
const auto* data_column = down_cast<StarRocksColumnType*>(column.get());
const auto& data = data_column->get_data();
const auto num_rows = column->size();
<<<<<<< HEAD
[[maybe_unused]] int precision = -1;
[[maybe_unused]] int scale = -1;
if constexpr (pt_is_decimal<PT>) {
Expand All @@ -207,6 +225,24 @@ struct ColumnToArrowConverter<PT, AT, is_nullable, ConvBinaryGuard<PT, AT>> {
}
for (auto i = 0; i < num_rows; ++i) {
ARROW_RETURN_NOT_OK(builder->Append(convert_datum(data[i], precision, scale)));
=======
if constexpr (lt_is_string<LT>) {
const auto& data = data_column->get_proxy_data();
for (auto i = 0; i < num_rows; ++i) {
ARROW_RETURN_NOT_OK(builder->Append(convert_datum(data[i], -1, -1)));
}
} else {
const auto& data = data_column->get_data();
[[maybe_unused]] int precision = -1;
[[maybe_unused]] int scale = -1;
if constexpr (lt_is_decimal<LT>) {
precision = data_column->precision();
scale = data_column->scale();
}
for (auto i = 0; i < num_rows; ++i) {
ARROW_RETURN_NOT_OK(builder->Append(convert_datum(data[i], precision, scale)));
}
>>>>>>> 2d86dd2346 ([Enhancement] No need to build_slice for get data from BinaryColumn (#30702))
}
}
return builder->Finish(&array);
Expand Down Expand Up @@ -295,7 +331,7 @@ Status convert_chunk_to_arrow_batch(Chunk* chunk, std::vector<ExprContext*>& _ou
std::vector<std::shared_ptr<arrow::Array>> arrays(result_num_column);

for (auto i = 0; i < result_num_column; ++i) {
ASSIGN_OR_RETURN(ColumnPtr column, _output_expr_ctxs[i]->evaluate(chunk));
ASSIGN_OR_RETURN(ColumnPtr column, _output_expr_ctxs[i]->evaluate(chunk))
Expr* expr = _output_expr_ctxs[i]->root();
if (column->is_constant()) {
column = vectorized::ColumnHelper::unfold_const_column(expr->type(), chunk->num_rows(), column);
Expand Down

0 comments on commit 0ba85cb

Please sign in to comment.