diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 94f8f0d1d5672f..d49edbd2abce1d 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -660,11 +660,8 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, for (int i = 0; i < tuple_desc->slots().size(); ++i) { const SlotDescriptor* slot_desc = tuple_desc->slots()[i]; - auto col_ptr = columns[i].get(); + auto* col_ptr = columns[i].get(); - if (!slot_desc->is_materialized()) { - continue; - } if (slot_desc->col_name() == FIELD_ID) { // actually this branch will not be reached, this is guaranteed by Doris FE. if (pure_doc_value) { diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index e48585b1aacdf2..babe6a11de520f 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -525,10 +525,7 @@ Status PushBrokerReader::_convert_to_output_block(vectorized::Block* block) { size_t rows = _src_block.rows(); auto filter_column = vectorized::ColumnUInt8::create(rows, 1); - for (auto slot_desc : _dest_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } + for (auto* slot_desc : _dest_tuple_desc->slots()) { int dest_index = ctx_idx++; vectorized::ColumnPtr column_ptr; @@ -612,9 +609,6 @@ Status PushBrokerReader::_init_expr_ctxes() { } bool has_slot_id_map = _params.__isset.dest_sid_to_src_sid_without_trans; for (auto slot_desc : _dest_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } auto it = _params.expr_of_dest_slot.find(slot_desc->id()); if (it == std::end(_params.expr_of_dest_slot)) { return Status::InternalError("No expr for dest slot, id={}, name={}", slot_desc->id(), diff --git a/be/src/pipeline/exec/es_scan_operator.cpp b/be/src/pipeline/exec/es_scan_operator.cpp index ba7e0f88d61591..79f53da66fc924 100644 --- a/be/src/pipeline/exec/es_scan_operator.cpp +++ b/be/src/pipeline/exec/es_scan_operator.cpp @@ -142,10 +142,7 @@ Status EsScanOperatorX::prepare(RuntimeState* state) { } // set up column name vector for ESScrollQueryBuilder - for (auto slot_desc : _tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } + for (auto* slot_desc : _tuple_desc->slots()) { _column_names.push_back(slot_desc->col_name()); } diff --git a/be/src/pipeline/exec/result_sink_operator.cpp b/be/src/pipeline/exec/result_sink_operator.cpp index 62fcc50fbd46b6..b1e06ee48bb24a 100644 --- a/be/src/pipeline/exec/result_sink_operator.cpp +++ b/be/src/pipeline/exec/result_sink_operator.cpp @@ -121,12 +121,6 @@ Status ResultSinkOperatorX::prepare(RuntimeState* state) { // prepare output_expr // From the thrift expressions create the real exprs. RETURN_IF_ERROR(vectorized::VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs)); - if (_fetch_option.use_two_phase_fetch) { - for (auto& expr_ctx : _output_vexpr_ctxs) { - // Must materialize if it a slot, or the slot column id will be -1 - expr_ctx->set_force_materialize_slot(); - } - } // Prepare the exprs to run. RETURN_IF_ERROR(vectorized::VExpr::prepare(_output_vexpr_ctxs, state, _row_desc)); diff --git a/be/src/runtime/descriptor_helper.h b/be/src/runtime/descriptor_helper.h index d61d887a5ab5c1..49ce2b53b364b8 100644 --- a/be/src/runtime/descriptor_helper.h +++ b/be/src/runtime/descriptor_helper.h @@ -77,7 +77,7 @@ class TSlotDescriptorBuilder { return type_desc; } - TSlotDescriptorBuilder() { _slot_desc.isMaterialized = true; } + TSlotDescriptorBuilder() = default; TSlotDescriptorBuilder& type(PrimitiveType type) { _slot_desc.slotType = get_common_type(to_thrift(type)); return *this; @@ -97,10 +97,6 @@ class TSlotDescriptorBuilder { _slot_desc.nullIndicatorByte = (nullable) ? 0 : -1; return *this; } - TSlotDescriptorBuilder& is_materialized(bool is_materialized) { - _slot_desc.isMaterialized = is_materialized; - return *this; - } TSlotDescriptorBuilder& column_name(const std::string& name) { _slot_desc.colName = name; return *this; @@ -135,10 +131,6 @@ class TSlotDescriptorBuilder { _slot_desc.slotIdx = slotIdx; return *this; } - TSlotDescriptorBuilder& set_isMaterialized(bool isMaterialized) { - _slot_desc.isMaterialized = isMaterialized; - return *this; - } TSlotDescriptorBuilder& set_colName(std::string colName) { _slot_desc.colName = colName; return *this; diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index d99cbdc038cbbd..005e6187aa628e 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -60,7 +60,6 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc) _col_unique_id(tdesc.col_unique_id), _slot_idx(tdesc.slotIdx), _field_idx(-1), - _is_materialized(tdesc.isMaterialized && tdesc.need_materialize), _is_key(tdesc.is_key), _column_paths(tdesc.column_paths), _is_auto_increment(tdesc.__isset.is_auto_increment ? tdesc.is_auto_increment : false), @@ -95,7 +94,6 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc) _col_unique_id(pdesc.col_unique_id()), _slot_idx(pdesc.slot_idx()), _field_idx(-1), - _is_materialized(pdesc.is_materialized()), _is_key(pdesc.is_key()), _column_paths(pdesc.column_paths().begin(), pdesc.column_paths().end()), _is_auto_increment(pdesc.is_auto_increment()) {} @@ -109,7 +107,6 @@ SlotDescriptor::SlotDescriptor() _col_unique_id(0), _slot_idx(0), _field_idx(-1), - _is_materialized(true), _is_key(false), _is_auto_increment(false) {} #endif @@ -124,7 +121,6 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const { pslot->set_null_indicator_bit(_type->is_nullable() ? 0 : -1); pslot->set_col_name(_col_name); pslot->set_slot_idx(_slot_idx); - pslot->set_is_materialized(_is_materialized); pslot->set_col_unique_id(_col_unique_id); pslot->set_is_key(_is_key); pslot->set_is_auto_increment(_is_auto_increment); @@ -366,15 +362,12 @@ TupleDescriptor::TupleDescriptor(const PTupleDescriptor& pdesc, bool own_slots) void TupleDescriptor::add_slot(SlotDescriptor* slot) { _slots.push_back(slot); + ++_num_materialized_slots; - if (slot->is_materialized()) { - ++_num_materialized_slots; - - if (is_complex_type(slot->type()->get_primitive_type()) || - is_var_len_object(slot->type()->get_primitive_type()) || - is_string_type(slot->type()->get_primitive_type())) { - _has_varlen_slots = true; - } + if (is_complex_type(slot->type()->get_primitive_type()) || + is_var_len_object(slot->type()->get_primitive_type()) || + is_string_type(slot->type()->get_primitive_type())) { + _has_varlen_slots = true; } } @@ -565,13 +558,10 @@ std::string RowDescriptor::debug_string() const { return ss.str(); } -int RowDescriptor::get_column_id(int slot_id, bool force_materialize_slot) const { +int RowDescriptor::get_column_id(int slot_id) const { int column_id_counter = 0; for (auto* const tuple_desc : _tuple_desc_map) { for (auto* const slot : tuple_desc->slots()) { - if (!force_materialize_slot && !slot->is_materialized()) { - continue; - } if (slot->id() == slot_id) { return column_id_counter; } diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index 0481c4ebfdbac7..973e8076b34ccc 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -25,8 +25,8 @@ #include #include #include -#include +#include #include #include #include @@ -42,6 +42,7 @@ #include "runtime/define_primitive_type.h" #include "runtime/types.h" #include "vec/data_types/data_type.h" + namespace google::protobuf { template class RepeatedField; @@ -57,14 +58,13 @@ class SlotDescriptor { public: MOCK_DEFINE(virtual ~SlotDescriptor() = default;) SlotId id() const { return _id; } - const vectorized::DataTypePtr type() const { return _type; } + vectorized::DataTypePtr type() const { return _type; } TupleId parent() const { return _parent; } // Returns the column index of this slot, including partition keys. // (e.g., col_pos - num_partition_keys = the table column this slot corresponds to) int col_pos() const { return _col_pos; } // Returns the field index in the generated llvm struct for this slot's tuple int field_idx() const { return _field_idx; } - bool is_materialized() const { return _is_materialized; } bool is_nullable() const; vectorized::DataTypePtr get_data_type_ptr() const; @@ -123,8 +123,6 @@ class SlotDescriptor { // leading null bytes. int _field_idx; - const bool _is_materialized; - const bool _is_key; const std::vector _column_paths; @@ -499,7 +497,7 @@ class RowDescriptor { std::string debug_string() const; - int get_column_id(int slot_id, bool force_materialize_slot = false) const; + int get_column_id(int slot_id) const; private: // Initializes tupleIdxMap during c'tor using the _tuple_desc_map. diff --git a/be/src/vec/common/sort/sorter.h b/be/src/vec/common/sort/sorter.h index 149939b9bd92bb..0c6150c0b8b999 100644 --- a/be/src/vec/common/sort/sorter.h +++ b/be/src/vec/common/sort/sorter.h @@ -56,8 +56,7 @@ class MergeSorterState { // create_empty_block should ignore invalid slots, unsorted_block // should be same structure with arrival block from child node // since block from child node may ignored these slots - : _unsorted_block(Block::create_unique( - VectorizedUtils::create_empty_block(row_desc, true /*ignore invalid slot*/))), + : _unsorted_block(Block::create_unique(VectorizedUtils::create_empty_block(row_desc))), _offset(offset) {} ~MergeSorterState() = default; diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index dbaf8ee023b835..3196d1134a7025 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -86,12 +86,8 @@ Block::Block(std::initializer_list il) : data {il} {} Block::Block(ColumnsWithTypeAndName data_) : data {std::move(data_)} {} -Block::Block(const std::vector& slots, size_t block_size, - bool ignore_trivial_slot) { +Block::Block(const std::vector& slots, size_t block_size) { for (auto* const slot_desc : slots) { - if (ignore_trivial_slot && !slot_desc->is_materialized()) { - continue; - } auto column_ptr = slot_desc->get_empty_mutable_column(); column_ptr->reserve(block_size); insert(ColumnWithTypeAndName(std::move(column_ptr), slot_desc->get_data_type_ptr(), @@ -99,13 +95,12 @@ Block::Block(const std::vector& slots, size_t block_size, } } -Block::Block(const std::vector& slots, size_t block_size, - bool ignore_trivial_slot) { +Block::Block(const std::vector& slots, size_t block_size) { std::vector slot_ptrs(slots.size()); for (size_t i = 0; i < slots.size(); ++i) { slot_ptrs[i] = const_cast(&slots[i]); } - *this = Block(slot_ptrs, block_size, ignore_trivial_slot); + *this = Block(slot_ptrs, block_size); } Status Block::deserialize(const PBlock& pblock, size_t* uncompressed_bytes, diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h index eb3849fa31a525..8a4b60bf2b10c0 100644 --- a/be/src/vec/core/block.h +++ b/be/src/vec/core/block.h @@ -79,10 +79,8 @@ class Block { Block() = default; Block(std::initializer_list il); Block(ColumnsWithTypeAndName data_); - Block(const std::vector& slots, size_t block_size, - bool ignore_trivial_slot = false); - Block(const std::vector& slots, size_t block_size, - bool ignore_trivial_slot = false); + Block(const std::vector& slots, size_t block_size); + Block(const std::vector& slots, size_t block_size); MOCK_FUNCTION ~Block() = default; Block(const Block& block) = default; diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index a78d6ea0ed6a76..ed3e83cceab5a5 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -986,9 +986,6 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val column_ptr->insert_default(); continue; } - if (!slot_desc->is_materialized()) { - continue; - } if (column_ptr->size() < cur_row_count + 1) { DCHECK(column_ptr->size() == cur_row_count); if (_should_process_skip_bitmap_col()) { @@ -1425,9 +1422,6 @@ Status NewJsonReader::_simdjson_write_columns_by_jsonpath( for (size_t i = 0; i < slot_descs.size(); i++) { auto* slot_desc = slot_descs[i]; - if (!slot_desc->is_materialized()) { - continue; - } auto* column_ptr = block.get_by_position(i).column->assume_mutable().get(); simdjson::ondemand::value json_value; Status st; diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index d72439e3c349a2..9e9b683c04e70d 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -2509,10 +2509,6 @@ Status OrcReader::on_string_dicts_loaded( int dict_pos = -1; int index = 0; for (const auto slot_desc : _tuple_descriptor->slots()) { - if (!slot_desc->is_materialized()) { - // should be ignored from reading - continue; - } if (slot_desc->id() == slot_id) { auto data_type = slot_desc->get_data_type_ptr(); if (data_type->is_nullable()) { diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp index c0a4b2a17046e0..f74fe7d2fec2a6 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp @@ -920,10 +920,6 @@ Status RowGroupReader::_rewrite_dict_predicates() { int dict_pos = -1; int index = 0; for (const auto slot_desc : _tuple_descriptor->slots()) { - if (!slot_desc->is_materialized()) { - // should be ignored from reading - continue; - } if (slot_desc->id() == slot_id) { auto data_type = slot_desc->get_data_type_ptr(); if (data_type->is_nullable()) { diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 2e1731aa36e020..bd70a2fdab7469 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -237,10 +237,6 @@ void FileScanner::_init_runtime_filter_partition_prune_ctxs() { void FileScanner::_init_runtime_filter_partition_prune_block() { // init block with empty column for (auto const* slot_desc : _real_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - // should be ignored from reading - continue; - } _runtime_filter_partition_prune_block.insert( ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), slot_desc->get_data_type_ptr(), slot_desc->col_name())); @@ -295,10 +291,6 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al size_t index = 0; bool first_column_filled = false; for (auto const* slot_desc : _real_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - // should be ignored from reading - continue; - } if (partition_slot_id_to_column.find(slot_desc->id()) != partition_slot_id_to_column.end()) { auto data_type = slot_desc->get_data_type_ptr(); @@ -775,10 +767,7 @@ Status FileScanner::_convert_to_output_block(Block* block) { // for (auto slot_desc : _output_tuple_desc->slots()) { for (int j = 0; j < mutable_output_columns.size(); ++j) { - auto slot_desc = _output_tuple_desc->slots()[j]; - if (!slot_desc->is_materialized()) { - continue; - } + auto* slot_desc = _output_tuple_desc->slots()[j]; int dest_index = ctx_idx; vectorized::ColumnPtr column_ptr; @@ -865,10 +854,7 @@ Status FileScanner::_truncate_char_or_varchar_columns(Block* block) { return Status::OK(); } int idx = 0; - for (auto slot_desc : _real_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } + for (auto* slot_desc : _real_tuple_desc->slots()) { const auto& type = slot_desc->type(); if (type->get_primitive_type() != TYPE_VARCHAR && type->get_primitive_type() != TYPE_CHAR) { ++idx; @@ -1590,11 +1576,8 @@ Status FileScanner::_generate_partition_columns() { Status FileScanner::_generate_missing_columns() { _missing_col_descs.clear(); if (!_missing_cols.empty()) { - for (auto slot_desc : _real_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } - if (_missing_cols.find(slot_desc->col_name()) == _missing_cols.end()) { + for (auto* slot_desc : _real_tuple_desc->slots()) { + if (!_missing_cols.contains(slot_desc->col_name())) { continue; } @@ -1654,8 +1637,7 @@ Status FileScanner::_init_expr_ctxes() { _file_col_names.push_back(it->second->col_name()); } - if (partition_name_to_key_index_map.find(it->second->col_name()) != - partition_name_to_key_index_map.end()) { + if (partition_name_to_key_index_map.contains(it->second->col_name())) { if (slot_info.is_file_slot) { // If there is slot which is both a partition column and a file column, // we should not fill the partition column from path. @@ -1679,10 +1661,7 @@ Status FileScanner::_init_expr_ctxes() { } // set column name to default value expr map - for (auto slot_desc : _real_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } + for (auto* slot_desc : _real_tuple_desc->slots()) { vectorized::VExprContextSPtr ctx; auto it = _params->default_value_of_src_slot.find(slot_desc->id()); if (it != std::end(_params->default_value_of_src_slot)) { @@ -1700,10 +1679,7 @@ Status FileScanner::_init_expr_ctxes() { // follow desc expr map is only for load task. bool has_slot_id_map = _params->__isset.dest_sid_to_src_sid_without_trans; int idx = 0; - for (auto slot_desc : _output_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } + for (auto* slot_desc : _output_tuple_desc->slots()) { auto it = _params->expr_of_dest_slot.find(slot_desc->id()); if (it == std::end(_params->expr_of_dest_slot)) { return Status::InternalError("No expr for dest slot, id={}, name={}", diff --git a/be/src/vec/exec/scan/meta_scanner.cpp b/be/src/vec/exec/scan/meta_scanner.cpp index 11c1d3ccde7ded..cb29a643cf7313 100644 --- a/be/src/vec/exec/scan/meta_scanner.cpp +++ b/be/src/vec/exec/scan/meta_scanner.cpp @@ -152,10 +152,6 @@ Status MetaScanner::_fill_block_with_remote_data(const std::vectorslots()[col_idx]; - // because the fe planner filter the non_materialize column - if (!slot_desc->is_materialized()) { - continue; - } for (int _row_idx = 0; _row_idx < _batch_data.size(); _row_idx++) { vectorized::IColumn* col_ptr = columns[col_idx].get(); diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp index e60bf769f2adc8..6031feccf88947 100644 --- a/be/src/vec/exec/scan/olap_scanner.cpp +++ b/be/src/vec/exec/scan/olap_scanner.cpp @@ -488,10 +488,7 @@ Status OlapScanner::_init_variant_columns() { return Status::OK(); } // Parent column has path info to distinction from each other - for (auto slot : _output_tuple_desc->slots()) { - if (!slot->is_materialized()) { - continue; - } + for (auto* slot : _output_tuple_desc->slots()) { if (slot->type()->get_primitive_type() == PrimitiveType::TYPE_VARIANT) { // Such columns are not exist in frontend schema info, so we need to // add them into tablet_schema for later column indexing. @@ -511,10 +508,6 @@ Status OlapScanner::_init_variant_columns() { Status OlapScanner::_init_return_columns() { for (auto* slot : _output_tuple_desc->slots()) { - if (!slot->is_materialized()) { - continue; - } - // variant column using path to index a column int32_t index = 0; auto& tablet_schema = _tablet_reader_params.tablet_schema; diff --git a/be/src/vec/exec/scan/scanner.cpp b/be/src/vec/exec/scan/scanner.cpp index d5ee62c2b791f8..3c71ea253845dd 100644 --- a/be/src/vec/exec/scan/scanner.cpp +++ b/be/src/vec/exec/scan/scanner.cpp @@ -94,10 +94,6 @@ Status Scanner::get_block(RuntimeState* state, Block* block, bool* eof) { int64_t rows_read_threshold = _num_rows_read + config::doris_scanner_row_num; if (!block->mem_reuse()) { for (auto* const slot_desc : _output_tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - // should be ignore from reading - continue; - } block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), slot_desc->get_data_type_ptr(), slot_desc->col_name())); diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp index beaf1f5f5cd8ed..916e14ff808547 100644 --- a/be/src/vec/exec/scan/scanner_context.cpp +++ b/be/src/vec/exec/scan/scanner_context.cpp @@ -237,8 +237,7 @@ vectorized::BlockUPtr ScannerContext::get_free_block(bool force) { // The caller of get_free_block will increase the memory usage } else if (_block_memory_usage < _max_bytes_in_queue || force) { _newly_create_free_blocks_num->update(1); - block = vectorized::Block::create_unique(_output_tuple_desc->slots(), 0, - true /*ignore invalid slots*/); + block = vectorized::Block::create_unique(_output_tuple_desc->slots(), 0); } return block; } @@ -377,9 +376,6 @@ Status ScannerContext::get_block_from_queue(RuntimeState* state, vectorized::Blo Status ScannerContext::validate_block_schema(Block* block) { size_t index = 0; for (auto& slot : _output_tuple_desc->slots()) { - if (!slot->is_materialized()) { - continue; - } auto& data = block->get_by_position(index++); if (data.column->is_nullable() != data.type->is_nullable()) { return Status::Error( diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index bce3c70d06d1d8..63116889f893b6 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -204,12 +204,7 @@ Status JdbcConnector::query() { return Status::InternalError("Query before open of JdbcConnector."); } // check materialize num equal - int materialize_num = 0; - for (int i = 0; i < _tuple_desc->slots().size(); ++i) { - if (_tuple_desc->slots()[i]->is_materialized()) { - materialize_num++; - } - } + auto materialize_num = _tuple_desc->slots().size(); JNIEnv* env = nullptr; RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); @@ -423,30 +418,27 @@ Status JdbcConnector::_get_reader_params(Block* block, JNIEnv* env, size_t colum for (int i = 0; i < column_size; ++i) { auto* slot = _tuple_desc->slots()[i]; - if (slot->is_materialized()) { - auto type = slot->type(); - // Record if column is nullable - columns_nullable << (slot->is_nullable() ? "true" : "false") << ","; - // Check column type and replace accordingly - std::string replace_type = "not_replace"; - if (type->get_primitive_type() == PrimitiveType::TYPE_BITMAP) { - replace_type = "bitmap"; - } else if (type->get_primitive_type() == PrimitiveType::TYPE_HLL) { - replace_type = "hll"; - } else if (type->get_primitive_type() == PrimitiveType::TYPE_JSONB) { - replace_type = "jsonb"; - } - columns_replace_string << replace_type << ","; - if (replace_type != "not_replace") { - block->get_by_position(i).column = std::make_shared() - ->create_column() - ->convert_to_full_column_if_const(); - block->get_by_position(i).type = std::make_shared(); - if (slot->is_nullable()) { - block->get_by_position(i).column = - make_nullable(block->get_by_position(i).column); - block->get_by_position(i).type = make_nullable(block->get_by_position(i).type); - } + auto type = slot->type(); + // Record if column is nullable + columns_nullable << (slot->is_nullable() ? "true" : "false") << ","; + // Check column type and replace accordingly + std::string replace_type = "not_replace"; + if (type->get_primitive_type() == PrimitiveType::TYPE_BITMAP) { + replace_type = "bitmap"; + } else if (type->get_primitive_type() == PrimitiveType::TYPE_HLL) { + replace_type = "hll"; + } else if (type->get_primitive_type() == PrimitiveType::TYPE_JSONB) { + replace_type = "jsonb"; + } + columns_replace_string << replace_type << ","; + if (replace_type != "not_replace") { + block->get_by_position(i).column = std::make_shared() + ->create_column() + ->convert_to_full_column_if_const(); + block->get_by_position(i).type = std::make_shared(); + if (slot->is_nullable()) { + block->get_by_position(i).column = make_nullable(block->get_by_position(i).column); + block->get_by_position(i).type = make_nullable(block->get_by_position(i).type); } } // Record required fields and column types @@ -473,10 +465,6 @@ Status JdbcConnector::_get_reader_params(Block* block, JNIEnv* env, size_t colum Status JdbcConnector::_cast_string_to_special(Block* block, JNIEnv* env, size_t column_size) { for (size_t column_index = 0; column_index < column_size; ++column_index) { auto* slot_desc = _tuple_desc->slots()[column_index]; - // because the fe planner filter the non_materialize column - if (!slot_desc->is_materialized()) { - continue; - } jint num_rows = env->CallNonvirtualIntMethod(_executor_obj, _executor_clazz, _executor_block_rows_id); diff --git a/be/src/vec/exprs/vexpr_context.h b/be/src/vec/exprs/vexpr_context.h index 74e37ad57ec7a6..bc57be586fc314 100644 --- a/be/src/vec/exprs/vexpr_context.h +++ b/be/src/vec/exprs/vexpr_context.h @@ -251,10 +251,6 @@ class VExprContext { void clone_fn_contexts(VExprContext* other); - bool force_materialize_slot() const { return _force_materialize_slot; } - - void set_force_materialize_slot() { _force_materialize_slot = true; } - VExprContext& operator=(const VExprContext& other) { if (this == &other) { return *this; @@ -333,10 +329,6 @@ class VExprContext { /// The depth of expression-tree. int _depth_num = 0; - // This flag only works on VSlotRef. - // Force to materialize even if the slot need_materialize is false, we just ignore need_materialize flag - bool _force_materialize_slot = false; - std::shared_ptr _index_context; size_t _memory_usage = 0; diff --git a/be/src/vec/exprs/virtual_slot_ref.cpp b/be/src/vec/exprs/virtual_slot_ref.cpp index 034f76cf9eb817..a2f090cf76f981 100644 --- a/be/src/vec/exprs/virtual_slot_ref.cpp +++ b/be/src/vec/exprs/virtual_slot_ref.cpp @@ -75,14 +75,7 @@ Status VirtualSlotRef::prepare(doris::RuntimeState* state, const doris::RowDescr _column_name = &slot_desc->col_name(); _column_data_type = slot_desc->get_data_type_ptr(); DCHECK(_column_data_type != nullptr); - if (!context->force_materialize_slot() && !slot_desc->is_materialized()) { - // slot should be ignored manually - _column_id = -1; - _prepare_finished = true; - return Status::OK(); - } - - _column_id = desc.get_column_id(_slot_id, context->force_materialize_slot()); + _column_id = desc.get_column_id(_slot_id); if (_column_id < 0) { return Status::Error( "VirtualSlotRef {} has invalid slot id: " diff --git a/be/src/vec/exprs/vslot_ref.cpp b/be/src/vec/exprs/vslot_ref.cpp index 086be05cea4e11..be6d96b2243079 100644 --- a/be/src/vec/exprs/vslot_ref.cpp +++ b/be/src/vec/exprs/vslot_ref.cpp @@ -21,7 +21,6 @@ #include #include -#include #include "common/status.h" #include "runtime/descriptors.h" @@ -29,13 +28,8 @@ #include "vec/core/block.h" #include "vec/exprs/vexpr_context.h" -namespace doris { -namespace vectorized { -class VExprContext; -} // namespace vectorized -} // namespace doris - namespace doris::vectorized { +class VExprContext; VSlotRef::VSlotRef(const doris::TExprNode& node) : VExpr(node), @@ -62,13 +56,7 @@ Status VSlotRef::prepare(doris::RuntimeState* state, const doris::RowDescriptor& state->desc_tbl().debug_string()); } _column_name = &slot_desc->col_name(); - if (!context->force_materialize_slot() && !slot_desc->is_materialized()) { - // slot should be ignored manually - _column_id = -1; - _prepare_finished = true; - return Status::OK(); - } - _column_id = desc.get_column_id(_slot_id, context->force_materialize_slot()); + _column_id = desc.get_column_id(_slot_id); if (_column_id < 0) { return Status::Error( "VSlotRef {} have invalid slot id: {}, desc: {}, slot_desc: {}, desc_tbl: {}", diff --git a/be/src/vec/utils/util.hpp b/be/src/vec/utils/util.hpp index ddbe0bf68b8f75..9081c0e05acaa9 100644 --- a/be/src/vec/utils/util.hpp +++ b/be/src/vec/utils/util.hpp @@ -72,9 +72,6 @@ class VectorizedUtils { ColumnsWithTypeAndName columns_with_type_and_name; for (const auto& tuple_desc : row_desc.tuple_descriptors()) { for (const auto& slot_desc : tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } columns_with_type_and_name.emplace_back(nullptr, slot_desc->get_data_type_ptr(), slot_desc->col_name()); } @@ -86,23 +83,16 @@ class VectorizedUtils { NameAndTypePairs name_with_types; for (const auto& tuple_desc : row_desc.tuple_descriptors()) { for (const auto& slot_desc : tuple_desc->slots()) { - if (!slot_desc->is_materialized()) { - continue; - } name_with_types.emplace_back(slot_desc->col_name(), slot_desc->get_data_type_ptr()); } } return name_with_types; } - static ColumnsWithTypeAndName create_empty_block(const RowDescriptor& row_desc, - bool ignore_trivial_slot = true) { + static ColumnsWithTypeAndName create_empty_block(const RowDescriptor& row_desc) { ColumnsWithTypeAndName columns_with_type_and_name; for (const auto& tuple_desc : row_desc.tuple_descriptors()) { for (const auto& slot_desc : tuple_desc->slots()) { - if (ignore_trivial_slot && !slot_desc->is_materialized()) { - continue; - } columns_with_type_and_name.emplace_back( slot_desc->get_data_type_ptr()->create_column(), slot_desc->get_data_type_ptr(), slot_desc->col_name()); diff --git a/be/test/exprs/virtual_slot_ref_test.cpp b/be/test/exprs/virtual_slot_ref_test.cpp index d1a378a06bbf46..633042718f5c4e 100644 --- a/be/test/exprs/virtual_slot_ref_test.cpp +++ b/be/test/exprs/virtual_slot_ref_test.cpp @@ -81,7 +81,6 @@ class VirtualSlotRefTest : public testing::Test { slot_desc->_id = SlotId(slot_id); slot_desc->_col_name = col_name; slot_desc->_type = data_type; - // Note: _is_materialized is const, so it's set during construction return slot_desc; } diff --git a/be/test/olap/vector_search/ann_topn_runtime_negative_test.cpp b/be/test/olap/vector_search/ann_topn_runtime_negative_test.cpp index a84d99ec7831d7..9cd38de7fed4d2 100644 --- a/be/test/olap/vector_search/ann_topn_runtime_negative_test.cpp +++ b/be/test/olap/vector_search/ann_topn_runtime_negative_test.cpp @@ -53,9 +53,6 @@ TEST_F(VectorSearchTest, AnnTopNRuntimePrepare_NoFunctionCall) { TSlotDescriptor slot0; slot0.id = 3000; slot0.parent = 2000; - slot0.isMaterialized = true; - slot0.need_materialize = true; - slot0.__isset.need_materialize = true; // type: DOUBLE (matches fixture) TTypeNode type_node; type_node.type = TTypeNodeType::type::SCALAR; diff --git a/be/test/olap/vector_search/vector_search_utils.h b/be/test/olap/vector_search/vector_search_utils.h index 2d8e621bde0ee0..909e1034566299 100644 --- a/be/test/olap/vector_search/vector_search_utils.h +++ b/be/test/olap/vector_search/vector_search_utils.h @@ -209,9 +209,6 @@ class VectorSearchTest : public ::testing::Test { TSlotDescriptor slot_desc; slot_desc.id = 0; slot_desc.parent = 0; - slot_desc.isMaterialized = true; - slot_desc.need_materialize = true; - slot_desc.__isset.need_materialize = true; TTypeNode type_node; type_node.type = TTypeNodeType::type::SCALAR; TScalarType scalar_type; diff --git a/be/test/olap/wal/wal_manager_test.cpp b/be/test/olap/wal/wal_manager_test.cpp index 15c4a94bde6462..16bbcbf7587be5 100644 --- a/be/test/olap/wal/wal_manager_test.cpp +++ b/be/test/olap/wal/wal_manager_test.cpp @@ -177,7 +177,6 @@ void WalManagerTest::_init_desc_table() { slot_desc.colName = "c1"; slot_desc.slotIdx = 1; slot_desc.col_unique_id = 0; - slot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(slot_desc); } @@ -205,7 +204,6 @@ void WalManagerTest::_init_desc_table() { slot_desc.colName = "c2"; slot_desc.slotIdx = 2; slot_desc.col_unique_id = 1; - slot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(slot_desc); } @@ -233,7 +231,6 @@ void WalManagerTest::_init_desc_table() { slot_desc.colName = "c3"; slot_desc.slotIdx = 3; slot_desc.col_unique_id = 2; - slot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(slot_desc); } diff --git a/be/test/pipeline/pipeline_test.cpp b/be/test/pipeline/pipeline_test.cpp index ac2fcc44b92305..0f185a49b967d3 100644 --- a/be/test/pipeline/pipeline_test.cpp +++ b/be/test/pipeline/pipeline_test.cpp @@ -204,7 +204,6 @@ TEST_F(PipelineTest, HAPPY_PATH) { .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) - .set_isMaterialized(true) .set_colName("test_column0") .build(); @@ -509,7 +508,6 @@ TEST_F(PipelineTest, PLAN_LOCAL_EXCHANGE) { .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) - .set_isMaterialized(true) .set_colName("test_column0") .build(); @@ -605,7 +603,6 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) - .set_isMaterialized(true) .set_colName("test_column0") .build(); @@ -624,7 +621,6 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) - .set_isMaterialized(true) .set_colName("test_column1") .build(); @@ -643,7 +639,6 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) - .set_isMaterialized(true) .set_colName("test_column0") .build(); TSlotDescriptor slot3 = @@ -660,7 +655,6 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .set_nullIndicatorBit(-1) .set_byteOffset(4) .set_slotIdx(1) - .set_isMaterialized(true) .set_colName("test_column1") .build(); diff --git a/be/test/runtime/descriptor_test.cpp b/be/test/runtime/descriptor_test.cpp index dc98b9afc4b508..20c2b1000c8303 100644 --- a/be/test/runtime/descriptor_test.cpp +++ b/be/test/runtime/descriptor_test.cpp @@ -42,7 +42,6 @@ class SlotDescriptorTest : public testing::Test { tdesc.__set_col_unique_id(slot_id); tdesc.__set_slotIdx(0); tdesc.__set_isMaterialized(true); - tdesc.__set_need_materialize(true); tdesc.__set_is_key(false); tdesc.__set_nullIndicatorBit(0); return tdesc; @@ -80,7 +79,6 @@ TEST_F(SlotDescriptorTest, BasicConstructor) { EXPECT_EQ(slot_desc.id(), 1); EXPECT_EQ(slot_desc.col_name(), "test_col"); EXPECT_EQ(slot_desc.col_unique_id(), 1); - EXPECT_TRUE(slot_desc.is_materialized()); EXPECT_FALSE(slot_desc.is_key()); EXPECT_EQ(slot_desc.get_virtual_column_expr(), nullptr); }); diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp index 9b316baca605c3..e0c9cb8247b17d 100644 --- a/be/test/vec/core/block_test.cpp +++ b/be/test/vec/core/block_test.cpp @@ -1101,10 +1101,7 @@ TEST(BlockTest, ctor) { tuple_builder .add_slot( TSlotDescriptorBuilder().type(PrimitiveType::TYPE_INT).nullable(false).build()) - .add_slot(TSlotDescriptorBuilder() - .type(PrimitiveType::TYPE_STRING) - .set_isMaterialized(false) - .build()); + .add_slot(TSlotDescriptorBuilder().type(PrimitiveType::TYPE_STRING).build()); tuple_builder.build(&builder); auto t_table = builder.desc_tbl(); diff --git a/be/test/vec/exec/format/parquet/parquet_expr_test.cpp b/be/test/vec/exec/format/parquet/parquet_expr_test.cpp index d61773ca7ccc62..8a6ab87a10fe72 100644 --- a/be/test/vec/exec/format/parquet/parquet_expr_test.cpp +++ b/be/test/vec/exec/format/parquet/parquet_expr_test.cpp @@ -324,7 +324,6 @@ class ParquetExprTest : public testing::Test { tslot_desc.nullIndicatorBit = -1; tslot_desc.colName = table_column_names[i]; tslot_desc.slotIdx = 0; - tslot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(tslot_desc); } } diff --git a/be/test/vec/exec/format/parquet/parquet_read_lines.cpp b/be/test/vec/exec/format/parquet/parquet_read_lines.cpp index 08864a86fef400..a9b4b48ad5ab20 100644 --- a/be/test/vec/exec/format/parquet/parquet_read_lines.cpp +++ b/be/test/vec/exec/format/parquet/parquet_read_lines.cpp @@ -93,7 +93,6 @@ static void read_parquet_lines(std::vector numeric_types, tslot_desc.nullIndicatorBit = -1; tslot_desc.colName = numeric_types[i]; tslot_desc.slotIdx = 0; - tslot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(tslot_desc); } } diff --git a/be/test/vec/exec/format/parquet/parquet_reader_test.cpp b/be/test/vec/exec/format/parquet/parquet_reader_test.cpp index 5d260334d1ef68..038d6012e537ec 100644 --- a/be/test/vec/exec/format/parquet/parquet_reader_test.cpp +++ b/be/test/vec/exec/format/parquet/parquet_reader_test.cpp @@ -91,7 +91,6 @@ static void create_table_desc(TDescriptorTable& t_desc_table, TTableDescriptor& tslot_desc.nullIndicatorBit = -1; tslot_desc.colName = table_column_names[i]; tslot_desc.slotIdx = 0; - tslot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(tslot_desc); } } diff --git a/be/test/vec/exec/sort/sort_test.cpp b/be/test/vec/exec/sort/sort_test.cpp index ceea6bb3bf653e..c4388a7365aeac 100644 --- a/be/test/vec/exec/sort/sort_test.cpp +++ b/be/test/vec/exec/sort/sort_test.cpp @@ -82,7 +82,7 @@ class SortTestParam { } void append_block(ColumnInt32::Ptr column) { - Block block = VectorizedUtils::create_empty_block(*row_desc, true /*ignore invalid slot*/); + Block block = VectorizedUtils::create_empty_block(*row_desc); block.get_by_position(0).column = column->clone(); EXPECT_TRUE(sorter->append_block(&block).ok()); } diff --git a/be/test/vec/exec/vfile_scanner_exception_test.cpp b/be/test/vec/exec/vfile_scanner_exception_test.cpp index 1a3ebd6e8986a5..e5d38b18c54e72 100644 --- a/be/test/vec/exec/vfile_scanner_exception_test.cpp +++ b/be/test/vec/exec/vfile_scanner_exception_test.cpp @@ -152,7 +152,6 @@ void VfileScannerExceptionTest::_init_desc_table() { slot_desc.colName = "c1"; slot_desc.slotIdx = 1; slot_desc.col_unique_id = 0; - slot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(slot_desc); } @@ -180,7 +179,6 @@ void VfileScannerExceptionTest::_init_desc_table() { slot_desc.colName = "c2"; slot_desc.slotIdx = 2; slot_desc.col_unique_id = 1; - slot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(slot_desc); } @@ -208,7 +206,6 @@ void VfileScannerExceptionTest::_init_desc_table() { slot_desc.colName = "c3"; slot_desc.slotIdx = 3; slot_desc.col_unique_id = 2; - slot_desc.isMaterialized = true; t_desc_table.slotDescriptors.push_back(slot_desc); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 9c3596a5b6521c..ad7700f77ab62d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -55,10 +55,6 @@ public class SlotDescriptor { // if false, this slot cannot be NULL private boolean isNullable; - - // If set to false, then such slots will be ignored during - // materialize them.Used to optimize to read less data and less memory usage - private boolean needMaterialize = true; private boolean isAutoInc = false; private Expr virtualColumn = null; @@ -78,14 +74,6 @@ public SlotDescriptor(SlotId id, TupleDescriptor parent, SlotDescriptor src) { this.sourceExprs.add(new SlotRef(src)); } - public void setNeedMaterialize(boolean needMaterialize) { - this.needMaterialize = needMaterialize; - } - - public boolean isInvalid() { - return !this.needMaterialize; - } - public SlotId getId() { return id; } @@ -181,7 +169,6 @@ public TSlotDescriptor toThrift() { TSlotDescriptor tSlotDescriptor = new TSlotDescriptor(id.asInt(), parent.getId().asInt(), type.toThrift(), -1, 0, 0, getIsNullable() ? 0 : -1, colName, -1, true); - tSlotDescriptor.setNeedMaterialize(needMaterialize); tSlotDescriptor.setIsAutoIncrement(isAutoInc); if (column != null) { if (LOG.isDebugEnabled()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index e2da4a05ed4a62..48cfbc20d8632e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -992,12 +992,6 @@ public PlanFragment visitPhysicalDeferMaterializeOlapScan( PlanFragment planFragment = visitPhysicalOlapScan(deferMaterializeOlapScan.getPhysicalOlapScan(), context); OlapScanNode olapScanNode = (OlapScanNode) planFragment.getPlanRoot(); TupleDescriptor tupleDescriptor = context.getTupleDesc(olapScanNode.getTupleId()); - for (SlotDescriptor slotDescriptor : tupleDescriptor.getSlots()) { - if (deferMaterializeOlapScan.getDeferMaterializeSlotIds() - .contains(context.findExprId(slotDescriptor.getId()))) { - slotDescriptor.setNeedMaterialize(false); - } - } context.createSlotDesc(tupleDescriptor, deferMaterializeOlapScan.getColumnIdSlot()); context.getTopnFilterContext().translateTarget(deferMaterializeOlapScan, olapScanNode, context); return planFragment; @@ -2464,13 +2458,6 @@ public PlanFragment visitPhysicalDeferMaterializeTopN(PhysicalDeferMaterializeTo if (context.getTopnFilterContext().isTopnFilterSource(topN)) { context.getTopnFilterContext().translateSource(topN, sortNode); } - TupleDescriptor tupleDescriptor = sortNode.getSortInfo().getSortTupleDescriptor(); - for (SlotDescriptor slotDescriptor : tupleDescriptor.getSlots()) { - if (topN.getDeferMaterializeSlotIds() - .contains(context.findExprId(slotDescriptor.getId()))) { - slotDescriptor.setNeedMaterialize(false); - } - } } return planFragment; } diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index 31008c9072545b..99ee06ca6a0a8b 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -33,7 +33,7 @@ message PSlotDescriptor { required int32 null_indicator_bit = 7; required string col_name = 8; required int32 slot_idx = 9; - optional bool is_materialized = 10; + optional bool is_materialized = 10 [deprecated = true]; optional int32 col_unique_id = 11; optional bool is_key = 12; optional bool is_auto_increment = 13; diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index b9fe52c59f2a4c..53ad8a46ec42b5 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -65,12 +65,12 @@ struct TSlotDescriptor { 7: required i32 nullIndicatorBit 8: required string colName; 9: required i32 slotIdx - 10: required bool isMaterialized + 10: required bool isMaterialized // deprecated 11: optional i32 col_unique_id = -1 12: optional bool is_key = false // If set to false, then such slots will be ignored during - // materialize them.Used to optmize to read less data and less memory usage - 13: optional bool need_materialize = true + // materialize them.Used to optimize to read less data and less memory usage + 13: optional bool need_materialize = true // deprecated 14: optional bool is_auto_increment = false; // subcolumn path info list for semi structure column(variant) 15: optional list column_paths