branch-4.0: [chore](cleanup) remove useless field is_materialized in SlotDescriptor (#58320)

be/src/exec/es/es_scroll_parser.cpp

@@ -660,11 +660,8 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc,
 
     for (int i = 0; i < tuple_desc->slots().size(); ++i) {
         const SlotDescriptor* slot_desc = tuple_desc->slots()[i];
-        auto col_ptr = columns[i].get();
+        auto* col_ptr = columns[i].get();
 
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
         if (slot_desc->col_name() == FIELD_ID) {
             // actually this branch will not be reached, this is guaranteed by Doris FE.
             if (pure_doc_value) {

be/src/olap/push_handler.cpp

@@ -525,10 +525,7 @@ Status PushBrokerReader::_convert_to_output_block(vectorized::Block* block) {
     size_t rows = _src_block.rows();
     auto filter_column = vectorized::ColumnUInt8::create(rows, 1);
 
-    for (auto slot_desc : _dest_tuple_desc->slots()) {
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
+    for (auto* slot_desc : _dest_tuple_desc->slots()) {
         int dest_index = ctx_idx++;
         vectorized::ColumnPtr column_ptr;
 
@@ -612,9 +609,6 @@ Status PushBrokerReader::_init_expr_ctxes() {
     }
     bool has_slot_id_map = _params.__isset.dest_sid_to_src_sid_without_trans;
     for (auto slot_desc : _dest_tuple_desc->slots()) {
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
         auto it = _params.expr_of_dest_slot.find(slot_desc->id());
         if (it == std::end(_params.expr_of_dest_slot)) {
             return Status::InternalError("No expr for dest slot, id={}, name={}", slot_desc->id(),

be/src/pipeline/exec/es_scan_operator.cpp

@@ -142,10 +142,7 @@ Status EsScanOperatorX::prepare(RuntimeState* state) {
     }
 
     // set up column name vector for ESScrollQueryBuilder
-    for (auto slot_desc : _tuple_desc->slots()) {
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
+    for (auto* slot_desc : _tuple_desc->slots()) {
         _column_names.push_back(slot_desc->col_name());
     }
 

be/src/pipeline/exec/result_sink_operator.cpp

@@ -121,12 +121,6 @@ Status ResultSinkOperatorX::prepare(RuntimeState* state) {
     // prepare output_expr
     // From the thrift expressions create the real exprs.
     RETURN_IF_ERROR(vectorized::VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs));
-    if (_fetch_option.use_two_phase_fetch) {
-        for (auto& expr_ctx : _output_vexpr_ctxs) {
-            // Must materialize if it a slot, or the slot column id will be -1
-            expr_ctx->set_force_materialize_slot();
-        }
-    }
     // Prepare the exprs to run.
     RETURN_IF_ERROR(vectorized::VExpr::prepare(_output_vexpr_ctxs, state, _row_desc));
 

be/src/runtime/descriptor_helper.h

@@ -77,7 +77,7 @@ class TSlotDescriptorBuilder {
         return type_desc;
     }
 
-    TSlotDescriptorBuilder() { _slot_desc.isMaterialized = true; }
+    TSlotDescriptorBuilder() = default;
     TSlotDescriptorBuilder& type(PrimitiveType type) {
         _slot_desc.slotType = get_common_type(to_thrift(type));
         return *this;
@@ -97,10 +97,6 @@ class TSlotDescriptorBuilder {
         _slot_desc.nullIndicatorByte = (nullable) ? 0 : -1;
         return *this;
     }
-    TSlotDescriptorBuilder& is_materialized(bool is_materialized) {
-        _slot_desc.isMaterialized = is_materialized;
-        return *this;
-    }
     TSlotDescriptorBuilder& column_name(const std::string& name) {
         _slot_desc.colName = name;
         return *this;
@@ -135,10 +131,6 @@ class TSlotDescriptorBuilder {
         _slot_desc.slotIdx = slotIdx;
         return *this;
     }
-    TSlotDescriptorBuilder& set_isMaterialized(bool isMaterialized) {
-        _slot_desc.isMaterialized = isMaterialized;
-        return *this;
-    }
     TSlotDescriptorBuilder& set_colName(std::string colName) {
         _slot_desc.colName = colName;
         return *this;

be/src/runtime/descriptors.cpp

@@ -60,7 +60,6 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc)
           _col_unique_id(tdesc.col_unique_id),
           _slot_idx(tdesc.slotIdx),
           _field_idx(-1),
-          _is_materialized(tdesc.isMaterialized && tdesc.need_materialize),
           _is_key(tdesc.is_key),
           _column_paths(tdesc.column_paths),
           _is_auto_increment(tdesc.__isset.is_auto_increment ? tdesc.is_auto_increment : false),
@@ -95,7 +94,6 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc)
           _col_unique_id(pdesc.col_unique_id()),
           _slot_idx(pdesc.slot_idx()),
           _field_idx(-1),
-          _is_materialized(pdesc.is_materialized()),
           _is_key(pdesc.is_key()),
           _column_paths(pdesc.column_paths().begin(), pdesc.column_paths().end()),
           _is_auto_increment(pdesc.is_auto_increment()) {}
@@ -109,7 +107,6 @@ SlotDescriptor::SlotDescriptor()
           _col_unique_id(0),
           _slot_idx(0),
           _field_idx(-1),
-          _is_materialized(true),
           _is_key(false),
           _is_auto_increment(false) {}
 #endif
@@ -124,7 +121,6 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const {
     pslot->set_null_indicator_bit(_type->is_nullable() ? 0 : -1);
     pslot->set_col_name(_col_name);
     pslot->set_slot_idx(_slot_idx);
-    pslot->set_is_materialized(_is_materialized);
     pslot->set_col_unique_id(_col_unique_id);
     pslot->set_is_key(_is_key);
     pslot->set_is_auto_increment(_is_auto_increment);
@@ -366,15 +362,12 @@ TupleDescriptor::TupleDescriptor(const PTupleDescriptor& pdesc, bool own_slots)
 
 void TupleDescriptor::add_slot(SlotDescriptor* slot) {
     _slots.push_back(slot);
+    ++_num_materialized_slots;
 
-    if (slot->is_materialized()) {
-        ++_num_materialized_slots;
-
-        if (is_complex_type(slot->type()->get_primitive_type()) ||
-            is_var_len_object(slot->type()->get_primitive_type()) ||
-            is_string_type(slot->type()->get_primitive_type())) {
-            _has_varlen_slots = true;
-        }
+    if (is_complex_type(slot->type()->get_primitive_type()) ||
+        is_var_len_object(slot->type()->get_primitive_type()) ||
+        is_string_type(slot->type()->get_primitive_type())) {
+        _has_varlen_slots = true;
     }
 }
 
@@ -565,13 +558,10 @@ std::string RowDescriptor::debug_string() const {
     return ss.str();
 }
 
-int RowDescriptor::get_column_id(int slot_id, bool force_materialize_slot) const {
+int RowDescriptor::get_column_id(int slot_id) const {
     int column_id_counter = 0;
     for (auto* const tuple_desc : _tuple_desc_map) {
         for (auto* const slot : tuple_desc->slots()) {
-            if (!force_materialize_slot && !slot->is_materialized()) {
-                continue;
-            }
             if (slot->id() == slot_id) {
                 return column_id_counter;
             }

be/src/runtime/descriptors.h

@@ -25,8 +25,8 @@
 #include <gen_cpp/Types_types.h>
 #include <glog/logging.h>
 #include <google/protobuf/stubs/port.h>
-#include <stdint.h>
 
+#include <cstdint>
 #include <ostream>
 #include <string>
 #include <unordered_map>
@@ -42,6 +42,7 @@
 #include "runtime/define_primitive_type.h"
 #include "runtime/types.h"
 #include "vec/data_types/data_type.h"
+
 namespace google::protobuf {
 template <typename Element>
 class RepeatedField;
@@ -57,14 +58,13 @@ class SlotDescriptor {
 public:
     MOCK_DEFINE(virtual ~SlotDescriptor() = default;)
     SlotId id() const { return _id; }
-    const vectorized::DataTypePtr type() const { return _type; }
+    vectorized::DataTypePtr type() const { return _type; }
     TupleId parent() const { return _parent; }
     // Returns the column index of this slot, including partition keys.
     // (e.g., col_pos - num_partition_keys = the table column this slot corresponds to)
     int col_pos() const { return _col_pos; }
     // Returns the field index in the generated llvm struct for this slot's tuple
     int field_idx() const { return _field_idx; }
-    bool is_materialized() const { return _is_materialized; }
     bool is_nullable() const;
     vectorized::DataTypePtr get_data_type_ptr() const;
 
@@ -123,8 +123,6 @@ class SlotDescriptor {
     // leading null bytes.
     int _field_idx;
 
-    const bool _is_materialized;
-
     const bool _is_key;
     const std::vector<std::string> _column_paths;
 
@@ -499,7 +497,7 @@ class RowDescriptor {
 
     std::string debug_string() const;
 
-    int get_column_id(int slot_id, bool force_materialize_slot = false) const;
+    int get_column_id(int slot_id) const;
 
 private:
     // Initializes tupleIdxMap during c'tor using the _tuple_desc_map.

be/src/vec/common/sort/sorter.h

@@ -56,8 +56,7 @@ class MergeSorterState {
             // create_empty_block should ignore invalid slots, unsorted_block
             // should be same structure with arrival block from child node
             // since block from child node may ignored these slots
-            : _unsorted_block(Block::create_unique(
-                      VectorizedUtils::create_empty_block(row_desc, true /*ignore invalid slot*/))),
+            : _unsorted_block(Block::create_unique(VectorizedUtils::create_empty_block(row_desc))),
               _offset(offset) {}
 
     ~MergeSorterState() = default;

be/src/vec/core/block.cpp

@@ -86,26 +86,21 @@ Block::Block(std::initializer_list<ColumnWithTypeAndName> il) : data {il} {}
 
 Block::Block(ColumnsWithTypeAndName data_) : data {std::move(data_)} {}
 
-Block::Block(const std::vector<SlotDescriptor*>& slots, size_t block_size,
-             bool ignore_trivial_slot) {
+Block::Block(const std::vector<SlotDescriptor*>& slots, size_t block_size) {
     for (auto* const slot_desc : slots) {
-        if (ignore_trivial_slot && !slot_desc->is_materialized()) {
-            continue;
-        }
         auto column_ptr = slot_desc->get_empty_mutable_column();
         column_ptr->reserve(block_size);
         insert(ColumnWithTypeAndName(std::move(column_ptr), slot_desc->get_data_type_ptr(),
                                      slot_desc->col_name()));
     }
 }
 
-Block::Block(const std::vector<SlotDescriptor>& slots, size_t block_size,
-             bool ignore_trivial_slot) {
+Block::Block(const std::vector<SlotDescriptor>& slots, size_t block_size) {
     std::vector<SlotDescriptor*> slot_ptrs(slots.size());
     for (size_t i = 0; i < slots.size(); ++i) {
         slot_ptrs[i] = const_cast<SlotDescriptor*>(&slots[i]);
     }
-    *this = Block(slot_ptrs, block_size, ignore_trivial_slot);
+    *this = Block(slot_ptrs, block_size);
 }
 
 Status Block::deserialize(const PBlock& pblock, size_t* uncompressed_bytes,

be/src/vec/core/block.h

@@ -79,10 +79,8 @@ class Block {
     Block() = default;
     Block(std::initializer_list<ColumnWithTypeAndName> il);
     Block(ColumnsWithTypeAndName data_);
-    Block(const std::vector<SlotDescriptor*>& slots, size_t block_size,
-          bool ignore_trivial_slot = false);
-    Block(const std::vector<SlotDescriptor>& slots, size_t block_size,
-          bool ignore_trivial_slot = false);
+    Block(const std::vector<SlotDescriptor*>& slots, size_t block_size);
+    Block(const std::vector<SlotDescriptor>& slots, size_t block_size);
 
     MOCK_FUNCTION ~Block() = default;
     Block(const Block& block) = default;

be/src/vec/exec/format/json/new_json_reader.cpp

@@ -986,9 +986,6 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
             column_ptr->insert_default();
             continue;
         }
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
         if (column_ptr->size() < cur_row_count + 1) {
             DCHECK(column_ptr->size() == cur_row_count);
             if (_should_process_skip_bitmap_col()) {
@@ -1425,9 +1422,6 @@ Status NewJsonReader::_simdjson_write_columns_by_jsonpath(
 
     for (size_t i = 0; i < slot_descs.size(); i++) {
         auto* slot_desc = slot_descs[i];
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
         auto* column_ptr = block.get_by_position(i).column->assume_mutable().get();
         simdjson::ondemand::value json_value;
         Status st;

be/src/vec/exec/format/orc/vorc_reader.cpp

@@ -2509,10 +2509,6 @@ Status OrcReader::on_string_dicts_loaded(
         int dict_pos = -1;
         int index = 0;
         for (const auto slot_desc : _tuple_descriptor->slots()) {
-            if (!slot_desc->is_materialized()) {
-                // should be ignored from reading
-                continue;
-            }
             if (slot_desc->id() == slot_id) {
                 auto data_type = slot_desc->get_data_type_ptr();
                 if (data_type->is_nullable()) {

be/src/vec/exec/format/parquet/vparquet_group_reader.cpp

@@ -920,10 +920,6 @@ Status RowGroupReader::_rewrite_dict_predicates() {
         int dict_pos = -1;
         int index = 0;
         for (const auto slot_desc : _tuple_descriptor->slots()) {
-            if (!slot_desc->is_materialized()) {
-                // should be ignored from reading
-                continue;
-            }
             if (slot_desc->id() == slot_id) {
                 auto data_type = slot_desc->get_data_type_ptr();
                 if (data_type->is_nullable()) {

be/src/vec/exec/scan/file_scanner.cpp

@@ -237,10 +237,6 @@ void FileScanner::_init_runtime_filter_partition_prune_ctxs() {
 void FileScanner::_init_runtime_filter_partition_prune_block() {
     // init block with empty column
     for (auto const* slot_desc : _real_tuple_desc->slots()) {
-        if (!slot_desc->is_materialized()) {
-            // should be ignored from reading
-            continue;
-        }
         _runtime_filter_partition_prune_block.insert(
                 ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(),
                                       slot_desc->get_data_type_ptr(), slot_desc->col_name()));
@@ -295,10 +291,6 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al
     size_t index = 0;
     bool first_column_filled = false;
     for (auto const* slot_desc : _real_tuple_desc->slots()) {
-        if (!slot_desc->is_materialized()) {
-            // should be ignored from reading
-            continue;
-        }
         if (partition_slot_id_to_column.find(slot_desc->id()) !=
             partition_slot_id_to_column.end()) {
             auto data_type = slot_desc->get_data_type_ptr();
@@ -775,10 +767,7 @@ Status FileScanner::_convert_to_output_block(Block* block) {
 
     // for (auto slot_desc : _output_tuple_desc->slots()) {
     for (int j = 0; j < mutable_output_columns.size(); ++j) {
-        auto slot_desc = _output_tuple_desc->slots()[j];
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
+        auto* slot_desc = _output_tuple_desc->slots()[j];
         int dest_index = ctx_idx;
         vectorized::ColumnPtr column_ptr;
 
@@ -865,10 +854,7 @@ Status FileScanner::_truncate_char_or_varchar_columns(Block* block) {
         return Status::OK();
     }
     int idx = 0;
-    for (auto slot_desc : _real_tuple_desc->slots()) {
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
+    for (auto* slot_desc : _real_tuple_desc->slots()) {
         const auto& type = slot_desc->type();
         if (type->get_primitive_type() != TYPE_VARCHAR && type->get_primitive_type() != TYPE_CHAR) {
             ++idx;
@@ -1590,11 +1576,8 @@ Status FileScanner::_generate_partition_columns() {
 Status FileScanner::_generate_missing_columns() {
     _missing_col_descs.clear();
     if (!_missing_cols.empty()) {
-        for (auto slot_desc : _real_tuple_desc->slots()) {
-            if (!slot_desc->is_materialized()) {
-                continue;
-            }
-            if (_missing_cols.find(slot_desc->col_name()) == _missing_cols.end()) {
+        for (auto* slot_desc : _real_tuple_desc->slots()) {
+            if (!_missing_cols.contains(slot_desc->col_name())) {
                 continue;
             }
 
@@ -1654,8 +1637,7 @@ Status FileScanner::_init_expr_ctxes() {
             _file_col_names.push_back(it->second->col_name());
         }
 
-        if (partition_name_to_key_index_map.find(it->second->col_name()) !=
-            partition_name_to_key_index_map.end()) {
+        if (partition_name_to_key_index_map.contains(it->second->col_name())) {
             if (slot_info.is_file_slot) {
                 // If there is slot which is both a partition column and a file column,
                 // we should not fill the partition column from path.
@@ -1679,10 +1661,7 @@ Status FileScanner::_init_expr_ctxes() {
     }
 
     // set column name to default value expr map
-    for (auto slot_desc : _real_tuple_desc->slots()) {
-        if (!slot_desc->is_materialized()) {
-            continue;
-        }
+    for (auto* slot_desc : _real_tuple_desc->slots()) {
         vectorized::VExprContextSPtr ctx;
         auto it = _params->default_value_of_src_slot.find(slot_desc->id());
         if (it != std::end(_params->default_value_of_src_slot)) {
@@ -1700,10 +1679,7 @@ Status FileScanner::_init_expr_ctxes() {
         // follow desc expr map is only for load task.
         bool has_slot_id_map = _params->__isset.dest_sid_to_src_sid_without_trans;
         int idx = 0;
-        for (auto slot_desc : _output_tuple_desc->slots()) {
-            if (!slot_desc->is_materialized()) {
-                continue;
-            }
+        for (auto* slot_desc : _output_tuple_desc->slots()) {
             auto it = _params->expr_of_dest_slot.find(slot_desc->id());
             if (it == std::end(_params->expr_of_dest_slot)) {
                 return Status::InternalError("No expr for dest slot, id={}, name={}",