Merge branch 'master' into fix-restore-schema

Author: xy720

be/src/olap/rowset/segment_v2/bloom_filter.h

@@ -194,7 +194,7 @@ class BloomFilter {
     /// Bloom filters must have equal size and seed.
     virtual bool contains(const BloomFilter& bf_) const { return true; };
 
-    virtual char* data() const { return _data; }
+    virtual const char* data() const { return _data; }
 
     size_t num_bytes() const { return _num_bytes; }
 

be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp

@@ -738,12 +738,12 @@ bool DorisRAMFSDirectory::list(std::vector<std::string>* names) const {
 
 bool DorisRAMFSDirectory::fileExists(const char* name) const {
     std::lock_guard<std::mutex> wlock(_this_lock);
-    return filesMap->exists((char*)name);
+    return filesMap->exists(name);
 }
 
 int64_t DorisRAMFSDirectory::fileModified(const char* name) const {
     std::lock_guard<std::mutex> wlock(_this_lock);
-    auto* f = filesMap->get((char*)name);
+    auto* f = filesMap->get(name);
     DBUG_EXECUTE_IF("DorisRAMFSDirectory::fileModified_file_not_found", { f = nullptr; })
     if (f == nullptr) {
         _CLTHROWA(CL_ERR_IO, fmt::format("NOT FOUND File {}.", name).c_str());
@@ -755,7 +755,7 @@ void DorisRAMFSDirectory::touchFile(const char* name) {
     lucene::store::RAMFile* file = nullptr;
     {
         std::lock_guard<std::mutex> wlock(_this_lock);
-        file = filesMap->get((char*)name);
+        file = filesMap->get(name);
         DBUG_EXECUTE_IF("DorisRAMFSDirectory::touchFile_file_not_found", { file = nullptr; })
         if (file == nullptr) {
             _CLTHROWA(CL_ERR_IO, fmt::format("NOT FOUND File {}.", name).c_str());
@@ -775,7 +775,7 @@ void DorisRAMFSDirectory::touchFile(const char* name) {
 
 int64_t DorisRAMFSDirectory::fileLength(const char* name) const {
     std::lock_guard<std::mutex> wlock(_this_lock);
-    auto* f = filesMap->get((char*)name);
+    auto* f = filesMap->get(name);
     DBUG_EXECUTE_IF("DorisRAMFSDirectory::fileLength_file_not_found", { f = nullptr; })
     if (f == nullptr) {
         _CLTHROWA(CL_ERR_IO, fmt::format("NOT FOUND File {}.", name).c_str());
@@ -786,7 +786,7 @@ int64_t DorisRAMFSDirectory::fileLength(const char* name) const {
 bool DorisRAMFSDirectory::openInput(const char* name, lucene::store::IndexInput*& ret,
                                     CLuceneError& error, int32_t bufferSize) {
     std::lock_guard<std::mutex> wlock(_this_lock);
-    auto* file = filesMap->get((char*)name);
+    auto* file = filesMap->get(name);
     DBUG_EXECUTE_IF("DorisRAMFSDirectory::openInput_file_not_found", { file = nullptr; })
     if (file == nullptr) {
         error.set(CL_ERR_IO,
@@ -805,7 +805,7 @@ void DorisRAMFSDirectory::close() {
 
 bool DorisRAMFSDirectory::doDeleteFile(const char* name) {
     std::lock_guard<std::mutex> wlock(_this_lock);
-    auto itr = filesMap->find((char*)name);
+    auto itr = filesMap->find(name);
     if (itr != filesMap->end()) {
         SCOPED_LOCK_MUTEX(this->THIS_LOCK);
         sizeInBytes -= itr->second->sizeInBytes;
@@ -821,15 +821,15 @@ bool DorisRAMFSDirectory::deleteDirectory() {
 
 void DorisRAMFSDirectory::renameFile(const char* from, const char* to) {
     std::lock_guard<std::mutex> wlock(_this_lock);
-    auto itr = filesMap->find((char*)from);
+    auto itr = filesMap->find(from);
 
     /* DSR:CL_BUG_LEAK:
     ** If a file named $to already existed, its old value was leaked.
     ** My inclination would be to prevent this implicit deletion with an
     ** exception, but it happens routinely in CLucene's internals (e.g., during
     ** IndexWriter.addIndexes with the file named 'segments'). */
-    if (filesMap->exists((char*)to)) {
-        auto itr1 = filesMap->find((char*)to);
+    if (filesMap->exists(to)) {
+        auto itr1 = filesMap->find(to);
         SCOPED_LOCK_MUTEX(this->THIS_LOCK);
         sizeInBytes -= itr1->second->sizeInBytes;
         filesMap->removeitr(itr1);
@@ -855,8 +855,8 @@ lucene::store::IndexOutput* DorisRAMFSDirectory::createOutput(const char* name)
     std::lock_guard<std::mutex> wlock(_this_lock);
 
     // get the actual pointer to the output name
-    char* n = nullptr;
-    auto itr = filesMap->find(const_cast<char*>(name));
+    const char* n = nullptr;
+    auto itr = filesMap->find(name);
     DBUG_EXECUTE_IF("DorisRAMFSDirectory::createOutput_itr_filesMap_end",
                     { itr = filesMap->end(); })
     if (itr != filesMap->end()) {

be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h

@@ -102,10 +102,10 @@ class CLUCENE_EXPORT DorisFSDirectory : public lucene::store::Directory {
 
 class CLUCENE_EXPORT DorisRAMFSDirectory : public DorisFSDirectory {
 protected:
-    using FileMap =
-            lucene::util::CLHashMap<char*, lucene::store::RAMFile*, lucene::util::Compare::Char,
-                                    lucene::util::Equals::Char, lucene::util::Deletor::acArray,
-                                    lucene::util::Deletor::Object<lucene::store::RAMFile>>;
+    using FileMap = lucene::util::CLHashMap<const char*, lucene::store::RAMFile*,
+                                            lucene::util::Compare::Char, lucene::util::Equals::Char,
+                                            lucene::util::Deletor::cacArray,
+                                            lucene::util::Deletor::Object<lucene::store::RAMFile>>;
 
     // unlike the java Hashtable, FileMap is not synchronized, and all access must be protected by a lock
     FileMap* filesMap;

be/src/olap/rowset/segment_v2/inverted_index_writer.cpp

@@ -542,7 +542,7 @@ Status InvertedIndexColumnWriter<field_type>::add_array_values(size_t field_size
             return Status::InternalError("field or index writer is null in inverted index writer");
         }
         for (int i = 0; i < count; ++i) {
-            auto* item_data_ptr = const_cast<CollectionValue*>(values)->mutable_data();
+            const auto* item_data_ptr = values->data();
             std::vector<std::string> strings;
 
             for (size_t j = 0; j < values->length(); ++j) {
@@ -561,7 +561,7 @@ Status InvertedIndexColumnWriter<field_type>::add_array_values(size_t field_size
         }
     } else if constexpr (field_is_numeric_type(field_type)) {
         for (int i = 0; i < count; ++i) {
-            auto* item_data_ptr = const_cast<CollectionValue*>(values)->mutable_data();
+            const auto* item_data_ptr = values->data();
 
             for (size_t j = 0; j < values->length(); ++j) {
                 const auto* p = reinterpret_cast<const CppType*>(item_data_ptr);

be/src/olap/rowset/segment_v2/ngram_bloom_filter.h

@@ -41,9 +41,7 @@ class NGramBloomFilter : public BloomFilter {
     void add_bytes(const char* data, size_t len) override;
     bool contains(const BloomFilter& bf_) const override;
     Status init(const char* buf, size_t size, HashStrategyPB strategy) override;
-    char* data() const override {
-        return reinterpret_cast<char*>(const_cast<uint64_t*>(filter.data()));
-    }
+    const char* data() const override { return reinterpret_cast<const char*>(filter.data()); }
     size_t size() const override { return _size; }
     void add_hash(uint64_t) override {}
     bool test_hash(uint64_t hash) const override { return true; }

be/src/olap/rowset/segment_v2/page_io.cpp

@@ -176,6 +176,7 @@ Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle
     if (opts.verify_checksum) {
         uint32_t expect = decode_fixed32_le((uint8_t*)page_slice.data + page_slice.size - 4);
         uint32_t actual = crc32c::Value(page_slice.data, page_slice.size - 4);
+        // here const_cast is used for testing.
         InjectionContext ctx = {&actual, const_cast<PageReadOptions*>(&opts)};
         (void)ctx;
         TEST_INJECTION_POINT_CALLBACK("PageIO::read_and_decompress_page:crc_failure_inj", &ctx);

be/src/olap/rowset/segment_v2/segment_writer.cpp

@@ -375,7 +375,7 @@ void SegmentWriter::_maybe_invalid_row_cache(const std::string& key) {
     }
 }
 
-void SegmentWriter::_serialize_block_to_row_column(vectorized::Block& block) {
+void SegmentWriter::_serialize_block_to_row_column(const vectorized::Block& block) {
     if (block.rows() == 0) {
         return;
     }
@@ -697,7 +697,7 @@ Status SegmentWriter::append_block(const vectorized::Block* block, size_t row_po
     // or it's schema change write(since column data type maybe changed, so we should reubild)
     if (_opts.write_type == DataWriteType::TYPE_DIRECT ||
         _opts.write_type == DataWriteType::TYPE_SCHEMA_CHANGE) {
-        _serialize_block_to_row_column(*const_cast<vectorized::Block*>(block));
+        _serialize_block_to_row_column(*block);
     }
 
     _olap_data_convertor->set_source_content(block, row_pos, num_rows);

be/src/olap/rowset/segment_v2/segment_writer.h

@@ -192,7 +192,7 @@ class SegmentWriter {
     void set_min_max_key(const Slice& key);
     void set_min_key(const Slice& key);
     void set_max_key(const Slice& key);
-    void _serialize_block_to_row_column(vectorized::Block& block);
+    void _serialize_block_to_row_column(const vectorized::Block& block);
     Status _generate_primary_key_index(
             const std::vector<const KeyCoder*>& primary_key_coders,
             const std::vector<vectorized::IOlapColumnDataAccessor*>& primary_key_columns,

be/src/olap/rowset/segment_v2/variant/hierarchical_data_iterator.cpp

@@ -381,18 +381,16 @@ Status HierarchicalDataIterator::_process_sparse_column(
                         // Case 1: subcolumn already created, append this row's value into it.
                         if (auto it = subcolumns_from_sparse_column.find(sub_path);
                             it != subcolumns_from_sparse_column.end()) {
-                            const auto& data = ColumnVariant::deserialize_from_sparse_column(
-                                    &src_sparse_data_values, lower_bound_index);
-                            it->second.insert(data.first, data.second);
+                            it->second.deserialize_from_sparse_column(&src_sparse_data_values,
+                                                                      lower_bound_index);
                         }
                         // Case 2: subcolumn not created yet and we still have quota → create it and insert.
                         else if (subcolumns_from_sparse_column.size() < count) {
                             // Initialize subcolumn with current logical row index i to align sizes.
                             ColumnVariant::Subcolumn subcolumn(/*size*/ i, /*is_nullable*/ true,
                                                                false);
-                            const auto& data = ColumnVariant::deserialize_from_sparse_column(
-                                    &src_sparse_data_values, lower_bound_index);
-                            subcolumn.insert(data.first, data.second);
+                            subcolumn.deserialize_from_sparse_column(&src_sparse_data_values,
+                                                                     lower_bound_index);
                             subcolumns_from_sparse_column.emplace(sub_path, std::move(subcolumn));
                         }
                         // Case 3: quota exhausted → keep the key/value in container's sparse column.
@@ -416,9 +414,8 @@ Status HierarchicalDataIterator::_process_sparse_column(
                             //     return Status::InternalError("Failed to add subcolumn for sparse column");
                             // }
                         }
-                        const auto& data = ColumnVariant::deserialize_from_sparse_column(
+                        container_variant.get_subcolumn({})->deserialize_from_sparse_column(
                                 &src_sparse_data_values, lower_bound_index);
-                        container_variant.get_subcolumn({})->insert(data.first, data.second);
                     }
                 }
                 // if root was created, and not seen in sparse data, insert default

be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp

@@ -355,7 +355,7 @@ void VerticalSegmentWriter::_maybe_invalid_row_cache(const std::string& key) con
     }
 }
 
-void VerticalSegmentWriter::_serialize_block_to_row_column(vectorized::Block& block) {
+void VerticalSegmentWriter::_serialize_block_to_row_column(const vectorized::Block& block) {
     if (block.rows() == 0) {
         return;
     }
@@ -948,7 +948,7 @@ Status VerticalSegmentWriter::write_batch() {
         _opts.write_type == DataWriteType::TYPE_SCHEMA_CHANGE) {
         for (auto& data : _batched_blocks) {
             // TODO: maybe we should pass range to this method
-            _serialize_block_to_row_column(*const_cast<vectorized::Block*>(data.block));
+            _serialize_block_to_row_column(*data.block);
         }
     }