【已解决】failed to do base compaction

Viewed 56

在2.1.1版本中,be后台一直报内存超限。
W20240416 19:27:29.919912 580810 compaction.cpp:374] fail to do base compaction. res=[MEM_LIMIT_EXCEEDED]PreCatch error code:11, [E11] Allocator sys memory check failed: Cannot alloc:3515, consuming tracker:BaseCompaction:14372, peak used 2619101, current used 2619101, exec node:<>, process memory used 75.21 GB exceed limit 62.52 GB or sys available memory 37.25 GB less than low water mark 1.60 GB.

    0#  doris::Exception::Exception(int, std::basic_string_view<char, std::char_traits<char> > const&) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173
    1#  Allocator<false, false, false>::sys_memory_check(unsigned long) const at /home/zcp/repo_center/doris_release/doris/be/src/vec/common/allocator.cpp:78
    2#  Allocator<false, false, false>::alloc_impl(unsigned long, unsigned long) at /home/zcp/repo_center/doris_release/doris/be/src/vec/common/allocator.cpp:173
    3#  doris::PageBase<Allocator<false, false, false> >::PageBase(unsigned long, std::shared_ptr<doris::MemTrackerLimiter> const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/page_cache.h:48
    4#  doris::segment_v2::PageIO::read_and_decompress_page(doris::segment_v2::PageReadOptions const&, doris::segment_v2::PageHandle*, doris::Slice*, doris::segment_v2::PageFooterPB*) at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/page_io.cpp:200
    5#  doris::segment_v2::FileColumnIterator::_read_data_page(doris::segment_v2::OrdinalPageIndexIterator const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/column_reader.cpp:0
    6#  doris::segment_v2::FileColumnIterator::seek_to_ordinal(unsigned long) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    7#  doris::segment_v2::SegmentIterator::_read_columns_by_index(unsigned int, unsigned int&, bool) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    8#  doris::segment_v2::SegmentIterator::_next_batch_internal(doris::vectorized::Block*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    9#  doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    10# doris::vectorized::VerticalMergeIteratorContext::_load_next_block() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    11# doris::vectorized::VerticalHeapMergeIterator::init(doris::StorageReadOptions const&) at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vertical_merge_iterator.cpp:0
    12# doris::vectorized::VerticalBlockReader::_init_collect_iter(doris::TabletReader::ReaderParams const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    13# doris::vectorized::VerticalBlockReader::init(doris::TabletReader::ReaderParams const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    14# doris::Merger::vertical_compact_one_group(std::shared_ptr<doris::Tablet>, doris::ReaderType, std::shared_ptr<doris::TabletSchema>, bool, std::vector<unsigned int, std::allocator<unsigned int> > const&, doris::vectorized::RowSourcesBuffer*, std::vector<std::shared_ptr<doris::RowsetReader>, std::allocator<std::shared_ptr<doris::RowsetReader> > > const&, doris::RowsetWriter*, long, doris::Merger::Statistics*, std::vector<unsigned int, std::allocator<unsigned int> >) at /home/zcp/repo_center/doris_release/doris/be/src/olap/merger.cpp:0
    15# doris::Merger::vertical_merge_rowsets(std::shared_ptr<doris::Tablet>, doris::ReaderType, std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetReader>, std::allocator<std::shared_ptr<doris::RowsetReader> > > const&, doris::RowsetWriter*, long, doris::Merger::Statistics*) at /home/zcp/repo_center/doris_release/doris/be/src/olap/merger.cpp:371
    16# doris::Compaction::do_compaction_impl(long) at /home/zcp/repo_center/doris_release/doris/be/src/olap/compaction.cpp:364
    17# doris::Compaction::do_compaction(long) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:1291
    18# doris::BaseCompaction::execute_compact_impl() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    19# doris::Compaction::execute_compact() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    20# doris::Tablet::execute_compaction(doris::Compaction&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    21# std::_Function_handler<void (), doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_1>::_M_invoke(std::_Any_data const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/olap_server.cpp:995
    22# doris::ThreadPool::dispatch_thread() at /home/zcp/repo_center/doris_release/doris/be/src/util/threadpool.cpp:0
    23# doris::Thread::supervise_thread(void*) at /var/local/ldb_toolchain/bin/../usr/include/pthread.h:562
    24# ?
    25# ?

, FILE:/home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp, LINE:2098, FUNCTION:auto doris::segment_v2::SegmentIterator::next_batch(vectorized::Block *)::(anonymous class)::operator()() const, tablet=14372, output_version=[0-11027]
W20240416 19:27:29.919965 580810 tablet.cpp:1938] failed to do base compaction, tablet=14372 : [MEM_LIMIT_EXCEEDED]PreCatch error code:11, [E11] Allocator sys memory check failed: Cannot alloc:3515, consuming tracker:BaseCompaction:14372, peak used 2619101, current used 2619101, exec node:<>, process memory used 75.21 GB exceed limit 62.52 GB or sys available memory 37.25 GB less than low water mark 1.60 GB.

    0#  doris::Exception::Exception(int, std::basic_string_view<char, std::char_traits<char> > const&) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173
    1#  Allocator<false, false, false>::sys_memory_check(unsigned long) const at /home/zcp/repo_center/doris_release/doris/be/src/vec/common/allocator.cpp:78
    2#  Allocator<false, false, false>::alloc_impl(unsigned long, unsigned long) at /home/zcp/repo_center/doris_release/doris/be/src/vec/common/allocator.cpp:173
    3#  doris::PageBase<Allocator<false, false, false> >::PageBase(unsigned long, std::shared_ptr<doris::MemTrackerLimiter> const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/page_cache.h:48
    4#  doris::segment_v2::PageIO::read_and_decompress_page(doris::segment_v2::PageReadOptions const&, doris::segment_v2::PageHandle*, doris::Slice*, doris::segment_v2::PageFooterPB*) at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/page_io.cpp:200
    5#  doris::segment_v2::FileColumnIterator::_read_data_page(doris::segment_v2::OrdinalPageIndexIterator const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/column_reader.cpp:0
    6#  doris::segment_v2::FileColumnIterator::seek_to_ordinal(unsigned long) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    7#  doris::segment_v2::SegmentIterator::_read_columns_by_index(unsigned int, unsigned int&, bool) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    8#  doris::segment_v2::SegmentIterator::_next_batch_internal(doris::vectorized::Block*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    9#  doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    10# doris::vectorized::VerticalMergeIteratorContext::_load_next_block() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    11# doris::vectorized::VerticalHeapMergeIterator::init(doris::StorageReadOptions const&) at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vertical_merge_iterator.cpp:0
    12# doris::vectorized::VerticalBlockReader::_init_collect_iter(doris::TabletReader::ReaderParams const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    13# doris::vectorized::VerticalBlockReader::init(doris::TabletReader::ReaderParams const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    14# doris::Merger::vertical_compact_one_group(std::shared_ptr<doris::Tablet>, doris::ReaderType, std::shared_ptr<doris::TabletSchema>, bool, std::vector<unsigned int, std::allocator<unsigned int> > const&, doris::vectorized::RowSourcesBuffer*, std::vector<std::shared_ptr<doris::RowsetReader>, std::allocator<std::shared_ptr<doris::RowsetReader> > > const&, doris::RowsetWriter*, long, doris::Merger::Statistics*, std::vector<unsigned int, std::allocator<unsigned int> >) at /home/zcp/repo_center/doris_release/doris/be/src/olap/merger.cpp:0
    15# doris::Merger::vertical_merge_rowsets(std::shared_ptr<doris::Tablet>, doris::ReaderType, std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetReader>, std::allocator<std::shared_ptr<doris::RowsetReader> > > const&, doris::RowsetWriter*, long, doris::Merger::Statistics*) at /home/zcp/repo_center/doris_release/doris/be/src/olap/merger.cpp:371
    16# doris::Compaction::do_compaction_impl(long) at /home/zcp/repo_center/doris_release/doris/be/src/olap/compaction.cpp:364
    17# doris::Compaction::do_compaction(long) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:1291
    18# doris::BaseCompaction::execute_compact_impl() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    19# doris::Compaction::execute_compact() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    20# doris::Tablet::execute_compaction(doris::Compaction&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    21# std::_Function_handler<void (), doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_1>::_M_invoke(std::_Any_data const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/olap_server.cpp:995
    22# doris::ThreadPool::dispatch_thread() at /home/zcp/repo_center/doris_release/doris/be/src/util/threadpool.cpp:0
    23# doris::Thread::supervise_thread(void*) at /var/local/ldb_toolchain/bin/../usr/include/pthread.h:562
    24# ?
    25# ?

, FILE:/home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp, LINE:2098, FUNCTION:auto doris::segment_v2::SegmentIterator::next_batch(vectorized::Block *)::(anonymous class)::operator()() const
W20240416 19:27:29.943230 580810 compaction.cpp:374] fail to do base compaction. res=[MEM_LIMIT_EXCEEDED]PreCatch error code:11, [E11] Allocator sys memory check failed: Cannot alloc:446, consuming tracker:BaseCompaction:14408, peak used 36864, current used 36864, exec node:<>, process memory used 75.21 GB exceed limit 62.52 GB or sys available memory 37.25 GB less than low water mark 1.60 GB.

    0#  doris::Exception::Exception(int, std::basic_string_view<char, std::char_traits<char> > const&) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173
    1#  Allocator<false, false, false>::sys_memory_check(unsigned long) const at /home/zcp/repo_center/doris_release/doris/be/src/vec/common/allocator.cpp:78
    2#  Allocator<false, false, false>::alloc_impl(unsigned long, unsigned long) at /home/zcp/repo_center/doris_release/doris/be/src/vec/common/allocator.cpp:173
    3#  doris::PageBase<Allocator<false, false, false> >::PageBase(unsigned long, std::shared_ptr<doris::MemTrackerLimiter> const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/page_cache.h:48
    4#  doris::segment_v2::PageIO::read_and_decompress_page(doris::segment_v2::PageReadOptions const&, doris::segment_v2::PageHandle*, doris::Slice*, doris::segment_v2::PageFooterPB*) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:155
    5#  doris::segment_v2::FileColumnIterator::_read_data_page(doris::segment_v2::OrdinalPageIndexIterator const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/column_reader.cpp:0
    6#  doris::segment_v2::FileColumnIterator::seek_to_ordinal(unsigned long) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    7#  doris::segment_v2::SegmentIterator::_read_columns_by_index(unsigned int, unsigned int&, bool) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    8#  doris::segment_v2::SegmentIterator::_next_batch_internal(doris::vectorized::Block*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    9#  doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    10# doris::vectorized::VerticalMergeIteratorContext::_load_next_block() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    11# doris::vectorized::VerticalHeapMergeIterator::init(doris::StorageReadOptions const&) at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vertical_merge_iterator.cpp:0
    12# doris::vectorized::VerticalBlockReader::_init_collect_iter(doris::TabletReader::ReaderParams const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    13# doris::vectorized::VerticalBlockReader::init(doris::TabletReader::ReaderParams const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    14# doris::Merger::vertical_compact_one_group(std::shared_ptr<doris::Tablet>, doris::ReaderType, std::shared_ptr<doris::TabletSchema>, bool, std::vector<unsigned int, std::allocator<unsigned int> > const&, doris::vectorized::RowSourcesBuffer*, std::vector<std::shared_ptr<doris::RowsetReader>, std::allocator<std::shared_ptr<doris::RowsetReader> > > const&, doris::RowsetWriter*, long, doris::Merger::Statistics*, std::vector<unsigned int, std::allocator<unsigned int> >) at /home/zcp/repo_center/doris_release/doris/be/src/olap/merger.cpp:0
    15# doris::Merger::vertical_merge_rowsets(std::shared_ptr<doris::Tablet>, doris::ReaderType, std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetReader>, std::allocator<std::shared_ptr<doris::RowsetReader> > > const&, doris::RowsetWriter*, long, doris::Merger::Statistics*) at /home/zcp/repo_center/doris_release/doris/be/src/olap/merger.cpp:371
    16# doris::Compaction::do_compaction_impl(long) at /home/zcp/repo_center/doris_release/doris/be/src/olap/compaction.cpp:364
    17# doris::Compaction::do_compaction(long) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:1291
    18# doris::BaseCompaction::execute_compact_impl() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    19# doris::Compaction::execute_compact() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    20# doris::Tablet::execute_compaction(doris::Compaction&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    21# std::_Function_handler<void (), doris::StorageEngine::_submit_compaction_task(std::shared_ptr<doris::Tablet>, doris::CompactionType, bool)::$_1>::_M_invoke(std::_Any_data const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/olap_server.cpp:995
    22# doris::ThreadPool::dispatch_thread() at /home/zcp/repo_center/doris_release/doris/be/src/util/threadpool.cpp:0
    23# doris::Thread::supervise_thread(void*) at /var/local/ldb_toolchain/bin/../usr/include/pthread.h:562
    24# ?
    25# ?

, FILE:/home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp, LINE:2098, FUNCTION:auto doris::segment_v2::SegmentIterator::next_batch(vectorized::Block *)::(anonymous class)::operator()() const, tablet=14408, output_version=[0-11028]
W20240416 19:27:29.943275 580810 tablet.cpp:1938] failed to do base compaction, tablet=14408 : [MEM_LIMIT_EXCEEDED]PreCatch error code:11, [E11] Allocator sys memory check failed: Cannot alloc:446, consuming tracker:BaseCompaction:14408, peak used 36864, current used 36864, exec node:<>, process memory used 75.21 GB exceed limit 62.52 GB or sys available memory 37.25 GB less than low water mark 1.60 GB.

    0#  doris::Exception::Exception(int, std::basic_string_view<char, std::char_traits<char> > const&) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173
    1#  Allocator<false, false, false>::sys_memory_check(unsigned long) const at /home/zcp/repo_center/doris_release/doris/be/src/vec/common/allocator.cpp:78
    2#  Allocator<false, false, false>::alloc_impl(unsigned long, unsigned long) at /home/zcp/repo_center/doris_release/doris/be/src/vec/common/allocator.cpp:173
    3#  doris::PageBase<Allocator<false, false, false> >::PageBase(unsigned long, std::shared_ptr<doris::MemTrackerLimiter> const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/page_cache.h:48
    4#  doris::segment_v2::PageIO::read_and_decompress_page(doris::segment_v2::PageReadOptions const&, doris::segment_v2::PageHandle*, doris::Slice*, doris::segment_v2::PageFooterPB*) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:155
    5#  doris::segment_v2::FileColumnIterator::_read_data_page(doris::segment_v2::OrdinalPageIndexIterator const&) at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/column_reader.cpp:0
    6#  doris::segment_v2::FileColumnIterator::seek_to_ordinal(unsigned long) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    7#  doris::segment_v2::SegmentIterator::_read_columns_by_index(unsigned int, unsigned int&, bool) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    8#  doris::segment_v2::SegmentIterator::_next_batch_internal(doris::vectorized::Block*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    9#  doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    10# doris::vectorized::VerticalMergeIteratorContext::_load_next_block() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    11# doris::vectorized::VerticalHeapMergeIterator::init(doris::StorageReadOptions const&) at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vertical_merge_iterator.cpp:0
    12# doris::vectorized::VerticalBlockReader::_init_collect_iter(doris::TabletReader::ReaderParams const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    13# doris::vectorized::VerticalBlockReader::init(doris::TabletReader::ReaderParams const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:449
    14# doris::Merger::vertical_compact_one_group(std::shared_ptr<doris::Tablet>, doris::ReaderType, std::shared_ptr<doris::TabletSchema>, bool, std::vector<unsigned int, std::allocator<unsigned int> > const&, doris::vectorized::RowSourcesBuffer*, std::vector<std::shared_ptr<doris::RowsetReader
1 Answers

要么扩容,要么把合并的那个表删除,使用force删除才不会合并