版本2.1.3 FE节点99、100、101 BE节点 88、89、99 报错节点 88;
前景描述:表约1000GB,分为900个tablet,[费用发生日期]为分区键,该列值有空值,在某次重启doris集群后,出现该问题,约10S一次,不间断输出日志。
/log/be.out未打印日志
/log/be.INFO日志如下
I20240807 10:15:42.880417 1920913 atorage_engine.cpp:1077) atart to delete unused rowset, size:0
120240807 10:15:42.880518 1920913 atorage_engine.cpp:1111] collected 0 unused rowgets to remove, skipped 0 rowsets due to use count > 1, skipped 0 rowsets due to don't need to delete file, skipped 0 rowse
I20240807 10:15:42.880530 1920913 storage_engine.cpp:1127] removed all collected unused rowseta
I20240807 10:15:43.219684 1920919 compaction.cpp:350] start base compaction. tablet=113937, output_veraion=[0-10],permits:9
120240807 10:15:43.2197291920917 compaction.cpp:350] start base compaction. tablet=113928, output_veraion=[0-10],permits:9
T20240807 1920917 merger.cpp:369] Start to do vertical compaction,tablet id: 113928
12024080710:15:43.219839 1920919 merger.cpp:369] Start to do vertical compaction, tablet_id:113937
I20240807 10:15:43.219684 1920916 compaction.cpp:350] start base compaction. tablet=113112,output_veraion=[0-3],permits: 3
12024080710:15:43.219756 1920918compaction.cpp;350] start base compaction. tablet=113925, output_version=[0-10],permits: 9
12024080710:15:43.219974 1920916 merger.cpp:369] start to do vertical compaction, tablet_id:113112
I2024080710:15:43.220000 1920918 merger.cpp:369] start to do vertical compaction, tablet id:113925
W20240807 10:15:43.220151 1920919 atatus.h:412] meet error status: [E-1903]fail to sub condition. condition=费用发生日期>=2020-07-01'
doria::DeleteHandler::parae_condition (std::_exx11::basic_string<char, std::char_traits,std::allocator > conats, doria::TCondition*)
1# doria::status doris::DeleteHandler::_parse_column_pred<atd::_cxx11::basic_atring<char, atd::char_traita,atd::allocator>>(atd::shared ptr,std::shared ptr<d2# doris:neletenandder: init (atd::shared _ptr, std::vector<atd::shared_ptr<doxin: :RowsetMeta>, atdttallocator atdr shared ptr > 2 cbamez, hong, bool)
17 3# doris::TabletReader::_init_delete_condition (doria::TabletReader::ReaderParams consts)
1 4# 5# doris::TabletReader::init(doris::TabletReader::ReaderParama conat&) doris::TabletReader!!_init_params(doria::TabletReader::ReaderParama conat&)
6# doris::vectorized::VerticalBlockReader::init(doxis::TabletReader::ReaderParams consts)
doris:icompaction::do _compaction_impl(ion;; hared _ptr, doria::Readertype, atd::mhared_ptr, atd::vector<std::shared ptr,sasiancator9#doris::Compaction::do compaction impl(long)10# doris!:Compactioniido_compaction(long)
11# doria::BaseCompaction: ;execute compact imp1()12# dorim::Compaction:texecute_compact()
中
13# dorin::Tablet::execute_compaction (doris::Compaction)
14# atd::_Punation_handler<void (),doris::atorageEngine:;_submit_compaction_task(atd::shared_ptr,doris::CompactionType,bool)::$_1>::_M_invoke(atd::_Any_data conats)15# doria::ThreadPool:;diapatch_thread ()16# dorim::Thread::supervige thread(void*)17#
W20240807 10:15:43,220216 1920916status.h:412] meet error atatus: [E-1903]fail to sub condition, condition=费用发生日期>=2020-07-01
35 o# doria::DeleteHandler:iparae_condition (atd::_exx11::basie_atring<char, std!tchar_traita,atdi:allocator > conats, doriaiiTCondition*)
36
3 2# doria::DeleteHandleriiinit(atd;!shared_ptr,atdiivector<atditshared_ptr,atdiiallocator<atd::shared_ptr>> eonath,long,bool)
dorisnistatasdoria:=DeleteHandler::parse_column pred<atd::exx11::basic_atring<char,stdr:char_traits,stdi:allocator>>(atd:!ahared_ptr<doria;;Tablet5chema>,atd:;shared_ptr<d dorisi:TabletReaderil init_delete_condition (doriai:TabletReader!1ReaderParama conat6) dorla:iTabletReader!i init parama(dorim::TabletReader::ReaderParama:conata) dorii:TabletReader:tinit(doria:iTabletReader::ReaderParama constl)
doriaiivectorized::VerticalBlockReaderitinit(dorist:TabletReader:;ReadecParama conste)7#
doriet:Meraerlivertical compact one qroup(atd::ahared ptr, doria::ReaderType,atd::ahared ptr, bool,atd::vectorcunmigned int, stdriallo0ator doris::Merger:iyertical merge_rowaeta(atd:iahared ptr,doxisi!ReaderType,std:iehared_ptr,atd::vector<atd:rshared ptr,atd::allocator<9# doris::Compactioniido_compaction impl(1ong)10# dorim::Compaction::do _compaction(long)
11# doriariBaseCompactioniiexecute compactimpl()12# doria::Compactionilexecute compact ()
40 13# dorim::Tablet::execute_compaction(dorim::Compactiong)
14# atdts Funetion handler<void O, doria::BtorageEngine::_aubmit _compaction ishared_ptr,doria::CompactionType, booll::$_1x1: Minvoke(stdi:_Any_data consts)
15# doris;:ThreadPool::dispatch thread O
w20240807 10:15:43.220548 1920917 tablet .opp;1949] failed to do bane compaction, tablet=113928 : [E-1903)fail to mub condition. condition=费用发生日期>=2020-07-01
dorin:ehtus doiss etennaotin parneollbanie_string<char, atd::char_traity, atd:;allocator > consts, doria::TCondition*)
1# doris::Deleterandieririnitiotd: aharea otr dorin crxlltitrng<chat,atd: char traitacntnr>,ncd:allocntor>>(ata::nhared_ptr, std::shaced 2# doriai:DeleteHandler:rinit(std:!shared_ptr,atd::vector<atd::shared_ptr,atd::allocator<atd::shared_ptr>> conats, long,bool)
doris::TabletReader::_init_delete_condition(doris::TabletReader::ReaderParama const&)4# doris::TabletReader:;_init_parama(doria::TabletReader::ReaderParama conat&)5# dorim::TabletRenderi:init(doria::TabletReader::ReaderParams conat6)
6 dorin::vectorized:;VerticalBlockReader::init(doris::TabletReader::ReaderParams conatG)
7# dorin::h gor: erticnl margn lowpec aed swhered pts esrsdoriabiets,o? doriah Reder?ype, atd::ahared ptr, bool, atd: :vector<unsigned int, atd::all. tor<unsigned 1
中
8# doria::Merger::vertical merge_rownets(atd:;shared_ptr,doria::ReaderType,atd::shared ptr,atd::vector<atd::ahared_ptr,atd::allocat9# dorin::Compaction::do_compaction_impl(long)10# dorim::Compaction::do_compaction (long)
12# dorim::Compaction:iexecute compact() 11# doria::BaseCompaction::execute_compact_imp1() 13# dorin::Tablet:;execute_compaction (doriar:Compactiong) I
14# atd::_Punction_handler<void (), doria::8torageEngine::Bubmit_compaction taak(atd::ohared_ptr<dorin;:Tablet>,doris::CompactionType, bool)::$ 1>:: M invoke(atd::_Any_data conat&)15# doris::ThreadPool::dispatch thread ()160 doria::Thread::supervise thread(void*)17#218#7
w20240807 10:15:43220526 1920918 tablet_reader.cpp:129] fail to init reader when init parama, rep: [E-1903]fail to aub condition. condition=费用发生日期>=2020-07-01
o# doria::DeleteHandler::parse_condition(atd::__cxx11:;banic_atring<char, std::char_traita,atd::allocator> conati, doria::TCondition*)1#
2# dorin::DeleteHandler::init(std:ishared_ptr, std::vector<std:;shared_ptr, atd:;allocator<atd:;shared_ptr>> conaté, long,bool)
dorim::atatus dorin::DeleteHandler::_parse_column_pred<atd::cxx11::basic string<char, std::char _traite,atd::allocator>>(std:1shared ptr,atd::mhared pt:3# dorin::TabletReader::_init_delete _condition(doriat!TabletReader!!ReaderParama conat6)
doris::TabletReader::_init_parama(doria::TabletRender:1ReaderParama consts)5# doris::TabletReader:!init(doria::TabletReader::ReaderParama conat6)
6# doris::vectorized::VerticalBlockReader::init(doria::TabletReader::ReaderParama conats)7
doriai:Merger:!vertical compact_one_group(std::ahared ptr, doriai1ReaderType,std::ahared_ptr, bool,atd::vector<unaigned int,atd::allocator<unsigned int doris::Merger::vertical_merge_rowgets(atd::shared_ptr, doria::ReaderType, atd::shared_ptr atd::vector<atd::ahared_ptr,atd::allocato: doris::Compaction::do_compaction_impl(1ong)10# doris::Compaction::do_compaction(long)
20240808补充
88、89、99BE节点,执行 tail -f /home/doris/be_log/be.INFO |grep failed 均打印出上述fail to sub condition;
找到其中一条问题日志
failed to do base compaction,tablet=328830 : [E-1903] fail to sub contidion.condition=费用发生日期>='2021-10-01 00:00:00'
执行 SELECT table_schema,table_name FROM information_schema.COLUMNS where COLUMN_NAME = '费用发生日期'
使用上述结果 通过 SHOW TABLES FROM table_schema.table_name 表 找到Tabletid对应的MetaUrl
使用浏览器访问该URL: xx.xx.xx.89:8040/api/meta/header/328830,获取该tablet对应的partition_id
通过 SHOW PARTITIONS FROM table_schema.table_name ,找到PartitionId对应的PartitionName
执行SHOW REPLICA STATUS FROM table_schema.table_name PARTITION(PartitionName),查看Status均为OK。