现象描述:
Apache doris 版本 2.1.2 ,ROUTINE LOAD 任务一直正常同步,修改完FE 和BE Conf 参数后重启集群之后,ROUTINE LOAD 任务状态一直显示Running , 但是Lag 一直不变化,数据也未写入。FE 和 BE 日志未发现报错信息,desired_concurrent_number参数设置为2 , 发现其中两个BE节点一直再刷新打印数据
任务语句:
CREATE ROUTINE LOAD ODS.ODS_LW_VEHICLE_PASS_INFO_JOB ON ODS_LW_VEHICLE_PASS_INFO
COLUMNS(gcsj,hphm,xjbh,xjmc,kkbh,kkmc,fxbh,gcsj1,cdh,jzcdh,hpzl,clsd,clxs,hpys,cllx,sbcx,clpp,clzpp,csys,zaqdjc,faqdjc,zzybjc,fzybjc,zsysj,qjtplj,tztplj,fztp1lj,fwqbh,wjfwqbh,tssj,tpsl,hxbbh,fxlx,rksj) , where gcsj >= date_add(now(),-90) and gcsj <= date_add(now(),90) and gcsj is not null and hphm is not null
PROPERTIES
(
"desired_concurrent_number"="2",
"max_batch_interval" = "30",
"max_batch_rows" = "300000",
"max_batch_size" = "209715200",
"max_filter_ratio" = "0.2" ,
"strict_mode" = "false",
"format" = "json",
"jsonpaths" = "[\"$.gcsj\",\"$.hphm\",\"$.xjbh\",\"$.xjmc\",\"$.kkbh\",\"$.kkmc\",\"$.fxbh\",\"$.gcsj1\",\"$.cdh\",\"$.jzcdh\",\"$.hpzl\",\"$.clsd\",\"$.clxs\",\"$.hpys\",\"$.cllx\",\"$.sbcx\",\"$.clpp\",\"$.clzpp\",\"$.csys\",\"$.zaqdjc\",\"$.faqdjc\",\"$.zzybjc\",\"$.fzybjc\",\"$.zsysj\",\"$.qjtplj\",\"$.tztplj\",\"$.fztp1lj\",\"$.fwqbh\",\"$.wjfwqbh\",\"$.tssj\",\"$.tpsl\",\"$.hxbbh\",\"$.fxlx\",\"$.rksj\"]",
"strip_outer_array" = "false"
)
FROM KAFKA
(
"kafka_broker_list" = "node1:9092",
"kafka_topic" = "vehicle_pass_detail",
"property.group.id" = "vehicle_pass_detail_0520",
"property.kafka_default_offsets" = "OFFSET_BEGINNING"
);
查看任务详情信息:
1837351 ODS_LW_VEHICLE_PASS_INFO_JOB 2024-05-20 08:34:47 ODS ODS_LW_VEHICLE_PASS_INFO false RUNNING KAFKA 2 {"max_batch_rows":"300000","timezone":"Asia/Shanghai","send_batch_parallelism":"1","load_to_single_tablet":"false","current_concurrent_number":"2","delete":"","partial_columns":"false","merge_type":"APPEND","exec_mem_limit":"2147483648","strict_mode":"false","jsonpaths":"["$.gcsj","$.hphm","$.xjbh","$.xjmc","$.kkbh","$.kkmc","$.fxbh","$.gcsj1","$.cdh","$.jzcdh","$.hpzl","$.clsd","$.clxs","$.hpys","$.cllx","$.sbcx","$.clpp","$.clzpp","$.csys","$.zaqdjc","$.faqdjc","$.zzybjc","$.fzybjc","$.zsysj","$.qjtplj","$.tztplj","$.fztp1lj","$.fwqbh","$.wjfwqbh","$.tssj","$.tpsl","$.hxbbh","$.fxlx","$.rksj"]","max_batch_interval":"30","max_batch_size":"209715200","fuzzy_parse":"false","partitions":"","columnToColumnExpr":"gcsj,hphm,xjbh,xjmc,kkbh,kkmc,fxbh,gcsj1,cdh,jzcdh,hpzl,clsd,clxs,hpys,cllx,sbcx,clpp,clzpp,csys,zaqdjc,faqdjc,zzybjc,fzybjc,zsysj,qjtplj,tztplj,fztp1lj,fwqbh,wjfwqbh,tssj,tpsl,hxbbh,fxlx,rksj","whereExpr":"(
gcsj
>= '2024-02-20 08:34:51') AND (gcsj
<= '2024-08-18 08:34:51') ANDgcsj
IS NOT NULL ANDhphm
IS NOT NULL","desired_concurrent_number":"2","precedingFilter":"*","format":"json","max_error_number":"0","max_filter_ratio":"0.2","json_root":"","strip_outer_array":"false","num_as_string":"false"} {"topic":"vehicle_pass_detail","currentKafkaPartitions":"0,1,2,3,4,5,6,7,8,9","brokerList":"node1:9092"} {"kafka_default_offsets":"OFFSET_BEGINNING","group.id":"vehicle_pass_detail_0520"} {"receivedBytes":0,"runningTxns":[5513757,5513758],"errorRows":0,"committedTaskNum":0,"loadedRows":0,"loadRowsRate":0,"abortedTaskNum":2,"errorRowsAfterResumed":0,"totalRows":0,"unselectedRows":0,"receivedBytesRate":0,"taskExecuteTimeMs":1} {"0":"OFFSET_BEGINNING","1":"OFFSET_BEGINNING","2":"OFFSET_BEGINNING","3":"OFFSET_BEGINNING","4":"OFFSET_BEGINNING","5":"OFFSET_BEGINNING","6":"OFFSET_BEGINNING","7":"OFFSET_BEGINNING","8":"OFFSET_BEGINNING","9":"OFFSET_BEGINNING"} {"0":296709634,"1":296698093,"2":296713853,"3":296703660,"4":296730066,"5":296677983,"6":296705336,"7":296718163,"8":296732344,"9":296713358} root