k8部署的3.0.3存算分离环境,使用的s3协议存储,在写数据时不定期出现报错,调了一些超时参数后,报错少了些,但仍然有,重跑调度任务后正常
2025-03-31 05:01:04,783 ERROR [o.a.s.c.s.SeaTunnel ] [main] -
===============================================================================
Exception in thread "main" org.apache.seatunnel.core.starter.exception.CommandExecuteException: SeaTunnel job executed failed
at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:228)
at org.apache.seatunnel.core.starter.SeaTunnel.run(SeaTunnel.java:40)
at org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient.main(SeaTunnelClient.java:34)
Caused by: org.apache.seatunnel.engine.common.exception.SeaTunnelEngineException: java.lang.RuntimeException: java.lang.RuntimeException: java.util.concurrent.ExecutionException: org.apache.seatunnel.connectors.doris.exception.DorisConnectorException: ErrorCode:[Doris-01], ErrorDescription:[stream load error] - stream load error: [INTERNAL_ERROR][INTERNAL_ERROR]close wait failed coz rpc error. VNodeChannel[3562688-10060], load_id=664571749dcddc92-eb66ac5f3694f8a6, txn_id=10910856817811456, node=test-disaggregated-cluster-cg1-2.test-disaggregated-cluster-cg1.doris.svc.lygk8s04.local:8060, add batch req success but status isn't ok, err: [INTERNAL_ERROR]PStatus: (test-disaggregated-cluster-cg1-2.test-disaggregated-cluster-cg1.doris.svc.lygk8s04.local)[INTERNAL_ERROR]failed to commit rowset: failed to save rowset meta, err=MaybeCommitted
0# doris::Status doris::Status::create<true>(doris::PStatus const&) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187
1# doris::vectorized::VNodeChannel::_add_block_success_callback(doris::PTabletWriterAddBlockResult const&, doris::vectorized::WriteBlockCallbackContext const&) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:501
2# std::_Function_handler<void (doris::PTabletWriterAddBlockResult const&, doris::vectorized::WriteBlockCallbackContext const&), doris::vectorized::VNodeChannel::init(doris::RuntimeState*)::$_1>::_M_invoke(std::_Any_data const&, doris::PTabletWriterAddBlockResult const&, doris::vectorized::WriteBlockCallbackContext const&) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/atomicity.h:98
3# doris::vectorized::WriteBlockCallback<doris::PTabletWriterAddBlockResult>::call() at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:0
4# doris::AutoReleaseClosure<doris::PTabletWriterAddBlockRequest, doris::vectorized::WriteBlockCallback<doris::PTabletWriterAddBlockResult> >::Run() at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/atomicity.h:98
5# brpc::Controller::EndRPC(brpc::Controller::CompletionInfo const&)
6# brpc::policy::ProcessRpcResponse(brpc::InputMessageBase*)
7# brpc::ProcessInputMessage(void*)
8# brpc::InputMessenger::InputMessageClosure::~InputMessageClosure()
9# brpc::InputMessenger::OnNewMessages(brpc::Socket*)
10# brpc::Socket::ProcessEvent(void*)
11# bthread::TaskGroup::task_runner(long)
12# bthread_make_fcontext
, host: test-disaggregated-cluster-cg1-2.test-disaggregated-cluster-cg1.doris.svc.lygk8s04.local, see more in null
at org.apache.seatunnel.engine.server.task.flow.SinkFlowLifeCycle.received(SinkFlowLifeCycle.java:302)
at org.apache.seatunnel.engine.server.task.flow.SinkFlowLifeCycle.received(SinkFlowLifeCycle.java:70)
at org.apache.seatunnel.engine.server.task.SeaTunnelTransformCollector.collect(SeaTunnelTransformCollector.java:39)
at org.apache.seatunnel.engine.server.task.SeaTunnelTransformCollector.collect(SeaTunnelTransformCollector.java:27)
at org.apache.seatunnel.engine.server.task.group.queue.IntermediateBlockingQueue.handleRecord(IntermediateBlockingQueue.java:70)
at org.apache.seatunnel.engine.server.task.group.queue.IntermediateBlockingQueue.collect(IntermediateBlockingQueue.java:50)
at org.apache.seatunnel.engine.server.task.flow.IntermediateQueueFlowLifeCycle.collect(IntermediateQueueFlowLifeCycle.java:51)
at org.apache.seatunnel.engine.server.task.TransformSeaTunnelTask.collect(TransformSeaTunnelTask.java:72)
at org.apache.seatunnel.engine.server.task.SeaTunnelTask.stateProcess(SeaTunnelTask.java:169)
at org.apache.seatunnel.engine.server.task.TransformSeaTunnelTask.call(TransformSeaTunnelTask.java:77)
at org.apache.seatunnel.engine.server.TaskExecutionService$BlockingWorker.run(TaskExecutionService.java:694)
at org.apache.seatunnel.engine.server.TaskExecutionService$NamedTaskWrapper.run(TaskExecutionService.java:1019)
at org.apache.seatunnel.api.tracing.MDCRunnable.run(MDCRunnable.java:43)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.lang.RuntimeException: java.util.concurrent.ExecutionException: org.apache.seatunnel.connectors.doris.exception.DorisConnectorException: ErrorCode:[Doris-01], ErrorDescription:[stream load error] - stream load error: [INTERNAL_ERROR][INTERNAL_ERROR]close wait failed coz rpc error. VNodeChannel[3562688-10060], load_id=664571749dcddc92-eb66ac5f3694f8a6, txn_id=10910856817811456, node=test-disaggregated-cluster-cg1-2.test-disaggregated-cluster-cg1.doris.svc.lygk8s04.local:8060, add batch req success but status isn't ok, err: [INTERNAL_ERROR]PStatus: (test-disaggregated-cluster-cg1-2.test-disaggregated-cluster-cg1.doris.svc.lygk8s04.local)[INTERNAL_ERROR]failed to commit rowset: failed to save rowset meta, err=MaybeCommitted