Flink CDC整库同步MySQL到Doris失败,状态RESTARTING

Viewed 133

版本信息
Doris-2.0.7
Flink-1.19.0
flink-cep-1.19.0.jar
flink-connector-files-1.19.0.jar
flink-csv-1.19.0.jar
flink-dist-1.19.0.jar
flink-doris-connector-1.19-1.6.0.jar
flink-json-1.19.0.jar
flink-scala_2.12-1.19.0.jar
flink-sql-connector-mysql-cdc-3.0.1.jar
flink-table-api-java-uber-1.19.0.jar
flink-table-planner-loader-1.19.0.jar
flink-table-runtime-1.19.0.jar

使用Flink run命令行提交作业时报错

java.lang.IllegalArgumentException: Self-suppression not permitted
	at java.lang.Throwable.addSuppressed(Throwable.java:1082) ~[?:1.8.0_391]
	at org.apache.flink.util.InstantiationUtil.serializeObject(InstantiationUtil.java:556) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.util.SerializedValue.<init>(SerializedValue.java:62) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.blob.BlobWriter.serializeAndTryOffload(BlobWriter.java:91) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.getTaskInformationOrBlobKey(ExecutionJobVertex.java:446) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.deployment.TaskDeploymentDescriptorFactory.createDeploymentDescriptor(TaskDeploymentDescriptorFactory.java:125) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.executiongraph.Execution.deploy(Execution.java:575) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.DefaultExecutionOperations.deploy(DefaultExecutionOperations.java:32) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.DefaultExecutionDeployer.deployTaskSafe(DefaultExecutionDeployer.java:323) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.DefaultExecutionDeployer.lambda$deployOrHandleError$6(DefaultExecutionDeployer.java:313) ~[flink-dist-1.19.0.jar:1.19.0]
	at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836) ~[?:1.8.0_391]
	at java.util.concurrent.CompletableFuture.uniHandleStage(CompletableFuture.java:848) ~[?:1.8.0_391]
	at java.util.concurrent.CompletableFuture.handle(CompletableFuture.java:2168) ~[?:1.8.0_391]
	at org.apache.flink.runtime.scheduler.DefaultExecutionDeployer.lambda$deployAll$3(DefaultExecutionDeployer.java:190) ~[flink-dist-1.19.0.jar:1.19.0]
	at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836) ~[?:1.8.0_391]
	at java.util.concurrent.CompletableFuture.uniHandleStage(CompletableFuture.java:848) ~[?:1.8.0_391]
	at java.util.concurrent.CompletableFuture.handle(CompletableFuture.java:2168) ~[?:1.8.0_391]
	at org.apache.flink.runtime.scheduler.DefaultExecutionDeployer.waitForAllSlotsAndDeploy(DefaultExecutionDeployer.java:154) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.DefaultExecutionDeployer.allocateSlotsAndDeploy(DefaultExecutionDeployer.java:104) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.DefaultScheduler.allocateSlotsAndDeploy(DefaultScheduler.java:471) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.strategy.PipelinedRegionSchedulingStrategy.scheduleRegion(PipelinedRegionSchedulingStrategy.java:287) ~[flink-dist-1.19.0.jar:1.19.0]
	at java.util.Collections$SingletonList.forEach(Collections.java:4824) ~[?:1.8.0_391]
	at org.apache.flink.runtime.scheduler.strategy.PipelinedRegionSchedulingStrategy.maybeScheduleRegions(PipelinedRegionSchedulingStrategy.java:233) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.strategy.PipelinedRegionSchedulingStrategy.restartTasks(PipelinedRegionSchedulingStrategy.java:208) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.DefaultScheduler.restartTasks(DefaultScheduler.java:413) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.scheduler.DefaultScheduler.lambda$null$2(DefaultScheduler.java:368) ~[flink-dist-1.19.0.jar:1.19.0]
	at java.util.concurrent.CompletableFuture.uniRun(CompletableFuture.java:719) ~[?:1.8.0_391]
	at java.util.concurrent.CompletableFuture$UniRun.tryFire(CompletableFuture.java:701) ~[?:1.8.0_391]
	at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456) ~[?:1.8.0_391]
	at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.lambda$handleRunAsync$4(PekkoRpcActor.java:451) ~[flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.flink.runtime.concurrent.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) ~[flink-dist-1.19.0.jar:1.19.0]
	at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.handleRunAsync(PekkoRpcActor.java:451) ~[flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.handleRpcMessage(PekkoRpcActor.java:218) ~[flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.flink.runtime.rpc.pekko.FencedPekkoRpcActor.handleRpcMessage(FencedPekkoRpcActor.java:85) ~[flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.handleMessage(PekkoRpcActor.java:168) ~[flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:33) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:29) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at scala.PartialFunction.applyOrElse(PartialFunction.scala:127) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:29) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:175) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:176) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:176) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.actor.Actor.aroundReceive(Actor.scala:547) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.actor.Actor.aroundReceive$(Actor.scala:545) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.actor.AbstractActor.aroundReceive(AbstractActor.scala:229) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.actor.ActorCell.receiveMessage(ActorCell.scala:590) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.actor.ActorCell.invoke(ActorCell.scala:557) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.dispatch.Mailbox.processMailbox(Mailbox.scala:280) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.dispatch.Mailbox.run(Mailbox.scala:241) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at org.apache.pekko.dispatch.Mailbox.exec(Mailbox.scala:253) [flink-rpc-akka46a443fa-0182-4990-a37d-c4b072c38dca.jar:1.19.0]
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_391]
	at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1067) [?:1.8.0_391]
	at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1703) [?:1.8.0_391]
	at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:172) [?:1.8.0_391]
Caused by: java.lang.OutOfMemoryError: Java heap space
2024-04-09 13:25:54,541 INFO  org.apache.flink.runtime.executiongraph.ExecutionGraph       [] - Discarding the results produced by task execution e208853ad7949b3c31b57174748292fd_cbc357ccb763df2852fee8c4fc7d55f2_0_9.
2024-04-09 13:25:54,541 INFO  org.apache.flink.runtime.source.coordinator.SourceCoordinator [] - Removing registered reader after failure for subtask 0 (#9) of source Source: MySQL Source.
2024-04-09 13:25:54,541 INFO  org.apache.flink.runtime.resourcemanager.slotmanager.FineGrainedSlotManager [] - Clearing resource requirements of job 40f1c5ed60b177519ecf6706b222cee2
2024-04-09 13:25:54,541 INFO  org.apache.flink.runtime.jobmaster.JobMaster                 [] - 1 tasks will be restarted to recover the failed task e208853ad7949b3c31b57174748292fd_cbc357ccb763df2852fee8c4fc7d55f2_0_9.
2024-04-09 13:25:54,541 INFO  org.apache.flink.runtime.executiongraph.ExecutionGraph       [] - Job MySQL-Doris Sync Database: srm_sit (40f1c5ed60b177519ecf6706b222cee2) switched from state RUNNING to RESTARTING.
2 Answers

频繁不,得具体看看jobmanager是不是有oom或ckp等异常

调了8个G还是报错,一个同步任务,1000张表,怎么估算调多大内存

java.util.concurrent.ExecutionException: Boxed Error
at scala.concurrent.impl.Promise$.resolver(Promise.scala:87)
at scala.concurrent.impl.Promise$.scala$concurrent$impl$Promise$$resolveTry(Promise.scala:79)
at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)
at org.apache.pekko.pattern.PromiseActorRef.$bang(AskSupport.scala:629)
at org.apache.pekko.remote.DefaultMessageDispatcher.dispatch(Endpoint.scala:128)
at org.apache.pekko.remote.EndpointReader$$anonfun$receive$2.applyOrElse(Endpoint.scala:1154)
at org.apache.pekko.actor.Actor.aroundReceive(Actor.scala:547)
at org.apache.pekko.actor.Actor.aroundReceive$(Actor.scala:545)
at org.apache.pekko.remote.EndpointActor.aroundReceive(Endpoint.scala:550)
at org.apache.pekko.actor.ActorCell.receiveMessage(ActorCell.scala:590)
at org.apache.pekko.actor.ActorCell.invoke(ActorCell.scala:557)
at org.apache.pekko.dispatch.Mailbox.processMailbox(Mailbox.scala:280)
at org.apache.pekko.dispatch.Mailbox.run(Mailbox.scala:241)
at org.apache.pekko.dispatch.Mailbox.exec(Mailbox.scala:253)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1067)
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1703)
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:172)
Caused by: java.lang.OutOfMemoryError: Java heap space
at java.lang.reflect.Array.newInstance(Array.java:75)
at java.io.ObjectInputStream.readArray(ObjectInputStream.java:2106)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1680)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:508)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:466)
at java.util.HashMap.readObject(HashMap.java:1418)
at sun.reflect.GeneratedMethodAccessor47.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1185)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2345)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2236)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1692)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2454)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2378)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2236)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1692)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2454)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2378)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2236)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1692)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:508)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:466)
at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:539)
at org.apache.flink.runtime.deployment.TaskDeploymentDescriptor.loadBigData(TaskDeploymentDescriptor.java:306)
at org.apache.flink.runtime.taskexecutor.TaskExecutor.submitTask(TaskExecutor.java:686)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.lambda$handleRpcInvocation$1(PekkoRpcActor.java:309)
at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor$$Lambda$619/1885079283.get(Unknown Source)