我本周做了个调整， 将jdk1.8.0_351换成 OpenJDK8U-jdk_x64_linux_hotspot_8u412b08 ，以前一直没有爆be内存彪高，本周已爆2次了，而且内存一直没有被回收，只能重启服务器。
日志如下：
consumption 201.84 MB, backend 10.166.1.137 process memory used 7.91 GB exceed limit 11.50 GB or sys available memory 1.42 GB less than low water mark 1.44 GB. Execute again after enough memory, details see be.INFO. at org.mariadb.jdbc.export.ExceptionFactory.createException(ExceptionFactory.java:287) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.export.ExceptionFactory.create(ExceptionFactory.java:358) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.message.ClientMessage.readPacket(ClientMessage.java:133) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.client.impl.StandardClient.readPacket(StandardClient.java:815) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.client.impl.StandardClient.readResults(StandardClient.java:754) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.client.impl.StandardClient.readResponse(StandardClient.java:673) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.client.impl.StandardClient.execute(StandardClient.java:616) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.Statement.executeInternal(Statement.java:906) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.Statement.execute(Statement.java:1031) ~[mariadb-java-client-3.0.4.jar:?] at org.mariadb.jdbc.Statement.execute(Statement.java:441) ~[mariadb-java-client-3.0.4.jar:?] at org.apache.doris.httpv2.util.StatementSubmitter$Worker.call(StatementSubmitter.java:120) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.httpv2.util.StatementSubmitter$Worker.call(StatementSubmitter.java:83) ~[doris-fe.jar:1.2-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_412] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_412] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_412] ... 1 more2024-04-26 12:55:03,203 WARN (REPLICA 10.166.1.137_9010_1698371205844(1)|70) [Env.notifyNewFETypeTransfer():2393] notify new FE type transfer: UNKNOWN2024-04-26 12:55:03,205 WARN (stateListener|82) [Env.transferToNonMaster():1541] FOLLOWER to UNKNOWN, still offer read service2024-04-26 12:55:03,239 WARN (UNKNOWN 10.166.1.137_9010_1698371205844(1)|70) [Env.notifyNewFETypeTransfer():2393] notify new FE type transfer: FOLLOWER2024-04-26 12:55:03,497 WARN (replayer|83) [Backend.handleHbResponse():689] Backend [id=10005, host=10.166.1.138, heartbeatPort=9050, alive=false, lastStartTime=\N, tags: {location=default}] is dead,2024-04-26 12:55:35,153 WARN (ForkJoinPool-2-worker-1|38947) [TabletStatMgr.lambda$null$0():71] task exec error. backend[10005]org.apache.thrift.transport.TTransportException: java.net.NoRouteToHostException: No route to host (Host unreachable) at org.apache.thrift.transport.TSocket.open(TSocket.java:255) ~[libthrift-0.16.0.jar:0.16.0] at org.apache.doris.common.GenericPool$ThriftClientFactory.create(GenericPool.java:143) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.common.GenericPool$ThriftClientFactory.create(GenericPool.java:126) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.commons.pool2.BaseKeyedPooledObjectFactory.makeObject(BaseKeyedPooledObjectFactory.java:62) ~[commons-pool2-2.2.jar:2.2] at org.apache.commons.pool2.impl.GenericKeyedObjectPool.create(GenericKeyedObjectPool.java:1012) ~[commons-pool2-2.2.jar:2.2] at org.apache.commons.pool2.impl.GenericKeyedObjectPool.borrowObject(GenericKeyedObjectPool.java:356) ~[commons-pool2-2.2.jar:2.2] at org.apache.commons.pool2.impl.GenericKeyedObjectPool.borrowObject(GenericKeyedObjectPool.java:277) ~[commons-pool2-2.2.jar:2.2] at org.apache.doris.common.GenericPool.borrowObject(GenericPool.java:95) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.catalog.TabletStatMgr.lambda$null$0(TabletStatMgr.java:64) ~[doris-fe.jar:1.2-SNAPSHOT] at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183) ~[?:1.8.0_412] at com.google.common.collect.CollectSpliterators$1WithCharacteristics.lambda$forEachRemaining$1(CollectSpliterators.java:72) ~[guava-32.1.2-jre.jar:?] at java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:110) ~[?:1.8.0_412] at com.google.common.collect.CollectSpliterators$1WithCharacteristics.forEachRemaining(CollectSpliterators.java:72) ~[guava-32.1.2-jre.jar:?] at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482) ~[?:1.8.0_412] at java.util.stream.ForEachOps$ForEachTask.compute(ForEachOps.java:290) ~[?:1.8.0_412] at java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:731) ~[?:1.8.0_412] at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_412] at java.util.concurrent.ForkJoinTask.doInvoke(ForkJoinTask.java:401) ~[?:1.8.0_412] at java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:734) ~[?:1.8.0_412] at java.util.stream.ForEachOps$ForEachOp.evaluateParallel(ForEachOps.java:159) ~[?:1.8.0_412] at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateParallel(ForEachOps.java:173) ~[?:1.8.0_412] at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233) ~[?:1.8.0_412] at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485) ~[?:1.8.0_412] at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:650) ~[?:1.8.0_412] at org.apache.doris.catalog.TabletStatMgr.lambda$runAfterCatalogReady$1(TabletStatMgr.java:58) ~[doris-fe.jar:1.2-SNAPSHOT] at java.util.concurrent.ForkJoinTask$AdaptedRunnableAction.exec(ForkJoinTask.java:1386) ~[?:1.8.0_412] at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_412] at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) ~[?:1.8.0_412] at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) ~[?:1.8.0_412] at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) ~[?:1.8.0_412]Caused by: java.net.NoRouteToHostException: No route to host (Host unreachable) at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_412] at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[?:1.8.0_412] at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[?:1.8.0_412] at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[?:1.8.0_412] at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:1.8.0_412] at java.net.Socket.connect(Socket.java:607) ~[?:1.8.0_412] at org.apache.thrift.transport.TSocket.open(TSocket.java:250) ~[libthrift-0.16.0.jar:0.16.0] ... 29 more2024-04-26 13:40:32,009 WARN (qtp1456817679-39201|39201) [ThreadPoolManager$LogDiscardPolicy.rejectedExecution():298] Task java.util.concurrent.FutureTask@3ee2f568 rejected from SQL submitter java.util.concurrent.ThreadPoolExecutor@189fc53a[Running, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 53572]2024-04-26 15:55:47,204 WARN (qtp1456817679-40333|40333) [ThreadPoolManager$LogDiscardPolicy.rejectedExecution():298] Task java.util.concurrent.FutureTask@7ce3f684 rejected from SQL submitter java.util.concurrent.ThreadPoolExecutor@189fc53a[Running, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 66268]2024-04-26 15:59:05,802 WARN (qtp1456817679-40550|40550) [ThreadPoolManager$LogDiscardPolicy.rejectedExecution():298] Task java.util.concurrent.FutureTask@4d1ff7d4 rejected from SQL submitter java.util.concurrent.ThreadPoolExecutor@189fc53a[Running, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 67693]2024-04-26 16:01:57,605 WARN (qtp1456817679-40706|40706) [ThreadPoolManager$LogDiscardPolicy.rejectedExecution():298] Task java.util.concurrent.FutureTask@5c919074 rejected from SQL submitter java.util.concurrent.ThreadPoolExecutor@189fc53a[Running, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 68773]2024-04-26 16:07:29,148 WARN (qtp1456817679-40761|40761) [ThreadPoolManager$LogDiscardPolicy.rejectedExecution():298] Task java.util.concurrent.FutureTask@752320ba rejected from SQL submitter java.util.concurrent.ThreadPoolExecutor@189fc53a[Running, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 69957]2024-04-26 16:09:14,750 WARN (qtp1456817679-40591|40591) [ThreadPoolManager$LogDiscardPolicy.rejectedExecution():298] Task java.util.concurrent.FutureTask@7cab9821 rejected from SQL submitter java.util.concurrent.ThreadPoolExecutor@189fc53a[Running, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 70455]2024-04-26 16:11:00,308 WARN (qtp1456817679-40866|40866) [ThreadPoolManager$LogDiscardPolicy.rejectedExecution():298] Task java.util.concurrent.FutureTask@1041b840 rejected from SQL submitter java.util.concurrent.ThreadPoolExecutor@189fc53a[Running, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 70923]2024-04-26 16:13:46,197 WARN (qtp1456817679-40927|40927) [ThreadPoolManager$LogDiscardPolicy.rejectedExecution():298] Task java.util.concurrent.FutureTask@1b75691d rejected from SQL submitter java.util.concurrent.ThreadPoolExecutor@189fc53a[Running, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 71790]2024-04-26 18:28:47,808 WARN (UNKNOWN 10.166.1.137_9010_1698371205844(-1)|1) [Env.notifyNewFETypeTransfer():2393] notify new FE type transfer: UNKNOWN2024-04-26 18:28:47,844 WARN (RepNode 10.166.1.137_9010_1698371205844(-1)|70) [Env.notifyNewFETypeTransfer():2393] notify new FE type transfer: FOLLOWER2024-04-26 18:28:49,350 WARN (replayer|85) [Backend.handleHbResponse():689] Backend [id=2598249, host=10.166.1.137, heartbeatPort=9050, alive=false, lastStartTime=\N, tags: {location=default}] is dead,2024-04-26 18:28:49,386 WARN (replayer|85) [Env.replayJournal():2563] replay journal cost too much time: 1357 replayedJournalId: 7998176
be的log目录生成了be.gc.log.20240426-182805文件，内容如下：
OpenJDK 64-Bit Server VM (25.412-b08) for linux-amd64 JRE (1.8.0_412-b08), built on Apr 17 2024 02:26:16 by "jenkins" with gcc 7.5.0
Memory: 4k page, physical 15077628k(13841348k free), swap 0k(0k free)
CommandLine flags: -XX:-CriticalJNINatives -XX:InitialHeapSize=241242048 -XX:MaxHeapSize=1073741824 -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+UseCompressedClassPointers -XX:+UseCompressedOops -XX:+UseParallelGC
2.079: [GC (Allocation Failure) 59392K->6640K(227840K), 0.0118630 secs]
2.336: [GC (Allocation Failure) 66032K->10416K(227840K), 0.0065120 secs]
2.381: [GC (Metadata GC Threshold) 16336K->5224K(227840K), 0.0106573 secs]
2.392: [Full GC (Metadata GC Threshold) 5224K->4276K(141312K), 0.0273969 secs]
2.548: [GC (Allocation Failure) 63668K->5108K(183296K), 0.0014227 secs]
2.701: [GC (Allocation Failure) 115188K->4712K(192000K), 0.0008729 secs]
2.829: [GC (Allocation Failure) 114792K->4516K(276480K), 0.0013281 secs]
3.007: [GC (Allocation Failure) 200100K->4500K(280576K), 0.0019370 secs]
3.147: [GC (Allocation Failure) 200084K->4612K(407040K), 0.0015146 secs]
操作系统是 x86 银河麒麟操作系统
Release:
Kylin Linux Advanced Server release V10 (Lance)
Kernel:
4.19.90-52.15.v2207.ky10.x86_64
Build:
Kylin Linux Advanced Server
release V10 (SP3) /(Lance)-x86_64-Build20/20221125
换JDK版本的考虑点是：jdk8.351 商用收费，所以换成 OpenJDK 商用免费版本。
