2.1.3升级到2.1.4后BE的CPU负载相较之前成倍增长

Viewed 134

image.png
图中上升的时间点为我操作升级的时间点,升级后BE负载就这个样子了,其实FE也有上涨不过没有BE明显。
image.png
这张是任意找的BE节点上be.WARINING的日志。
以下是FE节点的be.warn.log

2024-07-23 14:20:00,097 WARN (mysql-nio-pool-213|38829) [StmtExecutor.executeByLegacy():950] execute Exception. stmt[84192, 1cc195c41ac744d7-99403aa0f0a218df]
org.apache.doris.common.UserException: errCode = 2, detailMessage = Failed to prepare statementtry to set enable_server_side_prepared_statement = true
        at org.apache.doris.analysis.PrepareStmt.analyze(PrepareStmt.java:193) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.analyze(StmtExecutor.java:1123) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.executeByLegacy(StmtExecutor.java:838) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:588) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:514) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:313) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:204) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:258) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:286) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:339) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_381]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_381]
        at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_381]
2024-07-23 14:20:00,106 WARN (mysql-nio-pool-213|38829) [StmtExecutor.executeByLegacy():950] execute Exception. stmt[84194, 9b673fe16ccf4dff-90795d2364dd584c]
org.apache.doris.common.UserException: errCode = 2, detailMessage = Failed to prepare statementtry to set enable_server_side_prepared_statement = true
        at org.apache.doris.analysis.PrepareStmt.analyze(PrepareStmt.java:193) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.analyze(StmtExecutor.java:1123) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.executeByLegacy(StmtExecutor.java:838) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:588) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:514) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:313) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:204) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:258) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:286) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:339) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_381]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_381]
        at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_381]
2024-07-23 14:20:00,113 WARN (mysql-nio-pool-213|38829) [StmtExecutor.executeByLegacy():950] execute Exception. stmt[84196, c5dc03b5e8c048df-a5a6c630fbb3f5b2]
org.apache.doris.common.UserException: errCode = 2, detailMessage = Failed to prepare statementtry to set enable_server_side_prepared_statement = true
        at org.apache.doris.analysis.PrepareStmt.analyze(PrepareStmt.java:193) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.analyze(StmtExecutor.java:1123) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.executeByLegacy(StmtExecutor.java:838) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:588) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:514) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:313) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:204) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:258) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:286) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:339) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_381]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_381]
        at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_381]
2024-07-23 14:20:00,118 WARN (mysql-nio-pool-213|38829) [StmtExecutor.executeByLegacy():950] execute Exception. stmt[84198, be0102db1f3d4434-8cbd353d93b91daa]
org.apache.doris.common.UserException: errCode = 2, detailMessage = Failed to prepare statementtry to set enable_server_side_prepared_statement = true
        at org.apache.doris.analysis.PrepareStmt.analyze(PrepareStmt.java:193) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.analyze(StmtExecutor.java:1123) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.executeByLegacy(StmtExecutor.java:838) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:588) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:514) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:313) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:204) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:258) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:286) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:339) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_381]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_381]
        at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_381]
2024-07-23 14:20:00,141 WARN (mysql-nio-pool-215|39169) [FunctionCallExpr.analyzeImpl():1724] fn time_format(timediff(now(), convert_tz(now(), @@GLOBAL.time_zone, '+00:00')), '%H:%i') not exists
2024-07-23 14:20:00,141 WARN (mysql-nio-pool-215|39169) [StmtExecutor.executeByLegacy():950] execute Exception. stmt[84219, 151d0b8aea854f8b-ace8a6cdd7039725]
org.apache.doris.common.AnalysisException: errCode = 2, detailMessage = No matching function with signature: time_format(TIME, VARCHAR(65533)).
        at org.apache.doris.analysis.FunctionCallExpr.analyzeImpl(FunctionCallExpr.java:1725) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.analysis.Expr.analyze(Expr.java:434) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.analysis.SelectStmt.analyze(SelectStmt.java:589) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.analyzeAndGenerateQueryPlan(StmtExecutor.java:1290) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.analyze(StmtExecutor.java:1185) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.executeByLegacy(StmtExecutor.java:838) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:585) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:514) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:313) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:204) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:258) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:286) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:339) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_381]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_381]
        at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_381]
2024-07-23 14:21:34,790 WARN (leaderCheckpointer|93) [Env.replayJournal():2831] replay journal cost too much time: 1352 replayedJournalId: 178322431
2024-07-23 14:27:38,449 WARN (Routine load task scheduler|36) [RoutineLoadTaskScheduler.scheduleOneTask():226] failed to submit routine load task 7a834a7ad32b4068-9fc949dd00c63546 to BE: 14128, error: errCode = 2, detailMessage = failed to send task: Socket is closed by peer.
2024-07-23 14:32:49,469 WARN (Routine load task scheduler|36) [RoutineLoadTaskScheduler.scheduleOneTask():226] failed to submit routine load task c2cbdacc08184f1a-bd900bdab44e624f to BE: 14034, error: errCode = 2, detailMessage = failed to send task: Socket is closed by peer.
2024-07-23 14:33:42,155 WARN (leaderCheckpointer|93) [Env.replayJournal():2831] replay journal cost too much time: 1950 replayedJournalId: 178372431
2024-07-23 14:36:51,764 WARN (Routine load task scheduler|36) [RoutineLoadTaskScheduler.scheduleOneTask():226] failed to submit routine load task 5fe53ab3f2fd412b-9c914ffe67fd7d15 to BE: 14034, error: errCode = 2, detailMessage = failed to send task: Socket is closed by peer.
2 Answers

重新回退到2.1.3后恢复了

sy占用是不是比us占用更高