我的集群是按照官方建议最低配建的集群四台16核64G的机器,1fe 3be,我们的数据量不是很大,整个库数据量加起来可能都没超过1亿,但是我们数据写入更新非常频繁。查看fe监控是这样的
机器的内存情况是这样的
就是实际使用的比fe监控的内存要大的多,有大佬帮忙看看是什么原因吗
我的集群是按照官方建议最低配建的集群四台16核64G的机器,1fe 3be,我们的数据量不是很大,整个库数据量加起来可能都没超过1亿,但是我们数据写入更新非常频繁。查看fe监控是这样的
机器的内存情况是这样的
就是实际使用的比fe监控的内存要大的多,有大佬帮忙看看是什么原因吗
这种需要在 FE 内存升上去的时候打个 jmap 分析下了:
jmap -histo:live fe_pid > jmap2.log
jstack fe_pid > fe_js.txt
1: 835769 1470780368 [B (java.base@17.0.13)
2: 1064163 102702744 [Ljava.lang.Object; (java.base@17.0.13)
3: 308416 86356480 org.apache.doris.thrift.TExprNode
4: 269769 38846736 org.apache.doris.analysis.StringLiteral
5: 1123759 35960288 java.util.ArrayList (java.base@17.0.13)
6: 982548 31441536 com.google.common.base.Suppliers$NonSerializableMemoizingSupplier
7: 207745 30928736 [Ljava.util.HashMap$Node; (java.base@17.0.13)
8: 679115 21731680 java.lang.String (java.base@17.0.13)
9: 292045 21027240 org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral
10: 518257 20730280 java.util.HashMap$Node (java.base@17.0.13)
11: 369786 20708016 org.apache.doris.thrift.TTypeDesc
12: 293222 18766208 java.util.HashMap (java.base@17.0.13)
13: 369786 17749728 org.apache.doris.thrift.TTypeNode
14: 693961 16655064 java.lang.Long (java.base@17.0.13)
15: 369786 14791440 org.apache.doris.thrift.TScalarType
16: 375797 12025504 org.apache.doris.nereids.util.MutableState$SingleMutableState
17: 39998 11839408 org.apache.doris.statistics.AnalysisInfo
18: 450106 10802544 org.apache.doris.nereids.trees.expressions.Expression$$Lambda$342/0x00007f4aec846d70
19: 262144 8388608 org.apache.logging.log4j.core.async.AsyncLoggerConfigDisruptor$Log4jEventWrapper
20: 14964 8271776 [I (java.base@17.0.13)
21: 57317 7336576 org.apache.doris.nereids.trees.expressions.SlotReference
22: 303041 7272984 org.apache.doris.analysis.Expr$$Lambda$141/0x00007f4aec27b880
23: 180945 7237800 java.util.concurrent.ConcurrentHashMap$Node (java.base@17.0.13)
24: 61243 6369272 org.apache.doris.statistics.ColumnStatistic
25: 262582 6301968 org.apache.doris.thrift.TStringLiteral
26: 389712 6235392 java.lang.Integer (java.base@17.0.13)
27: 31823 6015656 java.lang.Class (java.base@17.0.13)
28: 31620 5818080 org.apache.doris.analysis.SlotRef
29: 103814 5813584 java.util.LinkedHashMap$Entry (java.base@17.0.13)
30: 59673 5728608 java.util.concurrent.ConcurrentHashMap (java.base@17.0.13)
Total 14664385 2240311872
"grpc_thread_pool-7552" #406597 daemon prio=5 os_prio=0 cpu=0.28ms elapsed=22.57s tid=0x00007f491c227e00 nid=0x382a waiting on condition [0x00007f48e7ceb000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for <0x00007f4bc0e3c3c0> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"grpc_thread_pool-7876" #406598 daemon prio=5 os_prio=0 cpu=0.35ms elapsed=21.42s tid=0x00007f493400cbb0 nid=0x382c waiting on condition [0x00007f48dd2d1000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for <0x00007f4bc0bea170> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"grpc_thread_pool-7877" #406599 daemon prio=5 os_prio=0 cpu=0.33ms elapsed=21.42s tid=0x00007f4930370ea0 nid=0x382d waiting on condition [0x00007f48ddedd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for <0x00007f4bc0bea170> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"grpc_thread_pool-7547" #406600 daemon prio=5 os_prio=0 cpu=0.20ms elapsed=20.55s tid=0x00007f4918233280 nid=0x382f waiting on condition [0x00007f48e9111000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for <0x00007f4bc09dc818> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"agent-task-pool-2301" #406601 daemon prio=5 os_prio=0 cpu=0.10ms elapsed=11.77s tid=0x00007f49dc38ba40 nid=0x383c waiting on condition [0x00007f48da7a6000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for <0x00007f4bc2915058> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"grpc_thread_pool-7618" #406602 daemon prio=5 os_prio=0 cpu=0.23ms elapsed=11.36s tid=0x00007f490027f560 nid=0x383d waiting on condition [0x00007f48d6665000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for <0x00007f4bc098a260> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)