我的集群是按照官方建议最低配建的集群四台16核64G的机器,1fe 3be,我们的数据量不是很大,整个库数据量加起来可能都没超过1亿,但是我们数据写入更新非常频繁。查看fe监控是这样的
机器的内存情况是这样的
就是实际使用的比fe监控的内存要大的多,有大佬帮忙看看是什么原因吗
我的集群是按照官方建议最低配建的集群四台16核64G的机器,1fe 3be,我们的数据量不是很大,整个库数据量加起来可能都没超过1亿,但是我们数据写入更新非常频繁。查看fe监控是这样的
机器的内存情况是这样的
就是实际使用的比fe监控的内存要大的多,有大佬帮忙看看是什么原因吗
这种需要在 FE 内存升上去的时候打个 jmap 分析下了:
jmap -histo:live fe_pid > jmap2.log
jstack fe_pid > fe_js.txt
1:        835769     1470780368  [B (java.base@17.0.13)
2:       1064163      102702744  [Ljava.lang.Object; (java.base@17.0.13)
3:        308416       86356480  org.apache.doris.thrift.TExprNode
4:        269769       38846736  org.apache.doris.analysis.StringLiteral
5:       1123759       35960288  java.util.ArrayList (java.base@17.0.13)
6:        982548       31441536  com.google.common.base.Suppliers$NonSerializableMemoizingSupplier
7:        207745       30928736  [Ljava.util.HashMap$Node; (java.base@17.0.13)
8:        679115       21731680  java.lang.String (java.base@17.0.13)
9:        292045       21027240  org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral
10:        518257       20730280  java.util.HashMap$Node (java.base@17.0.13)
11:        369786       20708016  org.apache.doris.thrift.TTypeDesc
12:        293222       18766208  java.util.HashMap (java.base@17.0.13)
13:        369786       17749728  org.apache.doris.thrift.TTypeNode
14:        693961       16655064  java.lang.Long (java.base@17.0.13)
15:        369786       14791440  org.apache.doris.thrift.TScalarType
16:        375797       12025504  org.apache.doris.nereids.util.MutableState$SingleMutableState
17:         39998       11839408  org.apache.doris.statistics.AnalysisInfo
18:        450106       10802544  org.apache.doris.nereids.trees.expressions.Expression$$Lambda$342/0x00007f4aec846d70
19:        262144        8388608  org.apache.logging.log4j.core.async.AsyncLoggerConfigDisruptor$Log4jEventWrapper
20:         14964        8271776  [I (java.base@17.0.13)
21:         57317        7336576  org.apache.doris.nereids.trees.expressions.SlotReference
22:        303041        7272984  org.apache.doris.analysis.Expr$$Lambda$141/0x00007f4aec27b880
23:        180945        7237800  java.util.concurrent.ConcurrentHashMap$Node (java.base@17.0.13)
24:         61243        6369272  org.apache.doris.statistics.ColumnStatistic
25:        262582        6301968  org.apache.doris.thrift.TStringLiteral
26:        389712        6235392  java.lang.Integer (java.base@17.0.13)
27:         31823        6015656  java.lang.Class (java.base@17.0.13)
28:         31620        5818080  org.apache.doris.analysis.SlotRef
29:        103814        5813584  java.util.LinkedHashMap$Entry (java.base@17.0.13)
30:         59673        5728608  java.util.concurrent.ConcurrentHashMap (java.base@17.0.13)
Total      14664385     2240311872
"grpc_thread_pool-7552" #406597 daemon prio=5 os_prio=0 cpu=0.28ms elapsed=22.57s tid=0x00007f491c227e00 nid=0x382a waiting on condition  [0x00007f48e7ceb000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for  <0x00007f4bc0e3c3c0> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"grpc_thread_pool-7876" #406598 daemon prio=5 os_prio=0 cpu=0.35ms elapsed=21.42s tid=0x00007f493400cbb0 nid=0x382c waiting on condition  [0x00007f48dd2d1000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for  <0x00007f4bc0bea170> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"grpc_thread_pool-7877" #406599 daemon prio=5 os_prio=0 cpu=0.33ms elapsed=21.42s tid=0x00007f4930370ea0 nid=0x382d waiting on condition  [0x00007f48ddedd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for  <0x00007f4bc0bea170> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"grpc_thread_pool-7547" #406600 daemon prio=5 os_prio=0 cpu=0.20ms elapsed=20.55s tid=0x00007f4918233280 nid=0x382f waiting on condition  [0x00007f48e9111000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for  <0x00007f4bc09dc818> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"agent-task-pool-2301" #406601 daemon prio=5 os_prio=0 cpu=0.10ms elapsed=11.77s tid=0x00007f49dc38ba40 nid=0x383c waiting on condition  [0x00007f48da7a6000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for  <0x00007f4bc2915058> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)
"grpc_thread_pool-7618" #406602 daemon prio=5 os_prio=0 cpu=0.23ms elapsed=11.36s tid=0x00007f490027f560 nid=0x383d waiting on condition  [0x00007f48d6665000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@17.0.13/Native Method)
- parking to wait for  <0x00007f4bc098a260> (a java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@17.0.13/LockSupport.java:252)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@17.0.13/SynchronousQueue.java:401)
at java.util.concurrent.SynchronousQueue.poll(java.base@17.0.13/SynchronousQueue.java:903)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@17.0.13/ThreadPoolExecutor.java:1061)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.13/ThreadPoolExecutor.java:1122)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.13/ThreadPoolExecutor.java:635)
at java.lang.Thread.run(java.base@17.0.13/Thread.java:842)