logstash写入doris,某个节点compaction score、cpu和内存很高,其它节点很低

Viewed 11

版本:3.0.6
集群:3节点,fe和be混合部署,配置:24C24G

想测试下logstash写入doris的性能,发现一个节点compaction score非常高,这个节点不是master fe节点。有什么方法可以排查?

image.png

建表语句如下:

CREATE TABLE `app_logs` (
  `log_time` datetime(3),
  `node_ip` varchar(20),
  `service_name` varchar(256),
  `log_level` varchar(10),
  `log_content` text,
  `stack_trace` text,
  `host_name` varchar(50),
  `class_method` varchar(300),
  `thread_name` varchar(256),
  `tx_id` varchar(80),
  `span_id` varchar(80),
  `log_file_path` varchar(500),
  `file_offset` bigint,
  `agent_version` varchar(20),
  `agent_id` varchar(60),
  `ecs_version` varchar(10),
  `raw_data` text,
  `doris_ingest_time` datetime,
  INDEX log_content_idx (`log_content`) USING INVERTED PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" = "true"),
  INDEX stack_trace_idx (`stack_trace`) USING INVERTED,
  INDEX class_method_idx (`class_method`) USING INVERTED
) ENGINE=OLAP
DUPLICATE KEY(`log_time`)
PARTITION BY RANGE(`log_time`)()
DISTRIBUTED BY RANDOM BUCKETS 18
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "DAY",
"dynamic_partition.time_zone" = "Asia/Shanghai",
"dynamic_partition.start" = "-31",
"dynamic_partition.end" = "7",
"dynamic_partition.prefix" = "p",
"dynamic_partition.replication_allocation" = "tag.location.default: 1",
"dynamic_partition.buckets" = "18",
"dynamic_partition.create_history_partition" = "false",
"dynamic_partition.history_partition_num" = "-1",
"dynamic_partition.hot_partition_num" = "0",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.storage_policy" = "",
"storage_medium" = "hdd",
"storage_format" = "V2",
"inverted_index_storage_format" = "V2",
"compression" = "ZSTD",
"light_schema_change" = "true",
"compaction_policy" = "time_series",
"time_series_compaction_goal_size_mbytes" = "1024",
"time_series_compaction_file_count_threshold" = "2000",
"time_series_compaction_time_threshold_seconds" = "3600",
"time_series_compaction_empty_rowsets_threshold" = "5",
"time_series_compaction_level_threshold" = "1",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
);
0 Answers