在尝试使用doris替换ES发现 在测试 logstash传输数据到doris时,上传带宽增加非常多
求带宽增加原因以及优化方案
logstash版本:8.15.1
配置:32H 68G 千兆网卡
selectDB版本:selectdb-doris-2.1.7-rc03-hotfix1-f4204ea9d8
配置:3FE,11BE,32H 128G 千兆网卡
logstash-output-doirs版本:1.0.0
数据量:每分钟大约 222,000条JSON日志,每条JSON日志约 3kb
查看logstash的带宽,发现logstash的带宽达到了将近 200Mb/s
查看 logstash 日志:doris传输速度为 最高为 11Mb/s
doris的上下带宽并不高,已优化compaction、开启tablet均衡策略,开启单副本导入等
logstash:
pipeline.workers: 32
pipeline.batch.size: 3000
pipeline.batch.delay: 10
output-dors配置
output {
doris {
http_hosts => ["http://117.24.xx.xxxx:xxxx","http://117.24.xx.xxxx:xxxx","http://117.24.xx.xxxx:xxxx"]
user => "xxxxxx"
password => "xxxxxxxxxx"
db => "xxxxx"
table => "full_logs_cs"
headers => {
"format" => "json"
"read_json_by_line" => "true"
"load_to_single_tablet" => "true"
}
mapping => {
"hostname" => "%{hostname}"
"request_uri" => "%{request_uri}"
"server_id" => "%{server_id}"
"ga_id" => "%{gamedun_id}"
"response_headers" => "%{response_headers}"
"http_referer" => "%{http_referer}"
"upstream_response_time" => "%{upstream_response_time}"
"http2_fingerprint" => "%{http2_fingerprint}"
"http_host" => "%{http_host}"
"remote_addr" => "%{remote_addr}"
"status" => "%{status}"
"request_time" => "%{request_time}"
"slice_range" => "%{slice_range}"
"tls_fingerprint" => "%{tls_fingerprint}"
"upstream_status" => "%{upstream_status}"
"upstream_response_length" => "%{upstream_response_length}"
"request_path" => "%{request_path}"
"request_method" => "%{request_method}"
"cache_key" => "%{cache_key}"
"bytes_sent" => "%{bytes_sent}"
"real_client_ip" => "%{real_client_ip}"
"logs_time" => "%{time}"
"http_user_agent" => "%{http_user_agent}"
"body_bytes_sent" => "%{body_bytes_sent}"
"headers" => "%{headers}"
"geoip.geo.country_name" => "%{[geoip][geo][country_name]}"
"geoip.geo.timezone" => "%{[geoip][geo][timezone]}"
"geoip.geo.region_iso_code" => "%{[geoip][geo][region_iso_code]}"
"geoip.geo.city_name" => "%{[geoip][geo][city_name]}"
"geoip.geo.region_name" => "%{[geoip][geo][region_name]}"
"geoip.geo.country_iso_code" => "%{[geoip][geo][country_iso_code]}"
"geoip.geo.continent_code" => "%{[geoip][geo][continent_code]}"
"geoip.ip" => "%{[geoip][ip]}"
"user_id" => "%{user_id}"
"http_x_forwarded_for" => "%{http_x_forwarded_for}"
"query_string" => "%{query_string}"
"scheme" => "%{scheme}"
"upstream_addr" => "%{upstream_addr}"
"remote_port" => "%{remote_port}"
"server_addr" => "%{server_addr}"
"cache_status" => "%{cache_status}"
"request_length" => "%{request_length}"
}
log_request => true
}
}