版本:v2.1
如题,springboot项目,我需要写入数据到doris表中,单次一般在7万条数据左右,所以不想用streamload的方式,streamload比较麻烦,还得转化文件格式,如果jdbc能达到官方说的1万条每秒,那完全够用了,但是问题是我现在测试发现批量sql,写5000条数据就得1分钟,才100条/秒,这样也太慢了,请教那位大神知道是什么情况吗?
- 已经参考官方说的开启Group Commit
url: jdbc:mysql://172.17.130.14:9030/uat?characterEncoding=UTF-8&characterSetResults=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&allowPublicKeyRetrieval=true&useTimezone=true&serverTimezone=Asia/Shanghai
&useServerPrepStmts=true
&useLocalSessionState=true
&rewriteBatchedStatements=true
&cachePrepStmts=true
&prepStmtCacheSqlLimit=99999
&prepStmtCacheSize=50
&sessionVariables=group_commit=async_mode,enable_nereids_planner=false
- 已经按照官方说的关闭事务
- 确实是通过insert into values (...),(...),(...)...的方式,示例sql如下
INSERT INTO cash_flow_collection_data ( fld_guid, fld_create_date, fld_create_user, fld_modify_date, fld_modify_user, fld_dq, fld_ywdy, fld_company, fld_xm, fld_area_guid, fld_area_name, fld_object_class_name, fld_butler_guid, fld_grid_no, fld_grid_name, fld_employee_guid, fld_employee_name, fld_object_count, fld_point_date, fld_cur_month_coll_cur_year_property_fee, fld_cur_month_coll_debt_property_fee, fld_cur_month_coll_next_year_property_fee, fld_cur_month_coll_future_property_fee, fld_cur_month_prepay_change_property_fee, fld_sub_total_property_fee, fld_cur_month_coll_cur_year_park_fee, fld_cur_month_coll_debt_park_fee, fld_cur_month_coll_next_year_park_fee, fld_cur_month_coll_future_park_fee, fld_cur_month_prepay_change_park_fee, fld_sub_total_park_fee, fld_cur_month_coll_cur_year_energy_fee, fld_cur_month_coll_debt_energy_fee, fld_cur_month_coll_next_year_energy_fee, fld_cur_month_coll_future_energy_fee, fld_cur_month_prepay_change_energy_fee, fld_sub_total_energy_fee, fld_cur_month_coll_cur_year_total, fld_cur_month_coll_debt_total, fld_cur_month_coll_next_year_total, fld_cur_month_coll_future_total, fld_cur_month_prepay_change_total, fld_fld_sub_total_total, fld_cur_month_coll_cur_year_new_property_fee, fld_cur_month_coll_debt_new_property_fee, fld_cur_month_coll_next_year_new_property_fee, fld_cur_month_coll_future_new_property_fee, fld_cur_month_prepay_change_new_property_fee, fld_sub_total_new_property_fee, fld_cur_month_coll_cur_year_new_park_fee, fld_cur_month_coll_debt_new_park_fee, fld_cur_month_coll_next_year_new_park_fee, fld_cur_month_coll_future_new_park_fee, fld_cur_month_prepay_change_new_park_fee, fld_sub_total_new_park_fee, fld_cur_month_coll_cur_year_new_total, fld_cur_month_coll_debt_new_total, fld_cur_month_coll_next_year_new_total, fld_cur_month_coll_future_new_total, fld_cur_month_prepay_change_new_total, fld_fld_sub_total_new_total, fld_cur_month_coll_cur_year_final_total, fld_cur_month_coll_debt_final_total, fld_cur_month_coll_next_year_final_total, fld_cur_month_coll_future_final_total, fld_cur_month_prepay_change_final_total, fld_fld_sub_total_final_total, fld_type )
VALUES
('6a3eea135dfd5308fc9d392e2c603ae7','2025-01-01 00:00:00','','2025-01-01 00:00:00','','彰泰融创物业合资公司','桂南公司','','崇左溯禾公园学府项目','16366241556631954481','崇左溯禾公园学府','产权车位','','','','','','0','2025-01-01 00:00:00',0,0,0,0,0,0,630.00,0,0,0,-210.00,420.00,0,0,0,0,0,0,630.00,0,0,0,-210.00,420.00,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,630.00,0,0,0,-210.00,420.00,0)
,('36c06886cb3377e1a71163c30817d53e','2025-01-01 00:00:00','','2025-01-01 00:00:00','','川云大区','环蓉公司','','自贡观溪樾项目','161933580400958342','自贡观溪樾','产权车位','31eea88049f8830fefdb288d7dfdba8f','22310','自贡观溪樾-网格B','ZONGQY1','宗庆英','0','2025-01-01 00:00:00',0,0,0,0,0,0,60.00,0,0,0,0,60.00,0,0,0,0,0,0,60.00,0,0,0,0,60.00,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,60.00,0,0,0,0,60.00,0)
,('b192f08d347b6b30ff05ea4bee433f2a','2025-01-01 00:00:00','','2025-01-01 00:00:00','','北京大区','北京公司','','北京融创亦庄壹号项目','162081109857608','北京融岸雅苑','商业','956165bdec0a9673d7680bf53155d7f1','23665','北京融创亦庄壹号-网格A','WANGX699','王鑫','0','Fri Jul 04 23:59:59 CST 2025',0,0,0,0,0,0,0,0,0,0,0,0,5067.50,0,0,0,0.00,5067.50,5067.50,0,0,0,0.00,5067.50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5067.50,0,0,0,0.00,5067.50,0)
- 这是官方说的jdbc测试速率