我在使用doris-flink-connector写doris双节点集群,写入速度一直在约40w/s左右无法提高了,be节点的cpu(48核)只有25%,磁盘和网络都有很大的富余,请问有什么地方提议进一步优化呢?或者我的配置哪里需要修改呢?谢谢
我的be节点配置:
push_worker_count_high_priority = 24
push_worker_count_normal_priority = 24
streaming_load_json_max_mb = 1024
number_tablet_writer_threads = 72
max_send_batch_parallelism_per_job = 48
enable_single_replica_load = true
#params for log write
enable_file_cache = true
file_cache_path = [{"path": "/data/doris_file_cache", "total_size":53687091200, "query_limit": "10737418240"}]
#write_buffer_size = 1073741824
write_buffer_size = 2147483648
max_tablet_version_num = 20000
max_cumu_compaction_threads = 12
inverted_index_compaction_enable = true
enable_segcompaction = false
enable_ordered_data_compaction = false
enable_compaction_priority_scheduling = false
total_permits_for_compaction_score = 200000
disable_storage_page_cache = true
inverted_index_searcher_cache_limit = 30%
inverted_index_cache_stale_sweep_time_sec = 3600
index_cache_entry_stay_time_after_lookup_s = 3600
enable_inverted_index_cache_on_cooldown = true
enable_write_index_searcher_cache = false
tablet_schema_cache_recycle_interval = 3600
segment_cache_capacity = 20000
inverted_index_ram_dir_enable = true
pipeline_executor_size = 36
doris_scanner_thread_pool_thread_num = 72
scan_thread_nice_value = 5
string_type_length_soft_limit_bytes = 10485760
trash_file_expire_time_sec = 300
path_gc_check_interval_second = 900
path_scan_interval_second = 900