求助:doris2.1写入速度无法提高

Viewed 54

我在使用doris-flink-connector写doris双节点集群,写入速度一直在约40w/s左右无法提高了,be节点的cpu(48核)只有25%,磁盘和网络都有很大的富余,请问有什么地方提议进一步优化呢?或者我的配置哪里需要修改呢?谢谢
我的be节点配置:
push_worker_count_high_priority = 24
push_worker_count_normal_priority = 24
streaming_load_json_max_mb = 1024
number_tablet_writer_threads = 72
max_send_batch_parallelism_per_job = 48
enable_single_replica_load = true

#params for log write
enable_file_cache = true
file_cache_path = [{"path": "/data/doris_file_cache", "total_size":53687091200, "query_limit": "10737418240"}]
#write_buffer_size = 1073741824
write_buffer_size = 2147483648
max_tablet_version_num = 20000
max_cumu_compaction_threads = 12
inverted_index_compaction_enable = true
enable_segcompaction = false
enable_ordered_data_compaction = false
enable_compaction_priority_scheduling = false
total_permits_for_compaction_score = 200000
disable_storage_page_cache = true
inverted_index_searcher_cache_limit = 30%
inverted_index_cache_stale_sweep_time_sec = 3600
index_cache_entry_stay_time_after_lookup_s = 3600
enable_inverted_index_cache_on_cooldown = true

enable_write_index_searcher_cache = false
tablet_schema_cache_recycle_interval = 3600
segment_cache_capacity = 20000
inverted_index_ram_dir_enable = true
pipeline_executor_size = 36
doris_scanner_thread_pool_thread_num = 72
scan_thread_nice_value = 5
string_type_length_soft_limit_bytes = 10485760
trash_file_expire_time_sec = 300
path_gc_check_interval_second = 900
path_scan_interval_second = 900

2 Answers

写入的是什么表模型,多少个bucket呢?方便贴一下表结构


已经建立联系,调整bucket后正在重新测试

你好,这是表结构,桶数我测试了2、24、48,写入速度没有明显差异
CREATE TABLE raw_log
(
time DATETIME(0),
cdn_property_id varchar(64),
client_dst_ep varchar(64),
client_src_ep varchar(64),
server_dst_ep varchar(64),
session_id varchar(64),
ttfb FLOAT,
service_time FLOAT,
debug_flags varchar(20480),
resp_code varchar(64),
client_bytes_sent BIGINT,
req_url varchar(20480),
cdn_node_id varchar(64),
cdn_traffic_type varchar(64),
file_time varchar(64),
service_name varchar(64),
cdn_node_hostname varchar(64),
full_message varchar(20480),
INDEX idx_time(time) USING INVERTED,
INDEX idx_cdn_property_id(cdn_property_id) USING INVERTED PROPERTIES("parser" = "english"),
INDEX idx_client_dst_ep(client_dst_ep) USING INVERTED PROPERTIES("parser" = "english", "char_filter_type" = "char_replace", "char_filter_pattern" = ".", "char_filter_replacement" = "1111"),
INDEX idx_client_src_ep(client_src_ep) USING INVERTED PROPERTIES("parser" = "english", "char_filter_type" = "char_replace", "char_filter_pattern" = ".", "char_filter_replacement" = "1111"),
INDEX idx_server_dst_ep(server_dst_ep) USING INVERTED PROPERTIES("parser" = "english", "char_filter_type" = "char_replace", "char_filter_pattern" = ".", "char_filter_replacement" = "1111"),
INDEX idx_resp_code(resp_code) USING INVERTED PROPERTIES("parser" = "english"),
INDEX idx_cdn_node_id(cdn_node_id) USING INVERTED PROPERTIES("parser" = "english"),
INDEX idx_debug_flags(debug_flags) USING INVERTED PROPERTIES("parser" = "english", "char_filter_type" = "char_replace", "char_filter_pattern" = ".-/", "char_filter_replacement" = "1111"),
INDEX idx_req_url(req_url) USING INVERTED PROPERTIES("parser" = "english", "char_filter_type" = "char_replace", "char_filter_pattern" = ".-", "char_filter_replacement" = "1111")
)
DUPLICATE KEY(time) -- 只用于指定排序列,相同的 KEY 行不会合并
PARTITION BY RANGE(time) ()
DISTRIBUTED BY HASH(time) BUCKETS 24
PROPERTIES (
"compression" = "zstd",
"compaction_policy" = "time_series",
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "DAY",
"dynamic_partition.start" = "-7",
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.create_history_partition" = "true",
"dynamic_partition.buckets" = "24",
"dynamic_partition.replication_num" = "2", -- 存算分离不需要
"replication_num" = "2" -- 存算分离不需要
);