Doris版本:2.1.3
问题描述:BE节点挂掉导致自动分区异常,分区字段由date_trunc(pdate
, 'day')变为date_trunc(a
.pdate
, 'day'),导致数据无法写入,错误信息如下:
WARN TaskSetManager: Lost task 129.1 in stage 4.0 (TID 462, 10.66.172.38, executor 7): org.apache.doris.spark.exception.StreamLoadException: stream load error, load status:Fail, response:StreamLoadResponse(200,OK,{
"TxnId": 16783,
"Label": "spark_streamload_20240702_114500_730929e6c56f49bfac7c034ab39abaa9",
"Comment": "",
"TwoPhaseCommit": "false",
"Status": "Fail",
"Message": "[ANALYSIS_ERROR]TStatus: errCode = 2, detailMessage = Unknown column 'pdate' in 'a'",
"NumberTotalRows": 0,
"NumberLoadedRows": 0,
"NumberFilteredRows": 0,
"NumberUnselectedRows": 0,
"LoadBytes": 0,
"LoadTimeMs": 0,
"BeginTxnTimeMs": 0,
"StreamLoadPutTimeMs": 6,
"ReadDataTimeMs": 0,
"WriteDataTimeMs": 0,
"CommitAndPublishTimeMs": 0
}
初始DDL:
CREATE TABLE bitnei_rtm_dwd.dwd_rtm_split_dd(
vin VARCHAR(100) comment '车辆vin',
vid VARCHAR(100) comment '车辆id',
segmenttype VARCHAR(10) comment '片段分类',
pdate date not null comment '分区日期')
DUPLICATE KEY(vin, vid, segmenttype)
AUTO PARTITION BY RANGE (date_trunc(`pdate`, 'day'))()
DISTRIBUTED BY HASH(vin) BUCKETS auto
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
"inverted_index_storage_format" = "V1",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
);
BE节点失败后DDL
CREATE TABLE bitnei_rtm_dwd.dwd_rtm_split_dd(
vin VARCHAR(100) comment '车辆vin',
vid VARCHAR(100) comment '车辆id',
segmenttype VARCHAR(10) comment '片段分类',
pdate date not null comment '分区日期')
DUPLICATE KEY(vin, vid, segmenttype)
AUTO PARTITION BY RANGE (date_trunc(`a`.`pdate`, 'day'))
(PARTITION p20240628000000 VALUES [('2024-06-28'), ('2024-06-29')),
PARTITION p20240701000000 VALUES [('2024-07-01'), ('2024-07-02')))
DISTRIBUTED BY HASH(`vin`) BUCKETS AUTO
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
"inverted_index_storage_format" = "V1",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
);