【已解决】自动分区异常,报错[ANALYSIS_ERROR]TStatus: errCode = 2, detailMessage = Unknown column 'pdate' in 'a'"

Viewed 150

Doris版本:2.1.3
问题描述:BE节点挂掉导致自动分区异常,分区字段由date_trunc(pdate, 'day')变为date_trunc(a.pdate, 'day'),导致数据无法写入,错误信息如下:

WARN TaskSetManager: Lost task 129.1 in stage 4.0 (TID 462, 10.66.172.38, executor 7): org.apache.doris.spark.exception.StreamLoadException: stream load error, load status:Fail, response:StreamLoadResponse(200,OK,{
      "TxnId": 16783,
      "Label": "spark_streamload_20240702_114500_730929e6c56f49bfac7c034ab39abaa9",
      "Comment": "",
      "TwoPhaseCommit": "false",
      "Status": "Fail",
      "Message": "[ANALYSIS_ERROR]TStatus: errCode = 2, detailMessage = Unknown column 'pdate' in 'a'",
      "NumberTotalRows": 0,
      "NumberLoadedRows": 0,
      "NumberFilteredRows": 0,
      "NumberUnselectedRows": 0,
      "LoadBytes": 0,
      "LoadTimeMs": 0,
      "BeginTxnTimeMs": 0,
      "StreamLoadPutTimeMs": 6,
      "ReadDataTimeMs": 0,
      "WriteDataTimeMs": 0,
      "CommitAndPublishTimeMs": 0
  }

初始DDL

CREATE TABLE bitnei_rtm_dwd.dwd_rtm_split_dd(
vin VARCHAR(100) comment '车辆vin',
vid VARCHAR(100) comment '车辆id',
segmenttype VARCHAR(10) comment '片段分类',
pdate date not null comment '分区日期')
DUPLICATE KEY(vin, vid, segmenttype)
AUTO PARTITION BY RANGE (date_trunc(`pdate`, 'day'))()
DISTRIBUTED BY HASH(vin) BUCKETS auto
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
"inverted_index_storage_format" = "V1",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
);

BE节点失败后DDL

CREATE TABLE bitnei_rtm_dwd.dwd_rtm_split_dd(
vin VARCHAR(100) comment '车辆vin',
vid VARCHAR(100) comment '车辆id',
segmenttype VARCHAR(10) comment '片段分类',
pdate date not null comment '分区日期')
DUPLICATE KEY(vin, vid, segmenttype)
AUTO PARTITION BY RANGE (date_trunc(`a`.`pdate`, 'day'))
(PARTITION p20240628000000 VALUES [('2024-06-28'), ('2024-06-29')),
PARTITION p20240701000000 VALUES [('2024-07-01'), ('2024-07-02')))
DISTRIBUTED BY HASH(`vin`) BUCKETS AUTO
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
"inverted_index_storage_format" = "V1",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
);
1 Answers