【问题】:通过华为云cdm组件将hudi数据写入doris中一直报错,报错信息为index out of bounds. 4466 is not between valid values of 1 and 4464,但是查看了写入的字段数量是对上没问题的,值得一提的是将插入字段数控制在十个以下,则可正常写入数据
【doris版本】:2.1.5
【报错截图】:
【CDM作业消息体】:
{
"service_transaction_id": null,
"task_type": "TASK",
"task_name": "dma_his_store_sale_flw_dtl_up_y_2021_p",
"task_id": "799618",
"sub_tasks": [
{
"reader": {
"stage_connections": [
{
"connection_id": "767504c90df64a379eb91ef83540691a",
"connection_type": "Hudi",
"connection_sub_type": "MRS_Hudi",
"database": "dma_his",
"tables": [
"dma_his_store_sale_flw_dtl_up_y_2021_p"
],
"databases": null,
"params": null
}
],
"stage_params": "{"columnMetaList":[{"type":"string","isCustomer":false,"transformer":[],"name":"month_id","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"province","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"store_code","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"store_name","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"comp_code","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"goods_code","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"goods_name","customerValue":""},{"type":"timestamp","isCustomer":false,"transformer":[],"name":"fildate","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"col_uuid","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"flowno","customerValue":""},{"type":"date","isCustomer":false,"transformer":[],"name":"dt","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"gdcode","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"spec","customerValue":""},{"type":"string","isCustomer":false,"transformer":[],"name":"brand","customerValue":""}]}",
"stage_type": null
},
"writer": {
"stage_connections": [
{
"database": "db_bigdata_saleflow",
"tables": [
"t_store_sale_flw_dtl_up_with_monthid_202101_yjstest2"
],
"connection_id": "0118a8655d0448a098bffdcbe6d74a82",
"connection_type": "MySQL",
"connection_sub_type": "Cloud_MySQL"
}
],
"stage_params": "{"beforeImportType":"shouldClearTable","onConflict":"EXCEPTION","useStageTable":false,"loaderConcurrency":"5","preSqlList":[],"postSqlList":[],"columnMetaList":[{"name":"month_id","type":"VARCHAR","selected":false},{"name":"province","type":"VARCHAR","selected":false},{"name":"store_code","type":"VARCHAR","selected":false},{"name":"store_name","type":"VARCHAR","selected":false},{"name":"comp_code","type":"VARCHAR","selected":false},{"name":"goods_code","type":"VARCHAR","selected":false},{"name":"goods_name","type":"VARCHAR","selected":false},{"name":"fildate","type":"DATETIME","selected":false},{"name":"col_uuid","type":"VARCHAR","selected":false},{"name":"flowno","type":"VARCHAR","selected":false},{"name":"dt","type":"DATETIME","selected":false},{"name":"gdcode","type":"VARCHAR","selected":false},{"name":"spec","type":"VARCHAR","selected":false},{"name":"brand","type":"VARCHAR","sortId":30}]}"
}
}
],
"connections": [
{
"connection_name": "real-hudi2",
"connection_id": "767504c90df64a379eb91ef83540691a",
"connection_type": "Hudi",
"connection_sub_type": "MRS_Hudi",
"user_credential": null,
"connection_params": null
},
{
"connection_name": "BIGDATA-Doris-db_bigdata_saleflow",
"connection_id": "0118a8655d0448a098bffdcbe6d74a82",
"connection_type": "MySQL",
"connection_sub_type": "Cloud_MySQL",
"user_credential": null,
"connection_params": null
}
],
"variables": {},
"settings": {
"throttlingConfig.numExtractors": "3",
"throttlingConfig.concurrentNumber": 1,
"throttlingConfig.splitRetryTime": 0,
"throttlingConfig.throttling": false,
"throttlingConfig.recordDirtyData": false
}
}
【Doris目标表结构】:
CREATE TABLE t_store_sale_flw_dtl_up_with_monthid_202101_yjstest2
(
col_uuid
VARCHAR(255) NULL COMMENT 'col_uuid',
dt
DATETIME NULL COMMENT '分区日期',
store_code
VARCHAR(255) NULL COMMENT '店号',
uuid
VARCHAR(255) NULL COMMENT 'UUID',
itemno
VARCHAR(255) NULL COMMENT '分区业务交易序号',
gid
INT NULL COMMENT '商品GID',
dealtime
DATETIME NULL COMMENT '流水从表2交易时间',
month_id
VARCHAR(255) NULL COMMENT '年份月份',
store_name
VARCHAR(255) NULL COMMENT '店名',
goods_code
VARCHAR(255) NULL COMMENT '商品代码',
goods_name
VARCHAR(255) NULL COMMENT '商品名称',
fildate
DATETIME NULL COMMENT '交易时间',
flowno
VARCHAR(255) NULL COMMENT '流水单号',
iqty
DECIMAL(18, 4) NULL COMMENT '销售数量',
realamt
DECIMAL(18, 4) NULL COMMENT '销售金额',
bd_lst_upd_time
DATETIME NULL COMMENT '大数据更新时间',
posno
VARCHAR(255) NULL COMMENT '流水主表Pos编号',
favamt
VARCHAR(255) NULL COMMENT '流水从表2优惠总额',
price
VARCHAR(255) NULL COMMENT '流水从表2商品单价',
tag
INT NULL COMMENT '流水从表2交易标记',
wrh
INT NULL COMMENT '仓位GID',
gdcode
VARCHAR(255) NULL COMMENT '流水从表2商品Code2或简码',
flag
INT NULL COMMENT '流水主表交易类型',
spec
VARCHAR(255) NULL COMMENT '商品规格',
code2
VARCHAR(255) NULL COMMENT '商品CODE2',
goodsstatus
VARCHAR(255) NULL COMMENT '商品状态',
munit
VARCHAR(255) NULL COMMENT '商品计量单位',
alc
VARCHAR(255) NULL COMMENT '商品配送方式',
rtlprc
VARCHAR(255) NULL COMMENT '商品核算售价',
inprc
VARCHAR(255) NULL COMMENT '商品核算进价',
brand
VARCHAR(255) NULL COMMENT '商品品牌',
taxrate
VARCHAR(255) NULL COMMENT '商品税率',
myj_goods_brand_chname
VARCHAR(255) NULL COMMENT '美宜佳商品品牌名称',
pradate
DATETIME NULL COMMENT '门店开业时间',
address
VARCHAR(255) NULL COMMENT '门店地址',
outdate
DATETIME NULL COMMENT '门店退出时间',
bandate
DATETIME NULL COMMENT '门店搬迁时间',
storestatus
VARCHAR(255) NULL COMMENT '门店状态',
bd_createtime
DATETIME NULL COMMENT '大数据更新时间',
province
VARCHAR(255) NULL COMMENT '省份',
comp_code
VARCHAR(255) NULL COMMENT '公司',
gd_l_class_code
VARCHAR(255) NULL COMMENT '商品大类code',
gd_l_class_name
VARCHAR(255) NULL COMMENT '商品大类名称',
gd_m_class_code
VARCHAR(255) NULL COMMENT '商品中类code',
gd_m_class_name
VARCHAR(255) NULL COMMENT '商品中类名称',
gd_s_class_code
VARCHAR(255) NULL COMMENT '商品小类code',
gd_s_class_name
VARCHAR(255) NULL COMMENT '商品小类名称'
) ENGINE=OLAP
DUPLICATE KEY(col_uuid
)
PARTITION BY RANGE(dt
)
(PARTITION p202101 VALUES [('0000-01-01 00:00:00'), ('2021-02-01 00:00:00')),
PARTITION p202102 VALUES [('2021-02-01 00:00:00'), ('2021-03-01 00:00:00')),
PARTITION p202103 VALUES [('2021-03-01 00:00:00'), ('2021-04-01 00:00:00')),
PARTITION p202104 VALUES [('2021-04-01 00:00:00'), ('2021-05-01 00:00:00')),
PARTITION p202105 VALUES [('2021-05-01 00:00:00'), ('2021-06-01 00:00:00')),
PARTITION p202106 VALUES [('2021-06-01 00:00:00'), ('2021-07-01 00:00:00')),
PARTITION p202107 VALUES [('2021-07-01 00:00:00'), ('2021-08-01 00:00:00')),
PARTITION p202108 VALUES [('2021-08-01 00:00:00'), ('2021-09-01 00:00:00')),
PARTITION p202109 VALUES [('2021-09-01 00:00:00'), ('2021-10-01 00:00:00')),
PARTITION p202110 VALUES [('2021-10-01 00:00:00'), ('2021-11-01 00:00:00')))
DISTRIBUTED BY HASH(store_code
) BUCKETS 10
PROPERTIES (
"replication_allocation" = "tag.location.default: 3",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
"inverted_index_storage_format" = "V1",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
);
【FE参数】: