2.1.3版本读取0.12.x版本的hudi表报错，在读取log文件是报：IOException: java.lang.RuntimeException: key not found: _hoodie_commit_time
=========================================================
SQL 错误 [1105] [HY000]: errCode = 2, detailMessage = (10.8.125.167)[CANCELLED]cur path: hdfs://XXXxxx/.862e7171-dc92-4761-b10b-3532035450c9-0_20240618181740438.log.1_0-9-19. failed to init reader for file hdfs://xxx/.862e7171-dc92-4761-b10b-3532035450c9-0_20240618181740438.log.1_0-9-19, err: [INTERNAL_ERROR]IOException: java.lang.RuntimeException: key not found: _hoodie_commit_time¶CAUSED BY: ExecutionException: java.lang.RuntimeException: key not found: _hoodie_commit_time¶CAUSED BY: RuntimeException: key not found: _hoodie_commit_time¶CAUSED BY: RuntimeException: key not found: _hoodie_commit_time¶CAUSED BY: NoSuchElementException: key not found: hoodie_commit_time¶¶ 0# doris::JniUtil::GetJniExceptionMsg(JNIEnv, bool, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&) at /home/zcp/repo_center/doris_release/doris/be/src/util/jni-util.h:117¶ 1# doris::vectorized::JniConnector::open(doris::RuntimeState, doris::RuntimeProfile*) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187¶ 2# doris::vectorized::HudiJniReader::init_reader(std::unordered_map<std::__cxx11::basic_string<char, std::char_traits, std::allocator >, std::variant<doris::ColumnValueRange<(doris::PrimitiveType)3>, doris::ColumnValueRange<(doris::PrimitiveType)4>, doris::ColumnValueRange<(doris::PrimitiveType)5>, doris::ColumnValueRange<(doris::PrimitiveType)6>, doris::ColumnValueRange<(doris::PrimitiveType)7>, doris::ColumnValueRange<(doris::PrimitiveType)15>, doris::ColumnValueRange<(doris::PrimitiveType)10>, doris::ColumnValueRange<(doris::PrimitiveType)23>, doris::ColumnValueRange<(doris::PrimitiveType)11>, doris::ColumnValueRange<(doris::PrimitiveType)25>, doris::ColumnValueRange<(doris::PrimitiveType)12>, doris::ColumnValueRange<(doris::PrimitiveType)26>, doris::ColumnValueRange<(doris::PrimitiveType)20>, doris::ColumnValueRange<(doris::PrimitiveType)2>, doris::ColumnValueRange<(doris::PrimitiveType)19>, doris::ColumnValueRange<(doris::PrimitiveType)28>, doris::ColumnValueRange<(doris::PrimitiveType)29>, doris::ColumnValueRange<(doris::PrimitiveType)30>, doris::ColumnValueRange<(doris::PrimitiveType)35> >, std::hash<std::__cxx11::basic_string<char, std::char_traits, std::allocator > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits, std::allocator > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits, std::allocator > const, std::variant<doris::ColumnValueRange<(doris::PrimitiveType)3>, doris::ColumnValueRange<(doris::PrimitiveType)4>, doris::ColumnValueRange<(doris::PrimitiveType)5>, doris::ColumnValueRange<(doris::PrimitiveType)6>, doris::ColumnValueRange<(doris::PrimitiveType)7>, doris::ColumnValueRange<(doris::PrimitiveType)15>, doris::ColumnValueRange<(doris::PrimitiveType)10>, doris::ColumnValueRange<(doris::PrimitiveType)23>, doris::ColumnValueRange<(doris::PrimitiveType)11>, doris::ColumnValueRange<(doris::PrimitiveType)25>, doris::ColumnValueRange<(doris::PrimitiveType)12>, doris::ColumnValueRange<(doris::PrimitiveType)26>, doris::ColumnValueRange<(doris::PrimitiveType)20>, doris::ColumnValueRange<(doris::PrimitiveType)2>, doris::ColumnValueRange<(doris::PrimitiveType)19>, doris::ColumnValueRange<(doris::PrimitiveType)28>, doris::ColumnValueRange<(doris::PrimitiveType)29>, doris::ColumnValueRange<(doris::PrimitiveType)30>, doris::ColumnValueRange<(doris::PrimitiveType)35> > > > >) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/format/table/hudi_jni_reader.cpp:103¶ 3# doris::vectorized::VFileScanner::_get_next_reader() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:377¶ 4# doris::vectorized::VFileScanner::_get_block_wrapped(doris::RuntimeState, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:481¶ 5# doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:481¶ 6# doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/vscanner.cpp:0¶ 7# doris::vectorized::VScanner::get_block_after_projects(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/vscanner.cpp:100¶ 8# doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr, std::shared_ptr) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:377¶ 9# std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr, std::shared_ptr)::$_3>::_M_invoke(std::_Any_data const&) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:701¶ 10# doris::ThreadPool::dispatch_thread() at /home/zcp/repo_center/doris_release/doris/be/src/util/threadpool.cpp:0¶ 11# doris::Thread::supervise_thread(void*) at /var/local/ldb_toolchain/bin/../usr/include/pthread.h:562¶ 12# start_thread¶ 13# clone¶
JNI LOG
2024-06-18 16:07:05 INFO Thread-9 org.apache.doris.hudi.HudiJniScanner.<clinit>(HudiJniScanner.java:91) - Create 16 daemon threads to load avro logs
2024-06-18 16:07:05 INFO Thread-9 org.apache.doris.hudi.HudiJniScanner.<clinit>(HudiJniScanner.java:98) - Get the resolved cache for avro reader
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Running Spark version 3.2.0
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - ==============================================================
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - No custom resources configured for spark.driver.
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - ==============================================================
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Submitted application: mock_sc
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Limiting resource is cpu
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Added ResourceProfile id: 0
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Changing view acls to: root,hive
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Changing modify acls to: root,hive
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Changing view acls groups to:
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Changing modify acls groups to:
2024-06-18 16:07:06 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root, hive); groups with view permissions: Set(); users with modify permissions: Set(root, hive); groups with modify permissions: Set()
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Successfully started service 'sparkDriver' on port 39674.
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Registering MapOutputTracker
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Registering BlockManagerMaster
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - BlockManagerMasterEndpoint up
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Registering BlockManagerMasterHeartbeat
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Created local directory at /tmp/blockmgr-0a1d1ad9-0750-4591-919f-7e3b6c56cc3d
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - MemoryStore started with capacity 366.3 MiB
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Registering OutputCommitCoordinator
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Starting executor ID driver on host dev002.xxxxx.com
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 40954.
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.network.netty.NettyBlockTransferService.init(NettyBlockTransferService.scala:82) - Server created on dev002.xxxxx.com:40954
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Registering BlockManager BlockManagerId(driver, dev002.xxxxx.com, 40954, None)
2024-06-18 16:07:07 INFO dispatcher-BlockManagerMaster org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Registering block manager dev002.xxxxx.com:40954 with 366.3 MiB RAM, BlockManagerId(driver, dev002.xxxxx.com, 40954, None)
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Registered BlockManager BlockManagerId(driver, dev002.xxxxx.com, 40954, None)
2024-06-18 16:07:07 INFO avro-log-reader-0 org.apache.spark.internal.Logging.logInfo(Logging.scala:57) - Initialized BlockManager: BlockManagerId(driver, dev002.xxxxx.com, 40954, None)
2024-06-18 16:07:07 INFO avro-log-reader-0 org.sparkproject.jetty.util.log.Log.initialized(Log.java:170) - Logging initialized @3191774ms to org.sparkproject.jetty.util.log.Slf4jLog
2024-06-18 16:07:08 ERROR avro-log-reader-0 org.apache.doris.hudi.HudiJniScanner.lambda$open$6(HudiJniScanner.java:178) - Failed to open hudi scanner, split params:
hudi_column_types=string#string#string#string#string#int#string#decimal(20,6)#timestamp(6)
hadoop_conf.dfs.namenode.rpc-address.xxxxx.nn2=dev002.xxxxx.com:8020
query_id=b9b8d90b3cce4428-82d0c2e05e769362
hadoop_conf.dfs.namenode.rpc-address.xxxxx.nn1=dev001.xxxxx.com:8020
serde=org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
hadoop_conf.create_time=2024-06-18 15:31:26.601
hadoop_conf.dfs.nameservices=xxxxx
hadoop_conf.dfs.ha.namenodes.xxxxx=nn1,nn2
base_path=hdfs://xxxxx/hoodie/xxxxx/xxxxx/hudi_mor_tbl
data_file_path=hdfs://xxxxx/hoodie/xxxxx/xxxxx/hudi_mor_tbl/37c84693-4e83-4570-a01c-0ac0ae59d2d9-0_0-14-10_20240301102052448.parquet
hadoop_conf.hadoop.username=hive
hadoop_conf.type=hms
input_format=org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat
instant_time=20240618160602142
hadoop_conf.ipc.client.fallback-to-simple-auth-allowed=true
hudi_column_names=_hoodie_commit_time,_hoodie_commit_seqno,_hoodie_record_key,_hoodie_partition_path,_hoodie_file_name,id,name,price,ts
required_fields=_hoodie_commit_time,_hoodie_commit_seqno,_hoodie_record_key,_hoodie_partition_path,_hoodie_file_name,id,name,price,ts
data_file_length=2405
delta_file_paths=hdfs://xxxxx/hoodie/xxxxx/xxxxx/hudi_mor_tbl/.37c84693-4e83-4570-a01c-0ac0ae59d2d9-0_20240301102052448.log.1_0-18-14
hadoop_conf.dfs.client.failover.proxy.provider.xxxxx=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
hadoop_conf.hive.metastore.uris=thrift://dev003.xxxxx.com:9083
hadoop_conf.use_meta_cache=false
java.lang.RuntimeException: key not found: _hoodie_commit_time
at org.apache.doris.common.security.authentication.HadoopUGI.ugiDoAs(HadoopUGI.java:120)
at org.apache.doris.hudi.HudiJniScanner.lambda$open$6(HudiJniScanner.java:168)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.util.NoSuchElementException: key not found: _hoodie_commit_time
at scala.collection.immutable.Map$Map4.apply(Map.scala:207)
at org.apache.doris.hudi.BaseSplitReader.$anonfun$projectSchema$2(BaseSplitReader.scala:343)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
at scala.collection.TraversableLike.map(TraversableLike.scala:238)
at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
at org.apache.doris.hudi.BaseSplitReader.projectSchema(BaseSplitReader.scala:342)
at org.apache.doris.hudi.BaseSplitReader.buildScanIterator(BaseSplitReader.scala:289)
at org.apache.doris.hudi.HudiJniScanner.lambda$null$4(HudiJniScanner.java:170)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
at org.apache.doris.common.security.authentication.HadoopUGI.ugiDoAs(HadoopUGI.java:115)
... 6 more
2024-06-18 16:08:13 ERROR avro-log-reader-1 org.apache.doris.hudi.HudiJniScanner.lambda$open$6(HudiJniScanner.java:178) - Failed to open hudi scanner, split params:
hudi_column_types=string#string#string#string#string#int#string#decimal(20,6)#timestamp(6)
hadoop_conf.dfs.namenode.rpc-address.xxxxx.nn2=dev002.xxxxx.com:8020
query_id=aa8436f46eb44150-a2a11bffc2a64370
hadoop_conf.dfs.namenode.rpc-address.xxxxx.nn1=dev001.xxxxx.com:8020
serde=org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
hadoop_conf.create_time=2024-06-18 15:31:26.601
hadoop_conf.dfs.nameservices=xxxxx
hadoop_conf.dfs.ha.namenodes.xxxxx=nn1,nn2
base_path=hdfs://xxxxx/hoodie/xxxxx/xxxxx/hudi_mor_tbl
data_file_path=hdfs://xxxxx/hoodie/xxxxx/xxxxx/hudi_mor_tbl/37c84693-4e83-4570-a01c-0ac0ae59d2d9-0_0-14-10_20240301102052448.parquet
hadoop_conf.hadoop.username=hive
hadoop_conf.type=hms
input_format=org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat
instant_time=20240618160602142
hadoop_conf.ipc.client.fallback-to-simple-auth-allowed=true
hudi_column_names=_hoodie_commit_time,_hoodie_commit_seqno,_hoodie_record_key,_hoodie_partition_path,_hoodie_file_name,id,name,price,ts
required_fields=_hoodie_commit_time,_hoodie_commit_seqno,_hoodie_record_key,_hoodie_partition_path,_hoodie_file_name,id,name,price,ts
data_file_length=2405
delta_file_paths=hdfs://xxxxx/hoodie/xxxxx/xxxxx/hudi_mor_tbl/.37c84693-4e83-4570-a01c-0ac0ae59d2d9-0_20240301102052448.log.1_0-18-14
hadoop_conf.dfs.client.failover.proxy.provider.xxxxx=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
hadoop_conf.hive.metastore.uris=thrift://dev003.xxxxx.com:9083
hadoop_conf.use_meta_cache=false
java.lang.RuntimeException: key not found: _hoodie_commit_time
at org.apache.doris.common.security.authentication.HadoopUGI.ugiDoAs(HadoopUGI.java:120)
at org.apache.doris.hudi.HudiJniScanner.lambda$open$6(HudiJniScanner.java:168)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.util.NoSuchElementException: key not found: _hoodie_commit_time
at scala.collection.immutable.Map$Map4.apply(Map.scala:207)
at org.apache.doris.hudi.BaseSplitReader.$anonfun$projectSchema$2(BaseSplitReader.scala:343)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
at scala.collection.TraversableLike.map(TraversableLike.scala:238)
at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
at org.apache.doris.hudi.BaseSplitReader.projectSchema(BaseSplitReader.scala:342)
at org.apache.doris.hudi.BaseSplitReader.buildScanIterator(BaseSplitReader.scala:289)
at org.apache.doris.hudi.HudiJniScanner.lambda$null$4(HudiJniScanner.java:170)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
at org.apache.doris.common.security.authentication.HadoopUGI.ugiDoAs(HadoopUGI.java:115)
... 6 more
2024-06-18 18:09:45 ERROR avro-log-reader-2 org.apache.doris.hudi.HudiJniScanner.lambda$open$6(HudiJniScanner.java:178) - Failed to open hudi scanner, split params:
hudi_column_types=string#string#string#string#string#bigint#string#bigint#int#decimal(20,6)#string#int#timestamp(6)#date
hadoop_conf.dfs.namenode.rpc-address.xxxxx.nn2=dev002.xxxxx.com:8020
query_id=b6cb160c563f44d7-ac55fd86992c0048
hadoop_conf.dfs.namenode.rpc-address.xxxxx.nn1=dev001.xxxxx.com:8020
serde=org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
hadoop_conf.create_time=2024-06-18 15:31:26.601
hadoop_conf.dfs.nameservices=xxxxx
hadoop_conf.dfs.ha.namenodes.xxxxx=nn1,nn2
base_path=hdfs://xxxxx/hoodie/xxxxx/xxxxx/pub_oms_order_goods_sale
data_file_path=
hadoop_conf.hadoop.username=hive
hadoop_conf.type=hms
input_format=org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat
instant_time=20240618174830243
hadoop_conf.ipc.client.fallback-to-simple-auth-allowed=true
hudi_column_names=_hoodie_commit_time,_hoodie_commit_seqno,_hoodie_record_key,_hoodie_partition_path,_hoodie_file_name,order_goods_id,order_sn,sku_id,number,price,precombine_field,sync_deleted,sync_time,partition_field
required_fields=_hoodie_commit_time,_hoodie_commit_seqno,_hoodie_record_key,_hoodie_partition_path,_hoodie_file_name,order_goods_id,order_sn,sku_id,number,price,precombine_field,sync_deleted,sync_time
data_file_length=0
delta_file_paths=hdfs://xxxxx/hoodie/xxxxx/xxxxx/pub_oms_order_goods_sale/partition_field=2024-06-01/.5e562654-7961-4ea7-b990-3dc496f065c7-0_20240618174818209.log.1_0-9-20
hadoop_conf.dfs.client.failover.proxy.provider.xxxxx=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
hadoop_conf.hive.metastore.uris=thrift://dev003.xxxxx.com:9083
hadoop_conf.use_meta_cache=false
java.lang.RuntimeException: key not found: _hoodie_commit_time
at org.apache.doris.common.security.authentication.HadoopUGI.ugiDoAs(HadoopUGI.java:120)
at org.apache.doris.hudi.HudiJniScanner.lambda$open$6(HudiJniScanner.java:168)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.util.NoSuchElementException: key not found: _hoodie_commit_time
at scala.collection.MapLike.default(MapLike.scala:235)
at scala.collection.MapLike.default$(MapLike.scala:234)
at scala.collection.AbstractMap.default(Map.scala:63)
at scala.collection.MapLike.apply(MapLike.scala:144)
at scala.collection.MapLike.apply$(MapLike.scala:143)
at scala.collection.AbstractMap.apply(Map.scala:63)
at org.apache.doris.hudi.BaseSplitReader.$anonfun$projectSchema$2(BaseSplitReader.scala:343)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
at scala.collection.TraversableLike.map(TraversableLike.scala:238)
at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
at org.apache.doris.hudi.BaseSplitReader.projectSchema(BaseSplitReader.scala:342)
at org.apache.doris.hudi.BaseSplitReader.buildScanIterator(BaseSplitReader.scala:289)
at org.apache.doris.hudi.HudiJniScanner.lambda$null$4(HudiJniScanner.java:170)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
at org.apache.doris.common.security.authentication.HadoopUGI.ugiDoAs(HadoopUGI.java:115)
... 6 more
2024-06-18 18:10:32 ERROR avro-log-reader-3 org.apache.doris.hudi.HudiJniScanner.lambda$open$6(HudiJniScanner.java:178) - Failed to open hudi scanner, split params:
hudi_column_types=string#string#string#string#string#bigint#string#bigint#int#decimal(20,6)#string#int#timestamp(6)#date
hadoop_conf.dfs.namenode.rpc-address.xxxxx.nn2=dev002.xxxxx.com:8020
query_id=86fece35c80649f6-8159a2230aa3bee1
hadoop_conf.dfs.namenode.rpc-address.xxxxx.nn1=dev001.xxxxx.com:8020
serde=org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
hadoop_conf.create_time=2024-06-18 15:31:26.601
hadoop_conf.dfs.nameservices=xxxxx
hadoop_conf.dfs.ha.namenodes.xxxxx=nn1,nn2
base_path=hdfs://xxxxx/hoodie/xxxxx/xxxxx/pub_oms_order_goods_sale
data_file_path=
hadoop_conf.hadoop.username=hive
hadoop_conf.type=hms
input_format=org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat
instant_time=20240618174830243
hadoop_conf.ipc.client.fallback-to-simple-auth-allowed=true
hudi_column_names=_hoodie_commit_time,_hoodie_commit_seqno,_hoodie_record_key,_hoodie_partition_path,_hoodie_file_name,order_goods_id,order_sn,sku_id,number,price,precombine_field,sync_deleted,sync_time,partition_field
required_fields=_hoodie_commit_time,_hoodie_commit_seqno,_hoodie_record_key,_hoodie_partition_path,_hoodie_file_name,order_goods_id,order_sn,sku_id,number,price,precombine_field,sync_deleted,sync_time
data_file_length=0
delta_file_paths=hdfs://xxxxx/hoodie/xxxxx/xxxxx/pub_oms_order_goods_sale/partition_field=2024-06-01/.5e562654-7961-4ea7-b990-3dc496f065c7-0_20240618174818209.log.1_0-9-20
hadoop_conf.dfs.client.failover.proxy.provider.xxxxx=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
hadoop_conf.hive.metastore.uris=thrift://dev003.xxxxx.com:9083
hadoop_conf.use_meta_cache=false
java.lang.RuntimeException: key not found: _hoodie_commit_time
at org.apache.doris.common.security.authentication.HadoopUGI.ugiDoAs(HadoopUGI.java:120)
at org.apache.doris.hudi.HudiJniScanner.lambda$open$6(HudiJniScanner.java:168)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.util.NoSuchElementException: key not found: _hoodie_commit_time
at scala.collection.MapLike.default(MapLike.scala:235)
at scala.collection.MapLike.default$(MapLike.scala:234)
at scala.collection.AbstractMap.default(Map.scala:63)
at scala.collection.MapLike.apply(MapLike.scala:144)
at scala.collection.MapLike.apply$(MapLike.scala:143)
at scala.collection.AbstractMap.apply(Map.scala:63)
at org.apache.doris.hudi.BaseSplitReader.$anonfun$projectSchema$2(BaseSplitReader.scala:343)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
at scala.collection.TraversableLike.map(TraversableLike.scala:238)
at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
at org.apache.doris.hudi.BaseSplitReader.projectSchema(BaseSplitReader.scala:342)
at org.apache.doris.hudi.BaseSplitReader.buildScanIterator(BaseSplitReader.scala:289)
at org.apache.doris.hudi.HudiJniScanner.lambda$null$4(HudiJniScanner.java:170)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
at org.apache.doris.common.security.authentication.HadoopUGI.ugiDoAs(HadoopUGI.java:115)
... 6 more
