Paimon异步物化视图刷新失败

Viewed 28
mtmv 3.1 物化视图

 创建paimon catalog

CREATE CATALOG `test` PROPERTIES (
"warehouse" = "s3://xxxx",
"type" = "paimon",
"schema.cache.ttl-second" = "0",
"s3.secret_key" = "*XXX",
"s3.region" = "us-east-1",
"s3.endpoint" = "http://xxxx",
"s3.access_key" = "xxxx",
"AWS_NEED_OVERRIDE_ENDPOINT" = "true"
);

基于catalog 创建物化视图

CREATE MATERIALIZED VIEW mv_xxx
BUILD IMMEDIATE REFRESH AUTO ON SCHEDULE EVERY 10 MINUTE
DUPLICATE KEY(id)
DISTRIBUTED BY RANDOM BUCKETS 5
PROPERTIES (
"file_cache_ttl_seconds" = "0",
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
"inverted_index_storage_format" = "V2",
"light_schema_change" = "true",
"storage_vault_id" = "1",
"storage_vault_name" = "s3_vault",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
)
AS  sql

问题

物化视图依赖的paimon表数据更新之后，物化视图刷新任务失败，但是同时使用创建物化视图的查询sql直接查询catalog可以正常查询到最新数据。手动refresh catalog之后，物化视图可以刷新成功。

报错信息

errCode = 2, detailMessage = (10.65.62.82)[INTERNAL_ERROR]failed to init reader, err: [RUNTIME_ERROR]RuntimeException: java.io.FileNotFoundException: File 's3://xxxxx/tablepath/bucket-0/data-dd20bdd3-2ffa-4b81-9dbd-eb84bee461cd-0.parquet' not found, Possible causes: 1.snapshot expires too fast, you can configure 'snapshot.time-retained' option with a larger value. 2.consumption is too slow, you can improve the performance of consumption (For example, increasing parallelism). | CAUSED BY: FileNotFoundException: File 's3://xxx/tablepath/bucket-0/data-dd20bdd3-2ffa-4b81-9dbd-eb84bee461cd-0.parquet' not found, Possible causes: 1.snapshot expires too fast, you can configure 'snapshot.time-retained' option with a larger value. 2.consumption is too slow, you can improve the performance of consumption (For example, increasing parallelism).

	0#  doris::JniUtil::GetJniExceptionMsg(JNIEnv_*, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) at /home/zcp/repo_center/doris_release/doris/be/src/util/jni-util.h:143
	1#  doris::vectorized::JniConnector::open(doris::RuntimeState*, doris::RuntimeProfile*) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187
	2#  doris::vectorized::PaimonJniReader::init_reader(std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::variant<doris::ColumnValueRange<(doris::PrimitiveType)3>, doris::ColumnValueRange<(doris::PrimitiveType)4>, doris::ColumnValueRange<(doris::PrimitiveType)5>, doris::ColumnValueRange<(doris::PrimitiveType)6>, doris::ColumnValueRange<(doris::PrimitiveType)7>, doris::ColumnValueRange<(doris::PrimitiveType)36>, doris::ColumnValueRange<(doris::PrimitiveType)37>, doris::ColumnValueRange<(doris::PrimitiveType)15>, doris::ColumnValueRange<(doris::PrimitiveType)10>, doris::ColumnValueRange<(doris::PrimitiveType)23>, doris::ColumnValueRange<(doris::PrimitiveType)11>, doris::ColumnValueRange<(doris::PrimitiveType)25>, doris::ColumnValueRange<(doris::PrimitiveType)12>, doris::ColumnValueRange<(doris::PrimitiveType)26>, doris::ColumnValueRange<(doris::PrimitiveType)20>, doris::ColumnValueRange<(doris::PrimitiveType)2>, doris::ColumnValueRange<(doris::PrimitiveType)19>, doris::ColumnValueRange<(doris::PrimitiveType)28>, doris::ColumnValueRange<(doris::PrimitiveType)29>, doris::ColumnValueRange<(doris::PrimitiveType)30>, doris::ColumnValueRange<(doris::PrimitiveType)35> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::variant<doris::ColumnValueRange<(doris::PrimitiveType)3>, doris::ColumnValueRange<(doris::PrimitiveType)4>, doris::ColumnValueRange<(doris::PrimitiveType)5>, doris::ColumnValueRange<(doris::PrimitiveType)6>, doris::ColumnValueRange<(doris::PrimitiveType)7>, doris::ColumnValueRange<(doris::PrimitiveType)36>, doris::ColumnValueRange<(doris::PrimitiveType)37>, doris::ColumnValueRange<(doris::PrimitiveType)15>, doris::ColumnValueRange<(doris::PrimitiveType)10>, doris::ColumnValueRange<(doris::PrimitiveType)23>, doris::ColumnValueRange<(doris::PrimitiveType)11>, doris::ColumnValueRange<(doris::PrimitiveType)25>, doris::ColumnValueRange<(doris::PrimitiveType)12>, doris::ColumnValueRange<(doris::PrimitiveType)26>, doris::ColumnValueRange<(doris::PrimitiveType)20>, doris::ColumnValueRange<(doris::PrimitiveType)2>, doris::ColumnValueRange<(doris::PrimitiveType)19>, doris::ColumnValueRange<(doris::PrimitiveType)28>, doris::ColumnValueRange<(doris::PrimitiveType)29>, doris::ColumnValueRange<(doris::PrimitiveType)30>, doris::ColumnValueRange<(doris::PrimitiveType)35> > > > > const*) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/format/table/paimon_jni_reader.cpp:105
	3#  doris::vectorized::VFileScanner::_get_next_reader() at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:392
	4#  doris::vectorized::VFileScanner::_get_block_wrapped(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:501
	5#  doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:506
	6#  doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/vscanner.cpp:0
	7#  doris::vectorized::VScanner::get_block_after_projects(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:506
	8#  doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) at /home/zcp/repo_center/doris_release/doris/be/src/common/status.h:392
	9#  std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}>::_M_invoke(std::_Any_data const&) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:701
	10# doris::ThreadPool::dispatch_thread() at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:730
	11# doris::Thread::supervise_thread(void*) at /var/local/ldb-toolchain/bin/../usr/include/pthread.h:562
	12# start_thread
	13# clone
. cur path: /data-dd20bdd3-2ffa-4b81-9dbd-eb84bee461cd-0.parquet
1 Answers

能把失败的task信息贴一下么（https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/table-valued-functions/tasks）？
还有是稳定复现么？

zhangdong281
Related Questions
Paimon异步物化视图刷新失败
1 answers
paimon外表catalog没法创建异步物化视图 Only support creating asynchronous materialized views in internal catalog
1 answers
什么时候doris出一个物化视图编排的管理工具
2 answers
物化视图创建失败，unable to find a suitable base table for partition
1 answers
3.1版本创建paimon的异步物化视图，按分区刷新不行
1 answers
异步物化视图不能透明改写
2 answers