环境
os: registry.access.redhat.com/ubi9
ldb-toolchain: 0.21
java: 17
doris: 3.0.3-rc04
构建脚本
os: registry.access.redhat.com/ubi9
ldb-toolchain: 0.21
java: 17
doris: 3.0.3-rc04
build script
FROM registry.access.redhat.com/ubi9
ARG LDB_TOOLCHAIN_VERSION=0.21
ARG PRODUCT_VERSION=3.0.3-rc04
ARG JAVA_VERSION=17
# install system requirements
RUN <<EOT
# https://adoptium.net/en-GB/installation/linux/#_centosrhelfedora_instructions
cat <<EOF > /etc/yum.repos.d/adoptium.repo
[Adoptium]
name=Adoptium
baseurl=https://packages.adoptium.net/artifactory/rpm/${DISTRIBUTION_NAME:-$(. /etc/os-release; echo $ID)}/\$releasever/\$basearch
enabled=1
gpgcheck=1
gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public
EOF
dnf update -y
dnf install -y \
automake \
bzip2 \
cmake \
cyrus-sasl-devel \
diffutils \
fuse-devel \
gcc \
gcc-c++ \
gettext \
git \
gzip \
hostname \
krb5-devel \
libcurl-devel \
libtool \
make \
maven \
nodejs \
openssl-devel \
patch \
pkg-config \
perl-CPAN \
tar \
temurin-17-jdk \
tzdata-java \
unzip \
wget \
which \
xz \
zlib-devel \
zip
microdnf clean all
rm -rf /var/cache/yum
# smoke test
java -version
EOT
ENV JAVA_HOME=/usr/lib/jvm/temurin-${JAVA_VERSION}-jdk
ENV JAVA_VERSION=${JAVA_VERSION}
# setup gettext
# Because of the gettext of dnf is lack autopoint, so we need to install gettext from source.
# Download gettext https://ftp.gnu.org/gnu/gettext/gettext-0.21.tar.gz
RUN <<EOF
set -ex
# Get already installed gettext version
GETTEXT_VERSION=$(gettext --version | head -n 1 | awk '{print $4}')
mkdir -p /build/gettext
mkdir -p /opt/gettext
pushd /build/gettext
curl -sSLf https://ftp.gnu.org/gnu/gettext/gettext-${GETTEXT_VERSION}.tar.gz \
| tar -xz --strip-components=1
./configure --prefix=/opt/gettext
make -j$(nproc)
make install
popd
# cleanup
rm -rf /build/gettext
EOF
ENV PATH="/opt/gettext/bin:$PATH"
# setup ldb-toolchain
# Download ldb_toolchain_gen.aarch64.sh
# https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.9.1/ldb_toolchain_gen.aarch64.sh
# or https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.9.1/ldb_toolchain_gen.sh
# Note:
# - In 0.9.1, ldb-toolchain containes curl, it conflicts with the system curl,
# when use it, it will cause dns resolution error
# - In 0.21, ldb-toolchain containes curl, it will verify the certificate
# file /etc/ssl/certs/ca-certificates.crt. But ca-certificates package generate a default
# file /etc/ssl/certs/ca-bundle.crt, so we need to create a symlink to fix it.
# - 0.14.1 aaarch64 version is error to build doris
RUN <<EOF
set -ex
ARCH=$(uname -m)
ARCH=${ARCH/arm64/aarch64}
release_file="ldb_toolchain_gen.sh"
if [ "$ARCH" == "aarch64" ]; then
release_file="ldb_toolchain_gen.aarch64.sh"
fi
mkdir -p /build/ldb-toolchain
pushd /build/ldb-toolchain
curl -sSLf https://github.com/amosbird/ldb_toolchain_gen/releases/download/v${LDB_TOOLCHAIN_VERSION}/${release_file} \
-o ldb_toolchain_gen.sh
chmod +x ldb_toolchain_gen.sh
./ldb_toolchain_gen.sh /opt/ldb-toolchain
popd
# create a symlink to bison, because doris need byacc
ln -s /opt/ldb-toolchain/bin/bison /opt/ldb-toolchain/bin/byacc
# fix /opt/ldb-toolchain/bin/curl certificate verify failed
ln -s /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
# cleanup
rm -rf /build/ldb-toolchain
EOF
ENV PATH="/opt/ldb-toolchain/bin:$PATH"
# build doris
# Download doris https://github.com/apache/doris/archive/refs/tags/3.0.3-rc04.tar.gz
RUN <<EOF
set -ex
# alias python3 to python
ln -s /usr/bin/python3 /usr/bin/python
mkdir -p /build/doris
pushd /build/doris
curl -sSLf https://github.com/apache/doris/archive/refs/tags/${PRODUCT_VERSION}.tar.gz \
| tar -xz --strip-components=1
# Add `set -x` after `set -euo pipefail`
sed -i '/set -eo pipefail/a set -x' /build/doris/build.sh
sed -i '/set -eo pipefail/a set -x' /build/doris/thirdparty/build-thirdparty.sh
# Patch submodules
# Patched sha from source by `git ls-remote`
sed -i "s|refs/heads/orc.tar.gz|db01184f765c03496e4107bd3ac37c077ac4bc5f/orc.tar.gz|g" build.sh
sed -i "s|refs/heads/clucene.tar.gz|48fa9cc4ec32b40bf3b02338d0a1b2cdbc6408cf/clucene.tar.gz|g" build.sh
# build doris, disable avx2
export USE_AVX2=OFF
export USE_UNWIND=OFF
# ./build.sh
nohup ./build.sh > /build/doris-build.log 2>&1 &
tail -f /build/doris-build.log
# TODO
EOF
## 错误信息
```text
[832/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/pb_convert.cpp.o
[833/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/cloud_tablet_mgr.cpp.o
[834/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/injection_point_action.cpp.o
[835/1371] Building CXX object src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o
FAILED: src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o
/opt/ldb-toolchain/bin/clang++ -DBOOST_CONTAINER_NO_LIB -DBOOST_DATE_TIME_NO_LIB -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_STACKTRACE_USE_BACKTRACE -DBOOST_SYSTEM_NO_DEPRECATED -DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1 -DBRPC_ENABLE_CPU_PROFILER -DGLOG_CUSTOM_PREFIX_SUPPORT -DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H -DLIBJVM -DS2_USE_GFLAGS -DS2_USE_GLOG -DUSE_HADOOP_HDFS -DUSE_JEMALLOC -DUSE_MEM_TRACKER -D__STDC_FORMAT_MACROS -Dindex_tool_EXPORTS -I/build/doris/be/src/apache-orc/c++/include -I/build/doris/be/build_Release/src/apache-orc/c++/include -I/build/doris/be/build_Release/src/clucene/src/shared -I/build/doris/be/src/clucene/src/core -I/build/doris/be/src/clucene/src/shared -I/build/doris/be/src/clucene/src/contribs-lib -I/build/doris/be/src -I/build/doris/be/test -I/usr/lib/jvm/temurin-17-jdk/include -I/usr/lib/jvm/temurin-17-jdk/include/linux -I/build/doris/be/../gensrc/build/common -isystem /build/doris/be/../common -isystem /build/doris/be/../gensrc/build -isystem /build/doris/thirdparty/installed/include -isystem /build/doris/thirdparty/installed/gperftools/include -isystem /build/doris/thirdparty/installed/include/lz4 -isystem /build/doris/thirdparty/installed/include/zstd -O3 -DNDEBUG -O3 -DNDEBUG -std=gnu++20 -D OS_LINUX -g -Wall -Wextra -Werror -pthread -fstrict-aliasing -fno-omit-frame-pointer -Wnon-virtual-dtor -Wno-unused-parameter -Wno-sign-compare -fcolor-diagnostics -Wpedantic -Wshadow-field -Wunused -Wunused-command-line-argument -Wunused-exception-parameter -Wunused-volatile-lvalue -Wunused-template -Wunused-member-function -Wunused-macros -Wconversion -Wno-gnu-statement-expression -Wno-implicit-float-conversion -Wno-implicit-int-conversion -Wno-sign-conversion -Wno-missing-field-initializers -Wno-unused-const-variable -Wno-shorten-64-to-32 -march=armv8-a+crc -Winvalid-pch -Xclang -include-pch -Xclang /build/doris/be/build_Release/CMakeFiles/pch.dir/cmake_pch.hxx.pch -Xclang -include -Xclang /build/doris/be/build_Release/CMakeFiles/pch.dir/cmake_pch.hxx -MD -MT src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o -MF src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o.d -o src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o -c /build/doris/be/src/index-tools/index_tool.cpp
/build/doris/be/src/index-tools/index_tool.cpp:565:35: error: no matching constructor for initialization of 'lucene::index::IndexWriter'
565 | auto indexwriter = _CLNEW lucene::index::IndexWriter(dir, analyzer, true, true);
| ^ ~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:579:12: note: candidate constructor not viable: no known conversion from 'std::shared_ptr<doris::segment_v2::DorisFSDirectory>' to 'lucene::store::Directory *' for 1st argument
579 | explicit IndexWriter(CL_NS(store)::Directory* d, CL_NS(analysis)::Analyzer* a, const bool create, const bool closeDirOnShutdown=false);
| ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:625:12: note: candidate constructor not viable: no known conversion from 'std::shared_ptr<doris::segment_v2::DorisFSDirectory>' to 'lucene::store::Directory *' for 1st argument
625 | explicit IndexWriter(CL_NS(store)::Directory* d, const bool autoCommit, CL_NS(analysis)::Analyzer* a, const bool create, IndexDeletionPolicy* deletionPolicy = NULL, const bool closeDirOnShutdown=false);
| ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:599:12: note: candidate constructor not viable: no known conversion from 'std::shared_ptr<doris::segment_v2::DorisFSDirectory>' to 'lucene::store::Directory *' for 1st argument
599 | explicit IndexWriter(CL_NS(store)::Directory* d, const bool autoCommit, CL_NS(analysis)::Analyzer* a, IndexDeletionPolicy* deletionPolicy = NULL, const bool closeDirOnShutdown=false);
| ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:557:12: note: candidate constructor not viable: requires 3 arguments, but 4 were provided
557 | explicit IndexWriter(const char* path, CL_NS(analysis)::Analyzer* a, const bool create);
| ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:187:22: note: candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 4 were provided
187 | class CLUCENE_EXPORT IndexWriter:LUCENE_BASE {
| ^~~~~~~~~~~
1 error generated.
[836/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/cloud_txn_delete_bitmap_cache.cpp.o
[837/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/cloud_warm_up_manager.cpp.o
[838/1371] Building CXX object src/util/CMakeFiles/Util.dir/arrow/block_convertor.cpp.o
[839/1371] Building CXX object src/tools/CMakeFiles/meta_tool.dir/meta_tool.cpp.o
[840/1371] Building CXX object src/olap/CMakeFiles/Olap.dir/delete_handler.cpp.o
[841/1371] Building CXX object src/olap/CMakeFiles/Olap.dir/tablet_reader.cpp.o
相关文件
来源 doris 3.0.3-rc04
https://github.com/apache/doris/blob/62a58bff4c2f640f1afcba8c754058d5f77d420f/be/src/index-tools/index_tool.cpp#L550-L566
auto fs = doris::io::global_local_filesystem();
auto index_file_writer = std::make_unique<InvertedIndexFileWriter>(
fs,
std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
doris::local_segment_path(file_dir, rowset_id, seg_id))},
rowset_id, seg_id, doris::InvertedIndexStorageFormatPB::V2);
auto st = index_file_writer->open(&index_meta);
if (!st.has_value()) {
std::cerr << "InvertedIndexFileWriter init error:" << st.error() << std::endl;
return -1;
}
using T = std::decay_t<decltype(st)>;
auto dir = std::forward<T>(st).value();
auto analyzer = _CLNEW lucene::analysis::standard95::StandardAnalyzer();
// auto analyzer = _CLNEW lucene::analysis::SimpleAnalyzer<char>();
auto indexwriter = _CLNEW lucene::index::IndexWriter(dir, analyzer, true, true);
indexwriter->setRAMBufferSizeMB(512);
来源 doris-thirdparty
https://github.com/apache/doris-thirdparty/blob/48fa9cc4ec32b40bf3b02338d0a1b2cdbc6408cf/src/core/CLucene/index/IndexWriter.h#L579
/**
* Constructs an IndexWriter for the index in <code>d</code>.
* Text will be analyzed with <code>a</code>. If <code>create</code>
* is true, then a new, empty index will be created in
* <code>d</code>, replacing the index already there, if any.
*
* @param d the index directory
* @param a the analyzer to use
* @param create <code>true</code> to create the index or overwrite
* the existing one; <code>false</code> to append to the existing
* index
* @throws CorruptIndexException if the index is corrupt
* @throws LockObtainFailedException if another writer
* has this index open (<code>write.lock</code> could not
* be obtained)
* @throws IOException if the directory cannot be read/written to, or
* if it does not exist and <code>create</code> is
* <code>false</code> or if there is any other low-level
* IO error
*/
explicit IndexWriter(CL_NS(store)::Directory* d, CL_NS(analysis)::Analyzer* a, const bool create, const bool closeDirOnShutdown=false);
附加说明
由于我不熟悉 cpp ，所以简单的让 copilot 分析了一下。
