Most of my struggles are documented here: llvm-tblgen and clang-tblgen: error while loading shared libraries: libc++.so.1 · Issue #53561 · llvm/llvm-project · GitHub
This would be reproducible in any non-bootstrap build as well, unfortunately, unless libc++ is already available in ldconfig.
Goal:
To build a distro that is almost exclusively dependent on internal LLVM runtimes (sans libc) and where everything is a shared object as much as possible.
Issue:
During a bootstrap build where runtimes and clang are built together and clang depends on shared libs libc++/abi/libunwind, the stage2 build will always fail due to none of the runtime shared libraries being available when building any of the stage2 binaries (-tblgen etc) for any of the projects (llvm, clang etc) being built, because those stage2 runtimes in turn depend on stage2 clang being available and thus expecting to build after clang in stage2. This is a circular dependency.
Ultimately, even though all the -tblgen use can be forced to be used from the bootstrap/host,
llvm.twostage.build/tools/clang/stage2-bins/bin/clang-ast-dump: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory will kill you and getting that binary bootstrapped is not a trivial matter.
The solution would be in a bootstrap build to always make runtime builds depend on bootstrap/stageN-1 clang and make stageN(n >= 2) projects to either always depend on that stage’s runtime or only when shared libraries are forced.
I’ve exhausted my CMake capabilities (that aren’t great to begin with) and hope someone has a decent quick fix that I’m missing?
# Common
set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld;lldb" CACHE STRING "")
set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
set(CLANG_BOOTSTRAP_PASSTHROUGH "CMAKE_INSTALL_PREFIX;PYTHON_HOME;PYTHON_EXECUTABLE;Python3_EXECUTABLE;LLVM_TARGETS_TO_BUILD;LLVM_PARALLEL_COMPILE_JOBS;LLVM_PARALLEL_LINK_JOBS" CACHE STRING "")
set(CLANG_BOOTSTRAP_TARGETS check-llvm check-clang check-all runtimes clang-resource-headers CACHE LIST "")
set(LLVM_PARALLEL_LINK_JOBS 1 CACHE STRING "")
# Stage 1
set(LLVM_BUILD_TOOLS OFF CACHE BOOL "")
set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")
set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g0" CACHE STRING "")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g0" CACHE STRING "")
set(CMAKE_ASM_FLAGS_RELWITHDEBINFO "-O2 -g0" CACHE STRING "")
# Stage 2
# set(BOOTSTRAP_LLVM_USE_HOST_TOOLS ON CACHE BOOL "")
# set(BOOTSTRAP_LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_BUILD_TOOLS OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_DUMP ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "")
set(BOOTSTRAP_CLANG_LINK_CLANG_DYLIB ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LTO THIN CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_TERMINFO OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LIBCXX ON CACHE BOOL "")
set(BOOTSTRAP_CLANG_DEFAULT_CXX_STDLIB libc++ CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_LINKER lld CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_RTLIB compiler-rt CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_UNWINDLIB libunwind CACHE STRING "")
set(BOOTSTRAP_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
set(BOOTSTRAP_SANITIZER_CXX_ABI libc++ CACHE STRING "")
set(BOOTSTRAP_SANITIZER_TEST_CXX libc++ CACHE STRING "")
set(BOOTSTRAP_LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "")
set(BOOTSTRAP_LIBCXX_USE_COMPILER_RT YES CACHE BOOL "")
set(BOOTSTRAP_LIBCXXABI_USE_COMPILER_RT YES CACHE BOOL "")
set(BOOTSTRAP_LIBCXXABI_USE_LLVM_UNWINDER YES CACHE BOOL "")
set(BOOTSTRAP_LLVM_INSTALL_BINUTILS_SYMLINKS ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_INSTALL_CCTOOLS_SYMLINKS ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_USE_SPLIT_DWARF ON CACHE BOOL "")
set(BOOTSTRAP_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
set(BOOTSTRAP_CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")
set(BOOTSTRAP_CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")
set(BOOTSTRAP_CMAKE_ASM_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")
Built with
SOURCE_DIR="$(readlink -nf ../llvm-project/llvm)"
BUILD_DIR="$(readlink -nf ../llvm.twostage.build)"
TARGET_DIR="$(readlink -nf ../llvm.twostage.bin)"
LLVM_DISTRO_CONF="$(readlink -nf twostage.cmake)"
rm -rf "$BUILD_DIR/"* || true
rm -rf "$TARGET_DIR/"* || true
cmake3 -G Ninja \
-Wno-dev \
-DCMAKE_INSTALL_PREFIX="$TARGET_DIR" \
-DPYTHON_HOME="$(python -c 'import sys; print(sys.exec_prefix)')" \
-DPYTHON_EXECUTABLE="$(python -c 'import sys; print(sys.executable)')" \
-DPython3_EXECUTABLE="$(python -c 'import sys; print(sys.executable)')" \
-DLLVM_PARALLEL_COMPILE_JOBS="$(python -c 'from subprocess import check_output; print(int(max(2, int(check_output(["nproc"], universal_newlines=True))/2 - 2)))')" \
-C "$LLVM_DISTRO_CONF" \
-S "$SOURCE_DIR" -B "$BUILD_DIR"
The whole clang-ast-dump thing is pretty broken at the moment; there’s a giant list of conditions in llvm-project/CMakeLists.txt at c87c61c52cad32576597fb6de764863f21b2ee7e · llvm/llvm-project · GitHub describing various situations in which we just skip the clang-ast-dump invocation. You could hack that. See ⚙ D93164 [AST] Add generator for source location introspection .
Alternatively, you can probably point LD_LIBRARY_PATH to the stage1 libc++.
Alternatively, you could treat this process as cross-compilation, instead of a bootstrap build. See How To Cross-Compile Clang/LLVM using Clang/LLVM — LLVM 16.0.0git documentation .
1 Like
Big progress with cross-compilation but no cigar.
[4506/4652] Performing configure step for 'builtins'
-- The C compiler identification is unknown
-- The ASM compiler identification is unknown
-- Found assembler: /home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/clang
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - failed
-- Warning: Did not find file Compiler/-ASM
-- Looking for unwind.h
-- Looking for unwind.h - not found
-- Looking for rpc/xdr.h
-- Looking for rpc/xdr.h - not found
/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/bin/llvm-config: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory
CMake Error at /home/user/Documents/src/llm/llvm-project/compiler-rt/cmake/Modules/CompilerRTUtils.cmake:299 (message):
llvm-config failed with status 127
Call Stack (most recent call first):
CMakeLists.txt:18 (load_llvm_config)
-- Configuring incomplete, errors occurred!
See also "/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/runtimes/builtins-bins/CMakeFiles/CMakeOutput.log".
See also "/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/runtimes/builtins-bins/CMakeFiles/CMakeError.log".
FAILED: runtimes/builtins-stamps/builtins-configure /home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/runtimes/builtins-stamps/builtins-configure
cd /home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/runtimes/builtins-bins && /usr/bin/cmake -DCMAKE_C_COMPILER=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/clang -DCMAKE_CXX_COMPILER=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/clang++ -DCMAKE_ASM_COMPILER=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/clang -DCMAKE_LINKER=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/ld.lld -DCMAKE_AR=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/llvm-ar -DCMAKE_RANLIB=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/llvm-ranlib -DCMAKE_NM=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/llvm-nm -DCMAKE_OBJDUMP=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/llvm-objdump -DCMAKE_OBJCOPY=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/llvm-objcopy -DCMAKE_STRIP=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin/llvm-strip -DCMAKE_C_COMPILER_TARGET=x86_64-unknown-linux-gnu -DCMAKE_CXX_COMPILER_TARGET=x86_64-unknown-linux-gnu -DCMAKE_ASM_COMPILER_TARGET=x86_64-unknown-linux-gnu -DCMAKE_INSTALL_PREFIX=/home/user/Documents/src/llm/llvm.twostage.bin -DLLVM_BINARY_DIR=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins -DLLVM_CONFIG_PATH=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/bin/llvm-config -DLLVM_ENABLE_WERROR=OFF -DLLVM_HOST_TRIPLE=x86_64-unknown-linux-gnu -DLLVM_HAVE_LINK_VERSION_SCRIPT=1 -DLLVM_USE_RELATIVE_PATHS_IN_DEBUG_INFO=OFF -DLLVM_USE_RELATIVE_PATHS_IN_FILES=OFF -DLLVM_LIT_ARGS=-sv -DLLVM_SOURCE_PREFIX= -DPACKAGE_VERSION=13.0.1 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_MAKE_PROGRAM=/usr/bin/ninja-build -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DLLVM_LIBRARY_OUTPUT_INTDIR=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./lib -DLLVM_RUNTIME_OUTPUT_INTDIR=/home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/./bin -DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-unknown-linux-gnu -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON -DCMAKE_C_COMPILER_WORKS=ON -DCMAKE_ASM_COMPILER_WORKS=ON -DCOMPILER_RT_USE_BUILTINS_LIBRARY=ON -GNinja /home/user/Documents/src/llm/llvm-project/llvm/runtimes/../../compiler-rt/lib/builtins && /usr/bin/cmake -E touch /home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins/runtimes/builtins-stamps//builtins-configure
[4514/4652] Building CXX object tools/lldb/source/API/CMakeFiles/liblldb.dir/SBTypeSynthetic.cpp.o
ninja: build stopped: subcommand failed.
FAILED: tools/clang/stage2-stamps/stage2-build /home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-stamps/stage2-build
cd /home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-bins && /usr/bin/cmake --build . && /usr/bin/cmake -E touch /home/user/Documents/src/llm/llvm.twostage.build/tools/clang/stage2-stamps//stage2-build
ninja: build stopped: subcommand failed.
LD_LIBRARY_PATH approach does not work at all. While individual utilities run that way, ninja-build stage2 still fails due to stage2 tooling (llvm-config) not working - I have a suspicion LD_LIBRARY_PATH gets overridden somewhere in CMake process.
Have you tried adding rpath arguments to the linker? It should work for the stage1 llvm-config.
Bootstrap works because I’m building it with host’s toolchain (although llvm-config isn’t there in bootstrap by default). It’s the stage2 artifacts that do not work at all as even if built, stage2 clang/llvm-config will not run as libc++.so isn’t in stage2 (yet) and building it requires runtimes to run stage2 clang/llvm-config.
So I’m looking right now at either adding cmake logic to add llvm-config to bootstrap and then force stage2 runtimes to use stage1 bins or use the same technique as with crosscompilation where -tblgen tools are built with build_tool. But with the latter I’d have to build the entire clang compiler as a “tool”, which in turn would overturn the entire bootstrap.
Reproducing the final solution thanks to @tobiashieta and https://github.com/jonhoo:
# Stage 1
cmake3 --build "$BUILD_DIR" --target clang-bootstrap-deps
# Stage 2 Prep
HOST_TARGET=$("$BUILD_DIR"/bin/llvm-config --host-target)
STAGE1_LIB="$BUILD_DIR/lib"
STAGE2_LIB="$BUILD_DIR/tools/clang/stage2-bins/lib"
#Stage 2
LDFLAGS="-L$STAGE2_LIB/$HOST_TARGET -L$STAGE2_LIB -L$STAGE1_LIB/$HOST_TARGET -L$STAGE1_LIB" \
LD_LIBRARY_PATH="$STAGE2_LIB/$HOST_TARGET:$STAGE2_LIB:$STAGE1_LIB/$HOST_TARGET:$STAGE1_LIB" \
cmake3 --build "$BUILD_DIR" --target stage2
This code also accounts for the need for fix to RUNPATH not regard LLVM_ENABLE_PER_TARGET_RUNTIME_DIR · Issue #54955 · llvm/llvm-project · GitHub
1 Like
Is there documentation that needs updating? What you’re doing is not exactly mainstream but it seems worth describing somewhere on the website.
I think some docs should definitely be updated here. I run into this when building a 2-stage DistributionExample on Linux and this post is the only resource I’ve found that gives me hope. 