diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py index ff43547c9bbe..17a2136a270d 100644 --- a/.ci/compute_projects.py +++ b/.ci/compute_projects.py @@ -52,6 +52,9 @@ "clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"}, "clang-tools-extra": {"libc"}, "mlir": {"flang"}, + # Test everything if ci scripts are changed. + # FIXME: Figure out what is missing and add here. + ".ci": {"llvm", "clang", "lld", "lldb"}, } DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}} @@ -130,12 +133,11 @@ def _add_dependencies(projects: Set[str]) -> Set[str]: def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]: projects_to_test = set() for modified_project in modified_projects: - # Skip all projects where we cannot run tests. - if modified_project not in PROJECT_CHECK_TARGETS: - continue if modified_project in RUNTIMES: continue - projects_to_test.add(modified_project) + # Skip all projects where we cannot run tests. + if modified_project in PROJECT_CHECK_TARGETS: + projects_to_test.add(modified_project) if modified_project not in DEPENDENTS_TO_TEST: continue for dependent_project in DEPENDENTS_TO_TEST[modified_project]: diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py index e787fd8133c8..f71b56d2525b 100644 --- a/.ci/compute_projects_test.py +++ b/.ci/compute_projects_test.py @@ -188,6 +188,23 @@ def test_exclude_gn(self): self.assertEqual(env_variables["runtimes_to_build"], "") self.assertEqual(env_variables["runtimes_check_targets"], "") + def test_ci(self): + env_variables = compute_projects.get_env_variables( + [".ci/compute_projects.py"], "Linux" + ) + self.assertEqual(env_variables["projects_to_build"], "clang;lld;llvm;lldb") + self.assertEqual( + env_variables["project_check_targets"], + "check-clang check-lld check-llvm check-lldb", + ) + self.assertEqual( + env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind" + ) + self.assertEqual( + env_variables["runtimes_check_targets"], + "check-cxx check-cxxabi check-unwind", + ) + if __name__ == "__main__": unittest.main() diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py index 73942caeb677..b05a77febb3b 100644 --- a/.ci/generate_test_report_lib.py +++ b/.ci/generate_test_report_lib.py @@ -92,7 +92,7 @@ def plural(num_tests): ] ) elif failures: - report.extend(["", "## Failed Tests", "(click to see output)"]) + report.extend(["", "## Failed Tests", "(click on a test name to see its output)"]) for testsuite_name, failures in failures.items(): report.extend(["", f"### {testsuite_name}"]) diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py index 0366f8dc70bb..fad6ab2bec89 100644 --- a/.ci/generate_test_report_lib_test.py +++ b/.ci/generate_test_report_lib_test.py @@ -2,6 +2,9 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# To run these tests: +# python -m unittest generate_test_report_lib_test.py + import unittest from io import StringIO from textwrap import dedent @@ -150,7 +153,7 @@ def test_report_single_file_single_testsuite(self): * 2 tests failed ## Failed Tests - (click to see output) + (click on a test name to see its output) ### Bar
@@ -182,7 +185,7 @@ def test_report_single_file_single_testsuite(self): * 2 tests failed ## Failed Tests - (click to see output) + (click on a test name to see its output) ### ABC
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index 13c7a93c364d..a56a8bcbffec 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -28,11 +28,16 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then ccache --clear fi +mkdir -p artifacts/reproducers + +# Make sure any clang reproducers will end up as artifacts. +export CLANG_CRASH_DIAGNOSTICS_DIR=`realpath artifacts/reproducers` + function at-exit { retcode=$? - mkdir -p artifacts ccache --print-stats > artifacts/ccache_stats.txt + cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log # If building fails there will be no results files. shopt -s nullglob @@ -57,9 +62,18 @@ export PIP_BREAK_SYSTEM_PACKAGES=1 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt + +# Set the system llvm-symbolizer as preferred. +export LLVM_SYMBOLIZER_PATH=`which llvm-symbolizer` +[[ ! -f "${LLVM_SYMBOLIZER_PATH}" ]] && echo "llvm-symbolizer not found!" + +# Set up all runtimes either way. libcxx is a dependency of LLDB. +# It will not be built unless it is used. cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_PROJECTS="${projects}" \ + -D LLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ -G Ninja \ + -D CMAKE_PREFIX_PATH="${HOME}/.local" \ -D CMAKE_BUILD_TYPE=Release \ -D LLVM_ENABLE_ASSERTIONS=ON \ -D LLVM_BUILD_EXAMPLES=ON \ @@ -68,7 +82,10 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_LLD=ON \ -D CMAKE_CXX_FLAGS=-gmlt \ -D LLVM_CCACHE_BUILD=ON \ + -D LIBCXX_CXX_ABI=libcxxabi \ -D MLIR_ENABLE_BINDINGS_PYTHON=ON \ + -D LLDB_ENABLE_PYTHON=ON \ + -D LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=ON \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" echo "--- ninja" diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh index 8ccc875609a3..2968bc8c8702 100755 --- a/.ci/monolithic-windows.sh +++ b/.ci/monolithic-windows.sh @@ -32,6 +32,7 @@ function at-exit { mkdir -p artifacts sccache --show-stats >> artifacts/sccache_stats.txt + cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log # If building fails there will be no results files. shopt -s nullglob diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml index c0c61748010d..1ec159bfaa09 100644 --- a/.github/new-prs-labeler.yml +++ b/.github/new-prs-labeler.yml @@ -7,6 +7,12 @@ ClangIR: - clang/tools/cir-*/**/* - clang/test/CIR/**/* +clang:bytecode: + - clang/docs/ConstantInterpreter.rst + - clang/lib/AST/ByteCode/**/* + - clang/test/AST/ByteCode/**/* + - clang/unittests/AST/ByteCode/**/* + clang:dataflow: - clang/include/clang/Analysis/FlowSensitive/**/* - clang/lib/Analysis/FlowSensitive/**/* diff --git a/.github/workflows/ci-post-commit-analyzer.yml b/.github/workflows/ci-post-commit-analyzer.yml index e0d30b66f66b..b8074859d23a 100644 --- a/.github/workflows/ci-post-commit-analyzer.yml +++ b/.github/workflows/ci-post-commit-analyzer.yml @@ -34,9 +34,9 @@ jobs: if: >- github.repository_owner == 'llvm' && github.event.action != 'closed' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: - image: 'ghcr.io/llvm/ci-ubuntu-22.04:latest' + image: 'ghcr.io/llvm/ci-ubuntu-24.04:latest' env: LLVM_VERSION: 18 steps: diff --git a/.github/workflows/commit-access-greeter.yml b/.github/workflows/commit-access-greeter.yml index ef489b4d3c9a..a5fbbbb94e22 100644 --- a/.github/workflows/commit-access-greeter.yml +++ b/.github/workflows/commit-access-greeter.yml @@ -12,6 +12,7 @@ jobs: commit-access-greeter: permissions: issues: write + pull-requests: read if: >- github.repository_owner == 'llvm' && github.event.label.name == 'infra:commit-access-request' diff --git a/.github/workflows/containers/github-action-ci/Dockerfile b/.github/workflows/containers/github-action-ci/Dockerfile index e1e9827737ad..60f47653a078 100644 --- a/.github/workflows/containers/github-action-ci/Dockerfile +++ b/.github/workflows/containers/github-action-ci/Dockerfile @@ -32,7 +32,7 @@ RUN cmake -B ./build -G Ninja ./llvm \ -DLLVM_ENABLE_RUNTIMES="compiler-rt" \ -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \ -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \ - -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build" \ + -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build;llvm-symbolizer" \ -DCLANG_DEFAULT_LINKER="lld" RUN ninja -C ./build stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution diff --git a/.github/workflows/libc-fullbuild-tests.yml b/.github/workflows/libc-fullbuild-tests.yml index 00d2696e7e58..54d0df99b687 100644 --- a/.github/workflows/libc-fullbuild-tests.yml +++ b/.github/workflows/libc-fullbuild-tests.yml @@ -19,13 +19,13 @@ jobs: include: - os: ubuntu-24.04 ccache-variant: sccache - c_compiler: clang - cpp_compiler: clang++ + c_compiler: clang-20 + cpp_compiler: clang++-20 # TODO: remove ccache logic when https://github.com/hendrikmuhs/ccache-action/issues/279 is resolved. - os: ubuntu-24.04-arm ccache-variant: ccache - c_compiler: clang - cpp_compiler: clang++ + c_compiler: clang-20 + cpp_compiler: clang++-20 # TODO: add back gcc build when it is fixed # - c_compiler: gcc # cpp_compiler: g++ @@ -51,6 +51,9 @@ jobs: # For more information, see https://wiki.debian.org/Multiarch/LibraryPathOverview - name: Prepare dependencies (Ubuntu) run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 20 sudo apt-get update sudo apt-get install -y libmpfr-dev libgmp-dev libmpc-dev ninja-build linux-libc-dev sudo ln -sf /usr/include/$(uname -p)-linux-gnu/asm /usr/include/asm diff --git a/.github/workflows/libclang-python-tests.yml b/.github/workflows/libclang-python-tests.yml index cbfb3e5cbabf..50ef4acf2feb 100644 --- a/.github/workflows/libclang-python-tests.yml +++ b/.github/workflows/libclang-python-tests.yml @@ -37,5 +37,5 @@ jobs: projects: clang # There is an issue running on "windows-2019". # See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082. - os_list: '["ubuntu-22.04"]' + os_list: '["ubuntu-24.04"]' python_version: ${{ matrix.python-version }} diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml index 326c6f288750..fce20d6d9de0 100644 --- a/.github/workflows/libcxx-build-and-test.yaml +++ b/.github/workflows/libcxx-build-and-test.yaml @@ -216,7 +216,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0 with: - xcode-version: 'latest' + # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md + xcode-version: '16.3' - uses: seanmiddleditch/gha-setup-ninja@3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6 - name: Build and test run: | diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml index 6f2a81291ff2..862f30c4801e 100644 --- a/.github/workflows/llvm-project-tests.yml +++ b/.github/workflows/llvm-project-tests.yml @@ -47,12 +47,7 @@ on: type: string # Use windows-2019 due to: # https://developercommunity.visualstudio.com/t/Prev-Issue---with-__assume-isnan-/1597317 - # Use ubuntu-22.04 rather than ubuntu-24.04 to match the ubuntu - # version in the CI container. Without this, setup-python tries - # to install a python version linked against a newer version of glibc. - # TODO(boomanaiden154): Bump the Ubuntu version once the version in the - # container is bumped. - default: '["ubuntu-22.04", "windows-2019", "macOS-13"]' + default: '["ubuntu-24.04", "windows-2019", "macOS-13"]' python_version: required: false @@ -72,7 +67,7 @@ jobs: name: Lit Tests runs-on: ${{ matrix.os }} container: - image: ${{(startsWith(matrix.os, 'ubuntu') && 'ghcr.io/llvm/ci-ubuntu-22.04:latest') || null}} + image: ${{(startsWith(matrix.os, 'ubuntu') && 'ghcr.io/llvm/ci-ubuntu-24.04:latest') || null}} strategy: fail-fast: false matrix: @@ -106,7 +101,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y python3-dev python3-pip - pip install -r mlir/python/requirements.txt + pip install --break-system-packages -r mlir/python/requirements.txt - name: Setup ccache uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17 with: diff --git a/.github/workflows/mlir-tests.yml b/.github/workflows/mlir-tests.yml index 7ac8003bab31..d66d7f7ad338 100644 --- a/.github/workflows/mlir-tests.yml +++ b/.github/workflows/mlir-tests.yml @@ -25,4 +25,4 @@ jobs: projects: mlir cache-key: amd-mlir extra_cmake_args: -DMLIR_ENABLE_BINDINGS_PYTHON=ON - os_list: '["ubuntu-22.04"]' + os_list: '["ubuntu-24.04"]' diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml index b7d2c7154762..dd442dc636c0 100644 --- a/.github/workflows/premerge.yaml +++ b/.github/workflows/premerge.yaml @@ -65,6 +65,14 @@ jobs: export CXX=/opt/llvm/bin/clang++ ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" + - name: "Upload artifact" + - name: Upload Artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: Premerge Artifacts (Linux) + path: artifacts/ + retention-days: 5 + include-hidden-files: 'true' premerge-checks-windows: name: Windows Premerge Checks (Test Only - Please Ignore Results) @@ -113,6 +121,13 @@ jobs: set MAX_PARALLEL_LINK_JOBS=64 call C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64 bash .ci/monolithic-windows.sh "${{ steps.vars.outputs.windows-projects }}" "${{ steps.vars.outputs.windows-check-targets }}" + - name: Upload Artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: Premerge Artifacts (Windows) + path: artifacts/ + retention-days: 5 + include-hidden-files: 'true' premerge-check-macos: name: MacOS Premerge Checks diff --git a/.github/workflows/spirv-tests.yml b/.github/workflows/spirv-tests.yml index ea466dc6c52e..f15ca1cb64ba 100644 --- a/.github/workflows/spirv-tests.yml +++ b/.github/workflows/spirv-tests.yml @@ -26,4 +26,4 @@ jobs: build_target: check-llvm-codegen-spirv projects: extra_cmake_args: '-DLLVM_TARGETS_TO_BUILD="SPIRV" -DLLVM_INCLUDE_SPIRV_TOOLS_TESTS=ON' - os_list: '["ubuntu-22.04"]' + os_list: '["ubuntu-24.04"]' diff --git a/.mailmap b/.mailmap index f5b220f892fc..a3f9dbcab019 100644 --- a/.mailmap +++ b/.mailmap @@ -29,7 +29,8 @@ - +Min Hsu +Min Hsu @@ -48,6 +49,6 @@ Ramkumar Ramachandra Ramkumar Ramachandra Saleem Abdulrasool Tommy Chiang -Wang Pengcheng -Wang Pengcheng <137158460+wangpc-pp@users.noreply.github.com> -Wang Pengcheng +Pengcheng Wang +Pengcheng Wang <137158460+wangpc-pp@users.noreply.github.com> +Pengcheng Wang diff --git a/bolt/CMakeLists.txt b/bolt/CMakeLists.txt index f5ffa8122706..52c796518ac0 100644 --- a/bolt/CMakeLists.txt +++ b/bolt/CMakeLists.txt @@ -82,7 +82,8 @@ endforeach() set(BOLT_ENABLE_RUNTIME_default OFF) if ((CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" - OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)$") + OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)$" + OR CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") AND (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Darwin") AND (NOT CMAKE_CROSSCOMPILING)) @@ -136,7 +137,7 @@ if (LLVM_INCLUDE_TESTS) endif() if (BOLT_ENABLE_RUNTIME) - message(STATUS "Building BOLT runtime libraries for X86") + message(STATUS "Building BOLT runtime libraries for ${CMAKE_SYSTEM_PROCESSOR}") set(extra_args "") if(CMAKE_SYSROOT) list(APPEND extra_args -DCMAKE_SYSROOT=${CMAKE_SYSROOT}) diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index d3d11f8c5fb7..a52998564ee1 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -1174,6 +1174,11 @@ class BinaryFunction { return getSecondaryEntryPointSymbol(BB.getLabel()); } + /// Remove a label from the secondary entry point map. + void removeSymbolFromSecondaryEntryPointMap(const MCSymbol *Label) { + SecondaryEntryPoints.erase(Label); + } + /// Return true if the basic block is an entry point into the function /// (either primary or secondary). bool isEntryPoint(const BinaryBasicBlock &BB) const { @@ -2126,6 +2131,10 @@ class BinaryFunction { return Islands && !Islands->DataOffsets.empty(); } + bool isStartOfConstantIsland(uint64_t Offset) const { + return hasConstantIsland() && Islands->DataOffsets.count(Offset); + } + /// Return true iff the symbol could be seen inside this function otherwise /// it is probably another function. bool isSymbolValidInScope(const SymbolRef &Symbol, uint64_t SymbolSize) const; diff --git a/bolt/include/bolt/Passes/PAuthGadgetScanner.h b/bolt/include/bolt/Passes/PAuthGadgetScanner.h index 622e6721dea5..ee69337637da 100644 --- a/bolt/include/bolt/Passes/PAuthGadgetScanner.h +++ b/bolt/include/bolt/Passes/PAuthGadgetScanner.h @@ -175,8 +175,8 @@ raw_ostream &operator<<(raw_ostream &OS, const MCInstReference &); namespace PAuthGadgetScanner { -class PacRetAnalysis; -struct State; +class SrcSafetyAnalysis; +struct SrcState; /// Description of a gadget kind that can be detected. Intended to be /// statically allocated to be attached to reports by reference. diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h index 56eb463fc98f..79a91861554d 100644 --- a/bolt/include/bolt/Profile/DataAggregator.h +++ b/bolt/include/bolt/Profile/DataAggregator.h @@ -223,7 +223,8 @@ class DataAggregator : public DataReader { bool recordExit(BinaryFunction &BF, uint64_t From, bool Mispred, uint64_t Count = 1) const; - /// Aggregation statistics + /// Branch stacks aggregation statistics + uint64_t NumTraces{0}; uint64_t NumInvalidTraces{0}; uint64_t NumLongRangeTraces{0}; /// Specifies how many samples were recorded in cold areas if we are dealing @@ -231,6 +232,7 @@ class DataAggregator : public DataReader { /// for the source of the branch to avoid counting cold activity twice (one /// for source and another for destination). uint64_t NumColdSamples{0}; + uint64_t NumTotalSamples{0}; /// Looks into system PATH for Linux Perf and set up the aggregator to use it void findPerfExecutable(); @@ -327,8 +329,8 @@ class DataAggregator : public DataReader { /// Parse a single LBR entry as output by perf script -Fbrstack ErrorOr parseLBREntry(); - /// Parse LBR sample, returns the number of traces. - uint64_t parseLBRSample(const PerfBranchSample &Sample, bool NeedsSkylakeFix); + /// Parse LBR sample. + void parseLBRSample(const PerfBranchSample &Sample, bool NeedsSkylakeFix); /// Parse and pre-aggregate branch events. std::error_code parseBranchEvents(); @@ -487,6 +489,13 @@ class DataAggregator : public DataReader { void dump(const PerfBranchSample &Sample) const; void dump(const PerfMemSample &Sample) const; + /// Profile diagnostics print methods + void printColdSamplesDiagnostic() const; + void printLongRangeTracesDiagnostic() const; + void printBranchSamplesDiagnostics() const; + void printBasicSamplesDiagnostics(uint64_t OutOfRangeSamples) const; + void printBranchStacksDiagnostics(uint64_t IgnoredSamples) const; + public: /// If perf.data was collected without build ids, the buildid-list may contain /// incomplete entries. Return true if the buffer containing diff --git a/bolt/lib/Core/BinaryBasicBlock.cpp b/bolt/lib/Core/BinaryBasicBlock.cpp index 2a2192b79bb4..311d5c15b8dc 100644 --- a/bolt/lib/Core/BinaryBasicBlock.cpp +++ b/bolt/lib/Core/BinaryBasicBlock.cpp @@ -372,8 +372,7 @@ void BinaryBasicBlock::updateJumpTableSuccessors() { [](const BinaryBasicBlock *BB1, const BinaryBasicBlock *BB2) { return BB1->getInputOffset() < BB2->getInputOffset(); }); - SuccessorBBs.erase(std::unique(SuccessorBBs.begin(), SuccessorBBs.end()), - SuccessorBBs.end()); + SuccessorBBs.erase(llvm::unique(SuccessorBBs), SuccessorBBs.end()); for (BinaryBasicBlock *BB : SuccessorBBs) addSuccessor(BB); diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp index 80b15d7b7961..82ede09c29dd 100644 --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -579,6 +579,21 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address, TrimmedSize = EntriesAsAddress->size(); }; + auto printEntryDiagnostics = [&](raw_ostream &OS, + const BinaryFunction *TargetBF) { + OS << "FAIL: function doesn't contain this address\n"; + if (!TargetBF) + return; + OS << " ! function containing this address: " << *TargetBF << '\n'; + if (!TargetBF->isFragment()) + return; + OS << " ! is a fragment with parents: "; + ListSeparator LS; + for (BinaryFunction *Parent : TargetBF->ParentFragments) + OS << LS << *Parent; + OS << '\n'; + }; + ErrorOr Section = getSectionForAddress(Address); if (!Section) return false; @@ -646,25 +661,9 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address, // Function or one of its fragments. const BinaryFunction *TargetBF = getBinaryFunctionContainingAddress(Value); - const bool DoesBelongToFunction = - BF.containsAddress(Value) || - (TargetBF && areRelatedFragments(TargetBF, &BF)); - if (!DoesBelongToFunction) { - LLVM_DEBUG({ - if (!BF.containsAddress(Value)) { - dbgs() << "FAIL: function doesn't contain this address\n"; - if (TargetBF) { - dbgs() << " ! function containing this address: " - << TargetBF->getPrintName() << '\n'; - if (TargetBF->isFragment()) { - dbgs() << " ! is a fragment"; - for (BinaryFunction *Parent : TargetBF->ParentFragments) - dbgs() << ", parent: " << Parent->getPrintName(); - dbgs() << '\n'; - } - } - } - }); + if (!TargetBF || !areRelatedFragments(TargetBF, &BF)) { + LLVM_DEBUG(printEntryDiagnostics(dbgs(), TargetBF)); + (void)printEntryDiagnostics; break; } @@ -703,10 +702,7 @@ void BinaryContext::populateJumpTables() { ++JTI) { JumpTable *JT = JTI->second; - bool NonSimpleParent = false; - for (BinaryFunction *BF : JT->Parents) - NonSimpleParent |= !BF->isSimple(); - if (NonSimpleParent) + if (!llvm::all_of(JT->Parents, std::mem_fn(&BinaryFunction::isSimple))) continue; uint64_t NextJTAddress = 0; @@ -840,33 +836,26 @@ BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address, assert(JT->Type == Type && "jump table types have to match"); assert(Address == JT->getAddress() && "unexpected non-empty jump table"); - // Prevent associating a jump table to a specific fragment twice. - if (!llvm::is_contained(JT->Parents, &Function)) { - assert(llvm::all_of(JT->Parents, - [&](const BinaryFunction *BF) { - return areRelatedFragments(&Function, BF); - }) && - "cannot re-use jump table of a different function"); - // Duplicate the entry for the parent function for easy access - JT->Parents.push_back(&Function); - if (opts::Verbosity > 2) { - this->outs() << "BOLT-INFO: Multiple fragments access same jump table: " - << JT->Parents[0]->getPrintName() << "; " - << Function.getPrintName() << "\n"; - JT->print(this->outs()); - } - Function.JumpTables.emplace(Address, JT); - for (BinaryFunction *Parent : JT->Parents) - Parent->setHasIndirectTargetToSplitFragment(true); - } + if (llvm::is_contained(JT->Parents, &Function)) + return JT->getFirstLabel(); - bool IsJumpTableParent = false; - (void)IsJumpTableParent; - for (BinaryFunction *Frag : JT->Parents) - if (Frag == &Function) - IsJumpTableParent = true; - assert(IsJumpTableParent && + // Prevent associating a jump table to a specific fragment twice. + auto isSibling = std::bind(&BinaryContext::areRelatedFragments, this, + &Function, std::placeholders::_1); + assert(llvm::all_of(JT->Parents, isSibling) && "cannot re-use jump table of a different function"); + (void)isSibling; + if (opts::Verbosity > 2) { + this->outs() << "BOLT-INFO: multiple fragments access the same jump table" + << ": " << *JT->Parents[0] << "; " << Function << '\n'; + JT->print(this->outs()); + } + if (JT->Parents.size() == 1) + JT->Parents.front()->setHasIndirectTargetToSplitFragment(true); + Function.setHasIndirectTargetToSplitFragment(true); + // Duplicate the entry for the parent function for easy access + JT->Parents.push_back(&Function); + Function.JumpTables.emplace(Address, JT); return JT->getFirstLabel(); } diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp index 1aad25242712..7b5cd276fee8 100644 --- a/bolt/lib/Core/BinaryEmitter.cpp +++ b/bolt/lib/Core/BinaryEmitter.cpp @@ -373,8 +373,10 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, Streamer.emitLabel(StartSymbol); } + const bool NeedsFDE = + Function.hasCFI() && !(Function.isPatch() && Function.isAnonymous()); // Emit CFI start - if (Function.hasCFI()) { + if (NeedsFDE) { Streamer.emitCFIStartProc(/*IsSimple=*/false); if (Function.getPersonalityFunction() != nullptr) Streamer.emitCFIPersonality(Function.getPersonalityFunction(), @@ -421,7 +423,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, Streamer.emitBytes(BC.MIB->getTrapFillValue()); // Emit CFI end - if (Function.hasCFI()) + if (NeedsFDE) Streamer.emitCFIEndProc(); MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum()); @@ -809,52 +811,50 @@ void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection, Streamer.switchSection(JT.Count > 0 ? HotSection : ColdSection); Streamer.emitValueToAlignment(Align(JT.EntrySize)); } - MCSymbol *LastLabel = nullptr; + MCSymbol *JTLabel = nullptr; uint64_t Offset = 0; for (MCSymbol *Entry : JT.Entries) { auto LI = JT.Labels.find(Offset); - if (LI != JT.Labels.end()) { - LLVM_DEBUG({ - dbgs() << "BOLT-DEBUG: emitting jump table " << LI->second->getName() - << " (originally was at address 0x" - << Twine::utohexstr(JT.getAddress() + Offset) - << (Offset ? ") as part of larger jump table\n" : ")\n"); - }); - if (!LabelCounts.empty()) { - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: " - << LabelCounts[LI->second] << '\n'); - if (LabelCounts[LI->second] > 0) - Streamer.switchSection(HotSection); - else - Streamer.switchSection(ColdSection); - Streamer.emitValueToAlignment(Align(JT.EntrySize)); - } - // Emit all labels registered at the address of this jump table - // to sync with our global symbol table. We may have two labels - // registered at this address if one label was created via - // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing - // this location) and another via getOrCreateJumpTable(). This - // creates a race where the symbols created by these two - // functions may or may not be the same, but they are both - // registered in our symbol table at the same address. By - // emitting them all here we make sure there is no ambiguity - // that depends on the order that these symbols were created, so - // whenever this address is referenced in the binary, it is - // certain to point to the jump table identified at this - // address. - if (BinaryData *BD = BC.getBinaryDataByName(LI->second->getName())) { - for (MCSymbol *S : BD->getSymbols()) - Streamer.emitLabel(S); - } else { - Streamer.emitLabel(LI->second); - } - LastLabel = LI->second; + if (LI == JT.Labels.end()) + goto emitEntry; + JTLabel = LI->second; + LLVM_DEBUG({ + dbgs() << "BOLT-DEBUG: emitting jump table " << JTLabel->getName() + << " (originally was at address 0x" + << Twine::utohexstr(JT.getAddress() + Offset) + << (Offset ? ") as part of larger jump table\n" : ")\n"); + }); + if (!LabelCounts.empty()) { + const uint64_t JTCount = LabelCounts[JTLabel]; + LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: " << JTCount << '\n'); + Streamer.switchSection(JTCount ? HotSection : ColdSection); + Streamer.emitValueToAlignment(Align(JT.EntrySize)); + } + // Emit all labels registered at the address of this jump table + // to sync with our global symbol table. We may have two labels + // registered at this address if one label was created via + // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing + // this location) and another via getOrCreateJumpTable(). This + // creates a race where the symbols created by these two + // functions may or may not be the same, but they are both + // registered in our symbol table at the same address. By + // emitting them all here we make sure there is no ambiguity + // that depends on the order that these symbols were created, so + // whenever this address is referenced in the binary, it is + // certain to point to the jump table identified at this + // address. + if (BinaryData *BD = BC.getBinaryDataByName(JTLabel->getName())) { + for (MCSymbol *S : BD->getSymbols()) + Streamer.emitLabel(S); + } else { + Streamer.emitLabel(JTLabel); } + emitEntry: if (JT.Type == JumpTable::JTT_NORMAL) { Streamer.emitSymbolValue(Entry, JT.OutputEntrySize); } else { // JTT_PIC const MCSymbolRefExpr *JTExpr = - MCSymbolRefExpr::create(LastLabel, Streamer.getContext()); + MCSymbolRefExpr::create(JTLabel, Streamer.getContext()); const MCSymbolRefExpr *E = MCSymbolRefExpr::create(Entry, Streamer.getContext()); const MCBinaryExpr *Value = diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index c4f4d234b30c..4624abadc701 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -1896,6 +1896,15 @@ void BinaryFunction::postProcessEntryPoints() { if (BC.isAArch64() && Offset == getSize()) continue; + // If we have grabbed a wrong code label which actually points to some + // constant island inside the function, ignore this label and remove it + // from the secondary entry point map. + if (isStartOfConstantIsland(Offset)) { + BC.SymbolToFunctionMap.erase(Label); + removeSymbolFromSecondaryEntryPointMap(Label); + continue; + } + BC.errs() << "BOLT-WARNING: reference in the middle of instruction " "detected in function " << *this << " at offset 0x" << Twine::utohexstr(Offset) << '\n'; @@ -2367,7 +2376,7 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) { // Without doing jump table value profiling we don't have a use for extra // (duplicate) branches. llvm::sort(TakenBranches); - auto NewEnd = std::unique(TakenBranches.begin(), TakenBranches.end()); + auto NewEnd = llvm::unique(TakenBranches); TakenBranches.erase(NewEnd, TakenBranches.end()); for (std::pair &Branch : TakenBranches) { diff --git a/bolt/lib/Core/DebugNames.cpp b/bolt/lib/Core/DebugNames.cpp index 366c22c38e61..82cabe689eb1 100644 --- a/bolt/lib/Core/DebugNames.cpp +++ b/bolt/lib/Core/DebugNames.cpp @@ -440,8 +440,7 @@ void DWARF5AcceleratorTable::computeBucketCount() { for (const auto &E : Entries) Uniques.push_back(E.second.HashValue); array_pod_sort(Uniques.begin(), Uniques.end()); - std::vector::iterator P = - std::unique(Uniques.begin(), Uniques.end()); + std::vector::iterator P = llvm::unique(Uniques); UniqueHashCount = std::distance(Uniques.begin(), P); diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp index a3be147a0906..7752079b6153 100644 --- a/bolt/lib/Core/MCPlusBuilder.cpp +++ b/bolt/lib/Core/MCPlusBuilder.cpp @@ -442,10 +442,10 @@ void MCPlusBuilder::getUsedRegs(const MCInst &Inst, BitVector &Regs) const { for (MCPhysReg ImplicitUse : InstInfo.implicit_uses()) Regs |= getAliases(ImplicitUse, /*OnlySmaller=*/true); - for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) { - if (!Inst.getOperand(I).isReg()) + for (const MCOperand &Operand : useOperands(Inst)) { + if (!Operand.isReg()) continue; - Regs |= getAliases(Inst.getOperand(I).getReg(), /*OnlySmaller=*/true); + Regs |= getAliases(Operand.getReg(), /*OnlySmaller=*/true); } } diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp index 4696a1f1f040..f099dfa46f3d 100644 --- a/bolt/lib/Core/Relocation.cpp +++ b/bolt/lib/Core/Relocation.cpp @@ -96,6 +96,7 @@ static bool isSupportedAArch64(uint32_t Type) { case ELF::R_AARCH64_MOVW_UABS_G2: case ELF::R_AARCH64_MOVW_UABS_G2_NC: case ELF::R_AARCH64_MOVW_UABS_G3: + case ELF::R_AARCH64_PLT32: return true; } } @@ -122,6 +123,7 @@ static bool isSupportedRISCV(uint32_t Type) { case ELF::R_RISCV_LO12_S: case ELF::R_RISCV_64: case ELF::R_RISCV_TLS_GOT_HI20: + case ELF::R_RISCV_TLS_GD_HI20: case ELF::R_RISCV_TPREL_HI20: case ELF::R_RISCV_TPREL_ADD: case ELF::R_RISCV_TPREL_LO12_I: @@ -202,6 +204,7 @@ static size_t getSizeForTypeAArch64(uint32_t Type) { case ELF::R_AARCH64_MOVW_UABS_G2_NC: case ELF::R_AARCH64_MOVW_UABS_G3: case ELF::R_AARCH64_ABS32: + case ELF::R_AARCH64_PLT32: return 4; case ELF::R_AARCH64_ABS64: case ELF::R_AARCH64_PREL64: @@ -234,6 +237,7 @@ static size_t getSizeForTypeRISCV(uint32_t Type) { case ELF::R_RISCV_64: case ELF::R_RISCV_GOT_HI20: case ELF::R_RISCV_TLS_GOT_HI20: + case ELF::R_RISCV_TLS_GD_HI20: // See extractValueRISCV for why this is necessary. return 8; } @@ -354,6 +358,7 @@ static uint64_t extractValueAArch64(uint32_t Type, uint64_t Contents, case ELF::R_AARCH64_PREL16: return static_cast(PC) + SignExtend64<16>(Contents & 0xffff); case ELF::R_AARCH64_PREL32: + case ELF::R_AARCH64_PLT32: return static_cast(PC) + SignExtend64<32>(Contents & 0xffffffff); case ELF::R_AARCH64_PREL64: return static_cast(PC) + Contents; @@ -488,6 +493,7 @@ static uint64_t extractValueRISCV(uint32_t Type, uint64_t Contents, return extractBImmRISCV(Contents); case ELF::R_RISCV_GOT_HI20: case ELF::R_RISCV_TLS_GOT_HI20: + case ELF::R_RISCV_TLS_GD_HI20: // We need to know the exact address of the GOT entry so we extract the // value from both the AUIPC and L[D|W]. We cannot rely on the symbol in the // relocation for this since it simply refers to the object that is stored @@ -676,6 +682,7 @@ static bool isPCRelativeAArch64(uint32_t Type) { case ELF::R_AARCH64_PREL16: case ELF::R_AARCH64_PREL32: case ELF::R_AARCH64_PREL64: + case ELF::R_AARCH64_PLT32: return true; } } @@ -703,6 +710,7 @@ static bool isPCRelativeRISCV(uint32_t Type) { case ELF::R_RISCV_RVC_BRANCH: case ELF::R_RISCV_32_PCREL: case ELF::R_RISCV_TLS_GOT_HI20: + case ELF::R_RISCV_TLS_GD_HI20: return true; } } diff --git a/bolt/lib/Passes/AsmDump.cpp b/bolt/lib/Passes/AsmDump.cpp index 08191669e72f..ffc920af1c41 100644 --- a/bolt/lib/Passes/AsmDump.cpp +++ b/bolt/lib/Passes/AsmDump.cpp @@ -134,13 +134,14 @@ void dumpFunction(const BinaryFunction &BF) { std::unique_ptr MAB( BC.TheTarget->createMCAsmBackend(*BC.STI, *BC.MRI, MCTargetOptions())); int AsmPrinterVariant = BC.AsmInfo->getAssemblerDialect(); - MCInstPrinter *InstructionPrinter(BC.TheTarget->createMCInstPrinter( - *BC.TheTriple, AsmPrinterVariant, *BC.AsmInfo, *BC.MII, *BC.MRI)); + std::unique_ptr InstructionPrinter( + BC.TheTarget->createMCInstPrinter(*BC.TheTriple, AsmPrinterVariant, + *BC.AsmInfo, *BC.MII, *BC.MRI)); auto FOut = std::make_unique(OS); FOut->SetUnbuffered(); - std::unique_ptr AsmStreamer( - createAsmStreamer(*LocalCtx, std::move(FOut), InstructionPrinter, - std::move(MCEInstance.MCE), std::move(MAB))); + std::unique_ptr AsmStreamer(createAsmStreamer( + *LocalCtx, std::move(FOut), std::move(InstructionPrinter), + std::move(MCEInstance.MCE), std::move(MAB))); AsmStreamer->initSections(true, *BC.STI); std::unique_ptr TM(BC.TheTarget->createTargetMachine( *BC.TheTriple, "", "", TargetOptions(), std::nullopt)); diff --git a/bolt/lib/Passes/PAuthGadgetScanner.cpp b/bolt/lib/Passes/PAuthGadgetScanner.cpp index a7c22b23e436..2a20bcd09c78 100644 --- a/bolt/lib/Passes/PAuthGadgetScanner.cpp +++ b/bolt/lib/Passes/PAuthGadgetScanner.cpp @@ -172,7 +172,7 @@ class TrackedRegisters { /// * RET (which is implicitly RET X30) is a protected return if and only if /// X30 is safe-to-dereference - the state computed for sub- and /// super-registers is not inspected. -struct State { +struct SrcState { /// A BitVector containing the registers that are either safe at function /// entry and were not clobbered yet, or those not clobbered since being /// authenticated. @@ -186,12 +186,12 @@ struct State { std::vector> LastInstWritingReg; /// Construct an empty state. - State() {} + SrcState() {} - State(unsigned NumRegs, unsigned NumRegsToTrack) + SrcState(unsigned NumRegs, unsigned NumRegsToTrack) : SafeToDerefRegs(NumRegs), LastInstWritingReg(NumRegsToTrack) {} - State &merge(const State &StateIn) { + SrcState &merge(const SrcState &StateIn) { if (StateIn.empty()) return *this; if (empty()) @@ -208,11 +208,11 @@ struct State { /// neither safe, nor unsafe ones. bool empty() const { return SafeToDerefRegs.empty(); } - bool operator==(const State &RHS) const { + bool operator==(const SrcState &RHS) const { return SafeToDerefRegs == RHS.SafeToDerefRegs && LastInstWritingReg == RHS.LastInstWritingReg; } - bool operator!=(const State &RHS) const { return !((*this) == RHS); } + bool operator!=(const SrcState &RHS) const { return !((*this) == RHS); } }; static void printLastInsts( @@ -228,8 +228,8 @@ static void printLastInsts( } } -raw_ostream &operator<<(raw_ostream &OS, const State &S) { - OS << "pacret-state<"; +raw_ostream &operator<<(raw_ostream &OS, const SrcState &S) { + OS << "src-state<"; if (S.empty()) { OS << "empty"; } else { @@ -240,18 +240,18 @@ raw_ostream &operator<<(raw_ostream &OS, const State &S) { return OS; } -class PacStatePrinter { +class SrcStatePrinter { public: - void print(raw_ostream &OS, const State &State) const; - explicit PacStatePrinter(const BinaryContext &BC) : BC(BC) {} + void print(raw_ostream &OS, const SrcState &State) const; + explicit SrcStatePrinter(const BinaryContext &BC) : BC(BC) {} private: const BinaryContext &BC; }; -void PacStatePrinter::print(raw_ostream &OS, const State &S) const { +void SrcStatePrinter::print(raw_ostream &OS, const SrcState &S) const { RegStatePrinter RegStatePrinter(BC); - OS << "pacret-state<"; + OS << "src-state<"; if (S.empty()) { assert(S.SafeToDerefRegs.empty()); assert(S.LastInstWritingReg.empty()); @@ -265,19 +265,19 @@ void PacStatePrinter::print(raw_ostream &OS, const State &S) const { OS << ">"; } -class PacRetAnalysis - : public DataflowAnalysis { +class SrcSafetyAnalysis + : public DataflowAnalysis { using Parent = - DataflowAnalysis; + DataflowAnalysis; friend Parent; public: - PacRetAnalysis(BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId, - const std::vector &RegsToTrackInstsFor) + SrcSafetyAnalysis(BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId, + const std::vector &RegsToTrackInstsFor) : Parent(BF, AllocId), NumRegs(BF.getBinaryContext().MRI->getNumRegs()), RegsToTrackInstsFor(RegsToTrackInstsFor) {} - virtual ~PacRetAnalysis() {} + virtual ~SrcSafetyAnalysis() {} protected: const unsigned NumRegs; @@ -285,12 +285,12 @@ class PacRetAnalysis /// must compute which the last set of instructions writing to it are. const TrackedRegisters RegsToTrackInstsFor; - SmallPtrSet &lastWritingInsts(State &S, + SmallPtrSet &lastWritingInsts(SrcState &S, MCPhysReg Reg) const { unsigned Index = RegsToTrackInstsFor.getIndex(Reg); return S.LastInstWritingReg[Index]; } - const SmallPtrSet &lastWritingInsts(const State &S, + const SmallPtrSet &lastWritingInsts(const SrcState &S, MCPhysReg Reg) const { unsigned Index = RegsToTrackInstsFor.getIndex(Reg); return S.LastInstWritingReg[Index]; @@ -298,30 +298,30 @@ class PacRetAnalysis void preflight() {} - State createEntryState() { - State S(NumRegs, RegsToTrackInstsFor.getNumTrackedRegisters()); + SrcState createEntryState() { + SrcState S(NumRegs, RegsToTrackInstsFor.getNumTrackedRegisters()); for (MCPhysReg Reg : BC.MIB->getTrustedLiveInRegs()) S.SafeToDerefRegs |= BC.MIB->getAliases(Reg, /*OnlySmaller=*/true); return S; } - State getStartingStateAtBB(const BinaryBasicBlock &BB) { + SrcState getStartingStateAtBB(const BinaryBasicBlock &BB) { if (BB.isEntryPoint()) return createEntryState(); - return State(); + return SrcState(); } - State getStartingStateAtPoint(const MCInst &Point) { return State(); } + SrcState getStartingStateAtPoint(const MCInst &Point) { return SrcState(); } - void doConfluence(State &StateOut, const State &StateIn) { - PacStatePrinter P(BC); + void doConfluence(SrcState &StateOut, const SrcState &StateIn) { + SrcStatePrinter P(BC); LLVM_DEBUG({ - dbgs() << " PacRetAnalysis::Confluence(\n"; - dbgs() << " State 1: "; + dbgs() << " SrcSafetyAnalysis::Confluence(\n"; + dbgs() << " State 1: "; P.print(dbgs(), StateOut); dbgs() << "\n"; - dbgs() << " State 2: "; + dbgs() << " State 2: "; P.print(dbgs(), StateIn); dbgs() << ")\n"; }); @@ -329,7 +329,7 @@ class PacRetAnalysis StateOut.merge(StateIn); LLVM_DEBUG({ - dbgs() << " merged state: "; + dbgs() << " merged state: "; P.print(dbgs(), StateOut); dbgs() << "\n"; }); @@ -354,7 +354,7 @@ class PacRetAnalysis // Returns all registers that can be treated as if they are written by an // authentication instruction. SmallVector getRegsMadeSafeToDeref(const MCInst &Point, - const State &Cur) const { + const SrcState &Cur) const { SmallVector Regs; const MCPhysReg NoReg = BC.MIB->getNoRegister(); @@ -378,10 +378,10 @@ class PacRetAnalysis return Regs; } - State computeNext(const MCInst &Point, const State &Cur) { - PacStatePrinter P(BC); + SrcState computeNext(const MCInst &Point, const SrcState &Cur) { + SrcStatePrinter P(BC); LLVM_DEBUG({ - dbgs() << " PacRetAnalysis::ComputeNext("; + dbgs() << " SrcSafetyAnalysis::ComputeNext("; BC.InstPrinter->printInst(&const_cast(Point), 0, "", *BC.STI, dbgs()); dbgs() << ", "; @@ -395,7 +395,7 @@ class PacRetAnalysis if (Cur.empty()) { LLVM_DEBUG( { dbgs() << "Skipping computeNext(Point, Cur) as Cur is empty.\n"; }); - return State(); + return SrcState(); } // First, compute various properties of the instruction, taking the state @@ -406,7 +406,7 @@ class PacRetAnalysis getRegsMadeSafeToDeref(Point, Cur); // Then, compute the state after this instruction is executed. - State Next = Cur; + SrcState Next = Cur; Next.SafeToDerefRegs.reset(Clobbered); // Keep track of this instruction if it writes to any of the registers we @@ -430,7 +430,7 @@ class PacRetAnalysis } LLVM_DEBUG({ - dbgs() << " .. result: ("; + dbgs() << " .. result: ("; P.print(dbgs(), Next); dbgs() << ")\n"; }); @@ -438,7 +438,7 @@ class PacRetAnalysis return Next; } - StringRef getAnnotationName() const { return StringRef("PacRetAnalysis"); } + StringRef getAnnotationName() const { return StringRef("SrcSafetyAnalysis"); } public: std::vector @@ -448,8 +448,8 @@ class PacRetAnalysis return {}; auto MaybeState = getStateBefore(Inst); if (!MaybeState) - llvm_unreachable("Expected State to be present"); - const State &S = *MaybeState; + llvm_unreachable("Expected state to be present"); + const SrcState &S = *MaybeState; // Due to aliasing registers, multiple registers may have been tracked. std::set LastWritingInsts; for (MCPhysReg TrackedReg : UsedDirtyRegs) { @@ -468,7 +468,7 @@ class PacRetAnalysis static std::shared_ptr shouldReportReturnGadget(const BinaryContext &BC, const MCInstReference &Inst, - const State &S) { + const SrcState &S) { static const GadgetKind RetKind("non-protected ret found"); if (!BC.MIB->isReturn(Inst)) return nullptr; @@ -496,7 +496,7 @@ shouldReportReturnGadget(const BinaryContext &BC, const MCInstReference &Inst, static std::shared_ptr shouldReportCallGadget(const BinaryContext &BC, const MCInstReference &Inst, - const State &S) { + const SrcState &S) { static const GadgetKind CallKind("non-protected call found"); if (!BC.MIB->isIndirectCall(Inst) && !BC.MIB->isIndirectBranch(Inst)) return nullptr; @@ -524,10 +524,11 @@ Analysis::findGadgets(BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocatorId) { FunctionAnalysisResult Result; - PacRetAnalysis PRA(BF, AllocatorId, {}); + SrcSafetyAnalysis PRA(BF, AllocatorId, {}); + LLVM_DEBUG({ dbgs() << "Running src register safety analysis...\n"; }); PRA.run(); LLVM_DEBUG({ - dbgs() << " After PacRetAnalysis:\n"; + dbgs() << "After src register safety analysis:\n"; BF.dump(); }); @@ -535,7 +536,7 @@ Analysis::findGadgets(BinaryFunction &BF, for (BinaryBasicBlock &BB : BF) { for (int64_t I = 0, E = BB.size(); I < E; ++I) { MCInstReference Inst(&BB, I); - const State &S = *PRA.getStateBefore(Inst); + const SrcState &S = *PRA.getStateBefore(Inst); // If non-empty state was never propagated from the entry basic block // to Inst, assume it to be unreachable and report a warning. @@ -570,10 +571,12 @@ void Analysis::computeDetailedInfo(BinaryFunction &BF, std::vector RegsToTrackVec(RegsToTrack.begin(), RegsToTrack.end()); // Re-compute the analysis with register tracking. - PacRetAnalysis PRWIA(BF, AllocatorId, RegsToTrackVec); + SrcSafetyAnalysis PRWIA(BF, AllocatorId, RegsToTrackVec); + LLVM_DEBUG( + { dbgs() << "\nRunning detailed src register safety analysis...\n"; }); PRWIA.run(); LLVM_DEBUG({ - dbgs() << " After detailed PacRetAnalysis:\n"; + dbgs() << "After detailed src register safety analysis:\n"; BF.dump(); }); diff --git a/bolt/lib/Passes/ProfileQualityStats.cpp b/bolt/lib/Passes/ProfileQualityStats.cpp index 332c78da8a1e..dfd74d3dd571 100644 --- a/bolt/lib/Passes/ProfileQualityStats.cpp +++ b/bolt/lib/Passes/ProfileQualityStats.cpp @@ -52,6 +52,16 @@ struct FlowInfo { FunctionFlowMapTy CallGraphIncomingFlows; }; +// When reporting exception handling stats, we only consider functions with at +// least MinLPECSum counts in landing pads to avoid false positives due to +// sampling noise +const uint16_t MinLPECSum = 50; + +// When reporting CFG flow conservation stats, we only consider blocks with +// execution counts > MinBlockCount when reporting the distribution of worst +// gaps. +const uint16_t MinBlockCount = 500; + template void printDistribution(raw_ostream &OS, std::vector &values, bool Fraction = false) { @@ -91,8 +101,12 @@ void printCFGContinuityStats(raw_ostream &OS, std::vector FractionECUnreachables; for (const BinaryFunction *Function : Functions) { - if (Function->size() <= 1) + if (Function->size() <= 1) { + NumUnreachables.push_back(0); + SumECUnreachables.push_back(0); + FractionECUnreachables.push_back(0.0); continue; + } // Compute the sum of all BB execution counts (ECs). size_t NumPosECBBs = 0; @@ -142,8 +156,10 @@ void printCFGContinuityStats(raw_ostream &OS, const size_t NumPosECBBsUnreachableFromEntry = NumPosECBBs - NumReachableBBs; const size_t SumUnreachableBBEC = SumAllBBEC - SumReachableBBEC; - const double FractionECUnreachable = - (double)SumUnreachableBBEC / SumAllBBEC; + + double FractionECUnreachable = 0.0; + if (SumAllBBEC > 0) + FractionECUnreachable = (double)SumUnreachableBBEC / SumAllBBEC; if (opts::Verbosity >= 2 && FractionECUnreachable >= 0.05) { OS << "Non-trivial CFG discontinuity observed in function " @@ -157,9 +173,6 @@ void printCFGContinuityStats(raw_ostream &OS, FractionECUnreachables.push_back(FractionECUnreachable); } - if (FractionECUnreachables.empty()) - return; - llvm::sort(FractionECUnreachables); const int Rank = int(FractionECUnreachables.size() * opts::PercentileForProfileQualityCheck / 100); @@ -187,8 +200,10 @@ void printCallGraphFlowConservationStats( std::vector CallGraphGaps; for (const BinaryFunction *Function : Functions) { - if (Function->size() <= 1 || !Function->isSimple()) + if (Function->size() <= 1 || !Function->isSimple()) { + CallGraphGaps.push_back(0.0); continue; + } const uint64_t FunctionNum = Function->getFunctionNumber(); std::vector &IncomingFlows = @@ -199,60 +214,63 @@ void printCallGraphFlowConservationStats( TotalFlowMap.CallGraphIncomingFlows; // Only consider functions that are not a program entry. - if (CallGraphIncomingFlows.find(FunctionNum) != + if (CallGraphIncomingFlows.find(FunctionNum) == CallGraphIncomingFlows.end()) { - uint64_t EntryInflow = 0; - uint64_t EntryOutflow = 0; - uint32_t NumConsideredEntryBlocks = 0; - - Function->forEachEntryPoint([&](uint64_t Offset, const MCSymbol *Label) { - const BinaryBasicBlock *EntryBB = - Function->getBasicBlockAtOffset(Offset); - if (!EntryBB || EntryBB->succ_size() == 0) - return true; - NumConsideredEntryBlocks++; - EntryInflow += IncomingFlows[EntryBB->getLayoutIndex()]; - EntryOutflow += OutgoingFlows[EntryBB->getLayoutIndex()]; + CallGraphGaps.push_back(0.0); + continue; + } + + uint64_t EntryInflow = 0; + uint64_t EntryOutflow = 0; + uint32_t NumConsideredEntryBlocks = 0; + + Function->forEachEntryPoint([&](uint64_t Offset, const MCSymbol *Label) { + const BinaryBasicBlock *EntryBB = Function->getBasicBlockAtOffset(Offset); + if (!EntryBB || EntryBB->succ_size() == 0) return true; - }); - - uint64_t NetEntryOutflow = 0; - if (EntryOutflow < EntryInflow) { - if (opts::Verbosity >= 2) { - // We expect entry blocks' CFG outflow >= inflow, i.e., it has a - // non-negative net outflow. If this is not the case, then raise a - // warning if requested. - OS << "BOLT WARNING: unexpected entry block CFG outflow < inflow " - "in function " - << Function->getPrintName() << "\n"; - if (opts::Verbosity >= 3) - Function->dump(); - } - } else { - NetEntryOutflow = EntryOutflow - EntryInflow; - } - if (NumConsideredEntryBlocks > 0) { - const uint64_t CallGraphInflow = - TotalFlowMap.CallGraphIncomingFlows[Function->getFunctionNumber()]; - const uint64_t Min = std::min(NetEntryOutflow, CallGraphInflow); - const uint64_t Max = std::max(NetEntryOutflow, CallGraphInflow); - const double CallGraphGap = 1 - (double)Min / Max; - - if (opts::Verbosity >= 2 && CallGraphGap >= 0.5) { - OS << "Nontrivial call graph gap of size " - << formatv("{0:P}", CallGraphGap) << " observed in function " - << Function->getPrintName() << "\n"; - if (opts::Verbosity >= 3) - Function->dump(); - } + NumConsideredEntryBlocks++; + EntryInflow += IncomingFlows[EntryBB->getLayoutIndex()]; + EntryOutflow += OutgoingFlows[EntryBB->getLayoutIndex()]; + return true; + }); - CallGraphGaps.push_back(CallGraphGap); + uint64_t NetEntryOutflow = 0; + if (EntryOutflow < EntryInflow) { + if (opts::Verbosity >= 2) { + // We expect entry blocks' CFG outflow >= inflow, i.e., it has a + // non-negative net outflow. If this is not the case, then raise a + // warning if requested. + OS << "BOLT WARNING: unexpected entry block CFG outflow < inflow " + "in function " + << Function->getPrintName() << "\n"; + if (opts::Verbosity >= 3) + Function->dump(); } + } else { + NetEntryOutflow = EntryOutflow - EntryInflow; } - } + if (NumConsideredEntryBlocks > 0) { + const uint64_t CallGraphInflow = + TotalFlowMap.CallGraphIncomingFlows[Function->getFunctionNumber()]; + const uint64_t Min = std::min(NetEntryOutflow, CallGraphInflow); + const uint64_t Max = std::max(NetEntryOutflow, CallGraphInflow); + double CallGraphGap = 0.0; + if (Max > 0) + CallGraphGap = 1 - (double)Min / Max; + + if (opts::Verbosity >= 2 && CallGraphGap >= 0.5) { + OS << "Non-trivial call graph gap of size " + << formatv("{0:P}", CallGraphGap) << " observed in function " + << Function->getPrintName() << "\n"; + if (opts::Verbosity >= 3) + Function->dump(); + } - if (CallGraphGaps.empty()) - return; + CallGraphGaps.push_back(CallGraphGap); + } else { + CallGraphGaps.push_back(0.0); + } + } llvm::sort(CallGraphGaps); const int Rank = @@ -265,18 +283,19 @@ void printCallGraphFlowConservationStats( } } -void printCFGFlowConservationStats(raw_ostream &OS, +void printCFGFlowConservationStats(const BinaryContext &BC, raw_ostream &OS, iterator_range &Functions, FlowInfo &TotalFlowMap) { std::vector CFGGapsWeightedAvg; std::vector CFGGapsWorst; std::vector CFGGapsWorstAbs; - // We only consider blocks with execution counts > MinBlockCount when - // reporting the distribution of worst gaps. - const uint16_t MinBlockCount = 500; for (const BinaryFunction *Function : Functions) { - if (Function->size() <= 1 || !Function->isSimple()) + if (Function->size() <= 1 || !Function->isSimple()) { + CFGGapsWeightedAvg.push_back(0.0); + CFGGapsWorst.push_back(0.0); + CFGGapsWorstAbs.push_back(0); continue; + } const uint64_t FunctionNum = Function->getFunctionNumber(); std::vector &MaxCountMaps = @@ -295,12 +314,34 @@ void printCFGFlowConservationStats(raw_ostream &OS, if (BB.isEntryPoint() || BB.succ_size() == 0) continue; + if (BB.getKnownExecutionCount() == 0 || BB.getNumNonPseudos() == 0) + continue; + + // We don't consider blocks that is a landing pad or has a + // positive-execution-count landing pad + if (BB.isLandingPad()) + continue; + + if (llvm::any_of(BB.landing_pads(), + std::mem_fn(&BinaryBasicBlock::getKnownExecutionCount))) + continue; + + // We don't consider blocks that end with a recursive call instruction + const MCInst *Inst = BB.getLastNonPseudoInstr(); + if (BC.MIB->isCall(*Inst)) { + const MCSymbol *DstSym = BC.MIB->getTargetSymbol(*Inst); + const BinaryFunction *DstFunc = + DstSym ? BC.getFunctionForSymbol(DstSym) : nullptr; + if (DstFunc == Function) + continue; + } + const uint64_t Max = MaxCountMaps[BB.getLayoutIndex()]; const uint64_t Min = MinCountMaps[BB.getLayoutIndex()]; - const double Gap = 1 - (double)Min / Max; + double Gap = 0.0; + if (Max > 0) + Gap = 1 - (double)Min / Max; double Weight = BB.getKnownExecutionCount() * BB.getNumNonPseudos(); - if (Weight == 0) - continue; // We use log to prevent the stats from being dominated by extremely hot // blocks Weight = log(Weight); @@ -316,39 +357,36 @@ void printCFGFlowConservationStats(raw_ostream &OS, BBWorstGapAbs = &BB; } } - if (WeightSum > 0) { - const double WeightedGap = WeightedGapSum / WeightSum; - if (opts::Verbosity >= 2 && (WeightedGap >= 0.1 || WorstGap >= 0.9)) { - OS << "Nontrivial CFG gap observed in function " - << Function->getPrintName() << "\n" - << "Weighted gap: " << formatv("{0:P}", WeightedGap) << "\n"; - if (BBWorstGap) - OS << "Worst gap: " << formatv("{0:P}", WorstGap) - << " at BB with input offset: 0x" - << Twine::utohexstr(BBWorstGap->getInputOffset()) << "\n"; - if (BBWorstGapAbs) - OS << "Worst gap (absolute value): " << WorstGapAbs << " at BB with " - << "input offset 0x" - << Twine::utohexstr(BBWorstGapAbs->getInputOffset()) << "\n"; - if (opts::Verbosity >= 3) - Function->dump(); - } - - CFGGapsWeightedAvg.push_back(WeightedGap); - CFGGapsWorst.push_back(WorstGap); - CFGGapsWorstAbs.push_back(WorstGapAbs); + double WeightedGap = WeightedGapSum; + if (WeightSum > 0) + WeightedGap /= WeightSum; + if (opts::Verbosity >= 2 && WorstGap >= 0.9) { + OS << "Non-trivial CFG gap observed in function " + << Function->getPrintName() << "\n" + << "Weighted gap: " << formatv("{0:P}", WeightedGap) << "\n"; + if (BBWorstGap) + OS << "Worst gap: " << formatv("{0:P}", WorstGap) + << " at BB with input offset: 0x" + << Twine::utohexstr(BBWorstGap->getInputOffset()) << "\n"; + if (BBWorstGapAbs) + OS << "Worst gap (absolute value): " << WorstGapAbs << " at BB with " + << "input offset 0x" + << Twine::utohexstr(BBWorstGapAbs->getInputOffset()) << "\n"; + if (opts::Verbosity >= 3) + Function->dump(); } + CFGGapsWeightedAvg.push_back(WeightedGap); + CFGGapsWorst.push_back(WorstGap); + CFGGapsWorstAbs.push_back(WorstGapAbs); } - if (CFGGapsWeightedAvg.empty()) - return; llvm::sort(CFGGapsWeightedAvg); const int RankWA = int(CFGGapsWeightedAvg.size() * opts::PercentileForProfileQualityCheck / 100); llvm::sort(CFGGapsWorst); const int RankW = int(CFGGapsWorst.size() * opts::PercentileForProfileQualityCheck / 100); - OS << formatv("CFG flow conservation gap {0:P} (weighted) {1:P} (worst)\n", + OS << formatv("CFG flow conservation gap {0:P} (weighted) {1:P} (worst); ", CFGGapsWeightedAvg[RankWA], CFGGapsWorst[RankW]); if (opts::Verbosity >= 1) { OS << "distribution of weighted CFG flow conservation gaps\n"; @@ -365,6 +403,74 @@ void printCFGFlowConservationStats(raw_ostream &OS, } } +void printExceptionHandlingStats(const BinaryContext &BC, raw_ostream &OS, + iterator_range &Functions) { + std::vector LPCountFractionsOfTotalBBEC; + std::vector LPCountFractionsOfTotalInvokeEC; + for (const BinaryFunction *Function : Functions) { + size_t LPECSum = 0; + size_t BBECSum = 0; + size_t InvokeECSum = 0; + for (BinaryBasicBlock &BB : *Function) { + const size_t BBEC = BB.getKnownExecutionCount(); + BBECSum += BBEC; + if (BB.isLandingPad()) + LPECSum += BBEC; + for (const MCInst &Inst : BB) { + if (!BC.MIB->isInvoke(Inst)) + continue; + const std::optional EHInfo = + BC.MIB->getEHInfo(Inst); + if (EHInfo->first) + InvokeECSum += BBEC; + } + } + + if (LPECSum <= MinLPECSum) { + LPCountFractionsOfTotalBBEC.push_back(0.0); + LPCountFractionsOfTotalInvokeEC.push_back(0.0); + continue; + } + double FracTotalBBEC = 0.0; + if (BBECSum > 0) + FracTotalBBEC = (double)LPECSum / BBECSum; + double FracTotalInvokeEC = 0.0; + if (InvokeECSum > 0) + FracTotalInvokeEC = (double)LPECSum / InvokeECSum; + LPCountFractionsOfTotalBBEC.push_back(FracTotalBBEC); + LPCountFractionsOfTotalInvokeEC.push_back(FracTotalInvokeEC); + + if (opts::Verbosity >= 2 && FracTotalInvokeEC >= 0.05) { + OS << "Non-trivial usage of exception handling observed in function " + << Function->getPrintName() << "\n" + << formatv( + "Fraction of total InvokeEC that goes to landing pads: {0:P}\n", + FracTotalInvokeEC); + if (opts::Verbosity >= 3) + Function->dump(); + } + } + + llvm::sort(LPCountFractionsOfTotalBBEC); + const int RankBBEC = int(LPCountFractionsOfTotalBBEC.size() * + opts::PercentileForProfileQualityCheck / 100); + llvm::sort(LPCountFractionsOfTotalInvokeEC); + const int RankInvoke = int(LPCountFractionsOfTotalInvokeEC.size() * + opts::PercentileForProfileQualityCheck / 100); + OS << formatv("exception handling usage {0:P} (of total BBEC) {1:P} (of " + "total InvokeEC)\n", + LPCountFractionsOfTotalBBEC[RankBBEC], + LPCountFractionsOfTotalInvokeEC[RankInvoke]); + if (opts::Verbosity >= 1) { + OS << "distribution of exception handling usage as a fraction of total " + "BBEC of each function\n"; + printDistribution(OS, LPCountFractionsOfTotalBBEC, /*Fraction=*/true); + OS << "distribution of exception handling usage as a fraction of total " + "InvokeEC of each function\n"; + printDistribution(OS, LPCountFractionsOfTotalInvokeEC, /*Fraction=*/true); + } +} + void computeFlowMappings(const BinaryContext &BC, FlowInfo &TotalFlowMap) { // Increment block inflow and outflow with CFG jump counts. TotalFlowMapTy &TotalIncomingFlows = TotalFlowMap.TotalIncomingFlows; @@ -519,8 +625,8 @@ void printAll(BinaryContext &BC, FunctionListType &ValidFunctions, 100 - opts::PercentileForProfileQualityCheck); printCFGContinuityStats(BC.outs(), Functions); printCallGraphFlowConservationStats(BC.outs(), Functions, TotalFlowMap); - printCFGFlowConservationStats(BC.outs(), Functions, TotalFlowMap); - + printCFGFlowConservationStats(BC, BC.outs(), Functions, TotalFlowMap); + printExceptionHandlingStats(BC, BC.outs(), Functions); // Print more detailed bucketed stats if requested. if (opts::Verbosity >= 1 && RealNumTopFunctions >= 5) { const size_t PerBucketSize = RealNumTopFunctions / 5; @@ -550,7 +656,8 @@ void printAll(BinaryContext &BC, FunctionListType &ValidFunctions, MaxFunctionExecutionCount); printCFGContinuityStats(BC.outs(), Functions); printCallGraphFlowConservationStats(BC.outs(), Functions, TotalFlowMap); - printCFGFlowConservationStats(BC.outs(), Functions, TotalFlowMap); + printCFGFlowConservationStats(BC, BC.outs(), Functions, TotalFlowMap); + printExceptionHandlingStats(BC, BC.outs(), Functions); } } } diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp index d20626bd5062..a8a187974418 100644 --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -634,7 +634,7 @@ bool DataAggregator::doSample(BinaryFunction &OrigFunc, uint64_t Address, uint64_t Count) { BinaryFunction *ParentFunc = getBATParentFunction(OrigFunc); BinaryFunction &Func = ParentFunc ? *ParentFunc : OrigFunc; - if (ParentFunc) + if (ParentFunc || (BAT && !BAT->isBATFunction(OrigFunc.getAddress()))) NumColdSamples += Count; auto I = NamesToSamples.find(Func.getOneName()); @@ -756,12 +756,13 @@ bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count, Addr = BAT->translate(Func->getAddress(), Addr, IsFrom); BinaryFunction *ParentFunc = getBATParentFunction(*Func); + if (IsFrom && + (ParentFunc || (BAT && !BAT->isBATFunction(Func->getAddress())))) + NumColdSamples += Count; + if (!ParentFunc) return std::pair{Func, IsRetOrCallCont}; - if (IsFrom) - NumColdSamples += Count; - return std::pair{ParentFunc, IsRetOrCallCont}; }; @@ -871,11 +872,6 @@ DataAggregator::getFallthroughsInTrace(BinaryFunction &BF, BinaryContext &BC = BF.getBinaryContext(); - if (!BF.isSimple()) - return std::nullopt; - - assert(BF.hasCFG() && "can only record traces in CFG state"); - // Offsets of the trace within this function. const uint64_t From = FirstLBR.To - BF.getAddress(); const uint64_t To = SecondLBR.From - BF.getAddress(); @@ -883,6 +879,20 @@ DataAggregator::getFallthroughsInTrace(BinaryFunction &BF, if (From > To) return std::nullopt; + // Accept fall-throughs inside pseudo functions (PLT/thunks). + // This check has to be above BF.empty as pseudo functions would pass it: + // pseudo => ignored => CFG not built => empty. + // If we return nullopt, trace would be reported as mismatching disassembled + // function contents which it is not. To avoid this, return an empty + // fall-through list instead. + if (BF.isPseudo()) + return Branches; + + if (!BF.isSimple()) + return std::nullopt; + + assert(BF.hasCFG() && "can only record traces in CFG state"); + const BinaryBasicBlock *FromBB = BF.getBasicBlockContainingOffset(From); const BinaryBasicBlock *ToBB = BF.getBasicBlockContainingOffset(To); @@ -1413,9 +1423,8 @@ std::error_code DataAggregator::printLBRHeatMap() { return std::error_code(); } -uint64_t DataAggregator::parseLBRSample(const PerfBranchSample &Sample, - bool NeedsSkylakeFix) { - uint64_t NumTraces{0}; +void DataAggregator::parseLBRSample(const PerfBranchSample &Sample, + bool NeedsSkylakeFix) { // LBRs are stored in reverse execution order. NextLBR refers to the next // executed branch record. const LBREntry *NextLBR = nullptr; @@ -1478,7 +1487,83 @@ uint64_t DataAggregator::parseLBRSample(const PerfBranchSample &Sample, ++Info.TakenCount; Info.MispredCount += LBR.Mispred; } - return NumTraces; +} + +void DataAggregator::printColdSamplesDiagnostic() const { + if (NumColdSamples > 0) { + const float ColdSamples = NumColdSamples * 100.0f / NumTotalSamples; + outs() << "PERF2BOLT: " << NumColdSamples + << format(" (%.1f%%)", ColdSamples) + << " samples recorded in cold regions of split functions.\n"; + if (ColdSamples > 5.0f) + outs() + << "WARNING: The BOLT-processed binary where samples were collected " + "likely used bad data or your service observed a large shift in " + "profile. You may want to audit this\n"; + } +} + +void DataAggregator::printLongRangeTracesDiagnostic() const { + outs() << "PERF2BOLT: out of range traces involving unknown regions: " + << NumLongRangeTraces; + if (NumTraces > 0) + outs() << format(" (%.1f%%)", NumLongRangeTraces * 100.0f / NumTraces); + outs() << "\n"; +} + +static float printColoredPct(uint64_t Numerator, uint64_t Denominator, float T1, + float T2) { + if (Denominator == 0) { + outs() << "\n"; + return 0; + } + float Percent = Numerator * 100.0f / Denominator; + outs() << " ("; + if (outs().has_colors()) { + if (Percent > T2) + outs().changeColor(raw_ostream::RED); + else if (Percent > T1) + outs().changeColor(raw_ostream::YELLOW); + else + outs().changeColor(raw_ostream::GREEN); + } + outs() << format("%.1f%%", Percent); + if (outs().has_colors()) + outs().resetColor(); + outs() << ")\n"; + return Percent; +} + +void DataAggregator::printBranchSamplesDiagnostics() const { + outs() << "PERF2BOLT: traces mismatching disassembled function contents: " + << NumInvalidTraces; + if (printColoredPct(NumInvalidTraces, NumTraces, 5, 10) > 10) + outs() << "\n !! WARNING !! This high mismatch ratio indicates the input " + "binary is probably not the same binary used during profiling " + "collection. The generated data may be ineffective for improving " + "performance\n\n"; + printLongRangeTracesDiagnostic(); + printColdSamplesDiagnostic(); +} + +void DataAggregator::printBasicSamplesDiagnostics( + uint64_t OutOfRangeSamples) const { + outs() << "PERF2BOLT: out of range samples recorded in unknown regions: " + << OutOfRangeSamples; + if (printColoredPct(OutOfRangeSamples, NumTotalSamples, 40, 60) > 80) + outs() << "\n !! WARNING !! This high mismatch ratio indicates the input " + "binary is probably not the same binary used during profiling " + "collection. The generated data may be ineffective for improving " + "performance\n\n"; + printColdSamplesDiagnostic(); +} + +void DataAggregator::printBranchStacksDiagnostics( + uint64_t IgnoredSamples) const { + outs() << "PERF2BOLT: ignored samples: " << IgnoredSamples; + if (printColoredPct(IgnoredSamples, NumTotalSamples, 20, 50) > 50) + errs() << "PERF2BOLT-WARNING: less than 50% of all recorded samples " + "were attributed to the input binary\n"; } std::error_code DataAggregator::parseBranchEvents() { @@ -1486,11 +1571,9 @@ std::error_code DataAggregator::parseBranchEvents() { NamedRegionTimer T("parseBranch", "Parsing branch events", TimerGroupName, TimerGroupDesc, opts::TimeAggregator); - uint64_t NumTotalSamples = 0; uint64_t NumEntries = 0; uint64_t NumSamples = 0; uint64_t NumSamplesNoLBR = 0; - uint64_t NumTraces = 0; bool NeedsSkylakeFix = false; while (hasData() && NumTotalSamples < opts::MaxSamples) { @@ -1517,7 +1600,7 @@ std::error_code DataAggregator::parseBranchEvents() { NeedsSkylakeFix = true; } - NumTraces += parseLBRSample(Sample, NeedsSkylakeFix); + parseLBRSample(Sample, NeedsSkylakeFix); } for (const Trace &Trace : llvm::make_first_range(BranchLBRs)) @@ -1525,22 +1608,6 @@ std::error_code DataAggregator::parseBranchEvents() { if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Addr)) BF->setHasProfileAvailable(); - auto printColored = [](raw_ostream &OS, float Percent, float T1, float T2) { - OS << " ("; - if (OS.has_colors()) { - if (Percent > T2) - OS.changeColor(raw_ostream::RED); - else if (Percent > T1) - OS.changeColor(raw_ostream::YELLOW); - else - OS.changeColor(raw_ostream::GREEN); - } - OS << format("%.1f%%", Percent); - if (OS.has_colors()) - OS.resetColor(); - OS << ")"; - }; - outs() << "PERF2BOLT: read " << NumSamples << " samples and " << NumEntries << " LBR entries\n"; if (NumTotalSamples) { @@ -1552,47 +1619,10 @@ std::error_code DataAggregator::parseBranchEvents() { "in no-LBR mode with -nl (the performance improvement in -nl " "mode may be limited)\n"; } else { - const uint64_t IgnoredSamples = NumTotalSamples - NumSamples; - const float PercentIgnored = 100.0f * IgnoredSamples / NumTotalSamples; - outs() << "PERF2BOLT: " << IgnoredSamples << " samples"; - printColored(outs(), PercentIgnored, 20, 50); - outs() << " were ignored\n"; - if (PercentIgnored > 50.0f) - errs() << "PERF2BOLT-WARNING: less than 50% of all recorded samples " - "were attributed to the input binary\n"; + printBranchStacksDiagnostics(NumTotalSamples - NumSamples); } } - outs() << "PERF2BOLT: traces mismatching disassembled function contents: " - << NumInvalidTraces; - float Perc = 0.0f; - if (NumTraces > 0) { - Perc = NumInvalidTraces * 100.0f / NumTraces; - printColored(outs(), Perc, 5, 10); - } - outs() << "\n"; - if (Perc > 10.0f) - outs() << "\n !! WARNING !! This high mismatch ratio indicates the input " - "binary is probably not the same binary used during profiling " - "collection. The generated data may be ineffective for improving " - "performance.\n\n"; - - outs() << "PERF2BOLT: out of range traces involving unknown regions: " - << NumLongRangeTraces; - if (NumTraces > 0) - outs() << format(" (%.1f%%)", NumLongRangeTraces * 100.0f / NumTraces); - outs() << "\n"; - - if (NumColdSamples > 0) { - const float ColdSamples = NumColdSamples * 100.0f / NumTotalSamples; - outs() << "PERF2BOLT: " << NumColdSamples - << format(" (%.1f%%)", ColdSamples) - << " samples recorded in cold regions of split functions.\n"; - if (ColdSamples > 5.0f) - outs() - << "WARNING: The BOLT-processed binary where samples were collected " - "likely used bad data or your service observed a large shift in " - "profile. You may want to audit this.\n"; - } + printBranchSamplesDiagnostics(); return std::error_code(); } @@ -1649,11 +1679,10 @@ void DataAggregator::processBasicEvents() { NamedRegionTimer T("processBasic", "Processing basic events", TimerGroupName, TimerGroupDesc, opts::TimeAggregator); uint64_t OutOfRangeSamples = 0; - uint64_t NumSamples = 0; for (auto &Sample : BasicSamples) { const uint64_t PC = Sample.first; const uint64_t HitCount = Sample.second; - NumSamples += HitCount; + NumTotalSamples += HitCount; BinaryFunction *Func = getBinaryFunctionContainingAddress(PC); if (!Func) { OutOfRangeSamples += HitCount; @@ -1662,33 +1691,9 @@ void DataAggregator::processBasicEvents() { doSample(*Func, PC, HitCount); } - outs() << "PERF2BOLT: read " << NumSamples << " samples\n"; + outs() << "PERF2BOLT: read " << NumTotalSamples << " samples\n"; - outs() << "PERF2BOLT: out of range samples recorded in unknown regions: " - << OutOfRangeSamples; - float Perc = 0.0f; - if (NumSamples > 0) { - outs() << " ("; - Perc = OutOfRangeSamples * 100.0f / NumSamples; - if (outs().has_colors()) { - if (Perc > 60.0f) - outs().changeColor(raw_ostream::RED); - else if (Perc > 40.0f) - outs().changeColor(raw_ostream::YELLOW); - else - outs().changeColor(raw_ostream::GREEN); - } - outs() << format("%.1f%%", Perc); - if (outs().has_colors()) - outs().resetColor(); - outs() << ")"; - } - outs() << "\n"; - if (Perc > 80.0f) - outs() << "\n !! WARNING !! This high mismatch ratio indicates the input " - "binary is probably not the same binary used during profiling " - "collection. The generated data may be ineffective for improving " - "performance.\n\n"; + printBasicSamplesDiagnostics(OutOfRangeSamples); } std::error_code DataAggregator::parseMemEvents() { @@ -1766,13 +1771,13 @@ void DataAggregator::processPreAggregated() { NamedRegionTimer T("processAggregated", "Processing aggregated branch events", TimerGroupName, TimerGroupDesc, opts::TimeAggregator); - uint64_t NumTraces = 0; for (const AggregatedLBREntry &AggrEntry : AggregatedLBRs) { switch (AggrEntry.EntryType) { case AggregatedLBREntry::BRANCH: case AggregatedLBREntry::TRACE: doBranch(AggrEntry.From.Offset, AggrEntry.To.Offset, AggrEntry.Count, AggrEntry.Mispreds); + NumTotalSamples += AggrEntry.Count; break; case AggregatedLBREntry::FT: case AggregatedLBREntry::FT_EXTERNAL_ORIGIN: { @@ -1790,37 +1795,7 @@ void DataAggregator::processPreAggregated() { outs() << "PERF2BOLT: read " << AggregatedLBRs.size() << " aggregated LBR entries\n"; - outs() << "PERF2BOLT: traces mismatching disassembled function contents: " - << NumInvalidTraces; - float Perc = 0.0f; - if (NumTraces > 0) { - outs() << " ("; - Perc = NumInvalidTraces * 100.0f / NumTraces; - if (outs().has_colors()) { - if (Perc > 10.0f) - outs().changeColor(raw_ostream::RED); - else if (Perc > 5.0f) - outs().changeColor(raw_ostream::YELLOW); - else - outs().changeColor(raw_ostream::GREEN); - } - outs() << format("%.1f%%", Perc); - if (outs().has_colors()) - outs().resetColor(); - outs() << ")"; - } - outs() << "\n"; - if (Perc > 10.0f) - outs() << "\n !! WARNING !! This high mismatch ratio indicates the input " - "binary is probably not the same binary used during profiling " - "collection. The generated data may be ineffective for improving " - "performance.\n\n"; - - outs() << "PERF2BOLT: Out of range traces involving unknown regions: " - << NumLongRangeTraces; - if (NumTraces > 0) - outs() << format(" (%.1f%%)", NumLongRangeTraces * 100.0f / NumTraces); - outs() << "\n"; + printBranchSamplesDiagnostics(); } std::optional DataAggregator::parseCommExecEvent() { diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp index e39485816356..2bdc7b65b480 100644 --- a/bolt/lib/Profile/YAMLProfileWriter.cpp +++ b/bolt/lib/Profile/YAMLProfileWriter.cpp @@ -133,12 +133,9 @@ std::vector YAMLProfileWriter::convertNodeProbes(NodeIdToProbes &NodeProbes) { struct BlockProbeInfoHasher { size_t operator()(const yaml::bolt::PseudoProbeInfo &BPI) const { - auto HashCombine = [](auto &Range) { - return llvm::hash_combine_range(Range.begin(), Range.end()); - }; - return llvm::hash_combine(HashCombine(BPI.BlockProbes), - HashCombine(BPI.CallProbes), - HashCombine(BPI.IndCallProbes)); + return llvm::hash_combine(llvm::hash_combine_range(BPI.BlockProbes), + llvm::hash_combine_range(BPI.CallProbes), + llvm::hash_combine_range(BPI.IndCallProbes)); } }; diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 23faa92642d0..69fb736d7bde 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -237,6 +237,12 @@ UseGnuStack("use-gnu-stack", cl::ZeroOrMore, cl::cat(BoltCategory)); +static cl::opt CustomAllocationVMA( + "custom-allocation-vma", + cl::desc("use a custom address at which new code will be put, " + "bypassing BOLT's logic to detect where to put code"), + cl::Hidden, cl::cat(BoltCategory)); + static cl::opt SequentialDisassembly("sequential-disassembly", cl::desc("performs disassembly sequentially"), @@ -592,13 +598,33 @@ Error RewriteInstance::discoverStorage() { FirstNonAllocatableOffset = NextAvailableOffset; + if (opts::CustomAllocationVMA) { + // If user specified a custom address where we should start writing new + // data, honor that. + NextAvailableAddress = opts::CustomAllocationVMA; + // Sanity check the user-supplied address and emit warnings if something + // seems off. + for (const ELF64LE::Phdr &Phdr : PHs) { + switch (Phdr.p_type) { + case ELF::PT_LOAD: + if (NextAvailableAddress >= Phdr.p_vaddr && + NextAvailableAddress < Phdr.p_vaddr + Phdr.p_memsz) { + BC->errs() << "BOLT-WARNING: user-supplied allocation vma 0x" + << Twine::utohexstr(NextAvailableAddress) + << " conflicts with ELF segment at 0x" + << Twine::utohexstr(Phdr.p_vaddr) << "\n"; + } + } + } + } NextAvailableAddress = alignTo(NextAvailableAddress, BC->PageAlign); NextAvailableOffset = alignTo(NextAvailableOffset, BC->PageAlign); // Hugify: Additional huge page from left side due to // weird ASLR mapping addresses (4KB aligned) - if (opts::Hugify && !BC->HasFixedLoadAddress) + if (opts::Hugify && !BC->HasFixedLoadAddress) { NextAvailableAddress += BC->PageAlign; + } if (!opts::UseGnuStack && !BC->IsLinuxKernel) { // This is where the black magic happens. Creating PHDR table in a segment @@ -1054,10 +1080,11 @@ void RewriteInstance::discoverFileObjects() { continue; } - if (!Section->isText()) { + if (!Section->isText() || Section->isVirtual()) { assert(SymbolType != SymbolRef::ST_Function && "unexpected function inside non-code section"); - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code\n"); + LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code or " + "is in nobits section\n"); registerName(SymbolSize); continue; } @@ -2603,7 +2630,9 @@ void RewriteInstance::readRelocations(const SectionRef &Section) { void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection, const RelocationRef &Rel) { const bool IsAArch64 = BC->isAArch64(); + const bool IsX86 = BC->isX86(); const bool IsFromCode = RelocatedSection.isText(); + const bool IsWritable = BinarySection(*BC, RelocatedSection).isWritable(); SmallString<16> TypeName; Rel.getTypeName(TypeName); @@ -2612,7 +2641,7 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection, return; // Adjust the relocation type as the linker might have skewed it. - if (BC->isX86() && (RType & ELF::R_X86_64_converted_reloc_bit)) { + if (IsX86 && (RType & ELF::R_X86_64_converted_reloc_bit)) { if (opts::Verbosity >= 1) dbgs() << "BOLT-WARNING: ignoring R_X86_64_converted_reloc_bit\n"; RType &= ~ELF::R_X86_64_converted_reloc_bit; @@ -2620,7 +2649,7 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection, if (Relocation::isTLS(RType)) { // No special handling required for TLS relocations on X86. - if (BC->isX86()) + if (IsX86) return; // The non-got related TLS relocations on AArch64 and RISC-V also could be @@ -2661,6 +2690,30 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection, return; } + if (!IsFromCode && !IsWritable && (IsX86 || IsAArch64) && + Relocation::isPCRelative(RType)) { + BinaryData *BD = BC->getBinaryDataContainingAddress(Rel.getOffset()); + if (BD && (BD->nameStartsWith("_ZTV") || // vtable + BD->nameStartsWith("_ZTCN"))) { // construction vtable + BinaryFunction *BF = BC->getBinaryFunctionContainingAddress( + SymbolAddress, /*CheckPastEnd*/ false, /*UseMaxSize*/ true); + if (!BF || BF->getAddress() != SymbolAddress) { + BC->errs() + << "BOLT-ERROR: the virtual function table entry at offset 0x" + << Twine::utohexstr(Rel.getOffset()); + if (BF) + BC->errs() << " points to the middle of a function @ 0x" + << Twine::utohexstr(BF->getAddress()) << "\n"; + else + BC->errs() << " does not point to any function\n"; + exit(1); + } + BC->addRelocation(Rel.getOffset(), BF->getSymbol(), RType, Addend, + ExtractedValue); + return; + } + } + const uint64_t Address = SymbolAddress + Addend; LLVM_DEBUG({ @@ -2724,7 +2777,7 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection, const bool IsToCode = ReferencedSection && ReferencedSection->isText(); // Special handling of PC-relative relocations. - if (BC->isX86() && Relocation::isPCRelative(RType)) { + if (IsX86 && Relocation::isPCRelative(RType)) { if (!IsFromCode && IsToCode) { // PC-relative relocations from data to code are tricky since the // original information is typically lost after linking, even with @@ -2899,12 +2952,12 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection, if (BinaryData *BD = BC->getBinaryDataContainingAddress(SymbolAddress)) { // Note: this assertion is trying to check sanity of BinaryData objects - // but AArch64 has inferred and incomplete object locations coming from - // GOT/TLS or any other non-trivial relocation (that requires creation - // of sections and whose symbol address is not really what should be - // encoded in the instruction). So we essentially disabled this check + // but AArch64 and RISCV has inferred and incomplete object locations + // coming from GOT/TLS or any other non-trivial relocation (that requires + // creation of sections and whose symbol address is not really what should + // be encoded in the instruction). So we essentially disabled this check // for AArch64 and live with bogus names for objects. - assert((IsAArch64 || IsSectionRelocation || + assert((IsAArch64 || BC->isRISCV() || IsSectionRelocation || BD->nameStartsWith(SymbolName) || BD->nameStartsWith("PG" + SymbolName) || (BD->nameStartsWith("ANONYMOUS") && @@ -5859,17 +5912,28 @@ void RewriteInstance::rewriteFile() { // Write all allocatable sections - reloc-mode text is written here as well for (BinarySection &Section : BC->allocatableSections()) { - if (!Section.isFinalized() || !Section.getOutputData()) + if (!Section.isFinalized() || !Section.getOutputData()) { + LLVM_DEBUG(if (opts::Verbosity > 1) { + dbgs() << "BOLT-INFO: new section is finalized or !getOutputData, skip " + << Section.getName() << '\n'; + }); continue; - if (Section.isLinkOnly()) + } + if (Section.isLinkOnly()) { + LLVM_DEBUG(if (opts::Verbosity > 1) { + dbgs() << "BOLT-INFO: new section is link only, skip " + << Section.getName() << '\n'; + }); continue; + } if (opts::Verbosity >= 1) BC->outs() << "BOLT: writing new section " << Section.getName() << "\n data at 0x" << Twine::utohexstr(Section.getAllocAddress()) << "\n of size " << Section.getOutputSize() << "\n at offset " - << Section.getOutputFileOffset() << '\n'; + << Section.getOutputFileOffset() << " with content size " + << Section.getOutputContents().size() << '\n'; OS.seek(Section.getOutputFileOffset()); Section.write(OS); } diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp index 106f0a880d78..e00d6a18b0f6 100644 --- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp +++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -2336,7 +2336,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder { std::optional createRelocation(const MCFixup &Fixup, const MCAsmBackend &MAB) const override { - const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind()); + MCFixupKindInfo FKI = MAB.getFixupKindInfo(Fixup.getKind()); assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected"); const uint64_t RelOffset = Fixup.getOffset(); diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp index 4320c679acd5..391c1866c810 100644 --- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp +++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp @@ -14,7 +14,9 @@ #include "MCTargetDesc/RISCVMCTargetDesc.h" #include "bolt/Core/MCPlusBuilder.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCContext.h" #include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/Support/ErrorHandling.h" @@ -72,6 +74,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder { case ELF::R_RISCV_LO12_I: case ELF::R_RISCV_LO12_S: case ELF::R_RISCV_TLS_GOT_HI20: + case ELF::R_RISCV_TLS_GD_HI20: return true; default: llvm_unreachable("Unexpected RISCV relocation type in code"); @@ -252,6 +255,11 @@ class RISCVMCPlusBuilder : public MCPlusBuilder { return createCall(RISCV::PseudoCALL, Inst, Target, Ctx); } + void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, + MCContext *Ctx) override { + createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true); + } + void createTailCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx) override { return createCall(RISCV::PseudoTAIL, Inst, Target, Ctx); @@ -424,6 +432,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder { return Expr; case ELF::R_RISCV_GOT_HI20: case ELF::R_RISCV_TLS_GOT_HI20: + case ELF::R_RISCV_TLS_GD_HI20: // The GOT is reused so no need to create GOT relocations case ELF::R_RISCV_PCREL_HI20: return RISCVMCExpr::create(Expr, RISCVMCExpr::VK_PCREL_HI, Ctx); @@ -483,6 +492,374 @@ class RISCVMCPlusBuilder : public MCPlusBuilder { return 2; return 4; } + + void createStackPointerIncrement( + MCInst &Inst, int imm, + bool NoFlagsClobber = false /*unused for RISCV*/) const override { + Inst = MCInstBuilder(RISCV::ADDI) + .addReg(RISCV::X2) + .addReg(RISCV::X2) + .addImm(-imm); + } + + void createStackPointerDecrement( + MCInst &Inst, int imm, + bool NoFlagsClobber = false /*unused for RISCV*/) const override { + Inst = MCInstBuilder(RISCV::ADDI) + .addReg(RISCV::X2) + .addReg(RISCV::X2) + .addImm(imm); + } + + void loadReg(MCInst &Inst, MCPhysReg To, MCPhysReg From, + int64_t offset) const { + Inst = MCInstBuilder(RISCV::LD).addReg(To).addReg(From).addImm(offset); + } + + void storeReg(MCInst &Inst, MCPhysReg From, MCPhysReg To, + int64_t offset) const { + Inst = MCInstBuilder(RISCV::SD).addReg(From).addReg(To).addImm(offset); + } + + void spillRegs(InstructionListType &Insts, + const SmallVector &Regs) const { + Insts.emplace_back(); + createStackPointerIncrement(Insts.back(), Regs.size() * 8); + + int64_t Offset = 0; + for (auto Reg : Regs) { + Insts.emplace_back(); + storeReg(Insts.back(), Reg, RISCV::X2, Offset); + Offset += 8; + } + } + + void reloadRegs(InstructionListType &Insts, + const SmallVector &Regs) const { + int64_t Offset = 0; + for (auto Reg : Regs) { + Insts.emplace_back(); + loadReg(Insts.back(), Reg, RISCV::X2, Offset); + Offset += 8; + } + + Insts.emplace_back(); + createStackPointerDecrement(Insts.back(), Regs.size() * 8); + } + + void atomicAdd(MCInst &Inst, MCPhysReg RegAtomic, MCPhysReg RegTo, + MCPhysReg RegCnt) const { + Inst = MCInstBuilder(RISCV::AMOADD_D) + .addReg(RegAtomic) + .addReg(RegTo) + .addReg(RegCnt); + } + + InstructionListType createRegCmpJE(MCPhysReg RegNo, MCPhysReg RegTmp, + const MCSymbol *Target, + MCContext *Ctx) const { + InstructionListType Insts; + Insts.emplace_back( + MCInstBuilder(RISCV::SUB).addReg(RegTmp).addReg(RegNo).addReg(RegNo)); + Insts.emplace_back(MCInstBuilder(RISCV::BEQ) + .addReg(RegNo) + .addReg(RegTmp) + .addExpr(MCSymbolRefExpr::create( + Target, MCSymbolRefExpr::VK_None, *Ctx))); + return Insts; + } + + void createTrap(MCInst &Inst) const override { + Inst.clear(); + Inst.setOpcode(RISCV::EBREAK); + } + + void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, + MCContext *Ctx, bool IsTailCall) override { + // The sequence of instructions we create here is the following: + // auipc a5, hi20(Target) + // addi a5, a5, low12(Target) + // jr x5 => jalr x0, x5, 0 + MCPhysReg Reg = RISCV::X5; + InstructionListType Insts = materializeAddress(Target, Ctx, Reg); + Insts.emplace_back(); + MCInst &Inst = Insts.back(); + Inst.clear(); + Inst = MCInstBuilder(RISCV::JALR).addReg(RISCV::X0).addReg(Reg).addImm(0); + if (IsTailCall) + setTailCall(Inst); + Seq.swap(Insts); + } + + InstructionListType createGetter(MCContext *Ctx, const char *name) const { + InstructionListType Insts(4); + MCSymbol *Locs = Ctx->getOrCreateSymbol(name); + InstructionListType Addr = materializeAddress(Locs, Ctx, RISCV::X10); + std::copy(Addr.begin(), Addr.end(), Insts.begin()); + loadReg(Insts[2], RISCV::X10, RISCV::X10, 0); + createReturn(Insts[3]); + return Insts; + } + + InstructionListType createIncMemory(MCPhysReg RegTo, MCPhysReg RegCnt, + MCPhysReg RegAtomic) const { + InstructionListType Insts; + Insts.emplace_back(); + Insts.back() = + MCInstBuilder(RISCV::ADDI).addReg(RegCnt).addReg(RegAtomic).addImm(1); + Insts.emplace_back(); + atomicAdd(Insts.back(), RegAtomic, RegTo, RegCnt); + return Insts; + } + + InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx, + MCPhysReg RegName, + int64_t Addend = 0) const override { + // Get the symbol address by auipc + addi + InstructionListType Insts(2); + MCSymbol *AuipcLabel = Ctx->createNamedTempSymbol("pcrel_hi"); + Insts[0] = MCInstBuilder(RISCV::AUIPC).addReg(RegName).addImm(0); + setOperandToSymbolRef(Insts[0], /* OpNum */ 1, Target, Addend, Ctx, + ELF::R_RISCV_PCREL_HI20); + setInstLabel(Insts[0], AuipcLabel); + + Insts[1] = + MCInstBuilder(RISCV::ADDI).addReg(RegName).addReg(RegName).addImm(0); + setOperandToSymbolRef(Insts[1], /* OpNum */ 2, AuipcLabel, Addend, Ctx, + ELF::R_RISCV_PCREL_LO12_I); + return Insts; + } + + InstructionListType + createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf, + unsigned CodePointerSize) const override { + // We need 2 scratch registers: one for the target address (x10), and one + // for the increment value (x11). + // addi sp, sp, -16 + // sd x10, 0(sp) + // sd x11, 8(sp) + // la x10, target # 1: auipc x10, %pcrel_hi(target) + // # addi x10, x10, %pcrel_lo(1b) + // li x11, 1 # addi x11, zero, 1 + // amoadd.d zero, x10, x11 + // ld x10, 0(sp) + // ld x11, 8(sp) + // addi sp, sp, 16 + + InstructionListType Insts; + spillRegs(Insts, {RISCV::X10, RISCV::X11}); + InstructionListType Addr = materializeAddress(Target, Ctx, RISCV::X10); + Insts.insert(Insts.end(), Addr.begin(), Addr.end()); + InstructionListType IncInsts = + createIncMemory(RISCV::X10, RISCV::X11, RISCV::X0); + Insts.insert(Insts.end(), IncInsts.begin(), IncInsts.end()); + reloadRegs(Insts, {RISCV::X10, RISCV::X11}); + return Insts; + } + + void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx, + bool IsTailCall) override { + Inst.setOpcode(RISCV::JAL); + Inst.clear(); + if (IsTailCall) { + Inst.addOperand(MCOperand::createReg(RISCV::X0)); + Inst.addOperand(MCOperand::createExpr(getTargetExprFor( + Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + *Ctx, 0))); + convertJmpToTailCall(Inst); + } else { + Inst.addOperand(MCOperand::createReg(RISCV::X1)); + Inst.addOperand(MCOperand::createExpr(getTargetExprFor( + Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + *Ctx, 0))); + } + } + + void createIndirectCallInst(MCInst &Inst, bool IsTailCall, MCPhysReg Reg, + int64_t Disp) const { + Inst.clear(); + Inst.setOpcode(RISCV::JALR); + Inst.clear(); + if (IsTailCall) { + Inst.addOperand(MCOperand::createReg(RISCV::X0)); + Inst.addOperand(MCOperand::createReg(Reg)); + Inst.addOperand(MCOperand::createImm(Disp)); + } else { + Inst.addOperand(MCOperand::createReg(RISCV::X1)); + Inst.addOperand(MCOperand::createReg(Reg)); + Inst.addOperand(MCOperand::createImm(Disp)); + } + } + + InstructionListType + createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, + const MCSymbol *IndCallHandler, + MCContext *Ctx) override { + // Code sequence used to check whether InstrTampoline was initialized + // and call it if so, returns via IndCallHandler + // sp -16(sp) + // sd x10, 0(sp) + // sd x11, 0(sp) + // la x10, InstrTrampoline -> auipc + addi + // ld x10, [x10] + // beq x10, x11, IndCallHandler + // sp -16(sp) + // sd x1, 0(sp) + // jalr x1,x10,0 + // ld x1, [sp], #16 + // sp 16(sp) + // jal x0, IndCallHandler + + InstructionListType Insts; + spillRegs(Insts, {RISCV::X10, RISCV::X11}); + InstructionListType Addr = + materializeAddress(InstrTrampoline, Ctx, RISCV::X10); + Insts.insert(Insts.end(), Addr.begin(), Addr.end()); + Insts.emplace_back(); + loadReg(Insts.back(), RISCV::X10, RISCV::X10, 0); + InstructionListType cmpJmp = + createRegCmpJE(RISCV::X10, RISCV::X11, IndCallHandler, Ctx); + Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end()); + Insts.emplace_back(); + createStackPointerIncrement(Insts.back(), 16); + Insts.emplace_back(); + storeReg(Insts.back(), RISCV::X1, RISCV::X2, 0); + Insts.emplace_back(); + createIndirectCallInst(Insts.back(), /*IsTailCall*/ false, RISCV::X10, 0); + Insts.emplace_back(); + loadReg(Insts.back(), RISCV::X1, RISCV::X2, 0); + Insts.emplace_back(); + createStackPointerDecrement(Insts.back(), 16); + Insts.emplace_back(); + createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true); + return Insts; + } + + InstructionListType createInstrumentedIndCallHandlerExitBB() const override { + InstructionListType Insts; + reloadRegs(Insts, {RISCV::X10, RISCV::X11}); + Insts.emplace_back(); + loadReg(Insts.back(), RISCV::X5, RISCV::X2, 0); + Insts.emplace_back(); + createStackPointerDecrement(Insts.back(), 16); + reloadRegs(Insts, {RISCV::X10, RISCV::X11}); + Insts.emplace_back(); + createIndirectCallInst(Insts.back(), /*IsTailCall*/ true, RISCV::X5, 0); + return Insts; + } + + InstructionListType + createInstrumentedIndTailCallHandlerExitBB() const override { + return createInstrumentedIndCallHandlerExitBB(); + } + + std::vector createSymbolTrampoline(const MCSymbol *TgtSym, + MCContext *Ctx) override { + std::vector Insts; + createShortJmp(Insts, TgtSym, Ctx, /*IsTailCall*/ true); + return Insts; + } + + InstructionListType createNumCountersGetter(MCContext *Ctx) const override { + return createGetter(Ctx, "__bolt_num_counters"); + } + + InstructionListType + createInstrLocationsGetter(MCContext *Ctx) const override { + return createGetter(Ctx, "__bolt_instr_locations"); + } + + InstructionListType createInstrTablesGetter(MCContext *Ctx) const override { + return createGetter(Ctx, "__bolt_instr_tables"); + } + + InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override { + return createGetter(Ctx, "__bolt_instr_num_funcs"); + } + + void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override { + bool IsTailCall = isTailCall(Inst); + if (IsTailCall) + removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); + Inst.setOpcode(RISCV::ADD); + Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); + Inst.insert(Inst.begin() + 1, MCOperand::createReg(RISCV::X0)); + return; + } + + InstructionListType createLoadImmediate(const MCPhysReg Dest, + uint64_t Imm) const override { + InstructionListType Insts; + // get IMM higher 32bit + Insts.emplace_back( + MCInstBuilder(RISCV::LUI).addReg(Dest).addImm((Imm >> 44) & 0xFFFFF)); + Insts.emplace_back(MCInstBuilder(RISCV::LUI) + .addReg(RISCV::X5) + .addImm((Imm >> 32) & 0xFFF)); + Insts.emplace_back(MCInstBuilder(RISCV::SRLI) + .addReg(RISCV::X5) + .addReg(RISCV::X5) + .addImm(12)); + Insts.emplace_back( + MCInstBuilder(RISCV::OR).addReg(Dest).addReg(Dest).addReg(RISCV::X5)); + Insts.emplace_back( + MCInstBuilder(RISCV::SLLI).addReg(Dest).addReg(Dest).addImm(32)); + + // get IMM lower 32bit + Insts.emplace_back(MCInstBuilder(RISCV::LUI) + .addReg(RISCV::X5) + .addImm((Imm >> 12) & 0xFFFFF)); + Insts.emplace_back( + MCInstBuilder(RISCV::LUI).addReg(RISCV::X6).addImm((Imm)&0xFFF)); + Insts.emplace_back(MCInstBuilder(RISCV::SRLI) + .addReg(RISCV::X6) + .addReg(RISCV::X6) + .addImm(12)); + Insts.emplace_back( + MCInstBuilder(RISCV::OR).addReg(RISCV::X5).addReg(RISCV::X5).addReg( + RISCV::X6)); + + // get 64bit IMM + Insts.emplace_back( + MCInstBuilder(RISCV::OR).addReg(Dest).addReg(Dest).addReg(RISCV::X5)); + return Insts; + } + + InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst, + MCSymbol *HandlerFuncAddr, + int CallSiteID, + MCContext *Ctx) override { + // Code sequence used to enter indirect call instrumentation helper: + // addi sp, sp, -0x10 + // sd a0, 0x0(sp) + // sd a1, 0x8(sp) + // mov target x0 convertIndirectCallToLoad -> add a0, zero, target + // mov x1 CallSiteID createLoadImmediate + // addi sp, sp, -0x10 + // sd a0, 0x0(sp) + // sd a1, 0x8(sp) + // la x0 *HandlerFuncAddr -> auipc + addi + // jalr x0 + + InstructionListType Insts; + spillRegs(Insts, {RISCV::X10, RISCV::X11}); + Insts.emplace_back(CallInst); + convertIndirectCallToLoad(Insts.back(), RISCV::X10); + InstructionListType LoadImm = createLoadImmediate(RISCV::X11, CallSiteID); + Insts.insert(Insts.end(), LoadImm.begin(), LoadImm.end()); + spillRegs(Insts, {RISCV::X10, RISCV::X11}); + InstructionListType Addr = + materializeAddress(HandlerFuncAddr, Ctx, RISCV::X5); + Insts.insert(Insts.end(), Addr.begin(), Addr.end()); + Insts.emplace_back(); + createIndirectCallInst(Insts.back(), isTailCall(CallInst), RISCV::X5, 0); + + // // Carry over metadata including tail call marker if present. + stripAnnotations(Insts.back()); + moveAnnotations(std::move(CallInst), Insts.back()); + + return Insts; + } }; } // end anonymous namespace diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp index 8e459e10244f..b909d7fb6bf2 100644 --- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -2441,7 +2441,7 @@ class X86MCPlusBuilder : public MCPlusBuilder { std::optional createRelocation(const MCFixup &Fixup, const MCAsmBackend &MAB) const override { - const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind()); + MCFixupKindInfo FKI = MAB.getFixupKindInfo(Fixup.getKind()); assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected"); const uint64_t RelOffset = Fixup.getOffset(); diff --git a/bolt/runtime/CMakeLists.txt b/bolt/runtime/CMakeLists.txt index 0deb69a27d43..87cc44812da1 100644 --- a/bolt/runtime/CMakeLists.txt +++ b/bolt/runtime/CMakeLists.txt @@ -35,15 +35,21 @@ set(BOLT_RT_FLAGS -fno-exceptions -fno-rtti -fno-stack-protector - -fPIC - -mgeneral-regs-only) + -fPIC) if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") - set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS} "-mno-sse") + set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS} + -mno-sse + -mgeneral-regs-only) +endif() +if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64") + set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS}) endif() if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") check_cxx_compiler_flag("-mno-outline-atomics" CXX_SUPPORTS_OUTLINE_ATOMICS) if (CXX_SUPPORTS_OUTLINE_ATOMICS) - set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS} "-mno-outline-atomics") + set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS} + -mno-outline-atomics + -mgeneral-regs-only) endif() endif() diff --git a/bolt/runtime/common.h b/bolt/runtime/common.h index 9b9965bae524..3461f8c545f9 100644 --- a/bolt/runtime/common.h +++ b/bolt/runtime/common.h @@ -151,10 +151,14 @@ struct timespec { uint64_t tv_nsec; /* nanoseconds */ }; -#if defined(__aarch64__) +#if defined(__aarch64__) || defined(__arm64__) #include "sys_aarch64.h" -#else +#elif defined(__riscv) +#include "sys_riscv64.h" +#elif defined(__x86_64__) #include "sys_x86_64.h" +#else +#error "For AArch64/ARM64,X86_64 AND RISCV64 only." #endif constexpr uint32_t BufSize = 10240; diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp index a89cba2243c4..672b04247dfa 100644 --- a/bolt/runtime/hugify.cpp +++ b/bolt/runtime/hugify.cpp @@ -6,7 +6,8 @@ // //===---------------------------------------------------------------------===// -#if defined (__x86_64__) && !defined(__APPLE__) +#if (defined(__x86_64__) || defined(__aarch64__) || defined(__arm64__)) && \ + !defined(__APPLE__) #include "common.h" @@ -73,8 +74,10 @@ static bool hasPagecacheTHPSupport() { if (Res < 0) return false; - if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]")) + if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]")) { + DEBUG(report("[hugify] THP support is not enabled.\n");) return false; + } struct KernelVersionTy { uint32_t major; @@ -167,12 +170,20 @@ extern "C" void __bolt_hugify_self_impl() { /// This is hooking ELF's entry, it needs to save all machine state. extern "C" __attribute((naked)) void __bolt_hugify_self() { + // clang-format off #if defined(__x86_64__) __asm__ __volatile__(SAVE_ALL "call __bolt_hugify_self_impl\n" RESTORE_ALL - "jmp __bolt_hugify_start_program\n" :: - :); + "jmp __bolt_hugify_start_program\n" + :::); +#elif defined(__aarch64__) || defined(__arm64__) + __asm__ __volatile__(SAVE_ALL "bl __bolt_hugify_self_impl\n" RESTORE_ALL + "adrp x16, __bolt_hugify_start_program\n" + "add x16, x16, #:lo12:__bolt_hugify_start_program\n" + "br x16\n" + :::); #else - exit(1); + __exit(1); #endif + // clang-format on } #endif diff --git a/bolt/runtime/instr.cpp b/bolt/runtime/instr.cpp index d1f8a216badc..ae356e71cbe4 100644 --- a/bolt/runtime/instr.cpp +++ b/bolt/runtime/instr.cpp @@ -1674,6 +1674,19 @@ extern "C" __attribute((naked)) void __bolt_instr_indirect_call() "ret\n" :::); // clang-format on +#elif defined(__riscv) + // clang-format off + __asm__ __volatile__( + SAVE_ALL + "addi sp, sp, 288\n" + "ld x10, 0(sp)\n" + "ld x11, 8(sp)\n" + "addi sp, sp, -288\n" + "jal x1, instrumentIndirectCall\n" + RESTORE_ALL + "ret\n" + :::); + // clang-format on #else // clang-format off __asm__ __volatile__(SAVE_ALL @@ -1698,6 +1711,18 @@ extern "C" __attribute((naked)) void __bolt_instr_indirect_tailcall() "ret\n" :::); // clang-format on +#elif defined(__riscv) + // clang-format off + __asm__ __volatile__(SAVE_ALL + "addi sp, sp, 288\n" + "ld x10, 0(sp)\n" + "ld x11, 8(sp)\n" + "addi sp, sp, -288\n" + "jal x1, instrumentIndirectCall\n" + RESTORE_ALL + "ret\n" + :::); + // clang-format on #else // clang-format off __asm__ __volatile__(SAVE_ALL @@ -1724,6 +1749,18 @@ extern "C" __attribute((naked)) void __bolt_instr_start() "br x16\n" :::); // clang-format on +#elif defined(__riscv) + // clang-format off + __asm__ __volatile__( + SAVE_ALL + "jal x1, __bolt_instr_setup\n" + RESTORE_ALL + "setup_symbol:\n" + "auipc x5, %%pcrel_hi(__bolt_start_trampoline)\n" + "addi x5, x5, %%pcrel_lo(setup_symbol)\n" + "jr x5\n" + :::); + // clang-format on #else // clang-format off __asm__ __volatile__(SAVE_ALL @@ -1746,6 +1783,17 @@ extern "C" void __bolt_instr_fini() { RESTORE_ALL :::); // clang-format on +#elif defined(__riscv) + // clang-format off + __asm__ __volatile__( + SAVE_ALL + "fini_symbol:\n" + "auipc x5, %%pcrel_hi(__bolt_fini_trampoline)\n" + "addi x5, x5, %%pcrel_lo(fini_symbol)\n" + "jalr x1, 0(x5)\n" + RESTORE_ALL + :::); + // clang-format on #else __asm__ __volatile__("call __bolt_fini_trampoline\n" :::); #endif diff --git a/bolt/runtime/sys_riscv64.h b/bolt/runtime/sys_riscv64.h new file mode 100644 index 000000000000..00a21e4945f0 --- /dev/null +++ b/bolt/runtime/sys_riscv64.h @@ -0,0 +1,460 @@ +#ifndef LLVM_TOOLS_LLVM_BOLT_SYS_RISCV +#define LLVM_TOOLS_LLVM_BOLT_SYS_RISCV + +// Save all registers while keeping 16B stack alignment +#define SAVE_ALL \ + "addi sp, sp, -16\n" \ + "sd x0, 0(sp)\n" \ + "sd x1, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x2, 0(sp)\n" \ + "sd x3, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x4, 0(sp)\n" \ + "sd x5, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x6, 0(sp)\n" \ + "sd x7, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x8, 0(sp)\n" \ + "sd x9, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x10, 0(sp)\n" \ + "sd x11, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x12, 0(sp)\n" \ + "sd x13, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x14, 0(sp)\n" \ + "sd x15, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x16, 0(sp)\n" \ + "sd x17, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x18, 0(sp)\n" \ + "sd x19, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x20, 0(sp)\n" \ + "sd x21, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x22, 0(sp)\n" \ + "sd x23, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x24, 0(sp)\n" \ + "sd x25, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x26, 0(sp)\n" \ + "sd x27, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x28, 0(sp)\n" \ + "sd x29, 8(sp)\n" \ + "addi sp, sp, -16\n" \ + "sd x30, 0(sp)\n" \ + "sd x31, 8(sp)\n" +// Mirrors SAVE_ALL +#define RESTORE_ALL \ + "ld x30, 0(sp)\n" \ + "ld x31, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x28, 0(sp)\n" \ + "ld x29, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x26, 0(sp)\n" \ + "ld x27, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x24, 0(sp)\n" \ + "ld x25, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x22, 0(sp)\n" \ + "ld x23, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x20, 0(sp)\n" \ + "ld x21, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x18, 0(sp)\n" \ + "ld x19, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x16, 0(sp)\n" \ + "ld x17, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x14, 0(sp)\n" \ + "ld x15, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x12, 0(sp)\n" \ + "ld x13, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x10, 0(sp)\n" \ + "ld x11, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x8, 0(sp)\n" \ + "ld x9, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x6, 0(sp)\n" \ + "ld x7, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x4, 0(sp)\n" \ + "ld x5, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x2, 0(sp)\n" \ + "ld x3, 8(sp)\n" \ + "addi sp, sp, 16\n" \ + "ld x0, 0(sp)\n" \ + "ld x1, 8(sp)\n" \ + "addi sp, sp, 16\n" + +// Anonymous namespace covering everything but our library entry point +namespace { + +// Get the difference between runtime addrress of .text section and +// static address in section header table. Can be extracted from arbitrary +// pc value recorded at runtime to get the corresponding static address, which +// in turn can be used to search for indirect call description. Needed because +// indirect call descriptions are read-only non-relocatable data. +uint64_t getTextBaseAddress() { + uint64_t DynAddr; + uint64_t StaticAddr; + __asm__ volatile("lla %0, __hot_end\n\t" + "lui %1, %%hi(__hot_end)\n\t" + "addi %1, %1, %%lo(__hot_end)\n\t" + : "=r"(DynAddr), "=r"(StaticAddr)); + return DynAddr - StaticAddr; +} + +uint64_t __read(uint64_t fd, const void *buf, uint64_t count) { + uint64_t ret; + register uint64_t a0 __asm__("a0") = fd; + register const void *a1 __asm__("a1") = buf; + register uint64_t a2 __asm__("a2") = count; + register uint64_t a7 __asm__("a7") = + 63; // Assuming 63 is the syscall number for read + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret) + : "r"(a0), "r"(a1), "r"(a2), "r"(a7) + : "memory"); + return ret; +} + +uint64_t __write(uint64_t fd, const void *buf, uint64_t count) { + uint64_t ret; + register uint64_t a0 __asm__("a0") = fd; + register const void *a1 __asm__("a1") = buf; + register uint64_t a2 __asm__("a2") = count; + register uint32_t a7 __asm__("a7") = + 64; // Assuming 64 is the syscall number for write + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret) + : "r"(a0), "r"(a1), "r"(a2), "r"(a7) + : "memory"); + return ret; +} + +void *__mmap(uint64_t addr, uint64_t size, uint64_t prot, uint64_t flags, + uint64_t fd, uint64_t offset) { + void *ret; + register uint64_t a0 __asm__("a0") = addr; + register uint64_t a1 __asm__("a1") = size; + register uint64_t a2 __asm__("a2") = prot; + register uint64_t a3 __asm__("a3") = flags; + register uint64_t a4 __asm__("a4") = fd; + register uint64_t a5 __asm__("a5") = offset; + register uint32_t a7 __asm__("a7") = + 222; // Assuming 222 is the syscall number for mmap + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5), + "r"(a7) + : "memory"); + return ret; +} + +uint64_t __munmap(void *addr, uint64_t size) { + uint64_t ret; + register void *a0 __asm__("a0") = addr; + register uint64_t a1 __asm__("a1") = size; + register uint32_t a7 __asm__("a7") = 215; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a7) + : "memory"); + return ret; +} + +uint64_t __exit(uint64_t code) { + uint64_t ret; + register uint64_t a0 __asm__("a0") = code; + register uint32_t a7 __asm__("a7") = 94; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0) + : "r"(a7) + : "memory"); + return ret; +} + +uint64_t __open(const char *pathname, uint64_t flags, uint64_t mode) { + uint64_t ret; + register int a0 __asm__("a0") = + -100; // Assuming -100 is an invalid file descriptor + register const char *a1 __asm__("a1") = pathname; + register uint64_t a2 __asm__("a2") = flags; + register uint64_t a3 __asm__("a3") = mode; + register uint64_t a7 __asm__("a7") = + 56; // Assuming 56 is the syscall number for open + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a7) + : "memory"); + return ret; +} + +long __getdents64(unsigned int fd, dirent64 *dirp, size_t count) { + long ret; + register unsigned int a0 __asm__("a0") = fd; + register dirent64 *a1 __asm__("a1") = dirp; + register size_t a2 __asm__("a2") = count; + register uint32_t a7 __asm__("a7") = 61; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a7) + : "memory"); + return ret; +} + +uint64_t __readlink(const char *pathname, char *buf, size_t bufsize) { + uint64_t ret; + register int a0 __asm__("a0") = -100; + register const char *a1 __asm__("a1") = pathname; + register char *a2 __asm__("a2") = buf; + register size_t a3 __asm__("a3") = bufsize; + register uint32_t a7 __asm__("a7") = 78; // readlinkat + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a3), "r"(a7) + : "memory"); + return ret; +} + +uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) { + uint64_t ret; + register uint64_t a0 __asm__("a0") = fd; + register uint64_t a1 __asm__("a1") = pos; + register uint64_t a2 __asm__("a2") = whence; + register uint32_t a7 __asm__("a7") = 62; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a7) + : "memory"); + return ret; +} + +int __ftruncate(uint64_t fd, uint64_t length) { + int ret; + register uint64_t a0 __asm__("a0") = fd; + register uint64_t a1 __asm__("a1") = length; + register uint32_t a7 __asm__("a7") = 46; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a7) + : "memory"); + return ret; +} + +int __close(uint64_t fd) { + int ret; + register uint64_t a0 __asm__("a0") = fd; + register uint32_t a7 __asm__("a7") = + 57; // Assuming 57 is the syscall number for close + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret) + : "r"(a0), "r"(a7) + : "memory"); + return ret; +} + +int __madvise(void *addr, size_t length, int advice) { + int ret; + register void *a0 __asm__("a0") = addr; + register size_t a1 __asm__("a1") = length; + register int a2 __asm__("a2") = advice; + register uint32_t a7 __asm__("a7") = 233; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a7) + : "memory"); + return ret; +} + +int __uname(struct UtsNameTy *buf) { + int ret; + register UtsNameTy *a0 __asm__("a0") = buf; + register uint32_t a7 __asm__("a7") = 160; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0) + : "r"(a7) + : "memory"); + return ret; +} + +uint64_t __nanosleep(const timespec *req, timespec *rem) { + uint64_t ret; + register const timespec *a0 __asm__("a0") = req; + register timespec *a1 __asm__("a1") = rem; + register uint32_t a7 __asm__("a7") = 101; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a7) + : "memory"); + return ret; +} + +int64_t __fork() { + uint64_t ret; + // clone instead of fork with flags + // "CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD" + register uint64_t a0 __asm__("a0") = 0x1200011; + register uint64_t a1 __asm__("a1") = 0; + register uint64_t a2 __asm__("a2") = 0; + register uint64_t a3 __asm__("a3") = 0; + register uint64_t a4 __asm__("a4") = 0; + register uint32_t a7 __asm__("a7") = 220; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a3), "r"(a4), "r"(a7) + : "memory"); + return ret; +} + +int __mprotect(void *addr, size_t len, int prot) { + int ret; + register void *a0 __asm__("a0") = addr; + register size_t a1 __asm__("a1") = len; + register int a2 __asm__("a2") = prot; + register uint32_t a7 __asm__("a7") = 226; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a7) + : "memory"); + return ret; +} + +uint64_t __getpid() { + uint64_t ret; + register uint32_t a7 __asm__("a7") = 172; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret) + : "r"(a7) + : "memory"); + return ret; +} + +uint64_t __getppid() { + uint64_t ret; + register uint32_t a7 __asm__("a7") = 173; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret) + : "r"(a7) + : "memory"); + return ret; +} + +int __setpgid(uint64_t pid, uint64_t pgid) { + int ret; + register uint64_t a0 __asm__("a0") = pid; + register uint64_t a1 __asm__("a1") = pgid; + register uint32_t a7 __asm__("a7") = 154; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a7) + : "memory"); + return ret; +} + +uint64_t __getpgid(uint64_t pid) { + uint64_t ret; + register uint64_t a0 __asm__("a0") = pid; + register uint32_t a7 __asm__("a7") = 155; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0) + : "r"(a7) + : "memory"); + return ret; +} + +int __kill(uint64_t pid, int sig) { + int ret; + register uint64_t a0 __asm__("a0") = pid; + register int a1 __asm__("a1") = sig; + register uint32_t a7 __asm__("a7") = 129; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a7) + : "memory"); + return ret; +} + +int __fsync(int fd) { + int ret; + register int a0 __asm__("a0") = fd; + register uint32_t a7 __asm__("a7") = 82; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0) + : "r"(a7) + : "memory"); + return ret; +} + +uint64_t __sigprocmask(int how, const void *set, void *oldset) { + uint64_t ret; + register int a0 __asm__("a0") = how; + register const void *a1 __asm__("a1") = set; + register void *a2 __asm__("a2") = oldset; + register long a3 asm("a3") = 8; + register uint32_t a7 __asm__("a7") = 135; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a3), "r"(a7) + : "memory"); + return ret; +} + +int __prctl(int option, unsigned long arg2, unsigned long arg3, + unsigned long arg4, unsigned long arg5) { + int ret; + register int a0 __asm__("a0") = option; + register unsigned long a1 __asm__("a1") = arg2; + register unsigned long a2 __asm__("a2") = arg3; + register unsigned long a3 __asm__("a3") = arg4; + register unsigned long a4 __asm__("a4") = arg5; + register uint32_t a7 __asm__("a7") = 167; + __asm__ __volatile__("ecall\n\t" + "mv %0, a0" + : "=r"(ret), "+r"(a0), "+r"(a1) + : "r"(a2), "r"(a3), "r"(a4), "r"(a7) + : "cc", "memory"); + return ret; +} +} // anonymous namespace + +#endif \ No newline at end of file diff --git a/bolt/test/AArch64/lite-mode.s b/bolt/test/AArch64/lite-mode.s index d81206089aae..a71edbe03466 100644 --- a/bolt/test/AArch64/lite-mode.s +++ b/bolt/test/AArch64/lite-mode.s @@ -11,6 +11,13 @@ # RUN: llvm-objdump -d --disassemble-symbols=cold_function %t.bolt \ # RUN: | FileCheck %s + +## Verify that the number of FDEs matches the number of functions in the output +## binary. There are three original functions and two optimized. +# RUN: llvm-readelf -u %t.bolt | grep -wc FDE \ +# RUN: | FileCheck --check-prefix=CHECK-FDE %s +# CHECK-FDE: 5 + ## In lite mode, optimized code will be separated from the original .text by ## over 128MB, making it impossible for call/bl instructions in cold functions ## to reach optimized functions directly. diff --git a/bolt/test/AArch64/validate-secondary-entry-point.s b/bolt/test/AArch64/validate-secondary-entry-point.s new file mode 100644 index 000000000000..0099a0ee4fe9 --- /dev/null +++ b/bolt/test/AArch64/validate-secondary-entry-point.s @@ -0,0 +1,34 @@ +# This test is to verify that BOLT won't take a label pointing to constant +# island as a secondary entry point (function `_start` doesn't have ELF size +# set originally) and the function won't otherwise be mistaken as non-simple. + +# RUN: %clang %cflags -pie %s -o %t.so -Wl,-q -Wl,--init=_foo -Wl,--fini=_foo +# RUN: llvm-bolt %t.so -o %t.bolt.so --print-cfg 2>&1 | FileCheck %s +# CHECK-NOT: BOLT-WARNING: reference in the middle of instruction detected \ +# CHECK-NOT: function _start at offset 0x{{[0-9a-f]+}} +# CHECK: Binary Function "_start" after building cfg + + .text + + .global _foo + .type _foo, %function +_foo: + ret + + .global _start + .type _start, %function +_start: + b _foo + + .balign 16 +_random_consts: + .long 0x12345678 + .long 0x90abcdef + + .global _bar + .type _bar, %function +_bar: + ret + + # Dummy relocation to force relocation mode + .reloc 0, R_AARCH64_NONE diff --git a/bolt/test/RISCV/validate-secondary-entry-point.s b/bolt/test/RISCV/validate-secondary-entry-point.s new file mode 100644 index 000000000000..0c29f5c97c68 --- /dev/null +++ b/bolt/test/RISCV/validate-secondary-entry-point.s @@ -0,0 +1,34 @@ +# This test is to verify that BOLT won't take a label pointing to constant +# island as a secondary entry point (function `_start` doesn't have ELF size +# set originally) and the function won't otherwise be mistaken as non-simple. + +# RUN: %clang %cflags -pie %s -o %t.so -Wl,-q -Wl,--init=_foo -Wl,--fini=_foo +# RUN: llvm-bolt %t.so -o %t.bolt.so --print-cfg 2>&1 | FileCheck %s +# CHECK-NOT: BOLT-WARNING: reference in the middle of instruction detected \ +# CHECK-NOT: function _start at offset 0x{{[0-9a-f]+}} +# CHECK: Binary Function "_start" after building cfg + + .text + + .global _foo + .type _foo, %function +_foo: + ret + + .global _start + .type _start, %function +_start: + j _foo + + .balign 16 +_random_consts: + .long 0x12345678 + .long 0x90abcdef + + .global _bar + .type _bar, %function +_bar: + ret + + # Dummy relocation to force relocation mode + .reloc 0, R_RISCV_NONE diff --git a/bolt/test/X86/callcont-fallthru.s b/bolt/test/X86/callcont-fallthru.s index 95cb4c5fc2df..ee72d8f62e03 100644 --- a/bolt/test/X86/callcont-fallthru.s +++ b/bolt/test/X86/callcont-fallthru.s @@ -9,6 +9,7 @@ # RUN: link_fdata %s %t %t.pa3 PREAGG3 # RUN: link_fdata %s %t %t.pat PREAGGT1 # RUN: link_fdata %s %t %t.pat2 PREAGGT2 +# RUN: link_fdata %s %t %t.patplt PREAGGPLT ## Check normal case: fallthrough is not LP or secondary entry. # RUN: llvm-strip --strip-unneeded %t -o %t.strip @@ -42,6 +43,12 @@ # RUN: llvm-bolt %t.strip --pa -p %t.pat2 -o %t.out \ # RUN: --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK3 +## Check pre-aggregated traces don't report zero-sized PLT fall-through as +## invalid trace +# RUN: llvm-bolt %t.strip --pa -p %t.patplt -o %t.out | FileCheck %s \ +# RUN: --check-prefix=CHECK-PLT +# CHECK-PLT: traces mismatching disassembled function contents: 0 + .globl foo .type foo, %function foo: @@ -65,7 +72,10 @@ main: movl $0x0, -0x4(%rbp) movl %edi, -0x8(%rbp) movq %rsi, -0x10(%rbp) +Ltmp0_br: callq puts@PLT +## Check PLT traces are accepted +# PREAGGPLT: T #Ltmp0_br# #puts@plt# #puts@plt# 3 ## Target is an external-origin call continuation # PREAGG1: B X:0 #Ltmp1# 2 0 # PREAGGT1: T X:0 #Ltmp1# #Ltmp4_br# 2 diff --git a/bolt/test/X86/high-segments.s b/bolt/test/X86/high-segments.s new file mode 100644 index 000000000000..dfddf18004c2 --- /dev/null +++ b/bolt/test/X86/high-segments.s @@ -0,0 +1,46 @@ +// Check that we are able to rewrite binaries when we fail to identify a +// suitable location to put new code and user supplies a custom one via +// --custom-allocation-vma. This happens more obviously if the binary has +// segments mapped to very high addresses. + +// In this example, my.reserved.section is mapped to a segment to be loaded +// at address 0x10000000000, while regular text should be at 0x200000. We +// pick a vma in the middle at 0x700000 to carve space for BOLT to put data, +// since BOLT's usual route of allocating after the last segment will put +// code far away and that will blow up relocations from main. + +// RUN: split-file %s %t +// RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %t/main.s -o %t.o +// RUN: %clang %cflags -no-pie %t.o -o %t.exe -Wl,-T %t/main.ls +// RUN: llvm-bolt %t.exe -o %t.bolt --custom-allocation-vma=0x700000 + +//--- main.s + .type reserved_space,@object + .section .my.reserved.section,"awx",@nobits + .globl reserved_space + .p2align 4, 0x0 +reserved_space: + .zero 0x80000000 + .size reserved_space, 0x80000000 + + .text + .globl main + .globl _start + .type main, %function +_start: +main: + .cfi_startproc + nop + nop + nop + retq + .cfi_endproc +.size main, .-main + +//--- main.ls +SECTIONS +{ + .my.reserved.section 1<<40 : { + *(.my.reserved.section); + } +} INSERT BEFORE .comment; diff --git a/bolt/test/X86/nobits-symbol.s b/bolt/test/X86/nobits-symbol.s new file mode 100644 index 000000000000..4110c79930b2 --- /dev/null +++ b/bolt/test/X86/nobits-symbol.s @@ -0,0 +1,24 @@ +## Check that llvm-bolt doesn't choke on symbols defined in nobits sections. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags %t.o -o %t.exe +# RUN: llvm-bolt %t.exe -o %t +# + + .type symbol_in_nobits,@object + .section .my.nobits.section,"awx",@nobits + .globl symbol_in_nobits + .p2align 4, 0x0 +symbol_in_nobits: + .zero 0x100000 + .size symbol_in_nobits, 0x100000 + + .text + .globl main + .type main, %function +main: + .cfi_startproc +.LBB06: + retq + .cfi_endproc +.size main, .-main diff --git a/bolt/test/X86/profile-quality-reporting-small-binary.s b/bolt/test/X86/profile-quality-reporting-small-binary.s new file mode 100644 index 000000000000..2b147c5eca81 --- /dev/null +++ b/bolt/test/X86/profile-quality-reporting-small-binary.s @@ -0,0 +1,35 @@ +## Test that BOLT-INFO is correctly formatted after profile quality reporting for +## a small binary. + +# RUN: llvm-mc --filetype=obj --triple x86_64-unknown-unknown %s -o %t.o +# RUN: link_fdata %s %t.o %t.fdata +# RUN: llvm-strip --strip-unneeded %t.o +# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.bolt --data=%t.fdata \ +# RUN: 2>&1 | FileCheck %s + +# CHECK: BOLT-INFO: profile quality metrics for the hottest 2 functions (reporting top 5% values): function CFG discontinuity 0.00%; call graph flow conservation gap 0.00%; CFG flow conservation gap 0.00% (weighted) 0.00% (worst); exception handling usage 0.00% (of total BBEC) 0.00% (of total InvokeEC) +# CHECK-NEXT: BOLT-INFO: + + .text + .globl func + .type func, @function +func: + pushq %rbp + ret +LLfunc_end: + .size func, LLfunc_end-func + + + .globl main + .type main, @function +main: + pushq %rbp + movq %rsp, %rbp +LLmain_func: + call func +# FDATA: 1 main #LLmain_func# 1 func 0 0 500 + movl $4, %edi + retq +.Lmain_end: + .size main, .Lmain_end-main diff --git a/bolt/test/X86/profile-quality-reporting.test b/bolt/test/X86/profile-quality-reporting.test index 2e15a6b245af..210d3e10a389 100644 --- a/bolt/test/X86/profile-quality-reporting.test +++ b/bolt/test/X86/profile-quality-reporting.test @@ -1,4 +1,4 @@ ## Check profile quality stats reporting RUN: yaml2obj %p/Inputs/blarge_new.yaml &> %t.exe RUN: llvm-bolt %t.exe -o %t.out --pa -p %p/Inputs/blarge_new.preagg.txt | FileCheck %s -CHECK: profile quality metrics for the hottest 5 functions (reporting top 5% values): function CFG discontinuity 100.00%; call graph flow conservation gap 60.00%; CFG flow conservation gap 45.53% (weighted) 96.87% (worst) +CHECK: profile quality metrics for the hottest 5 functions (reporting top 5% values): function CFG discontinuity 100.00%; call graph flow conservation gap 60.00%; CFG flow conservation gap 45.53% (weighted) 96.87% (worst); exception handling usage 0.00% (of total BBEC) 0.00% (of total InvokeEC) diff --git a/bolt/test/X86/split-func-jump-table-fragment-bidirection.s b/bolt/test/X86/split-func-jump-table-fragment-bidirection.s index 52c816ccd900..3d3247fc58f2 100644 --- a/bolt/test/X86/split-func-jump-table-fragment-bidirection.s +++ b/bolt/test/X86/split-func-jump-table-fragment-bidirection.s @@ -10,7 +10,7 @@ # RUN: %clang %cflags %t.o -o %t.exe -Wl,-q # RUN: llvm-bolt -print-cfg -v=3 %t.exe -o %t.out 2>&1 | FileCheck %s -# CHECK: BOLT-INFO: Multiple fragments access same jump table: main; main.cold.1 +# CHECK: BOLT-INFO: multiple fragments access the same jump table: main; main.cold.1 # CHECK: PIC Jump table JUMP_TABLE1 for function main, main.cold.1 at {{.*}} with a total count of 0: .text diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s b/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s index b271cda9da62..cca1efe69516 100644 --- a/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s +++ b/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s @@ -50,41 +50,42 @@ simple: // CHECK-NEXT: // CHECK-NEXT: End of Function "simple" // CHECK-EMPTY: -// CHECK-NEXT: PacRetAnalysis::ComputeNext( hint #25, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( stp x29, x30, [sp, #-0x10]!, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( b [[BB1]], pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::Confluence( -// CHECK-NEXT: State 1: pacret-state -// CHECK-NEXT: State 2: pacret-state) -// CHECK-NEXT: merged state: pacret-state -// CHECK-NEXT: PacRetAnalysis::ComputeNext( autiza x0, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( blr x0, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( ldp x29, x30, [sp], #0x10, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( hint #29, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( ret x30, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::Confluence( -// CHECK-NEXT: State 1: pacret-state -// CHECK-NEXT: State 2: pacret-state) -// CHECK-NEXT: merged state: pacret-state -// CHECK-NEXT: PacRetAnalysis::ComputeNext( autiza x0, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( blr x0, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( ldp x29, x30, [sp], #0x10, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( hint #29, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( ret x30, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: After PacRetAnalysis: +// CHECK-NEXT: Running src register safety analysis... +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( hint #25, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( stp x29, x30, [sp, #-0x10]!, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( b [[BB1]], src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::Confluence( +// CHECK-NEXT: State 1: src-state +// CHECK-NEXT: State 2: src-state) +// CHECK-NEXT: merged state: src-state +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( autiza x0, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( blr x0, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( ldp x29, x30, [sp], #0x10, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( hint #29, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( ret x30, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::Confluence( +// CHECK-NEXT: State 1: src-state +// CHECK-NEXT: State 2: src-state) +// CHECK-NEXT: merged state: src-state +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( autiza x0, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( blr x0, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( ldp x29, x30, [sp], #0x10, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( hint #29, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( ret x30, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: After src register safety analysis: // CHECK-NEXT: Binary Function "simple" { // CHECK-NEXT: Number : 1 // CHECK-NEXT: State : CFG constructed @@ -93,27 +94,27 @@ simple: // CHECK-NEXT: } // CHECK-NEXT: [[BB0]] (3 instructions, align : 1) // CHECK-NEXT: Entry Point -// CHECK-NEXT: 00000000: paciasp # PacRetAnalysis: pacret-state -// CHECK-NEXT: 00000004: stp x29, x30, [sp, #-0x10]! # PacRetAnalysis: pacret-state -// CHECK-NEXT: 00000008: b [[BB1]] # PacRetAnalysis: pacret-state +// CHECK-NEXT: 00000000: paciasp # SrcSafetyAnalysis: src-state +// CHECK-NEXT: 00000004: stp x29, x30, [sp, #-0x10]! # SrcSafetyAnalysis: src-state +// CHECK-NEXT: 00000008: b [[BB1]] # SrcSafetyAnalysis: src-state // CHECK-NEXT: Successors: [[BB1]] // CHECK-EMPTY: // CHECK-NEXT: [[BB1]] (5 instructions, align : 1) // CHECK-NEXT: Predecessors: [[BB0]] -// CHECK-NEXT: 0000000c: autiza x0 # PacRetAnalysis: pacret-state -// CHECK-NEXT: 00000010: blr x0 # PacRetAnalysis: pacret-state -// CHECK-NEXT: 00000014: ldp x29, x30, [sp], #0x10 # PacRetAnalysis: pacret-state -// CHECK-NEXT: 00000018: autiasp # PacRetAnalysis: pacret-state -// CHECK-NEXT: 0000001c: ret # PacRetAnalysis: pacret-state +// CHECK-NEXT: 0000000c: autiza x0 # SrcSafetyAnalysis: src-state +// CHECK-NEXT: 00000010: blr x0 # SrcSafetyAnalysis: src-state +// CHECK-NEXT: 00000014: ldp x29, x30, [sp], #0x10 # SrcSafetyAnalysis: src-state +// CHECK-NEXT: 00000018: autiasp # SrcSafetyAnalysis: src-state +// CHECK-NEXT: 0000001c: ret # SrcSafetyAnalysis: src-state // CHECK-EMPTY: // CHECK-NEXT: DWARF CFI Instructions: // CHECK-NEXT: // CHECK-NEXT: End of Function "simple" // CHECK-EMPTY: -// PAUTH-NEXT: Found call inst: 00000000: blr x0 # PacRetAnalysis: pacret-state +// PAUTH-NEXT: Found call inst: 00000000: blr x0 # SrcSafetyAnalysis: src-state // PAUTH-NEXT: Call destination reg: X0 // PAUTH-NEXT: SafeToDerefRegs: W0 X0 W0_HI{{[ \t]*$}} -// CHECK-NEXT: Found RET inst: 00000000: ret # PacRetAnalysis: pacret-state +// CHECK-NEXT: Found RET inst: 00000000: ret # SrcSafetyAnalysis: src-state // CHECK-NEXT: RetReg: LR // CHECK-NEXT: Authenticated reg: (none) // CHECK-NEXT: SafeToDerefRegs: LR W30 W30_HI{{[ \t]*$}} @@ -127,11 +128,12 @@ clobber: // CHECK-LABEL:Analyzing in function clobber, AllocatorId 1 // ... -// CHECK: PacRetAnalysis::ComputeNext( mov w30, #0x0, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( ret x30, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: After PacRetAnalysis: +// CHECK: Running src register safety analysis... +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( mov w30, #0x0, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( ret x30, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: After src register safety analysis: // CHECK-NEXT: Binary Function "clobber" { // ... // CHECK: End of Function "clobber" @@ -139,15 +141,17 @@ clobber: // The above output was printed after first run of analysis // CHECK-EMPTY: -// CHECK-NEXT: Found RET inst: 00000000: ret # PacRetAnalysis: pacret-state +// CHECK-NEXT: Found RET inst: 00000000: ret # SrcSafetyAnalysis: src-state // CHECK-NEXT: RetReg: LR // CHECK-NEXT: Authenticated reg: (none) // CHECK-NEXT: SafeToDerefRegs: W30_HI{{[ \t]*$}} -// CHECK-NEXT: PacRetAnalysis::ComputeNext( mov w30, #0x0, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: PacRetAnalysis::ComputeNext( ret x30, pacret-state) -// CHECK-NEXT: .. result: (pacret-state) -// CHECK-NEXT: After detailed PacRetAnalysis: +// CHECK-EMPTY: +// CHECK-NEXT: Running detailed src register safety analysis... +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( mov w30, #0x0, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: SrcSafetyAnalysis::ComputeNext( ret x30, src-state) +// CHECK-NEXT: .. result: (src-state) +// CHECK-NEXT: After detailed src register safety analysis: // CHECK-NEXT: Binary Function "clobber" { // ... // CHECK: End of Function "clobber" @@ -156,7 +160,7 @@ clobber: // Iterating over the reports and attaching clobbering info: // CHECK-EMPTY: -// CHECK-NEXT: Attaching clobbering info to: 00000000: ret # PacRetAnalysis: pacret-state +// CHECK-NEXT: Attaching clobbering info to: 00000000: ret # SrcSafetyAnalysis: src-state // CHECK-LABEL:Analyzing in function main, AllocatorId 1 diff --git a/bolt/test/link_fdata.py b/bolt/test/link_fdata.py index 028823a69ce0..bcf9a777922d 100755 --- a/bolt/test/link_fdata.py +++ b/bolt/test/link_fdata.py @@ -8,6 +8,8 @@ """ import argparse +import os +import shutil import subprocess import sys import re @@ -84,8 +86,16 @@ exit("ERROR: unexpected input:\n%s" % line) # Read nm output: +is_llvm_nm = os.path.basename(os.path.realpath(shutil.which(args.nmtool))) == "llvm-nm" nm_output = subprocess.run( - [args.nmtool, "--defined-only", args.objfile], text=True, capture_output=True + [ + args.nmtool, + "--defined-only", + "--special-syms" if is_llvm_nm else "--synthetic", + args.objfile, + ], + text=True, + capture_output=True, ).stdout # Populate symbol map symbols = {} diff --git a/bolt/test/runtime/X86/Inputs/user_func_order.txt b/bolt/test/runtime/Inputs/user_func_order.txt similarity index 100% rename from bolt/test/runtime/X86/Inputs/user_func_order.txt rename to bolt/test/runtime/Inputs/user_func_order.txt diff --git a/bolt/test/runtime/RISCV/basic-instrumentation.s b/bolt/test/runtime/RISCV/basic-instrumentation.s new file mode 100644 index 000000000000..e926f98cef43 --- /dev/null +++ b/bolt/test/runtime/RISCV/basic-instrumentation.s @@ -0,0 +1,33 @@ +# REQUIRES: system-linux,bolt-runtime + +# RUN: %clang %cflags -Wl,-q -o %t.exe %s +# RUN: llvm-bolt --instrument --instrumentation-file=%t.fdata -o %t.instr %t.exe + +## Run the profiled binary and check that the profile reports at least that `f` +## has been called. +# RUN: rm -f %t.fdata +# RUN: %t.instr +# RUN: cat %t.fdata | FileCheck %s +# CHECK: f 0 0 1{{$}} + +## Check BOLT works with this profile +# RUN: llvm-bolt --data %t.fdata --reorder-blocks=cache -o %t.bolt %t.exe + + .text + .globl main + .type main, @function +main: + addi sp, sp, -8 + sd ra, 0(sp) + call f + ld ra, 0(sp) + addi sp, sp, 8 + li a0, 0 + ret + .size main, .-main + + .globl f + .type f, @function +f: + ret + .size f, .-f diff --git a/bolt/test/runtime/RISCV/instrumentation-ind-call.c b/bolt/test/runtime/RISCV/instrumentation-ind-call.c new file mode 100644 index 000000000000..1fd49a774c94 --- /dev/null +++ b/bolt/test/runtime/RISCV/instrumentation-ind-call.c @@ -0,0 +1,39 @@ +/* +REQUIRES: system-linux,bolt-runtime + +RUN: %clang %cflags %s -o %t.exe -Wl,-q + +RUN: llvm-bolt %t.exe --instrument --instrumentation-file=%t.fdata \ +RUN: -o %t.instrumented + +# Instrumented program needs to finish returning zero +RUN: %t.instrumented | FileCheck %s -check-prefix=CHECK-OUTPUT + +# Test that the instrumented data makes sense +RUN: llvm-bolt %t.exe -o %t.bolted --data %t.fdata \ +RUN: --reorder-blocks=ext-tsp --reorder-functions=hfsort+ \ +RUN: --print-only=main --print-finalized | FileCheck %s + +RUN: %t.bolted | FileCheck %s -check-prefix=CHECK-OUTPUT + +CHECK-OUTPUT: The sum is: 30 + +# Check that our indirect call has 1 hit recorded in the fdata file and that +# this was processed correctly by BOLT +CHECK: jalr a2 # CallProfile: 1 (0 misses) : +CHECK-NEXT: { add: 1 (0 misses) } +*/ + +#include + +typedef int (*func_ptr)(int, int); + +int add(int a, int b) { return a + b; } + +int main() { + func_ptr fun; + fun = add; + int sum = fun(10, 20); // indirect call to 'add' + printf("The sum is: %d\n", sum); + return 0; +} diff --git a/bolt/test/runtime/RISCV/lit.local.cfg b/bolt/test/runtime/RISCV/lit.local.cfg new file mode 100644 index 000000000000..c0627d905ab3 --- /dev/null +++ b/bolt/test/runtime/RISCV/lit.local.cfg @@ -0,0 +1,2 @@ +if config.host_arch not in ["riscv64"]: + config.unsupported = True diff --git a/bolt/test/runtime/X86/hugify.c b/bolt/test/runtime/X86/hugify.c deleted file mode 100644 index cfc0cb62652b..000000000000 --- a/bolt/test/runtime/X86/hugify.c +++ /dev/null @@ -1,27 +0,0 @@ -// Make sure BOLT correctly processes --hugify option - -#include - -int main(int argc, char **argv) { - printf("Hello world\n"); - return 0; -} - -/* -REQUIRES: system-linux,bolt-runtime - -RUN: %clang %cflags -no-pie %s -o %t.nopie.exe -Wl,-q -RUN: %clang %cflags -fpic -pie %s -o %t.pie.exe -Wl,-q - -RUN: llvm-bolt %t.nopie.exe --lite=0 -o %t.nopie --hugify -RUN: llvm-bolt %t.pie.exe --lite=0 -o %t.pie --hugify - -RUN: %t.nopie | FileCheck %s -check-prefix=CHECK-NOPIE - -CHECK-NOPIE: Hello world - -RUN: %t.pie | FileCheck %s -check-prefix=CHECK-PIE - -CHECK-PIE: Hello world - -*/ diff --git a/bolt/test/runtime/X86/user-func-reorder.c b/bolt/test/runtime/X86/user-func-reorder.c deleted file mode 100644 index fcb92bca1625..000000000000 --- a/bolt/test/runtime/X86/user-func-reorder.c +++ /dev/null @@ -1,44 +0,0 @@ -/* Checks that BOLT correctly processes a user-provided function list file, - * reorder functions according to this list, update hot_start and hot_end - * symbols and insert a function to perform hot text mapping during program - * startup. - */ -#include - -int foo(int x) { - return x + 1; -} - -int fib(int x) { - if (x < 2) - return x; - return fib(x - 1) + fib(x - 2); -} - -int bar(int x) { - return x - 1; -} - -int main(int argc, char **argv) { - printf("fib(%d) = %d\n", argc, fib(argc)); - return 0; -} - -/* -REQUIRES: system-linux,bolt-runtime - -RUN: %clang %cflags -no-pie %s -o %t.exe -Wl,-q - -RUN: llvm-bolt %t.exe --relocs=1 --lite --reorder-functions=user \ -RUN: --hugify --function-order=%p/Inputs/user_func_order.txt -o %t -RUN: llvm-nm --numeric-sort --print-armap %t | \ -RUN: FileCheck %s -check-prefix=CHECK-NM -RUN: %t 1 2 3 | FileCheck %s -check-prefix=CHECK-OUTPUT - -CHECK-NM: W __hot_start -CHECK-NM: T main -CHECK-NM-NEXT: T fib -CHECK-NM-NEXT: W __hot_end - -CHECK-OUTPUT: fib(4) = 3 -*/ diff --git a/bolt/test/runtime/hugify.c b/bolt/test/runtime/hugify.c new file mode 100644 index 000000000000..a4a718a1160d --- /dev/null +++ b/bolt/test/runtime/hugify.c @@ -0,0 +1,38 @@ +// Make sure BOLT correctly processes --hugify option + +#include + +int main(int argc, char **argv) { + printf("Hello world\n"); + return 0; +} + +/* +REQUIRES: system-linux,bolt-runtime + +RUN: %clang %cflags -no-pie %s -o %t.nopie.exe -Wl,-q +RUN: %clang %cflags -fpic %s -o %t.pie.exe -Wl,-q + +RUN: llvm-bolt %t.nopie.exe --lite=0 -o %t.nopie --hugify +RUN: llvm-bolt %t.pie.exe --lite=0 -o %t.pie --hugify + +RUN: llvm-nm --numeric-sort --print-armap %t.nopie | \ +RUN: FileCheck %s -check-prefix=CHECK-NM +RUN: %t.nopie | FileCheck %s -check-prefix=CHECK-NOPIE + +RUN: llvm-nm --numeric-sort --print-armap %t.pie | \ +RUN: FileCheck %s -check-prefix=CHECK-NM +RUN: %t.pie | FileCheck %s -check-prefix=CHECK-PIE + +CHECK-NM: W __hot_start +CHECK-NM-NEXT: T _start +CHECK-NM: T main +CHECK-NM: W __hot_end +CHECK-NM: t __bolt_hugify_start_program +CHECK-NM-NEXT: W __bolt_runtime_start + +CHECK-NOPIE: Hello world + +CHECK-PIE: Hello world + +*/ diff --git a/bolt/test/runtime/relative-vftable.cpp b/bolt/test/runtime/relative-vftable.cpp new file mode 100644 index 000000000000..10de94a2e228 --- /dev/null +++ b/bolt/test/runtime/relative-vftable.cpp @@ -0,0 +1,57 @@ +// Test BOLT is able to handle relative virtual function table, i.e., when +// code is compiled with `-fexperimental-relative-c++-abi-vtables`. + +// REQUIRES: system-linux + +// RUN: split-file %s %t +// RUN: %clang -fuse-ld=lld -o %t/main.so %t/tt.cpp %t/main.cpp -Wl,-q \ +// RUN: -fno-rtti -fexperimental-relative-c++-abi-vtables +// RUN: %t/main.so | FileCheck %s + +// CHECK: derived_foo +// CHECK-NEXT: derived_bar +// CHECK-NEXT: derived_goo + +// RUN: llvm-bolt %t/main.so -o %t/main.bolted.so --trap-old-code +// RUN: %t/main.bolted.so | FileCheck %s + +;--- tt.h +#include + +class Base { +public: + virtual void foo(); + virtual void bar(); + virtual void goo(); +}; + +class Derived : public Base { +public: + virtual void foo() override; + virtual void bar() override; + virtual void goo() override; +}; + +;--- tt.cpp +#include "tt.h" +void Derived::goo() { printf("derived_goo\n"); } + +;--- main.cpp +#include "tt.h" +#pragma clang optimize off + +void Base::foo() { printf("base_foo\n"); } +void Base::bar() { printf("base_bar\n"); } +void Base::goo() { printf("base_goo\n"); } + +void Derived::foo() { printf("derived_foo\n"); } +void Derived::bar() { printf("derived_bar\n"); } + +int main() { + Derived D; + Base *ptr = &D; + ptr->foo(); + ptr->bar(); + ptr->goo(); + return 0; +} diff --git a/bolt/test/runtime/user-func-reorder.c b/bolt/test/runtime/user-func-reorder.c new file mode 100644 index 000000000000..c9e12ea65262 --- /dev/null +++ b/bolt/test/runtime/user-func-reorder.c @@ -0,0 +1,54 @@ +/* Checks that BOLT correctly processes a user-provided function list file, + * reorder functions according to this list, update hot_start and hot_end + * symbols and insert a function to perform hot text mapping during program + * startup. + */ +#include + +int foo(int x) { return x + 1; } + +int fib(int x) { + if (x < 2) + return x; + return fib(x - 1) + fib(x - 2); +} + +int bar(int x) { return x - 1; } + +int main(int argc, char **argv) { + printf("fib(%d) = %d\n", argc, fib(argc)); + return 0; +} + +/* +REQUIRES: system-linux,bolt-runtime + +RUN: %clang %cflags -no-pie %s -o %t.exe -Wl,-q + +RUN: llvm-bolt %t.exe --relocs=1 --lite --reorder-functions=user \ +RUN: --hugify --function-order=%p/Inputs/user_func_order.txt -o %t +RUN: llvm-bolt %t.exe --relocs=1 --lite --reorder-functions=user \ +RUN: --function-order=%p/Inputs/user_func_order.txt -o %t.nohugify +RUN: llvm-nm --numeric-sort --print-armap %t | \ +RUN: FileCheck %s -check-prefix=CHECK-NM +RUN: %t 1 2 3 | FileCheck %s -check-prefix=CHECK-OUTPUT +RUN: llvm-nm --numeric-sort --print-armap %t.nohugify | \ +RUN: FileCheck %s -check-prefix=CHECK-NM-NOHUGIFY +RUN: %t.nohugify 1 2 3 | FileCheck %s -check-prefix=CHECK-OUTPUT-NOHUGIFY + + +CHECK-NM: W __hot_start +CHECK-NM: T main +CHECK-NM-NEXT: T fib +CHECK-NM-NEXT: W __hot_end +CHECK-NM: t __bolt_hugify_start_program +CHECK-NM-NEXT: W __bolt_runtime_start + +CHECK-NM-NOHUGIFY: W __hot_start +CHECK-NM-NOHUGIFY: T main +CHECK-NM-NOHUGIFY-NEXT: T fib +CHECK-NM-NOHUGIFY-NEXT: W __hot_end + +CHECK-OUTPUT: fib(4) = 3 +CHECK-OUTPUT-NOHUGIFY: fib(4) = 3 +*/ diff --git a/bolt/unittests/Core/MCPlusBuilder.cpp b/bolt/unittests/Core/MCPlusBuilder.cpp index a3113cab3d33..7016dec0e357 100644 --- a/bolt/unittests/Core/MCPlusBuilder.cpp +++ b/bolt/unittests/Core/MCPlusBuilder.cpp @@ -8,6 +8,7 @@ #ifdef AARCH64_AVAILABLE #include "AArch64Subtarget.h" +#include "MCTargetDesc/AArch64MCTargetDesc.h" #endif // AARCH64_AVAILABLE #ifdef X86_AVAILABLE @@ -19,6 +20,7 @@ #include "bolt/Rewrite/RewriteInstance.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/Support/TargetSelect.h" #include "gtest/gtest.h" @@ -70,16 +72,28 @@ struct MCPlusBuilderTester : public testing::TestWithParam { BC->MRI.get(), BC->STI.get()))); } + void assertRegMask(const BitVector &RegMask, + std::initializer_list ExpectedRegs) { + ASSERT_EQ(RegMask.count(), ExpectedRegs.size()); + for (MCPhysReg Reg : ExpectedRegs) + ASSERT_TRUE(RegMask[Reg]) << "Expected " << BC->MRI->getName(Reg) << "."; + } + + void assertRegMask(std::function FillRegMask, + std::initializer_list ExpectedRegs) { + BitVector RegMask(BC->MRI->getNumRegs()); + FillRegMask(RegMask); + assertRegMask(RegMask, ExpectedRegs); + } + void testRegAliases(Triple::ArchType Arch, uint64_t Register, - uint64_t *Aliases, size_t Count, + std::initializer_list ExpectedAliases, bool OnlySmaller = false) { if (GetParam() != Arch) GTEST_SKIP(); const BitVector &BV = BC->MIB->getAliases(Register, OnlySmaller); - ASSERT_EQ(BV.count(), Count); - for (size_t I = 0; I < Count; ++I) - ASSERT_TRUE(BV[Aliases[I]]); + assertRegMask(BV, ExpectedAliases); } char ElfBuf[sizeof(typename ELF64LE::Ehdr)] = {}; @@ -94,17 +108,15 @@ INSTANTIATE_TEST_SUITE_P(AArch64, MCPlusBuilderTester, ::testing::Values(Triple::aarch64)); TEST_P(MCPlusBuilderTester, AliasX0) { - uint64_t AliasesX0[] = {AArch64::W0, AArch64::W0_HI, - AArch64::X0, AArch64::W0_W1, - AArch64::X0_X1, AArch64::X0_X1_X2_X3_X4_X5_X6_X7}; - size_t AliasesX0Count = sizeof(AliasesX0) / sizeof(*AliasesX0); - testRegAliases(Triple::aarch64, AArch64::X0, AliasesX0, AliasesX0Count); + testRegAliases(Triple::aarch64, AArch64::X0, + {AArch64::W0, AArch64::W0_HI, AArch64::X0, AArch64::W0_W1, + AArch64::X0_X1, AArch64::X0_X1_X2_X3_X4_X5_X6_X7}); } TEST_P(MCPlusBuilderTester, AliasSmallerX0) { - uint64_t AliasesX0[] = {AArch64::W0, AArch64::W0_HI, AArch64::X0}; - size_t AliasesX0Count = sizeof(AliasesX0) / sizeof(*AliasesX0); - testRegAliases(Triple::aarch64, AArch64::X0, AliasesX0, AliasesX0Count, true); + testRegAliases(Triple::aarch64, AArch64::X0, + {AArch64::W0, AArch64::W0_HI, AArch64::X0}, + /*OnlySmaller=*/true); } TEST_P(MCPlusBuilderTester, AArch64_CmpJE) { @@ -155,6 +167,100 @@ TEST_P(MCPlusBuilderTester, AArch64_CmpJNE) { ASSERT_EQ(Label, BB->getLabel()); } +TEST_P(MCPlusBuilderTester, testAccessedRegsImplicitDef) { + if (GetParam() != Triple::aarch64) + GTEST_SKIP(); + + // adds x0, x5, #42 + MCInst Inst = MCInstBuilder(AArch64::ADDSXri) + .addReg(AArch64::X0) + .addReg(AArch64::X5) + .addImm(42) + .addImm(0); + + assertRegMask([&](BitVector &BV) { BC->MIB->getClobberedRegs(Inst, BV); }, + {AArch64::NZCV, AArch64::W0, AArch64::X0, AArch64::W0_HI, + AArch64::X0_X1_X2_X3_X4_X5_X6_X7, AArch64::W0_W1, + AArch64::X0_X1}); + + assertRegMask( + [&](BitVector &BV) { BC->MIB->getTouchedRegs(Inst, BV); }, + {AArch64::NZCV, AArch64::W0, AArch64::W5, AArch64::X0, AArch64::X5, + AArch64::W0_HI, AArch64::W5_HI, AArch64::X0_X1_X2_X3_X4_X5_X6_X7, + AArch64::X2_X3_X4_X5_X6_X7_X8_X9, AArch64::X4_X5_X6_X7_X8_X9_X10_X11, + AArch64::W0_W1, AArch64::W4_W5, AArch64::X0_X1, AArch64::X4_X5}); + + assertRegMask([&](BitVector &BV) { BC->MIB->getWrittenRegs(Inst, BV); }, + {AArch64::NZCV, AArch64::W0, AArch64::X0, AArch64::W0_HI}); + + assertRegMask([&](BitVector &BV) { BC->MIB->getUsedRegs(Inst, BV); }, + {AArch64::W5, AArch64::X5, AArch64::W5_HI}); + + assertRegMask([&](BitVector &BV) { BC->MIB->getSrcRegs(Inst, BV); }, + {AArch64::W5, AArch64::X5, AArch64::W5_HI}); +} + +TEST_P(MCPlusBuilderTester, testAccessedRegsImplicitUse) { + if (GetParam() != Triple::aarch64) + GTEST_SKIP(); + + // b.eq