Remove split compile by default (#1722)

The split compile option has shown nondeterministic behavior. Disabling it for now.

Authors:
  - Nicolas Blin (https://github.com/Kh4ster)

Approvers:
  - Ramakrishna Prabhu (https://github.com/ramakrishnap-nv)
  - Akif ÇÖRDÜK (https://github.com/akifcorduk)

URL: #1722

Author: Kh4ster

build.sh

@@ -14,7 +14,7 @@ ARGS=$*
 REPODIR=$(cd "$(dirname "$0")"; pwd)
 LIBCUOPT_BUILD_DIR=${LIBCUOPT_BUILD_DIR:=${REPODIR}/cpp/build}
 
-VALIDARGS="clean codegen libcuopt cuopt_grpc_server cuopt cuopt_server cuopt_sh_client docs deb -a -b -g -fsanitize -tsan -msan -v -l= --verbose-pdlp --build-lp-only  --no-fetch-rapids --skip-c-python-adapters --skip-tests-build --skip-routing-build --skip-grpc-build --skip-fatbin-write --host-lineinfo [--cmake-args=\\\"<args>\\\"] [--cache-tool=<tool>] --install --allgpuarch --ci-only-arch --show_depr_warn -h --help"
+VALIDARGS="clean codegen libcuopt cuopt_grpc_server cuopt cuopt_server cuopt_sh_client docs deb -a -b -g -fsanitize -tsan -msan -v -l= --verbose-pdlp --build-lp-only  --no-fetch-rapids --skip-c-python-adapters --skip-tests-build --skip-routing-build --skip-grpc-build --skip-fatbin-write --host-lineinfo --split-compile [--cmake-args=\\\"<args>\\\"] [--cache-tool=<tool>] --install --allgpuarch --ci-only-arch --show_depr_warn -h --help"
 HELP="$0 [<target> ...] [<flag> ...]
  where <target> is:
    clean            - remove all existing build artifacts and configuration (start over)
@@ -45,6 +45,7 @@ HELP="$0 [<target> ...] [<flag> ...]
    --skip-grpc-build    - skip building gRPC and protobuf components (auto-enabled with -tsan)
    --skip-fatbin-write      - skip the fatbin write
    --host-lineinfo           - build with debug line information for host code
+   --split-compile           - opt in to nvcc split compilation; builds may be nondeterministic
    --cache-tool=<tool> - pass the build cache tool (eg: ccache, sccache, distcc) that will be used
                       to speedup the build process.
    --cmake-args=\\\"<args>\\\"   - pass arbitrary list of CMake configuration options (escape all quotes in argument)
@@ -260,6 +261,11 @@ fi
 if hasArg --host-lineinfo; then
     HOST_LINEINFO=1
 fi
+if hasArg --split-compile; then
+    # nvcc split compilation can produce nondeterministic cuOpt builds, so keep it opt-in.
+    echo "WARNING: nvcc split compilation may produce nondeterministic cuOpt builds."
+    export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS:+${NVCC_PREPEND_FLAGS} }--split-compile=0"
+fi
 
 function contains_string {
     local search_string="$1"

cpp/CMakeLists.txt

@@ -190,7 +190,7 @@ endif ()
 # Add jobserver flags for parallel compilation if PARALLEL_LEVEL is set
 if (PARALLEL_LEVEL AND NOT "${PARALLEL_LEVEL}" STREQUAL "")
     message(STATUS "Enabling nvcc parallel compilation support")
-    list(APPEND CUOPT_CUDA_FLAGS --threads=0 --split-compile=0)
+    list(APPEND CUOPT_CUDA_FLAGS --threads=0)
     if (USE_NVCC_JOBSERVER AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0)
         message(STATUS "Enabling nvcc jobserver support (NVCC >= 13.0)")
         list(APPEND CUOPT_CUDA_FLAGS --jobserver)
@@ -495,12 +495,6 @@ set(CUOPT_SRC_FILES)
 set(MPS_FAST_SRC_FILES)
 add_subdirectory(src)
 
-# nvcc 13.0.3 ICE (signal 11) compiling sliding_window.cu with 7 GPU architectures;
-# --split-compile breaks the codegen into per-arch sub-jobs to avoid the crash
-set_source_files_properties(
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/routing/local_search/sliding_window.cu
-        PROPERTIES COMPILE_OPTIONS "--split-compile=0")
-
 if (HOST_LINEINFO)
     set_source_files_properties(${CUOPT_SRC_FILES} DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTIES COMPILE_OPTIONS "-g1")
 endif ()