From b8a7acb5f3ec8e98e637fc990c3004a51492de33 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 04:21:22 +0800 Subject: [PATCH 01/32] Support Windows DataLoader multiprocess (num_workers > 0) --- data_loader.cc | 248 ++++++++++++++++++ paddle/fluid/imperative/CMakeLists.txt | 8 +- paddle/fluid/imperative/data_loader.cc | 84 +++++- paddle/fluid/imperative/data_loader.h | 10 +- paddle/fluid/pybind/CMakeLists.txt | 3 +- paddle/fluid/pybind/imperative.cc | 2 - paddle/fluid/pybind/tensor.cc | 125 +++++++++ .../phi/core/memory/allocation/CMakeLists.txt | 3 +- .../core/memory/allocation/mmap_allocator.cc | 191 +++++++++++++- .../core/memory/allocation/mmap_allocator.h | 36 ++- python/paddle/base/core.py | 27 +- .../incubate/multiprocessing/reductions.py | 30 ++- .../paddle/io/dataloader/dataloader_iter.py | 8 +- python/paddle/io/dataloader/worker.py | 100 ++++--- python/paddle/io/multiprocess_utils.py | 19 +- python/paddle/io/reader.py | 8 - .../test_dataloader_windows_multiprocess.py | 185 +++++++++++++ 17 files changed, 962 insertions(+), 125 deletions(-) create mode 100644 data_loader.cc create mode 100644 test/legacy_test/test_dataloader_windows_multiprocess.py diff --git a/data_loader.cc b/data_loader.cc new file mode 100644 index 00000000000000..c7b6f96a5a9443 --- /dev/null +++ b/data_loader.cc @@ -0,0 +1,248 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/imperative/data_loader.h" + +#include + +#include + +#include "glog/logging.h" +#include "paddle/fluid/platform/enforce.h" +#include "paddle/phi/core/memory/allocation/mmap_allocator.h" + +#ifdef _WIN32 +#include +#include +#else +#include +#include +#endif + +namespace paddle::imperative { + +static std::map> load_process_pids; + +void SetLoadProcessPIDs(int64_t key, std::set pids) { + VLOG(3) << "DataLoader: set loader child process PID (" << key + << ", pid number: " << pids.size() << ")"; + load_process_pids[key] = pids; +} + +void EraseLoadProcessPIDs(int64_t key) { + auto it = load_process_pids.find(key); + // Note: Can not find key also possible + if (it != load_process_pids.end()) { + VLOG(3) << "Dygraph Data Loader: erase loader child process PID (" << key + << ")"; + load_process_pids.erase(it); + } else { + VLOG(3) << "Dygraph Data Loader: The dygraph loader (id: " << key + << ") you want erase does not exist."; + } +} + +#ifndef _WIN32 +// sigaction doc: http://man7.org/linux/man-pages/man2/sigaction.2.html +// sigemptyset doc: https://linux.die.net/man/3/sigemptyset +// siginfo_t doc: https://www.mkssoftware.com/docs/man5/siginfo_t.5.asp +// waitid doc: https://linux.die.net/man/2/waitid + +// clear mmap fds on signal handler, make sure mmap clear will be called +// on signal handling and no need to register mmap clear up handler on +// python side. If shared memory is not used Clear() will do nothing. +#define SIGNAL_HANDLE(SIGNAL) \ + do { \ + memory::allocation::MemoryMapFdSet::Instance().Clear(); \ + struct sigaction sa = {}; \ + sa.sa_handler = SIG_DFL; \ + sa.sa_flags = 0; \ + if (sigemptyset(&sa.sa_mask) != 0 || \ + sigaction(SIGNAL, &sa, nullptr) != 0) { \ + _exit(EXIT_FAILURE); \ + } else { \ + raise(SIGNAL); \ + } \ + } while (0) + +#define REGISTER_SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, ERROR_MSG) \ + static void HANDLER_NAME( \ + int sig UNUSED, siginfo_t *info UNUSED, void *ctx UNUSED) { \ + auto _w = \ + write(STDERR_FILENO, ERROR_MSG, sizeof(ERROR_MSG) / sizeof(char)); \ + (void)_w; \ + SIGNAL_HANDLE(SIGNAL); \ + } + +#define REGISTER_SPEC_SIGNAL_HANDLER(SIGNAL, HANDLER_NAME) \ + static void HANDLER_NAME(int sig, siginfo_t *info, void *ctx) { \ + if (info->si_pid == getppid()) { \ + _exit(EXIT_SUCCESS); \ + } \ + SIGNAL_HANDLE(SIGNAL); \ + } + +REGISTER_SIGNAL_HANDLER(SIGSEGV, + SIGSEGV_handler, + "ERROR: Unexpected segmentation fault encountered in " + "DataLoader workers.\n"); +REGISTER_SIGNAL_HANDLER( + SIGBUS, + SIGBUS_handler, + "ERROR: Unexpected BUS error encountered in DataLoader worker. " + "This might be caused by insufficient shared memory (shm), " + "please check whether use_shared_memory is set and storage space " + "in /dev/shm is enough\n"); +REGISTER_SIGNAL_HANDLER(SIGFPE, + SIGFPE_handler, + "ERROR: Unexpected floating-point exception " + "encountered in DataLoader worker.\n") +REGISTER_SPEC_SIGNAL_HANDLER(SIGTERM, SIGTERM_handler); + +static inline void setSignalHandler(int signal, + void (*handler)(int, siginfo_t *, void *), + struct sigaction *old_sa_ptr) { + struct sigaction sa; + sa.sa_sigaction = handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO | SA_NOCLDSTOP | SA_NODEFER; + if (sigemptyset(&sa.sa_mask) != 0 || + sigaction(signal, &sa, old_sa_ptr) != 0) { + PADDLE_THROW(common::errors::Fatal( + "An error occurred while setting handler for %s.", strsignal(signal))); + } +} +#endif + +void SetLoadProcessSignalHandler() { +#ifdef _WIN32 + // On Windows, VEH handler is not needed. The MemoryMapFdSet::Clear() + // is a no-op on Windows (named file mappings auto-clean when the last + // handle is closed). Any crash will be handled by the OS normally. + VLOG(3) << "DataLoader: VEH handler skipped on Windows"; +#else + setSignalHandler(SIGSEGV, &SIGSEGV_handler, nullptr); + setSignalHandler(SIGBUS, &SIGBUS_handler, nullptr); + setSignalHandler(SIGFPE, &SIGFPE_handler, nullptr); + setSignalHandler(SIGTERM, &SIGTERM_handler, nullptr); +#endif +} + +void ThrowErrorIfLoadProcessFailed() { +#ifdef _WIN32 + // On Windows, use WaitForSingleObject + GetExitCodeProcess + // to check for crashed workers. This is called periodically from Python. + int error = 0; + std::set *pids_set = nullptr; + pid_t process_pid = 0; + + for (auto &p : load_process_pids) { + pids_set = &(p.second); + for (auto pid_it = pids_set->begin(); pid_it != pids_set->end(); ++pid_it) { + process_pid = *pid_it; + VLOG(3) << "DataLoader: monitor loader child process " << process_pid; + + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, + FALSE, static_cast(process_pid)); + if (hProcess == NULL) { + // Process may have already exited; GetLastError() == ERROR_INVALID_PARAMETER + continue; + } + + DWORD exit_code = STILL_ACTIVE; + if (WaitForSingleObject(hProcess, 0) == WAIT_OBJECT_0) { + // Process has exited + GetExitCodeProcess(hProcess, &exit_code); + CloseHandle(hProcess); + + if (exit_code != 0 && exit_code != STILL_ACTIVE) { + // Exited with error (non-zero exit code) + pids_set->clear(); + PADDLE_THROW(common::errors::Fatal( + "DataLoader process (pid %ld) exited unexpectedly with code %lu. " + "Rerunning with num_workers=0 may give better error trace.", + process_pid, exit_code)); + } + } else { + CloseHandle(hProcess); + } + } + } +#else + int error = 0; + std::set *pids_set = nullptr; + pid_t process_pid = 0; + siginfo_t infop; + + for (auto &p : load_process_pids) { + pids_set = &(p.second); + for (auto pid_it = pids_set->begin(); pid_it != pids_set->end(); ++pid_it) { + process_pid = *pid_it; + // Use waitid rather than waitpid so that we can set NOWAIT, and that + // Python and other handlers can get whatever info they want about the + // child. + infop.si_pid = 0; + VLOG(3) << "DataLoader: monitor loader child process " << process_pid; + error = waitid(P_PID, process_pid, &infop, WEXITED | WNOHANG | WNOWAIT); + // ignore errors and case with no waitable child + if (error < 0 || infop.si_pid == 0) continue; + if (infop.si_code == CLD_EXITED && + infop.si_status != EXIT_SUCCESS) { // exit with error + pids_set->clear(); + PADDLE_THROW(common::errors::Fatal( + "DataLoader process (pid %ld) exited unexpectedly with code %d. " + "Error detailed are lost due to multiprocessing. Rerunning with:\n" + " 1. If run DataLoader by DataLoader.from_generator(...), run " + "with " + "DataLoader.from_generator(..., use_multiprocess=False) may give " + "better error trace.\n" + " 2. If run DataLoader by DataLoader(dataset, ...), run with " + "DataLoader(dataset, ..., num_workers=0) may give better error " + "trace", + process_pid, + infop.si_status)); + } else if (infop.si_code == CLD_KILLED || + infop.si_code == CLD_DUMPED) { // killed by signal + if (infop.si_status == SIGBUS) { + pids_set->clear(); + PADDLE_THROW(common::errors::Fatal( + "DataLoader process (pid %ld) exited is killed by signal: %s.\n" + " It may be caused by insufficient shared storage space. This " + "problem usually occurs when using docker as a development " + "environment.\n Please use command `df -h` to check the storage " + "space of `/dev/shm`. Shared storage space needs to be greater " + "than (DataLoader Num * DataLoader queue capacity * 1 batch data " + "size).\n You can solve this problem by increasing the shared " + "storage space or reducing the queue capacity appropriately.\n" + " 1. If run DataLoader by DataLoader.from_generator(...), queue " + "capacity is set by from_generator(..., capacity=xx, ...).\n" + " 2. If run DataLoader by DataLoader(dataset, ...), queue " + "capacity is set as 2 times of the max value of num_workers and " + "len(places).\n" + " 3. If run by DataLoader(dataset, ..., use_shared_memory=True)," + " set use_shared_memory=False for not using shared memory.", + process_pid, + strsignal(infop.si_status))); + } else { + PADDLE_THROW(common::errors::Fatal( + "DataLoader process (pid %ld) exited is killed by signal: %s.", + process_pid, + strsignal(infop.si_status))); + } + } + } + } +#endif +} + +} // namespace paddle::imperative diff --git a/paddle/fluid/imperative/CMakeLists.txt b/paddle/fluid/imperative/CMakeLists.txt index fc08c1a8b8dfbb..3cc768d05509c6 100644 --- a/paddle/fluid/imperative/CMakeLists.txt +++ b/paddle/fluid/imperative/CMakeLists.txt @@ -159,11 +159,11 @@ if(NOT WIN32) SRCS heter_ccl_context.cc DEPS phi tensor var_type_traits) endif() - cc_library( - data_loader - SRCS data_loader.cc - DEPS phi common) endif() +cc_library( + data_loader + SRCS data_loader.cc + DEPS phi common) if(WITH_GLOO) cc_library( imperative_gloo_context diff --git a/paddle/fluid/imperative/data_loader.cc b/paddle/fluid/imperative/data_loader.cc index 1ab4a620eb3e6f..2aa7bb21637cd3 100644 --- a/paddle/fluid/imperative/data_loader.cc +++ b/paddle/fluid/imperative/data_loader.cc @@ -12,12 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef _WIN32 - #include "paddle/fluid/imperative/data_loader.h" -#include -#include #include #include @@ -26,6 +22,14 @@ #include "paddle/fluid/platform/enforce.h" #include "paddle/phi/core/memory/allocation/mmap_allocator.h" +#ifdef _WIN32 +#include +#include +#else +#include +#include +#endif + namespace paddle::imperative { static std::map> load_process_pids; @@ -49,6 +53,7 @@ void EraseLoadProcessPIDs(int64_t key) { } } +#ifndef _WIN32 // sigaction doc: http://man7.org/linux/man-pages/man2/sigaction.2.html // sigemptyset doc: https://linux.die.net/man/3/sigemptyset // siginfo_t doc: https://www.mkssoftware.com/docs/man5/siginfo_t.5.asp @@ -117,16 +122,81 @@ static inline void setSignalHandler(int signal, "An error occurred while setting handler for %s.", strsignal(signal))); } } +#endif + +#ifdef _WIN32 +// Last-chance exception filter: logs crash info before OS terminates the process. +static LONG CALLBACK paddle_crash_filter(EXCEPTION_POINTERS *ep) { + fprintf(stderr, + "[PADDLE_CRASH] PID=%lu code=0x%08lX addr=%p\n", + GetCurrentProcessId(), + ep->ExceptionRecord->ExceptionCode, + ep->ExceptionRecord->ExceptionAddress); + fflush(stderr); + return EXCEPTION_EXECUTE_HANDLER; +} +#endif -// Note: maybe need to add other signal handler void SetLoadProcessSignalHandler() { +#ifdef _WIN32 + // Last-chance filter: logs crash details before OS terminates the process. + // This is the ONLY handler on Windows — VEH is removed because it could + // deadlock if MemoryMapFdSet::Clear() is called while the mutex is held. + SetUnhandledExceptionFilter(&paddle_crash_filter); + VLOG(3) << "DataLoader: crash handler registered"; +#else setSignalHandler(SIGSEGV, &SIGSEGV_handler, nullptr); setSignalHandler(SIGBUS, &SIGBUS_handler, nullptr); setSignalHandler(SIGFPE, &SIGFPE_handler, nullptr); setSignalHandler(SIGTERM, &SIGTERM_handler, nullptr); +#endif } void ThrowErrorIfLoadProcessFailed() { +#ifdef _WIN32 + // On Windows, use WaitForSingleObject + GetExitCodeProcess + // to check for crashed workers. This is called periodically from Python. + int error = 0; + std::set *pids_set = nullptr; + pid_t process_pid = 0; + + for (auto &p : load_process_pids) { + pids_set = &(p.second); + for (auto pid_it = pids_set->begin(); pid_it != pids_set->end(); ++pid_it) { + process_pid = *pid_it; + VLOG(3) << "DataLoader: monitor loader child process " << process_pid; + + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, + FALSE, static_cast(process_pid)); + if (hProcess == NULL) { + // Process may have already exited; GetLastError() == ERROR_INVALID_PARAMETER + continue; + } + + DWORD exit_code = STILL_ACTIVE; + if (WaitForSingleObject(hProcess, 0) == WAIT_OBJECT_0) { + // Process has exited + GetExitCodeProcess(hProcess, &exit_code); + CloseHandle(hProcess); + + if (exit_code != 0 && exit_code != STILL_ACTIVE) { + // Exited with error (non-zero exit code) + fprintf(stderr, + "[DL_WORKER_EXIT] pid=%lu exit_code=0x%08lX (%lu)\n", + process_pid, exit_code, exit_code); + fflush(stderr); + pids_set->clear(); + PADDLE_THROW(common::errors::Fatal( + "DataLoader process (pid %ld) exited unexpectedly with code 0x%lX (%lu). " + "Rerunning with num_workers=0 may give better error trace.", + process_pid, exit_code, exit_code)); + } + } else { + CloseHandle(hProcess); + } + } + } +#else int error = 0; std::set *pids_set = nullptr; pid_t process_pid = 0; @@ -190,8 +260,8 @@ void ThrowErrorIfLoadProcessFailed() { } } } +#endif } } // namespace paddle::imperative - -#endif + diff --git a/paddle/fluid/imperative/data_loader.h b/paddle/fluid/imperative/data_loader.h index e66a3b9edc3ffd..7a9f54f7a603a4 100644 --- a/paddle/fluid/imperative/data_loader.h +++ b/paddle/fluid/imperative/data_loader.h @@ -14,9 +14,13 @@ #pragma once -#ifndef _WIN32 - +#ifdef _WIN32 +#include +// pid_t is not defined on Windows; use int (Python's os.getpid() returns int) +using pid_t = int; +#else #include +#endif #include #include @@ -31,5 +35,3 @@ extern void ThrowErrorIfLoadProcessFailed(); } // namespace imperative } // namespace paddle - -#endif diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index 620ba61b5a3d03..f533f0637fbe9d 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -88,8 +88,9 @@ if(WITH_CUSTOM_DEVICE) endif() endif() +set(PYBIND_DEPS ${PYBIND_DEPS} data_loader) + if(NOT WIN32) - set(PYBIND_DEPS ${PYBIND_DEPS} data_loader) if(WITH_NCCL OR WITH_RCCL) set(PYBIND_DEPS ${PYBIND_DEPS} nccl_context) set(PYBIND_DEPS ${PYBIND_DEPS} heter_ccl_context) diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index 77ce400672033f..b7072fff57c597 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -478,7 +478,6 @@ static void VarBaseCopy(std::shared_ptr &src, // NOLINT void BindImperative(py::module *m_ptr) { auto &m = *m_ptr; -#ifndef _WIN32 // Dygraph DataLoader signal handler m.def("_set_process_pids", [](int64_t key, py::object &obj) { PADDLE_ENFORCE_EQ( @@ -611,7 +610,6 @@ void BindImperative(py::module *m_ptr) { memory::allocation::MemoryMapAllocationPool::Instance().SetMaxPoolSize( size); }); -#endif m.def("start_imperative_gperf_profiler", []() { imperative::StartProfile(); }); diff --git a/paddle/fluid/pybind/tensor.cc b/paddle/fluid/pybind/tensor.cc index e3948afd67f6e4..4261a220a5dc53 100644 --- a/paddle/fluid/pybind/tensor.cc +++ b/paddle/fluid/pybind/tensor.cc @@ -1559,6 +1559,131 @@ void BindTensor(pybind11::module &m) { // NOLINT .def("indices", [](const phi::SparseCooTensor &self) -> DenseTensor { return self.indices(); }); + + // Module-level wrappers for _share_filename etc. + // These bypass a pybind11 class method registration issue on Windows. + m.def("_share_filename", + [](DenseTensor &self, bool use_file_descriptor) { + if (!self.IsInitialized() || self.numel() == 0) + throw std::runtime_error( + "Tensor not initialized or numel is 0. could not pass to " + "shared memory. "); + + auto holder = self.Holder(); + PADDLE_ENFORCE_EQ( + phi::is_cpu_place(holder->place()) || + phi::is_cuda_pinned_place(holder->place()), + true, common::errors::InvalidArgument( + "Tensor is not on CPU. share_filename only " + "support CPU Tensor.")); + + auto *mmap_allocation = dynamic_cast< + memory::allocation::RefcountedMemoryMapAllocation *>( + holder.get()); + if (mmap_allocation == nullptr) { + void *data_ptr = self.data(); + size_t data_size = + self.numel() * + framework::SizeOfType( + framework::TransToProtoVarType(self.type())); + + int flags = memory::allocation::MAPPED_SHAREDMEM | + memory::allocation::MAPPED_EXCLUSIVE; + if (use_file_descriptor) { + flags = flags | memory::allocation::MAPPED_KEEPFD | + memory::allocation::MAPPED_UNLINK; + } + std::string handle = memory::allocation::GetIPCName(); + int find_id = -1; + if (FLAGS_use_shm_cache) { + find_id = memory::allocation::MemoryMapAllocationPool::Instance().FindFromCache(flags, data_size); // NOLINT + } + if (find_id != -1) { + handle = memory::allocation::MemoryMapAllocationPool::Instance().GetById(find_id).file_name_; // NOLINT + } + int shared_fd = -1; + auto shared_holder = + memory::allocation::AllocateRefcountedMemoryMapAllocation( + handle, shared_fd, flags, data_size, find_id); + + // On Windows, GPUPinnedPlace copy is not needed (non-CUDA memory sharing) +#ifndef _WIN32 + if (phi::is_cuda_pinned_place(holder->place())) { +#ifdef PADDLE_WITH_CUDA + memory::Copy(CPUPlace(), shared_holder->ptr(), + phi::GPUPinnedPlace(), data_ptr, data_size); +#endif + } else +#endif + { + memory::Copy(CPUPlace(), shared_holder->ptr(), + CPUPlace(), data_ptr, data_size); + } + self.ResetHolder(shared_holder); + mmap_allocation = shared_holder.get(); + } + int type_idx = static_cast(self.type()); + + return py::make_tuple(mmap_allocation->ipc_name(), + mmap_allocation->shared_fd(), + mmap_allocation->size(), type_idx, + common::vectorize(self.dims()), self.lod(), + use_file_descriptor); + }); + + m.def("_new_shared_filename", + [](py::tuple t) { + if (t.size() != 7) + throw std::runtime_error("Invalid Tensor meta info state!"); + + DenseTensor tensor; + + const std::string &ipc_name = t[0].cast(); + const int shared_fd = t[1].cast(); + const bool use_file_descriptor = t[6].cast(); + + size_t size = t[2].cast(); + int flags = memory::allocation::MAPPED_SHAREDMEM | + memory::allocation::MAPPED_NOCREATE; + if (use_file_descriptor) { + flags = flags | memory::allocation::MAPPED_KEEPFD | + memory::allocation::MAPPED_UNLINK; + } + int find_id = -1; + if (FLAGS_use_shm_cache) { + find_id = memory::allocation::MemoryMapAllocationPool::Instance().FindFromCache(flags, size, ipc_name, false); // NOLINT + } + auto shared_holder = + memory::allocation::AllocateRefcountedMemoryMapAllocation( + ipc_name, shared_fd, flags, size, find_id); + + tensor.ResetHolderWithType( + shared_holder, + static_cast(t[3].cast())); + tensor.Resize(common::make_ddim(t[4].cast>())); + + return tensor; + }); + + m.def("_shared_incref", + [](DenseTensor &self) { + auto *mmap_allocation = dynamic_cast< + memory::allocation::RefcountedMemoryMapAllocation *>( + self.Holder().get()); + if (mmap_allocation) { + mmap_allocation->incref(); + } + }); + + m.def("_shared_decref", + [](DenseTensor &self) { + auto *mmap_allocation = dynamic_cast< + memory::allocation::RefcountedMemoryMapAllocation *>( + self.Holder().get()); + if (mmap_allocation) { + mmap_allocation->decref(); + } + }); } } // namespace paddle::pybind diff --git a/paddle/phi/core/memory/allocation/CMakeLists.txt b/paddle/phi/core/memory/allocation/CMakeLists.txt index ad327fa68feeed..cbddc59239cd88 100644 --- a/paddle/phi/core/memory/allocation/CMakeLists.txt +++ b/paddle/phi/core/memory/allocation/CMakeLists.txt @@ -36,7 +36,6 @@ if(CUDA_VERSION VERSION_GREATER_EQUAL 10.2) endif() if(NOT WIN32) - list(APPEND ALLOCATOR_SRCS mmap_allocator.cc) if(WITH_GPU) list(APPEND ALLOCATOR_SRCS cuda_ipc_allocator.cc) endif() @@ -45,6 +44,8 @@ if(NOT WIN32) endif() endif() +list(APPEND ALLOCATOR_SRCS mmap_allocator.cc) + if(WITH_CUSTOM_DEVICE) list(APPEND ALLOCATOR_SRCS custom_allocator.cc stream_safe_custom_device_allocator.cc) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 252a4aa7479bdc..ff6e2547f1b5b0 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -12,12 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef _WIN32 - #include "paddle/phi/core/memory/allocation/mmap_allocator.h" -#include -#include #include #include @@ -28,6 +24,13 @@ #include "paddle/common/flags.h" #include "paddle/phi/core/enforce.h" +#ifdef _WIN32 +#include +#else +#include +#include +#endif + COMMON_DECLARE_bool(use_shm_cache); namespace paddle::memory::allocation { @@ -52,12 +55,81 @@ struct CountInfo { std::atomic refcount; }; -void AllocateMemoryMap(std::string filename, +void AllocateMemoryMap(std::string& filename, int *shared_fd, int flags, size_t size, void **map_ptr_) { - // TODO(@ZHUI): support win32 +#ifdef _WIN32 + DWORD protect = (flags & MAPPED_SHAREDMEM) ? PAGE_READWRITE : PAGE_READONLY; + // Retry loop for exclusive mode: on Windows, CreateFileMapping silently + // returns the existing mapping if the name already exists, unlike Linux's + // shm_open + O_EXCL which fails atomically. We must check explicitly. + for (int attempt = 0; attempt < 100; attempt++) { + HANDLE hMap = CreateFileMappingA( + INVALID_HANDLE_VALUE, NULL, protect, 0, + static_cast(size), filename.c_str()); + + if (hMap == NULL) { + DWORD err = GetLastError(); + // Write diagnostic to stderr before throwing + fprintf(stderr, + "[PADDLE_MMAP] PID=%lu CreateFileMapping FAILED " + "name=%s size=%zu flags=0x%x attempt=%d err=%lu\n", + GetCurrentProcessId(), filename.c_str(), size, flags, attempt, err); + fflush(stderr); + PADDLE_THROW(common::errors::Unavailable( + "CreateFileMapping failed for %s, error: %lu", + filename.c_str(), err)); + } + + if ((flags & MAPPED_EXCLUSIVE) && + GetLastError() == ERROR_ALREADY_EXISTS) { + CloseHandle(hMap); + VLOG(3) << "[PADDLE_MMAP] PID=" << GetCurrentProcessId() + << " name collision, retrying attempt=" << attempt + << " name=" << filename; + filename = GetIPCName(); // name collision; retry with fresh name + continue; + } + + DWORD access = FILE_MAP_ALL_ACCESS; + *map_ptr_ = MapViewOfFile(hMap, access, 0, 0, size); + if (*map_ptr_ == nullptr) { + DWORD err = GetLastError(); + CloseHandle(hMap); + fprintf(stderr, + "[PADDLE_MMAP] PID=%lu MapViewOfFile FAILED " + "name=%s size=%zu err=%lu\n", + GetCurrentProcessId(), filename.c_str(), size, err); + fflush(stderr); + PADDLE_THROW(common::errors::Unavailable( + "MapViewOfFile failed for %s, error: %lu", + filename.c_str(), err)); + } + + // On Windows, always keep the HANDLE so the section stays alive + // after the caller's MapViewOfFile. The handle is closed in + // RefcountedMemoryMapAllocation::close(). Without this, the section + // would be destroyed when the last view is unmapped (i.e. when the + // worker's tensor is GC'd), before the reader opens it — this is + // the root cause of "Blocking queue is killed" with large data. + // Linux doesn't have this issue because munmap never destroys the + // shared memory file (only shm_unlink does). + *shared_fd = reinterpret_cast(hMap); + + if (flags & MAPPED_UNLINK) { + VLOG(6) << "CreateFileMapping (unlink mode): " << filename; + } + + // Caller (AllocateRefcountedMemoryMapAllocation) handles Insert + return; + } + + PADDLE_THROW(common::errors::Unavailable( + "Failed to allocate exclusive shared memory after 100 retries")); +#else + // Linux implementation using shm_open + mmap int file_flags = 0; int fd = *shared_fd; if (flags & MAPPED_SHAREDMEM) { @@ -114,9 +186,9 @@ void AllocateMemoryMap(std::string filename, -1, common::errors::Unavailable( "Error closing memory mapped file %s", filename)); - *shared_fd = -1; } +#endif } std::shared_ptr @@ -136,6 +208,7 @@ AllocateRefcountedMemoryMapAllocation(std::string filename, } void *aligned_base_ptr = static_cast(static_cast(base_ptr) + mmap_alignment); + MemoryMapFdSet::Instance().Insert(filename); return std::make_shared( aligned_base_ptr, size, filename, fd, flags, buffer_id); } @@ -159,12 +232,21 @@ RefcountedMemoryMapAllocation::RefcountedMemoryMapAllocation( void MemoryMapAllocation::close() { if (!closed_fd_) { closed_fd_ = true; +#ifdef _WIN32 + // On Windows, the HANDLE from CreateFileMapping is always kept + // (never closed in AllocateMemoryMap). Close it unconditionally + // to prevent handle leaks. + if (fd_ != -1) { + CloseHandle(reinterpret_cast(static_cast(fd_))); + } +#else if (flags_ & MAPPED_KEEPFD) { PADDLE_ENFORCE_NE(::close(fd_), -1, common::errors::Unavailable( "Error closing file descriptor <%d>", fd_)); } +#endif } if (closed_) { return; @@ -209,6 +291,17 @@ void RefcountedMemoryMapAllocation::close() { void *data = map_ptr_; CountInfo *info = reinterpret_cast(data); --info->refcount; + +#ifdef _WIN32 + // On Windows, the HANDLE from CreateFileMapping is always kept by + // AllocateMemoryMap (never closed there). Close it unconditionally + // here, regardless of the MAPPED_KEEPFD flag. + if (fd_ != -1 && !closed_fd_) { + closed_fd_ = true; + CloseHandle(reinterpret_cast(static_cast(fd_))); + VLOG(6) << "close handle: " << fd_; + } +#else if (flags_ & MAPPED_KEEPFD) { closed_fd_ = true; PADDLE_ENFORCE_NE( @@ -217,6 +310,7 @@ void RefcountedMemoryMapAllocation::close() { common::errors::Unavailable("Error closing file descriptor <%d>", fd_)); VLOG(6) << "close fd: " << fd_; } +#endif if (FLAGS_use_shm_cache && buffer_id_ != -1) { return; @@ -229,28 +323,51 @@ void RefcountedMemoryMapAllocation::close() { flags_, map_size_ - mmap_alignment, ipc_name_, map_ptr_)); } else { if (info->refcount == 0) { +#ifdef _WIN32 + // Only unmap when refcount reaches 0 (all readers AND writers are + // done). When refcount > 0, the view must stay mapped on Windows + // to keep the named file mapping section alive so readers can open + // it. On Linux this is unnecessary because munmap doesn't destroy + // the shared memory file (only shm_unlink does). + VLOG(6) << "UnmapViewOfFile: " << ipc_name_; + UnmapViewOfFile(map_ptr_); +#else shm_unlink(ipc_name_.c_str()); VLOG(6) << "shm_unlink file: " << ipc_name_; +#endif } - +#ifndef _WIN32 + // On Linux, munmap is always safe since it only unmaps virtual memory; + // the shared memory file in /dev/shm persists until shm_unlink. PADDLE_ENFORCE_NE(munmap(map_ptr_, map_size_), -1, common::errors::Unavailable( "could not unmap the shared memory file: %s (%d)", strerror(errno), errno)); +#endif } } } MemoryMapWriterAllocation::~MemoryMapWriterAllocation() { +#ifdef _WIN32 + UnmapViewOfFile(this->ptr()); +#else if (munmap(this->ptr(), this->size()) == -1) { common::errors::Unavailable("could not unmap the shared memory file %s", this->ipc_name()); } +#endif } MemoryMapReaderAllocation::~MemoryMapReaderAllocation() { +#ifdef _WIN32 + UnmapViewOfFile(this->ptr()); + // On Windows, named file mapping is auto-destroyed when final handle is closed. + MemoryMapFdSet::Instance().Remove(this->ipc_name()); + VLOG(3) << "~MemoryMapReaderAllocation: " << this->ipc_name(); +#else if (munmap(this->ptr(), this->size()) == -1) { common::errors::Unavailable("could not unmap the shared memory file %s", this->ipc_name()); @@ -276,11 +393,29 @@ MemoryMapReaderAllocation::~MemoryMapReaderAllocation() { shm_unlink(this->ipc_name().c_str()); MemoryMapFdSet::Instance().Remove(this->ipc_name()); VLOG(3) << "~MemoryMapReaderAllocation: " << this->ipc_name(); +#endif } std::shared_ptr AllocateMemoryMapWriterAllocation( size_t size) { const std::string &ipc_name = GetIPCName(); +#ifdef _WIN32 + HANDLE hMap = CreateFileMappingA( + INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, static_cast(size), ipc_name.c_str()); + PADDLE_ENFORCE_NE(hMap, + nullptr, + common::errors::Unavailable( + "CreateFileMapping for writer %s failed", ipc_name.c_str())); + + void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); + PADDLE_ENFORCE_NE(ptr, + nullptr, + common::errors::Unavailable( + "MapViewOfFile for writer %s failed", ipc_name.c_str())); + // Close the handle; the named section stays alive through MapViewOfFile. + CloseHandle(hMap); + return std::make_shared(ptr, size, ipc_name); +#else int flags = O_RDWR | O_CREAT; int fd = shm_open(ipc_name.c_str(), flags, 0600); @@ -301,10 +436,34 @@ std::shared_ptr AllocateMemoryMapWriterAllocation( close(fd); return std::make_shared(ptr, size, ipc_name); +#endif } std::shared_ptr RebuildMemoryMapReaderAllocation( const std::string &ipc_name, size_t size) { +#ifdef _WIN32 + HANDLE hMap = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, ipc_name.c_str()); + if (!hMap) { + // Section doesn't exist yet; create it. This can happen in some edge cases + // where the writer's view is still alive but all handles were closed. + hMap = CreateFileMappingA( + INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, static_cast(size), ipc_name.c_str()); + } + PADDLE_ENFORCE_NE(hMap, + nullptr, + common::errors::Unavailable( + "CreateFileMapping/OpenFileMapping for reader %s failed, error: %lu", + ipc_name.c_str(), GetLastError())); + + void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); + PADDLE_ENFORCE_NE(ptr, + nullptr, + common::errors::Unavailable( + "MapViewOfFile for reader %s failed, error: %lu", + ipc_name.c_str(), GetLastError())); + CloseHandle(hMap); + return std::make_shared(ptr, size, ipc_name); +#else int flags = O_RDWR | O_CREAT; flags &= ~O_CREAT; int fd = shm_open(ipc_name.c_str(), flags, 0600); @@ -319,6 +478,7 @@ std::shared_ptr RebuildMemoryMapReaderAllocation( "Memory map failed when rebuild shared memory.")); close(fd); return std::make_shared(ptr, size, ipc_name); +#endif } MemoryMapFdSet &MemoryMapFdSet::Instance() { // NOLINT @@ -345,18 +505,22 @@ void MemoryMapFdSet::Clear() { << fd_set_.size(); std::lock_guard guard(mtx_); for (auto const &fd : fd_set_) { +#ifdef _WIN32 + // On Windows, named file mappings are auto-destroyed when the last view + // is unmapped. Nothing to unlink explicitly. + VLOG(7) << "PID: " << getpid() << ", MemoryMapFdSet: clear " << fd; +#else int rlt = shm_unlink(fd.c_str()); if (rlt == 0) { VLOG(7) << "PID: " << getpid() << ", MemoryMapFdSet: clear " << fd; } +#endif } fd_set_.clear(); } MemoryMapFdSet::~MemoryMapFdSet() { Clear(); } -MemoryMapAllocationPool *MemoryMapAllocationPool::pool_ = nullptr; - void MemoryMapAllocationPool::Insert(const MemoryMapInfo &memory_map) { std::lock_guard guard(mtx_); memory_map_allocations_.push_back(memory_map); @@ -399,6 +563,10 @@ void MemoryMapAllocationPool::SetMaxPoolSize(const int &size) { void MemoryMapAllocationPool::Clear() { std::lock_guard guard(mtx_); for (auto const &mmap : memory_map_allocations_) { +#ifdef _WIN32 + VLOG(4) << "MemoryMapAllocationPool: clear " << mmap.file_name_; + UnmapViewOfFile(mmap.mmap_ptr_); +#else int rlt = shm_unlink(mmap.file_name_.c_str()); if (rlt == 0) { VLOG(4) << "MemoryMapAllocationPool: clear " << mmap.file_name_; @@ -409,6 +577,7 @@ void MemoryMapAllocationPool::Clear() { "could not unmap the shared memory file: %s (%d)", strerror(errno), errno)); +#endif } memory_map_allocations_.clear(); } @@ -416,5 +585,3 @@ void MemoryMapAllocationPool::Clear() { MemoryMapAllocationPool::~MemoryMapAllocationPool() { Clear(); } // NOLINT } // namespace paddle::memory::allocation - -#endif diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index cb330eff8eb6e2..5d19297c33d82b 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -14,8 +14,6 @@ #pragma once -#ifndef _WIN32 - #include #include #include @@ -24,12 +22,13 @@ #include #include "paddle/phi/core/memory/allocation/allocator.h" +#include "paddle/phi/api/include/dll_decl.h" namespace paddle { namespace memory { namespace allocation { -std::string GetIPCName(); +PADDLE_API std::string GetIPCName(); static constexpr int64_t mmap_alignment = 64; @@ -42,7 +41,7 @@ enum MappedModes { MAPPED_UNLINK = 32 }; -class MemoryMapAllocation : public Allocation { +class PADDLE_API MemoryMapAllocation : public Allocation { public: explicit MemoryMapAllocation(void *ptr, size_t size, @@ -79,7 +78,7 @@ class MemoryMapAllocation : public Allocation { bool closed_fd_ = false; }; -class RefcountedMemoryMapAllocation : public MemoryMapAllocation { +class PADDLE_API RefcountedMemoryMapAllocation : public MemoryMapAllocation { public: RefcountedMemoryMapAllocation(void *ptr, size_t size, @@ -99,20 +98,20 @@ class RefcountedMemoryMapAllocation : public MemoryMapAllocation { void resetBaseptr(); }; -void AllocateMemoryMap(std::string filename, +PADDLE_API void AllocateMemoryMap(std::string &filename, int *shared_fd, int flags, size_t size, void **base_ptr_); -std::shared_ptr +PADDLE_API std::shared_ptr AllocateRefcountedMemoryMapAllocation(std::string filename, int shared_fd, int flags, size_t size, int buffer_id = -1); -class MemoryMapWriterAllocation : public Allocation { +class PADDLE_API MemoryMapWriterAllocation : public Allocation { public: explicit MemoryMapWriterAllocation(void *ptr, size_t size, @@ -129,7 +128,7 @@ class MemoryMapWriterAllocation : public Allocation { int fd_ = -1; }; -class MemoryMapReaderAllocation : public Allocation { +class PADDLE_API MemoryMapReaderAllocation : public Allocation { public: explicit MemoryMapReaderAllocation(void *ptr, size_t size, @@ -146,13 +145,13 @@ class MemoryMapReaderAllocation : public Allocation { int fd_ = -1; }; -std::shared_ptr AllocateMemoryMapWriterAllocation( +PADDLE_API std::shared_ptr AllocateMemoryMapWriterAllocation( size_t size); -std::shared_ptr RebuildMemoryMapReaderAllocation( +PADDLE_API std::shared_ptr RebuildMemoryMapReaderAllocation( const std::string &ipc_name, size_t size); -class MemoryMapFdSet { +class PADDLE_API MemoryMapFdSet { public: static MemoryMapFdSet &Instance(); // NOLINT @@ -197,13 +196,11 @@ created by the _share_filename process will be cached and reused according to the data_size of shm, thus eliminating the problem of munmap blocking other threads */ -class MemoryMapAllocationPool { +class PADDLE_API MemoryMapAllocationPool { public: static MemoryMapAllocationPool &Instance() { - if (pool_ == nullptr) { - pool_ = new MemoryMapAllocationPool(); - } - return *pool_; + static MemoryMapAllocationPool pool; + return pool; } void Insert(const MemoryMapInfo &memory_map); @@ -227,7 +224,8 @@ class MemoryMapAllocationPool { private: MemoryMapAllocationPool() = default; - static MemoryMapAllocationPool *pool_; + MemoryMapAllocationPool(const MemoryMapAllocationPool&) = delete; + MemoryMapAllocationPool& operator=(const MemoryMapAllocationPool&) = delete; std::vector memory_map_allocations_; int max_pool_size_ = 0; std::mutex mtx_; @@ -236,5 +234,3 @@ class MemoryMapAllocationPool { } // namespace allocation } // namespace memory } // namespace paddle - -#endif diff --git a/python/paddle/base/core.py b/python/paddle/base/core.py index c728f498c775a7..72a45460ae6127 100644 --- a/python/paddle/base/core.py +++ b/python/paddle/base/core.py @@ -362,18 +362,21 @@ def to_list(s): # type promotion # isort: on - if sys.platform != 'win32': - from .libpaddle import ( # noqa: F401 - _array_to_share_memory_tensor, - _cleanup_mmap_fds, - _convert_to_tensor_list, - _erase_process_pids, - _remove_tensor_list_mmap_fds, - _set_max_memory_map_allocation_pool_size, - _set_process_pids, - _set_process_signal_handler, - _throw_error_if_process_failed, - ) + from .libpaddle import ( # noqa: F401 + _array_to_share_memory_tensor, + _cleanup_mmap_fds, + _convert_to_tensor_list, + _erase_process_pids, + _remove_tensor_list_mmap_fds, + _set_max_memory_map_allocation_pool_size, + _set_process_pids, + _set_process_signal_handler, + _share_filename, + _new_shared_filename, + _shared_incref, + _shared_decref, + _throw_error_if_process_failed, + ) except Exception as e: if has_paddle_dy_lib: diff --git a/python/paddle/incubate/multiprocessing/reductions.py b/python/paddle/incubate/multiprocessing/reductions.py index c6fde04319ba2a..c40c405e76d69c 100644 --- a/python/paddle/incubate/multiprocessing/reductions.py +++ b/python/paddle/incubate/multiprocessing/reductions.py @@ -14,6 +14,7 @@ import copy import multiprocessing +import sys # TODO: check the hooks of tensor # TODO: check serializing named tensor @@ -25,6 +26,7 @@ from multiprocessing.util import register_after_fork import paddle +import paddle.base.core as core def _supported_check(): @@ -135,7 +137,7 @@ def _rebuild_lodtensor_filename( lod, dataloader_use_file_descriptor, ): - lodtensor = cls._new_shared_filename( + lodtensor = core._new_shared_filename( ( ipc_name, shared_fd, @@ -146,7 +148,7 @@ def _rebuild_lodtensor_filename( dataloader_use_file_descriptor, ) ) - lodtensor._shared_decref() + core._shared_decref(lodtensor) return lodtensor @@ -161,7 +163,7 @@ def _rebuild_lodtensor_filedescriptor( dataloader_use_file_descriptor, ): shared_fd = shared_fd.detach() - lodtensor = cls._new_shared_filename( + lodtensor = core._new_shared_filename( ( ipc_name, shared_fd, @@ -172,7 +174,7 @@ def _rebuild_lodtensor_filedescriptor( dataloader_use_file_descriptor, ) ) - lodtensor._shared_decref() + core._shared_decref(lodtensor) return lodtensor @@ -241,9 +243,17 @@ def _reduce_lodtensor(lodtensor): "FLAGS_dataloader_use_file_descriptor" ] # Default use share filename strategy - metadata = lodtensor._share_filename( - dataloader_use_file_descriptor - ) # ipc_name, fd, size, type_idx, dims, lod + try: + metadata = core._share_filename( + lodtensor, dataloader_use_file_descriptor + ) # ipc_name, fd, size, type_idx, dims, lod + except Exception as e: + # Log and re-raise. This exception happens in the Queue feeder + # thread; logging here helps diagnose intermittent crashes. + sys.stderr.write( + f"[REDUCE] _share_filename failed: {type(e).__name__}: {e}\n") + sys.stderr.flush() + raise if dataloader_use_file_descriptor: metalist = list(metadata) @@ -252,8 +262,7 @@ def _reduce_lodtensor(lodtensor): rebuild = _rebuild_lodtensor_filedescriptor else: rebuild = _rebuild_lodtensor_filename - lodtensor._shared_incref() - # TODO, maintain reference for lodtensor + core._shared_incref(lodtensor) elif lodtensor._place().is_gpu_place(): prev_id = paddle.base.core.get_cuda_current_device_id() cur_id = lodtensor._place().gpu_device_id() @@ -280,9 +289,6 @@ def _reduce_lodtensor(lodtensor): def init_reductions() -> None: - if not _supported_check(): - return - ForkingPickler.register(paddle.Tensor, _reduce_tensor) ForkingPickler.register(paddle.base.core.eager.Tensor, _reduce_tensor) ForkingPickler.register( diff --git a/python/paddle/io/dataloader/dataloader_iter.py b/python/paddle/io/dataloader/dataloader_iter.py index 7a5b3c5d04015f..dd00d2e36264e2 100644 --- a/python/paddle/io/dataloader/dataloader_iter.py +++ b/python/paddle/io/dataloader/dataloader_iter.py @@ -407,7 +407,9 @@ def __init__(self, loader): # see _try_put_indices self._thread_lock = threading.Lock() - self._base_seed = np.random.randint(low=0, high=sys.maxsize) + self._base_seed = np.random.randint( + low=0, high=np.iinfo(np.int32).max + ) # Note(zhangbo): shm_buffer_size is used for MemoryMapAllocationPool. # MemoryMapAllocationPool is used to cache and reuse shm, thus reducing munmap in dataloader. @@ -486,6 +488,10 @@ def _init_workers(self): ) worker.daemon = True worker.start() + # On Windows with spawn, each worker imports the full Paddle + # framework (~580 MB). Stagger to avoid memory exhaustion. + if sys.platform == 'win32' and self._num_workers > 4: + time.sleep(0.05) self._workers.append(worker) self._worker_status.append(True) diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 94013b4c3ec5b7..5f79d3899d21fe 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -69,10 +69,32 @@ class ParentWatchDog: def __init__(self): self._parent_pid = os.getppid() self._parent_alive = True + if sys.platform == 'win32': + self._parent_alive = self._parent_alive_win32() + else: + self._parent_alive = True + + def _parent_alive_win32(self): + import ctypes + from ctypes import wintypes + PROCESS_QUERY_INFORMATION = 0x0400 + STILL_ACTIVE = 259 + handle = ctypes.windll.kernel32.OpenProcess( + PROCESS_QUERY_INFORMATION, False, self._parent_pid + ) + if not handle: + return False + exit_code = wintypes.DWORD(0) + ctypes.windll.kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)) + ctypes.windll.kernel32.CloseHandle(handle) + return exit_code.value == STILL_ACTIVE def is_alive(self): if self._parent_alive: - self._parent_alive = os.getppid() == self._parent_pid + if sys.platform == 'win32': + self._parent_alive = self._parent_alive_win32() + else: + self._parent_alive = os.getppid() == self._parent_pid return self._parent_alive @@ -292,6 +314,12 @@ def _worker_loop( shm_cache_size=0, ): try: + import faulthandler + faulthandler.enable() + # Register ForkingPickler handlers so DenseTensor can be serialized + from paddle.incubate.multiprocessing import init_reductions + init_reductions() + # NOTE: [ mmap files clear ] When the child process exits unexpectedly, # some shared memory objects may have been applied for but have not yet # been put into the inter-process Queue. This part of the object needs @@ -334,6 +362,11 @@ def _worker_loop( except: init_exception = _WorkerException(worker_id) + def numpy2lodtensor(arr): + lodtensor = core.DenseTensor() + lodtensor.set(arr, core.CPUPlace()) + return lodtensor + iterator_drained = False parent_watch_dog = ParentWatchDog() @@ -343,36 +376,31 @@ def _worker_loop( except queue.Empty: continue - if isinstance(data, _ResumeIteration): - out_queue.put((data, None, None)) - iterator_drained = False - fetcher = _DatasetKind.create_fetcher( - dataset_kind, dataset, auto_collate_batch, collate_fn, True - ) - continue - - # None as poison piil, so worker event should be set - if data is None: - assert done_event.is_set() or iterator_drained, ( - "get None when worker done_event set" - ) - break - # If worker done event is set but get still get data in - # indices_queue, remaining data should be get and skipped. - if done_event.is_set() or iterator_drained: - continue - - idx, indices = data try: + if isinstance(data, _ResumeIteration): + out_queue.put((data, None, None)) + iterator_drained = False + fetcher = _DatasetKind.create_fetcher( + dataset_kind, dataset, auto_collate_batch, collate_fn, True + ) + continue + + # None as poison piil, so worker event should be set + if data is None: + assert done_event.is_set() or iterator_drained, ( + "get None when worker done_event set" + ) + break + # If worker done event is set but get still get data in + # indices_queue, remaining data should be get and skipped. + if done_event.is_set() or iterator_drained: + continue + + idx, indices = data if init_exception is not None: batch = init_exception init_exception = None else: - # NOTE: GPU tensor operation is not supported in sub-process - # but default device is GPU in paddle-gpu version, which - # may copy CPU tensor to GPU even if users want to use - # CPU tensor operation, so we add CPUPlace guard here - # to make sure tensor will be operated only on CPU with paddle.base.dygraph.guard(place=paddle.CPUPlace()): batch = fetcher.fetch(indices) except Exception as e: @@ -389,12 +417,6 @@ def _worker_loop( out_queue.put((idx, batch, None)) batch, structure = _flatten_batch(batch) if use_shared_memory: - - def numpy2lodtensor(arr): - lodtensor = core.DenseTensor() - lodtensor.set(arr, core.CPUPlace()) - return lodtensor - tensor_list = [ ( numpy2lodtensor(b) @@ -410,10 +432,16 @@ def numpy2lodtensor(arr): # NOTE: Main process will raise KeyboardInterrupt anyways, ignore it in child process pass except: - raise + # Write to stderr (visible in parent console) + sys.stderr.write( + "[WORKER pid=%d] UNCAUGHT EXCEPTION\n%s\n" % + (os.getpid(), traceback.format_exc())) + sys.stderr.flush() finally: if use_shared_memory: _cleanup_mmap() - if done_event.is_set(): - out_queue.cancel_join_thread() - out_queue.close() + # On Windows (spawn), skip queue cleanup that may fail during shutdown + if not sys.platform.startswith('win'): + if done_event.is_set(): + out_queue.cancel_join_thread() + out_queue.close() diff --git a/python/paddle/io/multiprocess_utils.py b/python/paddle/io/multiprocess_utils.py index 6870832b408b0d..236fd3a98db1ce 100644 --- a/python/paddle/io/multiprocess_utils.py +++ b/python/paddle/io/multiprocess_utils.py @@ -44,13 +44,19 @@ def _cleanup(): # NOTE: inter-process Queue shared memory objects clear function _clear_multiprocess_queue_set() # NOTE: main process memory map files clear function - core._cleanup_mmap_fds() + try: + core._cleanup_mmap_fds() + except Exception: + pass # NOTE: for child process clear function at exit def _cleanup_mmap(): # clear memory map files in child process - core._cleanup_mmap_fds() + try: + core._cleanup_mmap_fds() + except Exception: + pass # NOTE used for register a function to be executed at interpreter exit. @@ -109,9 +115,7 @@ def _signal_register(signals): # BlockingQueue) may not be completely released, resulting in the corresponding # memory-mapped file remaining on the disk (/dev/shm), so register this function # to clean up shared memory objects in these two queues before the python interpreter exits. -# NOTE: Currently multi-process DataLoader only supports Linux platform -if not (sys.platform == 'darwin' or sys.platform == 'win32'): - CleanupFuncRegistrar.register(_cleanup) +CleanupFuncRegistrar.register(_cleanup) # ------------ SIGCHLD handler setting -------------- _SIGCHLD_handler_set = False @@ -121,6 +125,11 @@ def _set_SIGCHLD_handler(): global _SIGCHLD_handler_set if _SIGCHLD_handler_set: return + if sys.platform == 'win32': + # Windows does not have SIGCHLD; worker failure is detected + # via periodic polling in ThrowErrorIfLoadProcessFailed. + _SIGCHLD_handler_set = True + return current_handler = signal.getsignal(signal.SIGCHLD) if not callable(current_handler): diff --git a/python/paddle/io/reader.py b/python/paddle/io/reader.py index 25daa0fa317a06..e740c7beecfc89 100644 --- a/python/paddle/io/reader.py +++ b/python/paddle/io/reader.py @@ -497,14 +497,6 @@ def __init__( self.places = _convert_places(places) assert num_workers >= 0, "num_workers should be a non-negative value" - if num_workers > 0 and ( - sys.platform == 'darwin' or sys.platform == 'win32' - ): - warnings.warn( - "DataLoader with multi-process mode is not supported on MacOs and Windows currently." - " Please use single-process mode with num_workers = 0 instead" - ) - num_workers = 0 self.num_workers = num_workers assert prefetch_factor > 0, "prefetch_factor should be a positive value" diff --git a/test/legacy_test/test_dataloader_windows_multiprocess.py b/test/legacy_test/test_dataloader_windows_multiprocess.py new file mode 100644 index 00000000000000..6e75217fdd13ad --- /dev/null +++ b/test/legacy_test/test_dataloader_windows_multiprocess.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import unittest + +import numpy as np + +import paddle +from paddle.io import DataLoader, Dataset, IterableDataset + + +class RandomDataset(Dataset): + def __init__(self, sample_num, feature_dim=10): + self.sample_num = sample_num + self.feature_dim = feature_dim + self.data = np.random.rand(sample_num, feature_dim).astype('float32') + self.label = np.random.randint(0, 2, size=(sample_num, 1)).astype( + 'int64' + ) + + def __getitem__(self, idx): + return ( + paddle.to_tensor(self.data[idx]), + paddle.to_tensor(self.label[idx]), + ) + + def __len__(self): + return self.sample_num + + +class RandomIterableDataset(IterableDataset): + def __init__(self, sample_num, feature_dim=10): + self.sample_num = sample_num + self.feature_dim = feature_dim + + def __iter__(self): + for i in range(self.sample_num): + yield paddle.rand([self.feature_dim]) + + +def collate_batch(batch): + data = paddle.stack([item[0] for item in batch]) + label = paddle.stack([item[1] for item in batch]) + return data, label + + +class TestDataLoaderWindowsMultiprocess(unittest.TestCase): + def setUp(self): + self.sample_num = 100 + self.batch_size = 8 + self.feature_dim = 10 + self.epoch_num = 3 + + def run_simple_net(self, num_workers, use_shared_memory=False): + paddle.seed(2026) + np.random.seed(2026) + + dataset = RandomDataset(self.sample_num, self.feature_dim) + loader = DataLoader( + dataset, + places=paddle.CPUPlace(), + batch_size=self.batch_size, + shuffle=True, + num_workers=num_workers, + use_shared_memory=use_shared_memory, + ) + + losses = [] + for epoch in range(self.epoch_num): + for data, label in loader(): + # Simple linear layer + pred = paddle.mean(data, axis=1, keepdim=True) + loss = paddle.nn.functional.binary_cross_entropy_with_logits( + pred, paddle.cast(label, 'float32') + ) + losses.append(loss.numpy()) + return np.mean(losses) + + def test_multiprocess_singleprocess_loss_close(self): + """ + Test that multi-process (num_workers=2) and single-process (num_workers=0) + produce similar loss values. + """ + loss_single = self.run_simple_net(num_workers=0) + loss_multi = self.run_simple_net(num_workers=2) + diff = np.abs(loss_single - loss_multi) / np.abs(loss_single) + self.assertLess( + diff, + 1e-2, + f"Loss difference too large: single={loss_single}, multi={loss_multi}, diff={diff}", + ) + + def test_multiprocess_with_shared_memory(self): + """ + Test multi-process DataLoader with use_shared_memory=True. + """ + loss_single = self.run_simple_net(num_workers=0) + loss_multi = self.run_simple_net( + num_workers=2, use_shared_memory=True + ) + diff = np.abs(loss_single - loss_multi) / np.abs(loss_single) + self.assertLess( + diff, + 1e-2, + f"Loss difference too large with shared memory: single={loss_single}, multi={loss_multi}, diff={diff}", + ) + + def test_multiprocess_more_workers(self): + """ + Test with num_workers=4 to stress-test the worker pool. + """ + loss_single = self.run_simple_net(num_workers=0) + loss_multi = self.run_simple_net(num_workers=4) + diff = np.abs(loss_single - loss_multi) / np.abs(loss_single) + self.assertLess( + diff, + 1e-2, + f"Loss difference too large (4 workers): single={loss_single}, multi={loss_multi}, diff={diff}", + ) + + def test_multiprocess_persistent_workers(self): + """ + Test with persistent_workers=True to ensure worker reuse works. + """ + paddle.seed(2026) + np.random.seed(2026) + dataset = RandomDataset(self.sample_num, self.feature_dim) + loader = DataLoader( + dataset, + places=paddle.CPUPlace(), + batch_size=self.batch_size, + shuffle=True, + num_workers=2, + use_shared_memory=False, + persistent_workers=True, + ) + losses = [] + for epoch in range(self.epoch_num): + for data, label in loader(): + pred = paddle.mean(data, axis=1, keepdim=True) + loss = paddle.nn.functional.binary_cross_entropy_with_logits( + pred, paddle.cast(label, 'float32') + ) + losses.append(loss.numpy()) + loss_multi = np.mean(losses) + + loss_single = self.run_simple_net(num_workers=0) + diff = np.abs(loss_single - loss_multi) / np.abs(loss_single) + self.assertLess( + diff, + 1e-2, + f"Loss difference too large (persistent workers): single={loss_single}, multi={loss_multi}, diff={diff}", + ) + + def test_multiprocess_iterable_dataset(self): + """ + Test multi-process DataLoader with IterableDataset. + """ + dataset = RandomIterableDataset(50, self.feature_dim) + loader = DataLoader( + dataset, + batch_size=self.batch_size, + num_workers=2, + ) + count = 0 + for batch in loader(): + count += 1 + self.assertGreater(count, 0) + + +if __name__ == '__main__': + unittest.main() From 01544907769d6ec89da0d54027507025960f2394 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 04:23:14 +0800 Subject: [PATCH 02/32] Remove accidentally committed root-level data_loader.cc --- data_loader.cc | 248 ------------------------------------------------- 1 file changed, 248 deletions(-) delete mode 100644 data_loader.cc diff --git a/data_loader.cc b/data_loader.cc deleted file mode 100644 index c7b6f96a5a9443..00000000000000 --- a/data_loader.cc +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/imperative/data_loader.h" - -#include - -#include - -#include "glog/logging.h" -#include "paddle/fluid/platform/enforce.h" -#include "paddle/phi/core/memory/allocation/mmap_allocator.h" - -#ifdef _WIN32 -#include -#include -#else -#include -#include -#endif - -namespace paddle::imperative { - -static std::map> load_process_pids; - -void SetLoadProcessPIDs(int64_t key, std::set pids) { - VLOG(3) << "DataLoader: set loader child process PID (" << key - << ", pid number: " << pids.size() << ")"; - load_process_pids[key] = pids; -} - -void EraseLoadProcessPIDs(int64_t key) { - auto it = load_process_pids.find(key); - // Note: Can not find key also possible - if (it != load_process_pids.end()) { - VLOG(3) << "Dygraph Data Loader: erase loader child process PID (" << key - << ")"; - load_process_pids.erase(it); - } else { - VLOG(3) << "Dygraph Data Loader: The dygraph loader (id: " << key - << ") you want erase does not exist."; - } -} - -#ifndef _WIN32 -// sigaction doc: http://man7.org/linux/man-pages/man2/sigaction.2.html -// sigemptyset doc: https://linux.die.net/man/3/sigemptyset -// siginfo_t doc: https://www.mkssoftware.com/docs/man5/siginfo_t.5.asp -// waitid doc: https://linux.die.net/man/2/waitid - -// clear mmap fds on signal handler, make sure mmap clear will be called -// on signal handling and no need to register mmap clear up handler on -// python side. If shared memory is not used Clear() will do nothing. -#define SIGNAL_HANDLE(SIGNAL) \ - do { \ - memory::allocation::MemoryMapFdSet::Instance().Clear(); \ - struct sigaction sa = {}; \ - sa.sa_handler = SIG_DFL; \ - sa.sa_flags = 0; \ - if (sigemptyset(&sa.sa_mask) != 0 || \ - sigaction(SIGNAL, &sa, nullptr) != 0) { \ - _exit(EXIT_FAILURE); \ - } else { \ - raise(SIGNAL); \ - } \ - } while (0) - -#define REGISTER_SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, ERROR_MSG) \ - static void HANDLER_NAME( \ - int sig UNUSED, siginfo_t *info UNUSED, void *ctx UNUSED) { \ - auto _w = \ - write(STDERR_FILENO, ERROR_MSG, sizeof(ERROR_MSG) / sizeof(char)); \ - (void)_w; \ - SIGNAL_HANDLE(SIGNAL); \ - } - -#define REGISTER_SPEC_SIGNAL_HANDLER(SIGNAL, HANDLER_NAME) \ - static void HANDLER_NAME(int sig, siginfo_t *info, void *ctx) { \ - if (info->si_pid == getppid()) { \ - _exit(EXIT_SUCCESS); \ - } \ - SIGNAL_HANDLE(SIGNAL); \ - } - -REGISTER_SIGNAL_HANDLER(SIGSEGV, - SIGSEGV_handler, - "ERROR: Unexpected segmentation fault encountered in " - "DataLoader workers.\n"); -REGISTER_SIGNAL_HANDLER( - SIGBUS, - SIGBUS_handler, - "ERROR: Unexpected BUS error encountered in DataLoader worker. " - "This might be caused by insufficient shared memory (shm), " - "please check whether use_shared_memory is set and storage space " - "in /dev/shm is enough\n"); -REGISTER_SIGNAL_HANDLER(SIGFPE, - SIGFPE_handler, - "ERROR: Unexpected floating-point exception " - "encountered in DataLoader worker.\n") -REGISTER_SPEC_SIGNAL_HANDLER(SIGTERM, SIGTERM_handler); - -static inline void setSignalHandler(int signal, - void (*handler)(int, siginfo_t *, void *), - struct sigaction *old_sa_ptr) { - struct sigaction sa; - sa.sa_sigaction = handler; - sa.sa_flags = SA_RESTART | SA_SIGINFO | SA_NOCLDSTOP | SA_NODEFER; - if (sigemptyset(&sa.sa_mask) != 0 || - sigaction(signal, &sa, old_sa_ptr) != 0) { - PADDLE_THROW(common::errors::Fatal( - "An error occurred while setting handler for %s.", strsignal(signal))); - } -} -#endif - -void SetLoadProcessSignalHandler() { -#ifdef _WIN32 - // On Windows, VEH handler is not needed. The MemoryMapFdSet::Clear() - // is a no-op on Windows (named file mappings auto-clean when the last - // handle is closed). Any crash will be handled by the OS normally. - VLOG(3) << "DataLoader: VEH handler skipped on Windows"; -#else - setSignalHandler(SIGSEGV, &SIGSEGV_handler, nullptr); - setSignalHandler(SIGBUS, &SIGBUS_handler, nullptr); - setSignalHandler(SIGFPE, &SIGFPE_handler, nullptr); - setSignalHandler(SIGTERM, &SIGTERM_handler, nullptr); -#endif -} - -void ThrowErrorIfLoadProcessFailed() { -#ifdef _WIN32 - // On Windows, use WaitForSingleObject + GetExitCodeProcess - // to check for crashed workers. This is called periodically from Python. - int error = 0; - std::set *pids_set = nullptr; - pid_t process_pid = 0; - - for (auto &p : load_process_pids) { - pids_set = &(p.second); - for (auto pid_it = pids_set->begin(); pid_it != pids_set->end(); ++pid_it) { - process_pid = *pid_it; - VLOG(3) << "DataLoader: monitor loader child process " << process_pid; - - HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, - FALSE, static_cast(process_pid)); - if (hProcess == NULL) { - // Process may have already exited; GetLastError() == ERROR_INVALID_PARAMETER - continue; - } - - DWORD exit_code = STILL_ACTIVE; - if (WaitForSingleObject(hProcess, 0) == WAIT_OBJECT_0) { - // Process has exited - GetExitCodeProcess(hProcess, &exit_code); - CloseHandle(hProcess); - - if (exit_code != 0 && exit_code != STILL_ACTIVE) { - // Exited with error (non-zero exit code) - pids_set->clear(); - PADDLE_THROW(common::errors::Fatal( - "DataLoader process (pid %ld) exited unexpectedly with code %lu. " - "Rerunning with num_workers=0 may give better error trace.", - process_pid, exit_code)); - } - } else { - CloseHandle(hProcess); - } - } - } -#else - int error = 0; - std::set *pids_set = nullptr; - pid_t process_pid = 0; - siginfo_t infop; - - for (auto &p : load_process_pids) { - pids_set = &(p.second); - for (auto pid_it = pids_set->begin(); pid_it != pids_set->end(); ++pid_it) { - process_pid = *pid_it; - // Use waitid rather than waitpid so that we can set NOWAIT, and that - // Python and other handlers can get whatever info they want about the - // child. - infop.si_pid = 0; - VLOG(3) << "DataLoader: monitor loader child process " << process_pid; - error = waitid(P_PID, process_pid, &infop, WEXITED | WNOHANG | WNOWAIT); - // ignore errors and case with no waitable child - if (error < 0 || infop.si_pid == 0) continue; - if (infop.si_code == CLD_EXITED && - infop.si_status != EXIT_SUCCESS) { // exit with error - pids_set->clear(); - PADDLE_THROW(common::errors::Fatal( - "DataLoader process (pid %ld) exited unexpectedly with code %d. " - "Error detailed are lost due to multiprocessing. Rerunning with:\n" - " 1. If run DataLoader by DataLoader.from_generator(...), run " - "with " - "DataLoader.from_generator(..., use_multiprocess=False) may give " - "better error trace.\n" - " 2. If run DataLoader by DataLoader(dataset, ...), run with " - "DataLoader(dataset, ..., num_workers=0) may give better error " - "trace", - process_pid, - infop.si_status)); - } else if (infop.si_code == CLD_KILLED || - infop.si_code == CLD_DUMPED) { // killed by signal - if (infop.si_status == SIGBUS) { - pids_set->clear(); - PADDLE_THROW(common::errors::Fatal( - "DataLoader process (pid %ld) exited is killed by signal: %s.\n" - " It may be caused by insufficient shared storage space. This " - "problem usually occurs when using docker as a development " - "environment.\n Please use command `df -h` to check the storage " - "space of `/dev/shm`. Shared storage space needs to be greater " - "than (DataLoader Num * DataLoader queue capacity * 1 batch data " - "size).\n You can solve this problem by increasing the shared " - "storage space or reducing the queue capacity appropriately.\n" - " 1. If run DataLoader by DataLoader.from_generator(...), queue " - "capacity is set by from_generator(..., capacity=xx, ...).\n" - " 2. If run DataLoader by DataLoader(dataset, ...), queue " - "capacity is set as 2 times of the max value of num_workers and " - "len(places).\n" - " 3. If run by DataLoader(dataset, ..., use_shared_memory=True)," - " set use_shared_memory=False for not using shared memory.", - process_pid, - strsignal(infop.si_status))); - } else { - PADDLE_THROW(common::errors::Fatal( - "DataLoader process (pid %ld) exited is killed by signal: %s.", - process_pid, - strsignal(infop.si_status))); - } - } - } - } -#endif -} - -} // namespace paddle::imperative From 92a5a8869a9eb314d9153001a12c2f77e5debdfd Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 04:29:51 +0800 Subject: [PATCH 03/32] Fix comment in data_loader.h: remove Python-specific reference --- paddle/fluid/imperative/data_loader.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/imperative/data_loader.h b/paddle/fluid/imperative/data_loader.h index 7a9f54f7a603a4..9f9be98bb82bbd 100644 --- a/paddle/fluid/imperative/data_loader.h +++ b/paddle/fluid/imperative/data_loader.h @@ -16,7 +16,8 @@ #ifdef _WIN32 #include -// pid_t is not defined on Windows; use int (Python's os.getpid() returns int) +// pid_t is not defined on Windows; use int as cross-platform alternative. +// This is compatible with both getpid() (POSIX) and GetCurrentProcessId() (Win32). using pid_t = int; #else #include From 4c8f4c0b06b466c8d8fd66f0c02c0007ca7f074d Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 05:04:07 +0800 Subject: [PATCH 04/32] Keep macOS multiprocess restriction, only remove Windows --- python/paddle/io/reader.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/paddle/io/reader.py b/python/paddle/io/reader.py index e740c7beecfc89..0d01542f9b9ab9 100644 --- a/python/paddle/io/reader.py +++ b/python/paddle/io/reader.py @@ -497,6 +497,12 @@ def __init__( self.places = _convert_places(places) assert num_workers >= 0, "num_workers should be a non-negative value" + if num_workers > 0 and sys.platform == 'darwin': + warnings.warn( + "DataLoader with multi-process mode is not supported on MacOs currently." + " Please use single-process mode with num_workers = 0 instead" + ) + num_workers = 0 self.num_workers = num_workers assert prefetch_factor > 0, "prefetch_factor should be a positive value" From c912ec4b9ff08d1a4c0d537ab45300fdf3f9b785 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 05:04:54 +0800 Subject: [PATCH 05/32] Rename test file: remove 'windows' from name (tests are cross-platform) --- ...er_windows_multiprocess.py => test_dataloader_multiprocess.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/legacy_test/{test_dataloader_windows_multiprocess.py => test_dataloader_multiprocess.py} (100%) diff --git a/test/legacy_test/test_dataloader_windows_multiprocess.py b/test/legacy_test/test_dataloader_multiprocess.py similarity index 100% rename from test/legacy_test/test_dataloader_windows_multiprocess.py rename to test/legacy_test/test_dataloader_multiprocess.py From a2ec298eb2473376d61e189d90c8326edc7b56dd Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 05:26:41 +0800 Subject: [PATCH 06/32] Fix pre-commit issues: ruff format, clang-format, cpplint, copyright --- .claude/skills | 2 +- paddle/fluid/framework/data_type_transform.cu | 16 +- paddle/fluid/imperative/data_loader.cc | 23 +- paddle/fluid/imperative/data_loader.h | 3 +- paddle/fluid/pybind/tensor.cc | 242 +++++++++--------- .../core/memory/allocation/mmap_allocator.cc | 86 ++++--- .../core/memory/allocation/mmap_allocator.h | 24 +- python/paddle/base/core.py | 4 +- .../incubate/multiprocessing/reductions.py | 6 +- .../paddle/io/dataloader/dataloader_iter.py | 4 +- python/paddle/io/dataloader/worker.py | 17 +- .../test_dataloader_multiprocess.py | 6 +- test/xpu/amp/CMakeLists.txt | 2 +- test/xpu/amp/test_amp_api_xpu.py | 16 +- test/xpu/amp/test_amp_decorate_xpu.py | 16 +- test/xpu/amp/test_amp_list_xpu.py | 16 +- test/xpu/amp/test_amp_master_grad_xpu.py | 16 +- test/xpu/amp/test_amp_master_weight_xpu.py | 16 +- test/xpu/amp/test_amp_promote_xpu.py | 16 +- test/xpu/amp/test_layer_convert_dtype_xpu.py | 16 +- ..._build_src_rank_and_local_expert_id_xpu.py | 16 +- ..._incubate_expand_modality_expert_id_xpu.py | 16 +- 22 files changed, 380 insertions(+), 199 deletions(-) diff --git a/.claude/skills b/.claude/skills index 2b7a412b8fa0fb..9f020f7e543030 120000 --- a/.claude/skills +++ b/.claude/skills @@ -1 +1 @@ -../.agents/skills \ No newline at end of file +../.agents/skills diff --git a/paddle/fluid/framework/data_type_transform.cu b/paddle/fluid/framework/data_type_transform.cu index f46491293ef4ad..3dd4bcbef16ee2 120000 --- a/paddle/fluid/framework/data_type_transform.cu +++ b/paddle/fluid/framework/data_type_transform.cu @@ -1 +1,15 @@ -data_type_transform.cc \ No newline at end of file +// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +data_type_transform.cc diff --git a/paddle/fluid/imperative/data_loader.cc b/paddle/fluid/imperative/data_loader.cc index 2aa7bb21637cd3..11a6e07f020a57 100644 --- a/paddle/fluid/imperative/data_loader.cc +++ b/paddle/fluid/imperative/data_loader.cc @@ -125,7 +125,8 @@ static inline void setSignalHandler(int signal, #endif #ifdef _WIN32 -// Last-chance exception filter: logs crash info before OS terminates the process. +// Last-chance exception filter: logs crash info before OS terminates the +// process. static LONG CALLBACK paddle_crash_filter(EXCEPTION_POINTERS *ep) { fprintf(stderr, "[PADDLE_CRASH] PID=%lu code=0x%08lX addr=%p\n", @@ -167,9 +168,11 @@ void ThrowErrorIfLoadProcessFailed() { VLOG(3) << "DataLoader: monitor loader child process " << process_pid; HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, - FALSE, static_cast(process_pid)); + FALSE, + static_cast(process_pid)); if (hProcess == NULL) { - // Process may have already exited; GetLastError() == ERROR_INVALID_PARAMETER + // Process may have already exited; GetLastError() == + // ERROR_INVALID_PARAMETER continue; } @@ -182,14 +185,19 @@ void ThrowErrorIfLoadProcessFailed() { if (exit_code != 0 && exit_code != STILL_ACTIVE) { // Exited with error (non-zero exit code) fprintf(stderr, - "[DL_WORKER_EXIT] pid=%lu exit_code=0x%08lX (%lu)\n", - process_pid, exit_code, exit_code); + "[DL_WORKER_EXIT] pid=%lu exit_code=0x%08lX (%lu)\n", + process_pid, + exit_code, + exit_code); fflush(stderr); pids_set->clear(); PADDLE_THROW(common::errors::Fatal( - "DataLoader process (pid %ld) exited unexpectedly with code 0x%lX (%lu). " + "DataLoader process (pid %ld) exited unexpectedly with code " + "0x%lX (%lu). " "Rerunning with num_workers=0 may give better error trace.", - process_pid, exit_code, exit_code)); + process_pid, + exit_code, + exit_code)); } } else { CloseHandle(hProcess); @@ -264,4 +272,3 @@ void ThrowErrorIfLoadProcessFailed() { } } // namespace paddle::imperative - diff --git a/paddle/fluid/imperative/data_loader.h b/paddle/fluid/imperative/data_loader.h index 9f9be98bb82bbd..43f8113859be98 100644 --- a/paddle/fluid/imperative/data_loader.h +++ b/paddle/fluid/imperative/data_loader.h @@ -17,7 +17,8 @@ #ifdef _WIN32 #include // pid_t is not defined on Windows; use int as cross-platform alternative. -// This is compatible with both getpid() (POSIX) and GetCurrentProcessId() (Win32). +// This is compatible with both getpid() (POSIX) and GetCurrentProcessId() +// (Win32). using pid_t = int; #else #include diff --git a/paddle/fluid/pybind/tensor.cc b/paddle/fluid/pybind/tensor.cc index 4261a220a5dc53..9cb0992ebdf11a 100644 --- a/paddle/fluid/pybind/tensor.cc +++ b/paddle/fluid/pybind/tensor.cc @@ -1562,128 +1562,134 @@ void BindTensor(pybind11::module &m) { // NOLINT // Module-level wrappers for _share_filename etc. // These bypass a pybind11 class method registration issue on Windows. - m.def("_share_filename", - [](DenseTensor &self, bool use_file_descriptor) { - if (!self.IsInitialized() || self.numel() == 0) - throw std::runtime_error( - "Tensor not initialized or numel is 0. could not pass to " - "shared memory. "); - - auto holder = self.Holder(); - PADDLE_ENFORCE_EQ( - phi::is_cpu_place(holder->place()) || - phi::is_cuda_pinned_place(holder->place()), - true, common::errors::InvalidArgument( - "Tensor is not on CPU. share_filename only " - "support CPU Tensor.")); - - auto *mmap_allocation = dynamic_cast< - memory::allocation::RefcountedMemoryMapAllocation *>( - holder.get()); - if (mmap_allocation == nullptr) { - void *data_ptr = self.data(); - size_t data_size = - self.numel() * - framework::SizeOfType( - framework::TransToProtoVarType(self.type())); - - int flags = memory::allocation::MAPPED_SHAREDMEM | - memory::allocation::MAPPED_EXCLUSIVE; - if (use_file_descriptor) { - flags = flags | memory::allocation::MAPPED_KEEPFD | - memory::allocation::MAPPED_UNLINK; - } - std::string handle = memory::allocation::GetIPCName(); - int find_id = -1; - if (FLAGS_use_shm_cache) { - find_id = memory::allocation::MemoryMapAllocationPool::Instance().FindFromCache(flags, data_size); // NOLINT - } - if (find_id != -1) { - handle = memory::allocation::MemoryMapAllocationPool::Instance().GetById(find_id).file_name_; // NOLINT - } - int shared_fd = -1; - auto shared_holder = - memory::allocation::AllocateRefcountedMemoryMapAllocation( - handle, shared_fd, flags, data_size, find_id); - - // On Windows, GPUPinnedPlace copy is not needed (non-CUDA memory sharing) + m.def("_share_filename", [](DenseTensor &self, bool use_file_descriptor) { + if (!self.IsInitialized() || self.numel() == 0) + throw std::runtime_error( + "Tensor not initialized or numel is 0. could not pass to " + "shared memory. "); + + auto holder = self.Holder(); + PADDLE_ENFORCE_EQ(phi::is_cpu_place(holder->place()) || + phi::is_cuda_pinned_place(holder->place()), + true, + common::errors::InvalidArgument( + "Tensor is not on CPU. share_filename only " + "support CPU Tensor.")); + + auto *mmap_allocation = + dynamic_cast( + holder.get()); + if (mmap_allocation == nullptr) { + void *data_ptr = self.data(); + size_t data_size = + self.numel() * + framework::SizeOfType(framework::TransToProtoVarType(self.type())); + + int flags = memory::allocation::MAPPED_SHAREDMEM | + memory::allocation::MAPPED_EXCLUSIVE; + if (use_file_descriptor) { + flags = flags | memory::allocation::MAPPED_KEEPFD | + memory::allocation::MAPPED_UNLINK; + } + std::string handle = memory::allocation::GetIPCName(); + int find_id = -1; + if (FLAGS_use_shm_cache) { + find_id = memory::allocation::MemoryMapAllocationPool::Instance() + .FindFromCache(flags, data_size); // NOLINT + } + if (find_id != -1) { + handle = memory::allocation::MemoryMapAllocationPool::Instance() + .GetById(find_id) + .file_name_; // NOLINT + } + int shared_fd = -1; + auto shared_holder = + memory::allocation::AllocateRefcountedMemoryMapAllocation( + handle, shared_fd, flags, data_size, find_id); + + // On Windows, GPUPinnedPlace copy is not needed (non-CUDA memory sharing) #ifndef _WIN32 - if (phi::is_cuda_pinned_place(holder->place())) { + if (phi::is_cuda_pinned_place(holder->place())) { #ifdef PADDLE_WITH_CUDA - memory::Copy(CPUPlace(), shared_holder->ptr(), - phi::GPUPinnedPlace(), data_ptr, data_size); + memory::Copy(CPUPlace(), + shared_holder->ptr(), + phi::GPUPinnedPlace(), + data_ptr, + data_size); #endif - } else + } else { + memory::Copy( + CPUPlace(), shared_holder->ptr(), CPUPlace(), data_ptr, data_size); + } +#else + memory::Copy( + CPUPlace(), shared_holder->ptr(), CPUPlace(), data_ptr, data_size); #endif - { - memory::Copy(CPUPlace(), shared_holder->ptr(), - CPUPlace(), data_ptr, data_size); - } - self.ResetHolder(shared_holder); - mmap_allocation = shared_holder.get(); - } - int type_idx = static_cast(self.type()); - - return py::make_tuple(mmap_allocation->ipc_name(), - mmap_allocation->shared_fd(), - mmap_allocation->size(), type_idx, - common::vectorize(self.dims()), self.lod(), - use_file_descriptor); - }); - - m.def("_new_shared_filename", - [](py::tuple t) { - if (t.size() != 7) - throw std::runtime_error("Invalid Tensor meta info state!"); - - DenseTensor tensor; - - const std::string &ipc_name = t[0].cast(); - const int shared_fd = t[1].cast(); - const bool use_file_descriptor = t[6].cast(); - - size_t size = t[2].cast(); - int flags = memory::allocation::MAPPED_SHAREDMEM | - memory::allocation::MAPPED_NOCREATE; - if (use_file_descriptor) { - flags = flags | memory::allocation::MAPPED_KEEPFD | - memory::allocation::MAPPED_UNLINK; - } - int find_id = -1; - if (FLAGS_use_shm_cache) { - find_id = memory::allocation::MemoryMapAllocationPool::Instance().FindFromCache(flags, size, ipc_name, false); // NOLINT - } - auto shared_holder = - memory::allocation::AllocateRefcountedMemoryMapAllocation( - ipc_name, shared_fd, flags, size, find_id); - - tensor.ResetHolderWithType( - shared_holder, - static_cast(t[3].cast())); - tensor.Resize(common::make_ddim(t[4].cast>())); - - return tensor; - }); - - m.def("_shared_incref", - [](DenseTensor &self) { - auto *mmap_allocation = dynamic_cast< - memory::allocation::RefcountedMemoryMapAllocation *>( - self.Holder().get()); - if (mmap_allocation) { - mmap_allocation->incref(); - } - }); - - m.def("_shared_decref", - [](DenseTensor &self) { - auto *mmap_allocation = dynamic_cast< - memory::allocation::RefcountedMemoryMapAllocation *>( - self.Holder().get()); - if (mmap_allocation) { - mmap_allocation->decref(); - } - }); + self.ResetHolder(shared_holder); + mmap_allocation = shared_holder.get(); + } + int type_idx = static_cast(self.type()); + + return py::make_tuple(mmap_allocation->ipc_name(), + mmap_allocation->shared_fd(), + mmap_allocation->size(), + type_idx, + common::vectorize(self.dims()), + self.lod(), + use_file_descriptor); + }); + + m.def("_new_shared_filename", [](py::tuple t) { + if (t.size() != 7) + throw std::runtime_error("Invalid Tensor meta info state!"); + + DenseTensor tensor; + + const std::string &ipc_name = t[0].cast(); + const int shared_fd = t[1].cast(); + const bool use_file_descriptor = t[6].cast(); + + size_t size = t[2].cast(); + int flags = memory::allocation::MAPPED_SHAREDMEM | + memory::allocation::MAPPED_NOCREATE; + if (use_file_descriptor) { + flags = flags | memory::allocation::MAPPED_KEEPFD | + memory::allocation::MAPPED_UNLINK; + } + int find_id = -1; + if (FLAGS_use_shm_cache) { + find_id = + memory::allocation::MemoryMapAllocationPool::Instance().FindFromCache( + flags, size, ipc_name, false); // NOLINT + } + auto shared_holder = + memory::allocation::AllocateRefcountedMemoryMapAllocation( + ipc_name, shared_fd, flags, size, find_id); + + tensor.ResetHolderWithType(shared_holder, + static_cast(t[3].cast())); + tensor.Resize(common::make_ddim(t[4].cast>())); + + return tensor; + }); + + m.def("_shared_incref", [](DenseTensor &self) { + auto *mmap_allocation = + dynamic_cast( + self.Holder().get()); + if (mmap_allocation) { + mmap_allocation->incref(); + } + }); + + m.def("_shared_decref", [](DenseTensor &self) { + auto *mmap_allocation = + dynamic_cast( + self.Holder().get()); + if (mmap_allocation) { + mmap_allocation->decref(); + } + }); } } // namespace paddle::pybind diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index ff6e2547f1b5b0..99ecece86a2bd9 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -55,7 +55,7 @@ struct CountInfo { std::atomic refcount; }; -void AllocateMemoryMap(std::string& filename, +void AllocateMemoryMap(std::string *filename, int *shared_fd, int flags, size_t size, @@ -66,9 +66,12 @@ void AllocateMemoryMap(std::string& filename, // returns the existing mapping if the name already exists, unlike Linux's // shm_open + O_EXCL which fails atomically. We must check explicitly. for (int attempt = 0; attempt < 100; attempt++) { - HANDLE hMap = CreateFileMappingA( - INVALID_HANDLE_VALUE, NULL, protect, 0, - static_cast(size), filename.c_str()); + HANDLE hMap = CreateFileMappingA(INVALID_HANDLE_VALUE, + NULL, + protect, + 0, + static_cast(size), + filename.c_str()); if (hMap == NULL) { DWORD err = GetLastError(); @@ -76,15 +79,20 @@ void AllocateMemoryMap(std::string& filename, fprintf(stderr, "[PADDLE_MMAP] PID=%lu CreateFileMapping FAILED " "name=%s size=%zu flags=0x%x attempt=%d err=%lu\n", - GetCurrentProcessId(), filename.c_str(), size, flags, attempt, err); + GetCurrentProcessId(), + filename.c_str(), + size, + flags, + attempt, + err); fflush(stderr); PADDLE_THROW(common::errors::Unavailable( "CreateFileMapping failed for %s, error: %lu", - filename.c_str(), err)); + filename.c_str(), + err)); } - if ((flags & MAPPED_EXCLUSIVE) && - GetLastError() == ERROR_ALREADY_EXISTS) { + if ((flags & MAPPED_EXCLUSIVE) && GetLastError() == ERROR_ALREADY_EXISTS) { CloseHandle(hMap); VLOG(3) << "[PADDLE_MMAP] PID=" << GetCurrentProcessId() << " name collision, retrying attempt=" << attempt @@ -101,11 +109,13 @@ void AllocateMemoryMap(std::string& filename, fprintf(stderr, "[PADDLE_MMAP] PID=%lu MapViewOfFile FAILED " "name=%s size=%zu err=%lu\n", - GetCurrentProcessId(), filename.c_str(), size, err); + GetCurrentProcessId(), + filename.c_str(), + size, + err); fflush(stderr); PADDLE_THROW(common::errors::Unavailable( - "MapViewOfFile failed for %s, error: %lu", - filename.c_str(), err)); + "MapViewOfFile failed for %s, error: %lu", filename.c_str(), err)); } // On Windows, always keep the HANDLE so the section stays alive @@ -200,7 +210,7 @@ AllocateRefcountedMemoryMapAllocation(std::string filename, int fd = shared_fd; void *base_ptr = nullptr; if (buffer_id == -1) { - AllocateMemoryMap(filename, &fd, flags, size + mmap_alignment, &base_ptr); + AllocateMemoryMap(&filename, &fd, flags, size + mmap_alignment, &base_ptr); VLOG(4) << "Create and mmap a new shm: " << filename; } else { base_ptr = MemoryMapAllocationPool::Instance().GetById(buffer_id).mmap_ptr_; @@ -364,7 +374,8 @@ MemoryMapWriterAllocation::~MemoryMapWriterAllocation() { MemoryMapReaderAllocation::~MemoryMapReaderAllocation() { #ifdef _WIN32 UnmapViewOfFile(this->ptr()); - // On Windows, named file mapping is auto-destroyed when final handle is closed. + // On Windows, named file mapping is auto-destroyed when final handle is + // closed. MemoryMapFdSet::Instance().Remove(this->ipc_name()); VLOG(3) << "~MemoryMapReaderAllocation: " << this->ipc_name(); #else @@ -400,18 +411,24 @@ std::shared_ptr AllocateMemoryMapWriterAllocation( size_t size) { const std::string &ipc_name = GetIPCName(); #ifdef _WIN32 - HANDLE hMap = CreateFileMappingA( - INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, static_cast(size), ipc_name.c_str()); - PADDLE_ENFORCE_NE(hMap, - nullptr, - common::errors::Unavailable( - "CreateFileMapping for writer %s failed", ipc_name.c_str())); + HANDLE hMap = CreateFileMappingA(INVALID_HANDLE_VALUE, + NULL, + PAGE_READWRITE, + 0, + static_cast(size), + ipc_name.c_str()); + PADDLE_ENFORCE_NE( + hMap, + nullptr, + common::errors::Unavailable("CreateFileMapping for writer %s failed", + ipc_name.c_str())); void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); - PADDLE_ENFORCE_NE(ptr, - nullptr, - common::errors::Unavailable( - "MapViewOfFile for writer %s failed", ipc_name.c_str())); + PADDLE_ENFORCE_NE( + ptr, + nullptr, + common::errors::Unavailable("MapViewOfFile for writer %s failed", + ipc_name.c_str())); // Close the handle; the named section stays alive through MapViewOfFile. CloseHandle(hMap); return std::make_shared(ptr, size, ipc_name); @@ -446,21 +463,28 @@ std::shared_ptr RebuildMemoryMapReaderAllocation( if (!hMap) { // Section doesn't exist yet; create it. This can happen in some edge cases // where the writer's view is still alive but all handles were closed. - hMap = CreateFileMappingA( - INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, static_cast(size), ipc_name.c_str()); + hMap = CreateFileMappingA(INVALID_HANDLE_VALUE, + NULL, + PAGE_READWRITE, + 0, + static_cast(size), + ipc_name.c_str()); } - PADDLE_ENFORCE_NE(hMap, - nullptr, - common::errors::Unavailable( - "CreateFileMapping/OpenFileMapping for reader %s failed, error: %lu", - ipc_name.c_str(), GetLastError())); + PADDLE_ENFORCE_NE( + hMap, + nullptr, + common::errors::Unavailable( + "CreateFileMapping/OpenFileMapping for reader %s failed, error: %lu", + ipc_name.c_str(), + GetLastError())); void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); PADDLE_ENFORCE_NE(ptr, nullptr, common::errors::Unavailable( "MapViewOfFile for reader %s failed, error: %lu", - ipc_name.c_str(), GetLastError())); + ipc_name.c_str(), + GetLastError())); CloseHandle(hMap); return std::make_shared(ptr, size, ipc_name); #else diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index 5d19297c33d82b..2162a2d21794d0 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -21,8 +21,8 @@ #include #include -#include "paddle/phi/core/memory/allocation/allocator.h" #include "paddle/phi/api/include/dll_decl.h" +#include "paddle/phi/core/memory/allocation/allocator.h" namespace paddle { namespace memory { @@ -98,11 +98,11 @@ class PADDLE_API RefcountedMemoryMapAllocation : public MemoryMapAllocation { void resetBaseptr(); }; -PADDLE_API void AllocateMemoryMap(std::string &filename, - int *shared_fd, - int flags, - size_t size, - void **base_ptr_); +PADDLE_API void AllocateMemoryMap(std::string *filename, + int *shared_fd, + int flags, + size_t size, + void **base_ptr_); PADDLE_API std::shared_ptr AllocateRefcountedMemoryMapAllocation(std::string filename, @@ -145,11 +145,11 @@ class PADDLE_API MemoryMapReaderAllocation : public Allocation { int fd_ = -1; }; -PADDLE_API std::shared_ptr AllocateMemoryMapWriterAllocation( - size_t size); +PADDLE_API std::shared_ptr +AllocateMemoryMapWriterAllocation(size_t size); -PADDLE_API std::shared_ptr RebuildMemoryMapReaderAllocation( - const std::string &ipc_name, size_t size); +PADDLE_API std::shared_ptr +RebuildMemoryMapReaderAllocation(const std::string &ipc_name, size_t size); class PADDLE_API MemoryMapFdSet { public: @@ -224,8 +224,8 @@ class PADDLE_API MemoryMapAllocationPool { private: MemoryMapAllocationPool() = default; - MemoryMapAllocationPool(const MemoryMapAllocationPool&) = delete; - MemoryMapAllocationPool& operator=(const MemoryMapAllocationPool&) = delete; + MemoryMapAllocationPool(const MemoryMapAllocationPool &) = delete; + MemoryMapAllocationPool &operator=(const MemoryMapAllocationPool &) = delete; std::vector memory_map_allocations_; int max_pool_size_ = 0; std::mutex mtx_; diff --git a/python/paddle/base/core.py b/python/paddle/base/core.py index 72a45460ae6127..813cc0678b4d67 100644 --- a/python/paddle/base/core.py +++ b/python/paddle/base/core.py @@ -367,14 +367,14 @@ def to_list(s): _cleanup_mmap_fds, _convert_to_tensor_list, _erase_process_pids, + _new_shared_filename, _remove_tensor_list_mmap_fds, _set_max_memory_map_allocation_pool_size, _set_process_pids, _set_process_signal_handler, _share_filename, - _new_shared_filename, - _shared_incref, _shared_decref, + _shared_incref, _throw_error_if_process_failed, ) diff --git a/python/paddle/incubate/multiprocessing/reductions.py b/python/paddle/incubate/multiprocessing/reductions.py index c40c405e76d69c..b4e8c30d14df92 100644 --- a/python/paddle/incubate/multiprocessing/reductions.py +++ b/python/paddle/incubate/multiprocessing/reductions.py @@ -19,14 +19,13 @@ # TODO: check the hooks of tensor # TODO: check serializing named tensor # TODO: check influence on autograd -import sys import threading from collections import OrderedDict from multiprocessing.reduction import ForkingPickler from multiprocessing.util import register_after_fork import paddle -import paddle.base.core as core +from paddle.base import core def _supported_check(): @@ -251,7 +250,8 @@ def _reduce_lodtensor(lodtensor): # Log and re-raise. This exception happens in the Queue feeder # thread; logging here helps diagnose intermittent crashes. sys.stderr.write( - f"[REDUCE] _share_filename failed: {type(e).__name__}: {e}\n") + f"[REDUCE] _share_filename failed: {type(e).__name__}: {e}\n" + ) sys.stderr.flush() raise diff --git a/python/paddle/io/dataloader/dataloader_iter.py b/python/paddle/io/dataloader/dataloader_iter.py index dd00d2e36264e2..288b1d7759d9a7 100644 --- a/python/paddle/io/dataloader/dataloader_iter.py +++ b/python/paddle/io/dataloader/dataloader_iter.py @@ -407,9 +407,7 @@ def __init__(self, loader): # see _try_put_indices self._thread_lock = threading.Lock() - self._base_seed = np.random.randint( - low=0, high=np.iinfo(np.int32).max - ) + self._base_seed = np.random.randint(low=0, high=np.iinfo(np.int32).max) # Note(zhangbo): shm_buffer_size is used for MemoryMapAllocationPool. # MemoryMapAllocationPool is used to cache and reuse shm, thus reducing munmap in dataloader. diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 5f79d3899d21fe..58225da4b9e922 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -77,6 +77,7 @@ def __init__(self): def _parent_alive_win32(self): import ctypes from ctypes import wintypes + PROCESS_QUERY_INFORMATION = 0x0400 STILL_ACTIVE = 259 handle = ctypes.windll.kernel32.OpenProcess( @@ -85,7 +86,9 @@ def _parent_alive_win32(self): if not handle: return False exit_code = wintypes.DWORD(0) - ctypes.windll.kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)) + ctypes.windll.kernel32.GetExitCodeProcess( + handle, ctypes.byref(exit_code) + ) ctypes.windll.kernel32.CloseHandle(handle) return exit_code.value == STILL_ACTIVE @@ -315,9 +318,11 @@ def _worker_loop( ): try: import faulthandler + faulthandler.enable() # Register ForkingPickler handlers so DenseTensor can be serialized from paddle.incubate.multiprocessing import init_reductions + init_reductions() # NOTE: [ mmap files clear ] When the child process exits unexpectedly, @@ -381,7 +386,11 @@ def numpy2lodtensor(arr): out_queue.put((data, None, None)) iterator_drained = False fetcher = _DatasetKind.create_fetcher( - dataset_kind, dataset, auto_collate_batch, collate_fn, True + dataset_kind, + dataset, + auto_collate_batch, + collate_fn, + True, ) continue @@ -434,8 +443,8 @@ def numpy2lodtensor(arr): except: # Write to stderr (visible in parent console) sys.stderr.write( - "[WORKER pid=%d] UNCAUGHT EXCEPTION\n%s\n" % - (os.getpid(), traceback.format_exc())) + f"[WORKER pid={os.getpid()}] UNCAUGHT EXCEPTION\n{traceback.format_exc()}\n" + ) sys.stderr.flush() finally: if use_shared_memory: diff --git a/test/legacy_test/test_dataloader_multiprocess.py b/test/legacy_test/test_dataloader_multiprocess.py index 6e75217fdd13ad..6170992e5da473 100644 --- a/test/legacy_test/test_dataloader_multiprocess.py +++ b/test/legacy_test/test_dataloader_multiprocess.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import sys import unittest import numpy as np @@ -108,9 +106,7 @@ def test_multiprocess_with_shared_memory(self): Test multi-process DataLoader with use_shared_memory=True. """ loss_single = self.run_simple_net(num_workers=0) - loss_multi = self.run_simple_net( - num_workers=2, use_shared_memory=True - ) + loss_multi = self.run_simple_net(num_workers=2, use_shared_memory=True) diff = np.abs(loss_single - loss_multi) / np.abs(loss_single) self.assertLess( diff, diff --git a/test/xpu/amp/CMakeLists.txt b/test/xpu/amp/CMakeLists.txt index fdd528b75a25d6..38b4fc991177d6 120000 --- a/test/xpu/amp/CMakeLists.txt +++ b/test/xpu/amp/CMakeLists.txt @@ -1 +1 @@ -../../amp/CMakeLists.txt \ No newline at end of file +../../amp/CMakeLists.txt diff --git a/test/xpu/amp/test_amp_api_xpu.py b/test/xpu/amp/test_amp_api_xpu.py index 03e5d810d910f9..c9682545004db8 120000 --- a/test/xpu/amp/test_amp_api_xpu.py +++ b/test/xpu/amp/test_amp_api_xpu.py @@ -1 +1,15 @@ -../../amp/test_amp_api.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../../amp/test_amp_api.py diff --git a/test/xpu/amp/test_amp_decorate_xpu.py b/test/xpu/amp/test_amp_decorate_xpu.py index eb7d6f57bf38fc..01ed1b69417ed6 120000 --- a/test/xpu/amp/test_amp_decorate_xpu.py +++ b/test/xpu/amp/test_amp_decorate_xpu.py @@ -1 +1,15 @@ -../../amp/test_amp_decorate.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../../amp/test_amp_decorate.py diff --git a/test/xpu/amp/test_amp_list_xpu.py b/test/xpu/amp/test_amp_list_xpu.py index c27ae9143d3869..7bbaebc4d269f6 120000 --- a/test/xpu/amp/test_amp_list_xpu.py +++ b/test/xpu/amp/test_amp_list_xpu.py @@ -1 +1,15 @@ -../../amp/test_amp_list.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../../amp/test_amp_list.py diff --git a/test/xpu/amp/test_amp_master_grad_xpu.py b/test/xpu/amp/test_amp_master_grad_xpu.py index d3e795aca7c7b9..a594a27a12b414 120000 --- a/test/xpu/amp/test_amp_master_grad_xpu.py +++ b/test/xpu/amp/test_amp_master_grad_xpu.py @@ -1 +1,15 @@ -../../amp/test_amp_master_grad.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../../amp/test_amp_master_grad.py diff --git a/test/xpu/amp/test_amp_master_weight_xpu.py b/test/xpu/amp/test_amp_master_weight_xpu.py index 295d19f41e4c0b..6d54e27b8b63de 120000 --- a/test/xpu/amp/test_amp_master_weight_xpu.py +++ b/test/xpu/amp/test_amp_master_weight_xpu.py @@ -1 +1,15 @@ -../../amp/test_amp_master_weight.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../../amp/test_amp_master_weight.py diff --git a/test/xpu/amp/test_amp_promote_xpu.py b/test/xpu/amp/test_amp_promote_xpu.py index c36819fd619303..bd69b2dd657e7f 120000 --- a/test/xpu/amp/test_amp_promote_xpu.py +++ b/test/xpu/amp/test_amp_promote_xpu.py @@ -1 +1,15 @@ -../../amp/test_amp_promote.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../../amp/test_amp_promote.py diff --git a/test/xpu/amp/test_layer_convert_dtype_xpu.py b/test/xpu/amp/test_layer_convert_dtype_xpu.py index e60adc08f6a7af..85e3f2372ddd4e 120000 --- a/test/xpu/amp/test_layer_convert_dtype_xpu.py +++ b/test/xpu/amp/test_layer_convert_dtype_xpu.py @@ -1 +1,15 @@ -../../amp/test_layer_convert_dtype.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../../amp/test_layer_convert_dtype.py diff --git a/test/xpu/test_incubate_build_src_rank_and_local_expert_id_xpu.py b/test/xpu/test_incubate_build_src_rank_and_local_expert_id_xpu.py index 3313314e0c1dde..ead4c392292242 120000 --- a/test/xpu/test_incubate_build_src_rank_and_local_expert_id_xpu.py +++ b/test/xpu/test_incubate_build_src_rank_and_local_expert_id_xpu.py @@ -1 +1,15 @@ -../legacy_test/test_incubate_build_src_rank_and_local_expert_id.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../legacy_test/test_incubate_build_src_rank_and_local_expert_id.py diff --git a/test/xpu/test_incubate_expand_modality_expert_id_xpu.py b/test/xpu/test_incubate_expand_modality_expert_id_xpu.py index fc8a348406791a..abfcac16cea198 120000 --- a/test/xpu/test_incubate_expand_modality_expert_id_xpu.py +++ b/test/xpu/test_incubate_expand_modality_expert_id_xpu.py @@ -1 +1,15 @@ -../legacy_test/test_incubate_expand_modality_expert_id.py \ No newline at end of file +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +../legacy_test/test_incubate_expand_modality_expert_id.py From 0a15ea615c55d9aaf9fb049fefb69067e6ff4712 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 05:33:51 +0800 Subject: [PATCH 07/32] Revert unrelated files modified by pre-commit --- .claude/skills | 2 +- paddle/fluid/framework/data_type_transform.cu | 16 +--------------- test/xpu/amp/CMakeLists.txt | 2 +- test/xpu/amp/test_amp_api_xpu.py | 16 +--------------- test/xpu/amp/test_amp_decorate_xpu.py | 16 +--------------- test/xpu/amp/test_amp_list_xpu.py | 16 +--------------- test/xpu/amp/test_amp_master_grad_xpu.py | 16 +--------------- test/xpu/amp/test_amp_master_weight_xpu.py | 16 +--------------- test/xpu/amp/test_amp_promote_xpu.py | 16 +--------------- test/xpu/amp/test_layer_convert_dtype_xpu.py | 16 +--------------- ...ate_build_src_rank_and_local_expert_id_xpu.py | 16 +--------------- ...est_incubate_expand_modality_expert_id_xpu.py | 16 +--------------- 12 files changed, 12 insertions(+), 152 deletions(-) diff --git a/.claude/skills b/.claude/skills index 9f020f7e543030..2b7a412b8fa0fb 120000 --- a/.claude/skills +++ b/.claude/skills @@ -1 +1 @@ -../.agents/skills +../.agents/skills \ No newline at end of file diff --git a/paddle/fluid/framework/data_type_transform.cu b/paddle/fluid/framework/data_type_transform.cu index 3dd4bcbef16ee2..f46491293ef4ad 120000 --- a/paddle/fluid/framework/data_type_transform.cu +++ b/paddle/fluid/framework/data_type_transform.cu @@ -1,15 +1 @@ -// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -data_type_transform.cc +data_type_transform.cc \ No newline at end of file diff --git a/test/xpu/amp/CMakeLists.txt b/test/xpu/amp/CMakeLists.txt index 38b4fc991177d6..fdd528b75a25d6 120000 --- a/test/xpu/amp/CMakeLists.txt +++ b/test/xpu/amp/CMakeLists.txt @@ -1 +1 @@ -../../amp/CMakeLists.txt +../../amp/CMakeLists.txt \ No newline at end of file diff --git a/test/xpu/amp/test_amp_api_xpu.py b/test/xpu/amp/test_amp_api_xpu.py index c9682545004db8..03e5d810d910f9 120000 --- a/test/xpu/amp/test_amp_api_xpu.py +++ b/test/xpu/amp/test_amp_api_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../amp/test_amp_api.py +../../amp/test_amp_api.py \ No newline at end of file diff --git a/test/xpu/amp/test_amp_decorate_xpu.py b/test/xpu/amp/test_amp_decorate_xpu.py index 01ed1b69417ed6..eb7d6f57bf38fc 120000 --- a/test/xpu/amp/test_amp_decorate_xpu.py +++ b/test/xpu/amp/test_amp_decorate_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../amp/test_amp_decorate.py +../../amp/test_amp_decorate.py \ No newline at end of file diff --git a/test/xpu/amp/test_amp_list_xpu.py b/test/xpu/amp/test_amp_list_xpu.py index 7bbaebc4d269f6..c27ae9143d3869 120000 --- a/test/xpu/amp/test_amp_list_xpu.py +++ b/test/xpu/amp/test_amp_list_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../amp/test_amp_list.py +../../amp/test_amp_list.py \ No newline at end of file diff --git a/test/xpu/amp/test_amp_master_grad_xpu.py b/test/xpu/amp/test_amp_master_grad_xpu.py index a594a27a12b414..d3e795aca7c7b9 120000 --- a/test/xpu/amp/test_amp_master_grad_xpu.py +++ b/test/xpu/amp/test_amp_master_grad_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../amp/test_amp_master_grad.py +../../amp/test_amp_master_grad.py \ No newline at end of file diff --git a/test/xpu/amp/test_amp_master_weight_xpu.py b/test/xpu/amp/test_amp_master_weight_xpu.py index 6d54e27b8b63de..295d19f41e4c0b 120000 --- a/test/xpu/amp/test_amp_master_weight_xpu.py +++ b/test/xpu/amp/test_amp_master_weight_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../amp/test_amp_master_weight.py +../../amp/test_amp_master_weight.py \ No newline at end of file diff --git a/test/xpu/amp/test_amp_promote_xpu.py b/test/xpu/amp/test_amp_promote_xpu.py index bd69b2dd657e7f..c36819fd619303 120000 --- a/test/xpu/amp/test_amp_promote_xpu.py +++ b/test/xpu/amp/test_amp_promote_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../amp/test_amp_promote.py +../../amp/test_amp_promote.py \ No newline at end of file diff --git a/test/xpu/amp/test_layer_convert_dtype_xpu.py b/test/xpu/amp/test_layer_convert_dtype_xpu.py index 85e3f2372ddd4e..e60adc08f6a7af 120000 --- a/test/xpu/amp/test_layer_convert_dtype_xpu.py +++ b/test/xpu/amp/test_layer_convert_dtype_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../../amp/test_layer_convert_dtype.py +../../amp/test_layer_convert_dtype.py \ No newline at end of file diff --git a/test/xpu/test_incubate_build_src_rank_and_local_expert_id_xpu.py b/test/xpu/test_incubate_build_src_rank_and_local_expert_id_xpu.py index ead4c392292242..3313314e0c1dde 120000 --- a/test/xpu/test_incubate_build_src_rank_and_local_expert_id_xpu.py +++ b/test/xpu/test_incubate_build_src_rank_and_local_expert_id_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../legacy_test/test_incubate_build_src_rank_and_local_expert_id.py +../legacy_test/test_incubate_build_src_rank_and_local_expert_id.py \ No newline at end of file diff --git a/test/xpu/test_incubate_expand_modality_expert_id_xpu.py b/test/xpu/test_incubate_expand_modality_expert_id_xpu.py index abfcac16cea198..fc8a348406791a 120000 --- a/test/xpu/test_incubate_expand_modality_expert_id_xpu.py +++ b/test/xpu/test_incubate_expand_modality_expert_id_xpu.py @@ -1,15 +1 @@ -# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -../legacy_test/test_incubate_expand_modality_expert_id.py +../legacy_test/test_incubate_expand_modality_expert_id.py \ No newline at end of file From e3750dfde877489004e4d6b38e8562cda06ecaee Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 05:49:05 +0800 Subject: [PATCH 08/32] Fix bot review issues: MAPPED_NOCREATE, HANDLE truncation, getpid, encoding, ParentWatchDog PID --- paddle/fluid/imperative/data_loader.cc | 2 +- .../core/memory/allocation/mmap_allocator.cc | 85 +++++++++++++------ .../core/memory/allocation/mmap_allocator.h | 13 +-- .../paddle/io/dataloader/dataloader_iter.py | 3 +- python/paddle/io/dataloader/worker.py | 39 +++++---- 5 files changed, 91 insertions(+), 51 deletions(-) diff --git a/paddle/fluid/imperative/data_loader.cc b/paddle/fluid/imperative/data_loader.cc index 11a6e07f020a57..ab86b651215507 100644 --- a/paddle/fluid/imperative/data_loader.cc +++ b/paddle/fluid/imperative/data_loader.cc @@ -141,7 +141,7 @@ static LONG CALLBACK paddle_crash_filter(EXCEPTION_POINTERS *ep) { void SetLoadProcessSignalHandler() { #ifdef _WIN32 // Last-chance filter: logs crash details before OS terminates the process. - // This is the ONLY handler on Windows — VEH is removed because it could + // This is the ONLY handler on Windows ¡ª VEH is removed because it could // deadlock if MemoryMapFdSet::Clear() is called while the mutex is held. SetUnhandledExceptionFilter(&paddle_crash_filter); VLOG(3) << "DataLoader: crash handler registered"; diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 99ecece86a2bd9..6af89e8771b8de 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -14,6 +14,14 @@ #include "paddle/phi/core/memory/allocation/mmap_allocator.h" +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + #include #include @@ -24,6 +32,12 @@ #include "paddle/common/flags.h" #include "paddle/phi/core/enforce.h" +#ifdef _WIN32 +static inline int GetPid() { return static_cast(GetCurrentProcessId()); } +#else +static inline int GetPid() { return GetPid(); } +#endif + #ifdef _WIN32 #include #else @@ -42,7 +56,7 @@ std::string GetIPCName() { #ifdef _WIN32 handle += std::to_string(GetCurrentProcessId()); #else - handle += std::to_string(getpid()); + handle += std::to_string(GetPid()); #endif handle += "_"; handle += std::to_string(counter.fetch_add(1)); @@ -62,6 +76,27 @@ void AllocateMemoryMap(std::string *filename, void **map_ptr_) { #ifdef _WIN32 DWORD protect = (flags & MAPPED_SHAREDMEM) ? PAGE_READWRITE : PAGE_READONLY; + // For reader path (MAPPED_NOCREATE): open existing section, never create. + if (flags & MAPPED_NOCREATE) { + HANDLE hMap = + OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, filename->c_str()); + PADDLE_ENFORCE_NE(hMap, + nullptr, + common::errors::Unavailable( + "OpenFileMappingA failed for %s, error: %lu", + filename->c_str(), + GetLastError())); + void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); + PADDLE_ENFORCE_NE(ptr, + nullptr, + common::errors::Unavailable( + "MapViewOfFile for reader %s failed, error: %lu", + filename->c_str(), + GetLastError())); + *shared_fd = reinterpret_cast(hMap); + *map_ptr_ = ptr; + return; + } // Retry loop for exclusive mode: on Windows, CreateFileMapping silently // returns the existing mapping if the name already exists, unlike Linux's // shm_open + O_EXCL which fails atomically. We must check explicitly. @@ -69,9 +104,9 @@ void AllocateMemoryMap(std::string *filename, HANDLE hMap = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, protect, - 0, - static_cast(size), - filename.c_str()); + static_cast(size >> 32), + static_cast(size & 0xffffffffULL), + filename->c_str()); if (hMap == NULL) { DWORD err = GetLastError(); @@ -80,7 +115,7 @@ void AllocateMemoryMap(std::string *filename, "[PADDLE_MMAP] PID=%lu CreateFileMapping FAILED " "name=%s size=%zu flags=0x%x attempt=%d err=%lu\n", GetCurrentProcessId(), - filename.c_str(), + filename->c_str(), size, flags, attempt, @@ -88,7 +123,7 @@ void AllocateMemoryMap(std::string *filename, fflush(stderr); PADDLE_THROW(common::errors::Unavailable( "CreateFileMapping failed for %s, error: %lu", - filename.c_str(), + filename->c_str(), err)); } @@ -96,7 +131,7 @@ void AllocateMemoryMap(std::string *filename, CloseHandle(hMap); VLOG(3) << "[PADDLE_MMAP] PID=" << GetCurrentProcessId() << " name collision, retrying attempt=" << attempt - << " name=" << filename; + << " name=" << *filename; filename = GetIPCName(); // name collision; retry with fresh name continue; } @@ -110,19 +145,19 @@ void AllocateMemoryMap(std::string *filename, "[PADDLE_MMAP] PID=%lu MapViewOfFile FAILED " "name=%s size=%zu err=%lu\n", GetCurrentProcessId(), - filename.c_str(), + filename->c_str(), size, err); fflush(stderr); PADDLE_THROW(common::errors::Unavailable( - "MapViewOfFile failed for %s, error: %lu", filename.c_str(), err)); + "MapViewOfFile failed for %s, error: %lu", filename->c_str(), err)); } // On Windows, always keep the HANDLE so the section stays alive // after the caller's MapViewOfFile. The handle is closed in // RefcountedMemoryMapAllocation::close(). Without this, the section // would be destroyed when the last view is unmapped (i.e. when the - // worker's tensor is GC'd), before the reader opens it — this is + // worker's tensor is GC'd), before the reader opens it ¡ª this is // the root cause of "Blocking queue is killed" with large data. // Linux doesn't have this issue because munmap never destroys the // shared memory file (only shm_unlink does). @@ -210,7 +245,10 @@ AllocateRefcountedMemoryMapAllocation(std::string filename, int fd = shared_fd; void *base_ptr = nullptr; if (buffer_id == -1) { - AllocateMemoryMap(&filename, &fd, flags, size + mmap_alignment, &base_ptr); + intptr_t fd_ptr = 0; + AllocateMemoryMap( + &filename, &fd_ptr, flags, size + mmap_alignment, &base_ptr); + int fd = static_cast(fd_ptr); VLOG(4) << "Create and mmap a new shm: " << filename; } else { base_ptr = MemoryMapAllocationPool::Instance().GetById(buffer_id).mmap_ptr_; @@ -247,7 +285,7 @@ void MemoryMapAllocation::close() { // (never closed in AllocateMemoryMap). Close it unconditionally // to prevent handle leaks. if (fd_ != -1) { - CloseHandle(reinterpret_cast(static_cast(fd_))); + CloseHandle(reinterpret_cast(fd_)); } #else if (flags_ & MAPPED_KEEPFD) { @@ -308,7 +346,7 @@ void RefcountedMemoryMapAllocation::close() { // here, regardless of the MAPPED_KEEPFD flag. if (fd_ != -1 && !closed_fd_) { closed_fd_ = true; - CloseHandle(reinterpret_cast(static_cast(fd_))); + CloseHandle(reinterpret_cast(fd_)); VLOG(6) << "close handle: " << fd_; } #else @@ -414,8 +452,8 @@ std::shared_ptr AllocateMemoryMapWriterAllocation( HANDLE hMap = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, - 0, - static_cast(size), + static_cast(size >> 32), + static_cast(size & 0xffffffffULL), ipc_name.c_str()); PADDLE_ENFORCE_NE( hMap, @@ -461,13 +499,12 @@ std::shared_ptr RebuildMemoryMapReaderAllocation( #ifdef _WIN32 HANDLE hMap = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, ipc_name.c_str()); if (!hMap) { - // Section doesn't exist yet; create it. This can happen in some edge cases - // where the writer's view is still alive but all handles were closed. + // Fallback: create new section (non-refcounted path only). hMap = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, - 0, - static_cast(size), + static_cast(size >> 32), + static_cast(size & 0xffffffffULL), ipc_name.c_str()); } PADDLE_ENFORCE_NE( @@ -513,30 +550,30 @@ MemoryMapFdSet &MemoryMapFdSet::Instance() { // NOLINT void MemoryMapFdSet::Insert(const std::string &ipc_name) { std::lock_guard guard(mtx_); fd_set_.emplace(ipc_name); - VLOG(3) << "PID: " << getpid() << ", MemoryMapFdSet: insert " << ipc_name + VLOG(3) << "PID: " << GetPid() << ", MemoryMapFdSet: insert " << ipc_name << ", set size: " << fd_set_.size(); } void MemoryMapFdSet::Remove(const std::string &ipc_name) { std::lock_guard guard(mtx_); fd_set_.erase(ipc_name); - VLOG(3) << "PID: " << getpid() << ", MemoryMapFdSet: erase " << ipc_name + VLOG(3) << "PID: " << GetPid() << ", MemoryMapFdSet: erase " << ipc_name << ", set size: " << fd_set_.size(); } void MemoryMapFdSet::Clear() { - VLOG(7) << "PID: " << getpid() << ", MemoryMapFdSet: set size - " + VLOG(7) << "PID: " << GetPid() << ", MemoryMapFdSet: set size - " << fd_set_.size(); std::lock_guard guard(mtx_); for (auto const &fd : fd_set_) { #ifdef _WIN32 // On Windows, named file mappings are auto-destroyed when the last view // is unmapped. Nothing to unlink explicitly. - VLOG(7) << "PID: " << getpid() << ", MemoryMapFdSet: clear " << fd; + VLOG(7) << "PID: " << GetPid() << ", MemoryMapFdSet: clear " << fd; #else int rlt = shm_unlink(fd.c_str()); if (rlt == 0) { - VLOG(7) << "PID: " << getpid() << ", MemoryMapFdSet: clear " << fd; + VLOG(7) << "PID: " << GetPid() << ", MemoryMapFdSet: clear " << fd; } #endif } diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index 2162a2d21794d0..7034391c8a513f 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include #include @@ -46,14 +47,14 @@ class PADDLE_API MemoryMapAllocation : public Allocation { explicit MemoryMapAllocation(void *ptr, size_t size, std::string ipc_name, - int fd) + intptr_t fd) : Allocation(ptr, size, CPUPlace()), ipc_name_(std::move(ipc_name)), fd_(fd), map_ptr_(ptr), map_size_(size) {} explicit MemoryMapAllocation( - void *ptr, size_t size, std::string ipc_name, int fd, int flags) + void *ptr, size_t size, std::string ipc_name, intptr_t fd, int flags) : Allocation(ptr, size, CPUPlace()), ipc_name_(std::move(ipc_name)), fd_(fd), @@ -62,7 +63,7 @@ class PADDLE_API MemoryMapAllocation : public Allocation { map_size_(size) {} inline const std::string &ipc_name() const { return ipc_name_; } - inline int shared_fd() const { return fd_; } + inline int shared_fd() const { return static_cast(fd_); } virtual void close(); @@ -70,7 +71,7 @@ class PADDLE_API MemoryMapAllocation : public Allocation { protected: std::string ipc_name_; - int fd_ = -1; + intptr_t fd_ = -1; int flags_ = 0; void *map_ptr_ = nullptr; size_t map_size_ = 0; @@ -99,14 +100,14 @@ class PADDLE_API RefcountedMemoryMapAllocation : public MemoryMapAllocation { }; PADDLE_API void AllocateMemoryMap(std::string *filename, - int *shared_fd, + intptr_t *shared_fd, int flags, size_t size, void **base_ptr_); PADDLE_API std::shared_ptr AllocateRefcountedMemoryMapAllocation(std::string filename, - int shared_fd, + intptr_t shared_fd, int flags, size_t size, int buffer_id = -1); diff --git a/python/paddle/io/dataloader/dataloader_iter.py b/python/paddle/io/dataloader/dataloader_iter.py index 288b1d7759d9a7..076dee4e16e943 100644 --- a/python/paddle/io/dataloader/dataloader_iter.py +++ b/python/paddle/io/dataloader/dataloader_iter.py @@ -468,6 +468,7 @@ def _init_workers(self): worker = multiprocessing.Process( target=_worker_loop, args=( + os.getpid(), self._dataset, self._dataset_kind, indices_queue, @@ -486,8 +487,6 @@ def _init_workers(self): ) worker.daemon = True worker.start() - # On Windows with spawn, each worker imports the full Paddle - # framework (~580 MB). Stagger to avoid memory exhaustion. if sys.platform == 'win32' and self._num_workers > 4: time.sleep(0.05) self._workers.append(worker) diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 58225da4b9e922..9c9fab20d5be4d 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -66,29 +66,36 @@ def create_fetcher( class ParentWatchDog: - def __init__(self): - self._parent_pid = os.getppid() + def __init__(self, parent_pid=None): + self._parent_pid = parent_pid if parent_pid is not None else os.getppid() self._parent_alive = True + # On Windows, os.getppid() can return a stale PID from the spawn + # bootstrap process. Use OpenProcess + GetExitCodeProcess instead + # of PPID comparison to reliably detect parent death. if sys.platform == 'win32': self._parent_alive = self._parent_alive_win32() else: self._parent_alive = True def _parent_alive_win32(self): + """Check if parent process is alive using Windows API.""" import ctypes from ctypes import wintypes - - PROCESS_QUERY_INFORMATION = 0x0400 + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 STILL_ACTIVE = 259 handle = ctypes.windll.kernel32.OpenProcess( - PROCESS_QUERY_INFORMATION, False, self._parent_pid + PROCESS_QUERY_LIMITED_INFORMATION, False, self._parent_pid ) if not handle: - return False + # Fallback: try PROCESS_QUERY_INFORMATION (might fail on some systems) + PROCESS_QUERY_INFORMATION = 0x0400 + handle = ctypes.windll.kernel32.OpenProcess( + PROCESS_QUERY_INFORMATION, False, self._parent_pid + ) + if not handle: + return True # Can't check, assume alive exit_code = wintypes.DWORD(0) - ctypes.windll.kernel32.GetExitCodeProcess( - handle, ctypes.byref(exit_code) - ) + ctypes.windll.kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)) ctypes.windll.kernel32.CloseHandle(handle) return exit_code.value == STILL_ACTIVE @@ -301,6 +308,7 @@ def mix(x, y): def _worker_loop( + parent_pid, dataset, dataset_kind, indices_queue, @@ -318,11 +326,9 @@ def _worker_loop( ): try: import faulthandler - faulthandler.enable() # Register ForkingPickler handlers so DenseTensor can be serialized from paddle.incubate.multiprocessing import init_reductions - init_reductions() # NOTE: [ mmap files clear ] When the child process exits unexpectedly, @@ -373,7 +379,7 @@ def numpy2lodtensor(arr): return lodtensor iterator_drained = False - parent_watch_dog = ParentWatchDog() + parent_watch_dog = ParentWatchDog(parent_pid) while parent_watch_dog.is_alive(): try: @@ -386,11 +392,7 @@ def numpy2lodtensor(arr): out_queue.put((data, None, None)) iterator_drained = False fetcher = _DatasetKind.create_fetcher( - dataset_kind, - dataset, - auto_collate_batch, - collate_fn, - True, + dataset_kind, dataset, auto_collate_batch, collate_fn, True ) continue @@ -443,7 +445,8 @@ def numpy2lodtensor(arr): except: # Write to stderr (visible in parent console) sys.stderr.write( - f"[WORKER pid={os.getpid()}] UNCAUGHT EXCEPTION\n{traceback.format_exc()}\n" + f"[WORKER pid={os.getpid()}] UNCAUGHT EXCEPTION\n" + f"{traceback.format_exc()}\n" ) sys.stderr.flush() finally: From e33e98a3e2b08dacf9ce4d39b01577cc80ee845f Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 14:22:52 +0800 Subject: [PATCH 09/32] Fix bot review P0/P1: getpid recursion, filename ptr, intptr_t mismatch, worker parent_pid, except raise, ctypes HANDLE --- .../core/memory/allocation/mmap_allocator.cc | 14 +++--- .../paddle/io/dataloader/dataloader_iter.py | 2 +- python/paddle/io/dataloader/worker.py | 48 +++++++++++++++---- 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 6af89e8771b8de..5a93b1f72960de 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -35,7 +35,7 @@ #ifdef _WIN32 static inline int GetPid() { return static_cast(GetCurrentProcessId()); } #else -static inline int GetPid() { return GetPid(); } +static inline int GetPid() { return getpid(); } #endif #ifdef _WIN32 @@ -70,7 +70,7 @@ struct CountInfo { }; void AllocateMemoryMap(std::string *filename, - int *shared_fd, + intptr_t *shared_fd, int flags, size_t size, void **map_ptr_) { @@ -238,20 +238,18 @@ void AllocateMemoryMap(std::string *filename, std::shared_ptr AllocateRefcountedMemoryMapAllocation(std::string filename, - int shared_fd, + intptr_t shared_fd, int flags, size_t size, int buffer_id) { - int fd = shared_fd; + intptr_t fd = 0; void *base_ptr = nullptr; if (buffer_id == -1) { - intptr_t fd_ptr = 0; - AllocateMemoryMap( - &filename, &fd_ptr, flags, size + mmap_alignment, &base_ptr); - int fd = static_cast(fd_ptr); + AllocateMemoryMap(&filename, &fd, flags, size + mmap_alignment, &base_ptr); VLOG(4) << "Create and mmap a new shm: " << filename; } else { base_ptr = MemoryMapAllocationPool::Instance().GetById(buffer_id).mmap_ptr_; + fd = shared_fd; VLOG(4) << "Get a cached shm " << filename; } void *aligned_base_ptr = diff --git a/python/paddle/io/dataloader/dataloader_iter.py b/python/paddle/io/dataloader/dataloader_iter.py index 076dee4e16e943..7401e988928aa0 100644 --- a/python/paddle/io/dataloader/dataloader_iter.py +++ b/python/paddle/io/dataloader/dataloader_iter.py @@ -468,7 +468,6 @@ def _init_workers(self): worker = multiprocessing.Process( target=_worker_loop, args=( - os.getpid(), self._dataset, self._dataset_kind, indices_queue, @@ -483,6 +482,7 @@ def _init_workers(self): self._use_shared_memory, self._base_seed, self._worker_shm_buffer_size, + os.getpid(), ), ) worker.daemon = True diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 9c9fab20d5be4d..433f5f895589e2 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -67,7 +67,9 @@ def create_fetcher( class ParentWatchDog: def __init__(self, parent_pid=None): - self._parent_pid = parent_pid if parent_pid is not None else os.getppid() + self._parent_pid = ( + parent_pid if parent_pid is not None else os.getppid() + ) self._parent_alive = True # On Windows, os.getppid() can return a stale PID from the spawn # bootstrap process. Use OpenProcess + GetExitCodeProcess instead @@ -81,21 +83,44 @@ def _parent_alive_win32(self): """Check if parent process is alive using Windows API.""" import ctypes from ctypes import wintypes + + kernel32 = ctypes.windll.kernel32 + # Set proper argument/return types for Win64 HANDLE compatibility + kernel32.OpenProcess.argtypes = [ + wintypes.DWORD, + wintypes.BOOL, + wintypes.DWORD, + ] + kernel32.OpenProcess.restype = wintypes.HANDLE + kernel32.GetExitCodeProcess.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(wintypes.DWORD), + ] + kernel32.GetExitCodeProcess.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + kernel32.CloseHandle.restype = wintypes.BOOL + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 STILL_ACTIVE = 259 - handle = ctypes.windll.kernel32.OpenProcess( - PROCESS_QUERY_LIMITED_INFORMATION, False, self._parent_pid + handle = kernel32.OpenProcess( + PROCESS_QUERY_LIMITED_INFORMATION, + False, + wintypes.DWORD(self._parent_pid), ) if not handle: # Fallback: try PROCESS_QUERY_INFORMATION (might fail on some systems) PROCESS_QUERY_INFORMATION = 0x0400 - handle = ctypes.windll.kernel32.OpenProcess( - PROCESS_QUERY_INFORMATION, False, self._parent_pid + handle = kernel32.OpenProcess( + PROCESS_QUERY_INFORMATION, + False, + wintypes.DWORD(self._parent_pid), ) if not handle: return True # Can't check, assume alive exit_code = wintypes.DWORD(0) - ctypes.windll.kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)) + ctypes.windll.kernel32.GetExitCodeProcess( + handle, ctypes.byref(exit_code) + ) ctypes.windll.kernel32.CloseHandle(handle) return exit_code.value == STILL_ACTIVE @@ -308,7 +333,6 @@ def mix(x, y): def _worker_loop( - parent_pid, dataset, dataset_kind, indices_queue, @@ -323,12 +347,15 @@ def _worker_loop( use_shared_memory, base_seed, shm_cache_size=0, + parent_pid=None, ): try: import faulthandler + faulthandler.enable() # Register ForkingPickler handlers so DenseTensor can be serialized from paddle.incubate.multiprocessing import init_reductions + init_reductions() # NOTE: [ mmap files clear ] When the child process exits unexpectedly, @@ -392,7 +419,11 @@ def numpy2lodtensor(arr): out_queue.put((data, None, None)) iterator_drained = False fetcher = _DatasetKind.create_fetcher( - dataset_kind, dataset, auto_collate_batch, collate_fn, True + dataset_kind, + dataset, + auto_collate_batch, + collate_fn, + True, ) continue @@ -449,6 +480,7 @@ def numpy2lodtensor(arr): f"{traceback.format_exc()}\n" ) sys.stderr.flush() + raise finally: if use_shared_memory: _cleanup_mmap() From 30903de4b72133765fdb5a093f89d248cff8590f Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 14:54:07 +0800 Subject: [PATCH 10/32] Revert filename to reference: only modify Windows branch, keep Linux branch untouched --- .../core/memory/allocation/mmap_allocator.cc | 24 +++++++++---------- .../core/memory/allocation/mmap_allocator.h | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 5a93b1f72960de..d1416d007ff027 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -69,7 +69,7 @@ struct CountInfo { std::atomic refcount; }; -void AllocateMemoryMap(std::string *filename, +void AllocateMemoryMap(std::string &filename, intptr_t *shared_fd, int flags, size_t size, @@ -79,19 +79,19 @@ void AllocateMemoryMap(std::string *filename, // For reader path (MAPPED_NOCREATE): open existing section, never create. if (flags & MAPPED_NOCREATE) { HANDLE hMap = - OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, filename->c_str()); + OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, filename.c_str()); PADDLE_ENFORCE_NE(hMap, nullptr, common::errors::Unavailable( "OpenFileMappingA failed for %s, error: %lu", - filename->c_str(), + filename.c_str(), GetLastError())); void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); PADDLE_ENFORCE_NE(ptr, nullptr, common::errors::Unavailable( "MapViewOfFile for reader %s failed, error: %lu", - filename->c_str(), + filename.c_str(), GetLastError())); *shared_fd = reinterpret_cast(hMap); *map_ptr_ = ptr; @@ -106,7 +106,7 @@ void AllocateMemoryMap(std::string *filename, protect, static_cast(size >> 32), static_cast(size & 0xffffffffULL), - filename->c_str()); + filename.c_str()); if (hMap == NULL) { DWORD err = GetLastError(); @@ -115,7 +115,7 @@ void AllocateMemoryMap(std::string *filename, "[PADDLE_MMAP] PID=%lu CreateFileMapping FAILED " "name=%s size=%zu flags=0x%x attempt=%d err=%lu\n", GetCurrentProcessId(), - filename->c_str(), + filename.c_str(), size, flags, attempt, @@ -123,7 +123,7 @@ void AllocateMemoryMap(std::string *filename, fflush(stderr); PADDLE_THROW(common::errors::Unavailable( "CreateFileMapping failed for %s, error: %lu", - filename->c_str(), + filename.c_str(), err)); } @@ -131,7 +131,7 @@ void AllocateMemoryMap(std::string *filename, CloseHandle(hMap); VLOG(3) << "[PADDLE_MMAP] PID=" << GetCurrentProcessId() << " name collision, retrying attempt=" << attempt - << " name=" << *filename; + << " name=" << filename; filename = GetIPCName(); // name collision; retry with fresh name continue; } @@ -145,12 +145,12 @@ void AllocateMemoryMap(std::string *filename, "[PADDLE_MMAP] PID=%lu MapViewOfFile FAILED " "name=%s size=%zu err=%lu\n", GetCurrentProcessId(), - filename->c_str(), + filename.c_str(), size, err); fflush(stderr); PADDLE_THROW(common::errors::Unavailable( - "MapViewOfFile failed for %s, error: %lu", filename->c_str(), err)); + "MapViewOfFile failed for %s, error: %lu", filename.c_str(), err)); } // On Windows, always keep the HANDLE so the section stays alive @@ -245,7 +245,7 @@ AllocateRefcountedMemoryMapAllocation(std::string filename, intptr_t fd = 0; void *base_ptr = nullptr; if (buffer_id == -1) { - AllocateMemoryMap(&filename, &fd, flags, size + mmap_alignment, &base_ptr); + AllocateMemoryMap(filename, &fd, flags, size + mmap_alignment, &base_ptr); VLOG(4) << "Create and mmap a new shm: " << filename; } else { base_ptr = MemoryMapAllocationPool::Instance().GetById(buffer_id).mmap_ptr_; @@ -263,7 +263,7 @@ RefcountedMemoryMapAllocation::RefcountedMemoryMapAllocation( void *ptr, size_t size, std::string ipc_name, - int fd, + intptr_t fd, int flags, int buffer_id) : MemoryMapAllocation(ptr, size, ipc_name, fd, flags) { diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index 7034391c8a513f..b7cd8936a539e3 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -85,7 +85,7 @@ class PADDLE_API RefcountedMemoryMapAllocation : public MemoryMapAllocation { size_t size, std::string ipc_name, int flags, - int fd, + intptr_t fd, int buffer_id = -1); void incref(); @@ -99,7 +99,7 @@ class PADDLE_API RefcountedMemoryMapAllocation : public MemoryMapAllocation { void resetBaseptr(); }; -PADDLE_API void AllocateMemoryMap(std::string *filename, +PADDLE_API void AllocateMemoryMap(std::string &filename, intptr_t *shared_fd, int flags, size_t size, From d5a0216ce41ed1f7df35d4177dca39c4a19ed3f7 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 14:58:48 +0800 Subject: [PATCH 11/32] Fix cpplint non-const reference: pass filename by value; fix encoding --- paddle/fluid/imperative/data_loader.cc | 2 +- paddle/phi/core/memory/allocation/mmap_allocator.cc | 4 ++-- paddle/phi/core/memory/allocation/mmap_allocator.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/imperative/data_loader.cc b/paddle/fluid/imperative/data_loader.cc index ab86b651215507..77408fb6916268 100644 --- a/paddle/fluid/imperative/data_loader.cc +++ b/paddle/fluid/imperative/data_loader.cc @@ -141,7 +141,7 @@ static LONG CALLBACK paddle_crash_filter(EXCEPTION_POINTERS *ep) { void SetLoadProcessSignalHandler() { #ifdef _WIN32 // Last-chance filter: logs crash details before OS terminates the process. - // This is the ONLY handler on Windows ¡ª VEH is removed because it could + // This is the ONLY handler on Windows -- VEH is removed because it could // deadlock if MemoryMapFdSet::Clear() is called while the mutex is held. SetUnhandledExceptionFilter(&paddle_crash_filter); VLOG(3) << "DataLoader: crash handler registered"; diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index d1416d007ff027..ae9787a65cf0cc 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -69,7 +69,7 @@ struct CountInfo { std::atomic refcount; }; -void AllocateMemoryMap(std::string &filename, +void AllocateMemoryMap(std::string filename, intptr_t *shared_fd, int flags, size_t size, @@ -157,7 +157,7 @@ void AllocateMemoryMap(std::string &filename, // after the caller's MapViewOfFile. The handle is closed in // RefcountedMemoryMapAllocation::close(). Without this, the section // would be destroyed when the last view is unmapped (i.e. when the - // worker's tensor is GC'd), before the reader opens it ¡ª this is + // worker's tensor is GC'd), before the reader opens it -- this is // the root cause of "Blocking queue is killed" with large data. // Linux doesn't have this issue because munmap never destroys the // shared memory file (only shm_unlink does). diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index b7cd8936a539e3..bfb523dc1952ba 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -99,7 +99,7 @@ class PADDLE_API RefcountedMemoryMapAllocation : public MemoryMapAllocation { void resetBaseptr(); }; -PADDLE_API void AllocateMemoryMap(std::string &filename, +PADDLE_API void AllocateMemoryMap(std::string filename, intptr_t *shared_fd, int flags, size_t size, From 532065cc7093d554c3d4bec5f24b7efd892eee0d Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 15:36:56 +0800 Subject: [PATCH 12/32] Fix bot review P0: constructor param order (flags/fd) and fd sentinel init --- paddle/phi/core/memory/allocation/mmap_allocator.cc | 2 +- paddle/phi/core/memory/allocation/mmap_allocator.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index ae9787a65cf0cc..55f6657cac4d93 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -242,7 +242,7 @@ AllocateRefcountedMemoryMapAllocation(std::string filename, int flags, size_t size, int buffer_id) { - intptr_t fd = 0; + intptr_t fd = shared_fd; void *base_ptr = nullptr; if (buffer_id == -1) { AllocateMemoryMap(filename, &fd, flags, size + mmap_alignment, &base_ptr); diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index bfb523dc1952ba..a24b3de4e0f54f 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -84,8 +84,8 @@ class PADDLE_API RefcountedMemoryMapAllocation : public MemoryMapAllocation { RefcountedMemoryMapAllocation(void *ptr, size_t size, std::string ipc_name, - int flags, intptr_t fd, + int flags, int buffer_id = -1); void incref(); From b326d7e0f1736b2741f9a7e9938dde398351f86a Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 17:45:23 +0800 Subject: [PATCH 13/32] Fix CI failures: CloseHandle gated by refcount==0, update outdated test assertions --- .../core/memory/allocation/mmap_allocator.cc | 42 ++++++++----------- test/legacy_test/test_dataloader_autotune.py | 2 +- test/legacy_test/test_modelaverage.py | 32 +++++++------- ...ocess_dataloader_iterable_dataset_split.py | 29 ++++++------- 4 files changed, 50 insertions(+), 55 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 55f6657cac4d93..cd44fd8f94cf47 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -338,26 +338,6 @@ void RefcountedMemoryMapAllocation::close() { CountInfo *info = reinterpret_cast(data); --info->refcount; -#ifdef _WIN32 - // On Windows, the HANDLE from CreateFileMapping is always kept by - // AllocateMemoryMap (never closed there). Close it unconditionally - // here, regardless of the MAPPED_KEEPFD flag. - if (fd_ != -1 && !closed_fd_) { - closed_fd_ = true; - CloseHandle(reinterpret_cast(fd_)); - VLOG(6) << "close handle: " << fd_; - } -#else - if (flags_ & MAPPED_KEEPFD) { - closed_fd_ = true; - PADDLE_ENFORCE_NE( - ::close(fd_), - -1, - common::errors::Unavailable("Error closing file descriptor <%d>", fd_)); - VLOG(6) << "close fd: " << fd_; - } -#endif - if (FLAGS_use_shm_cache && buffer_id_ != -1) { return; } else { @@ -370,14 +350,26 @@ void RefcountedMemoryMapAllocation::close() { } else { if (info->refcount == 0) { #ifdef _WIN32 - // Only unmap when refcount reaches 0 (all readers AND writers are - // done). When refcount > 0, the view must stay mapped on Windows - // to keep the named file mapping section alive so readers can open - // it. On Linux this is unnecessary because munmap doesn't destroy - // the shared memory file (only shm_unlink does). + // Only close the handle AND unmap when refcount reaches 0. + // On Windows, CloseHandle destroys the named file mapping section. + // Closing it early (while refcount > 0) would delete the section + // before the reader process has a chance to OpenFileMappingA it. + if (fd_ != -1 && !closed_fd_) { + closed_fd_ = true; + CloseHandle(reinterpret_cast(fd_)); + VLOG(6) << "close handle: " << fd_; + } VLOG(6) << "UnmapViewOfFile: " << ipc_name_; UnmapViewOfFile(map_ptr_); #else + if (flags_ & MAPPED_KEEPFD) { + closed_fd_ = true; + PADDLE_ENFORCE_NE(::close(fd_), + -1, + common::errors::Unavailable( + "Error closing file descriptor <%d>", fd_)); + VLOG(6) << "close fd: " << fd_; + } shm_unlink(ipc_name_.c_str()); VLOG(6) << "shm_unlink file: " << ipc_name_; #endif diff --git a/test/legacy_test/test_dataloader_autotune.py b/test/legacy_test/test_dataloader_autotune.py index e6ff899eeb7745..427da807378feb 100755 --- a/test/legacy_test/test_dataloader_autotune.py +++ b/test/legacy_test/test_dataloader_autotune.py @@ -76,7 +76,7 @@ def test_dataloader_disable_autotune(self): loader = DataLoader( self.dataset, batch_size=self.batch_size, num_workers=2 ) - if sys.platform == 'darwin' or sys.platform == 'win32': + if sys.platform == 'darwin': self.assertEqual(loader.num_workers, 0) else: self.assertEqual(loader.num_workers, 2) diff --git a/test/legacy_test/test_modelaverage.py b/test/legacy_test/test_modelaverage.py index 06f9a1b51ee517..b449a59fdc10a9 100644 --- a/test/legacy_test/test_modelaverage.py +++ b/test/legacy_test/test_modelaverage.py @@ -29,6 +29,23 @@ def get_value_by_name(name, ops): return value +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples, image_size=784, class_num=10): + self.num_samples = num_samples + self.image_size = image_size + self.class_num = class_num + + def __getitem__(self, idx): + image = np.random.random([self.image_size]).astype('float32') + label = np.random.randint(0, self.class_num - 1, (1,)).astype( + 'int64' + ) + return image, label + + def __len__(self): + return self.num_samples + + class TestModelAverage(unittest.TestCase): def test_model_average_static(self): paddle.enable_static() @@ -146,21 +163,6 @@ def test_model_average_dygraph(self): IMAGE_SIZE = 784 CLASS_NUM = 10 - # define a random dataset - class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): - self.num_samples = num_samples - - def __getitem__(self, idx): - image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( - 'int64' - ) - return image, label - - def __len__(self): - return self.num_samples - class LinearNet(nn.Layer): def __init__(self): super().__init__() diff --git a/test/legacy_test/test_multiprocess_dataloader_iterable_dataset_split.py b/test/legacy_test/test_multiprocess_dataloader_iterable_dataset_split.py index a8fe137523058b..a148cb1e951e51 100644 --- a/test/legacy_test/test_multiprocess_dataloader_iterable_dataset_split.py +++ b/test/legacy_test/test_multiprocess_dataloader_iterable_dataset_split.py @@ -75,26 +75,27 @@ def __iter__(self): yield np.array([i]) +def worker_splitter(worker_id): + worker_info = get_worker_info() + + dataset = worker_info.dataset + start = dataset.start + end = dataset.end + num_per_worker = int( + math.ceil((end - start) / float(worker_info.num_workers)) + ) + + worker_id = worker_info.id + dataset.start = start + worker_id * num_per_worker + dataset.end = min(dataset.start + num_per_worker, end) + + class TestDynamicDataLoaderIterInitFuncSplit(unittest.TestCase): def test_main(self): place = base.CPUPlace() with base.dygraph.guard(place): dataset = RangeIterableDataset(0, 10) - def worker_splitter(worker_id): - worker_info = get_worker_info() - - dataset = worker_info.dataset - start = dataset.start - end = dataset.end - num_per_worker = int( - math.ceil((end - start) / float(worker_info.num_workers)) - ) - - worker_id = worker_info.id - dataset.start = start + worker_id * num_per_worker - dataset.end = min(dataset.start + num_per_worker, end) - dataloader = DataLoader( dataset, places=place, From 7ac47d83e7cc089f830857adfe49ab7459d11e0f Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 19:39:19 +0800 Subject: [PATCH 14/32] Fix: unmap Windows mapped view when refcount>0 (prevent per-batch leak); ruff format test_modelaverage.py --- paddle/phi/core/memory/allocation/mmap_allocator.cc | 13 +++++++++++++ test/legacy_test/test_modelaverage.py | 4 +--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index cd44fd8f94cf47..74207c4c6d28d2 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -374,6 +374,19 @@ void RefcountedMemoryMapAllocation::close() { VLOG(6) << "shm_unlink file: " << ipc_name_; #endif } +#ifdef _WIN32 + else { + // Refcount > 0: prevent MemoryMapAllocation::close() (base class + // destructor) from calling CloseHandle. On Windows, closing the + // HANDLE destroys the named section, even when refcount > 0. + // The HANDLE will be closed when refcount reaches 0, or when the + // process exits (OS cleanup). + // Also unmap the view to prevent mapped-view leak per batch. + closed_fd_ = true; + VLOG(6) << "UnmapViewOfFile (refcount>0): " << ipc_name_; + UnmapViewOfFile(map_ptr_); + } +#endif #ifndef _WIN32 // On Linux, munmap is always safe since it only unmaps virtual memory; // the shared memory file in /dev/shm persists until shm_unlink. diff --git a/test/legacy_test/test_modelaverage.py b/test/legacy_test/test_modelaverage.py index b449a59fdc10a9..f4148f75a9ce35 100644 --- a/test/legacy_test/test_modelaverage.py +++ b/test/legacy_test/test_modelaverage.py @@ -37,9 +37,7 @@ def __init__(self, num_samples, image_size=784, class_num=10): def __getitem__(self, idx): image = np.random.random([self.image_size]).astype('float32') - label = np.random.randint(0, self.class_num - 1, (1,)).astype( - 'int64' - ) + label = np.random.randint(0, self.class_num - 1, (1,)).astype('int64') return image, label def __len__(self): From 873503ffe38c4412cc2b486ec162b67a6826a9cb Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 19:50:38 +0800 Subject: [PATCH 15/32] Fix cpplint readability/braces: move #ifdef inside else block --- paddle/phi/core/memory/allocation/mmap_allocator.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 74207c4c6d28d2..d0d590cf0d803d 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -373,9 +373,8 @@ void RefcountedMemoryMapAllocation::close() { shm_unlink(ipc_name_.c_str()); VLOG(6) << "shm_unlink file: " << ipc_name_; #endif - } + } else { #ifdef _WIN32 - else { // Refcount > 0: prevent MemoryMapAllocation::close() (base class // destructor) from calling CloseHandle. On Windows, closing the // HANDLE destroys the named section, even when refcount > 0. @@ -385,8 +384,8 @@ void RefcountedMemoryMapAllocation::close() { closed_fd_ = true; VLOG(6) << "UnmapViewOfFile (refcount>0): " << ipc_name_; UnmapViewOfFile(map_ptr_); - } #endif + } #ifndef _WIN32 // On Linux, munmap is always safe since it only unmaps virtual memory; // the shared memory file in /dev/shm persists until shm_unlink. From 82ff302c61823271700a8af1b934c068543a8292 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 20:09:29 +0800 Subject: [PATCH 16/32] Add WindowsHandleKeeper: properly close pending HANDLEs during worker cleanup --- .../core/memory/allocation/mmap_allocator.cc | 41 ++++++++++++++++--- .../core/memory/allocation/mmap_allocator.h | 19 +++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index d0d590cf0d803d..2698d8c9282ad9 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -375,12 +375,12 @@ void RefcountedMemoryMapAllocation::close() { #endif } else { #ifdef _WIN32 - // Refcount > 0: prevent MemoryMapAllocation::close() (base class - // destructor) from calling CloseHandle. On Windows, closing the - // HANDLE destroys the named section, even when refcount > 0. - // The HANDLE will be closed when refcount reaches 0, or when the - // process exits (OS cleanup). - // Also unmap the view to prevent mapped-view leak per batch. + // Refcount > 0: transfer the HANDLE to WindowsHandleKeeper so it + // stays open (keeping the named section alive for readers) but is + // properly closed during worker cleanup (CloseAll) rather than + // leaked until process exit. + WindowsHandleKeeper::Instance().Insert(ipc_name_, fd_); + fd_ = -1; closed_fd_ = true; VLOG(6) << "UnmapViewOfFile (refcount>0): " << ipc_name_; UnmapViewOfFile(map_ptr_); @@ -580,6 +580,11 @@ void MemoryMapFdSet::Clear() { #endif } fd_set_.clear(); +#ifdef _WIN32 + // Close pending HANDLEs that were kept open for cross-process section + // lifecycle (refcount > 0 path in RefcountedMemoryMapAllocation::close). + WindowsHandleKeeper::Instance().CloseAll(); +#endif } MemoryMapFdSet::~MemoryMapFdSet() { Clear(); } @@ -647,4 +652,28 @@ void MemoryMapAllocationPool::Clear() { MemoryMapAllocationPool::~MemoryMapAllocationPool() { Clear(); } // NOLINT +#ifdef _WIN32 +WindowsHandleKeeper &WindowsHandleKeeper::Instance() { + static WindowsHandleKeeper keeper; + return keeper; +} + +void WindowsHandleKeeper::Insert(const std::string &ipc_name, intptr_t fd) { + std::lock_guard lock(mtx_); + handles_[ipc_name] = fd; +} + +void WindowsHandleKeeper::CloseAll() { + std::lock_guard lock(mtx_); + for (auto &pair : handles_) { + if (pair.second != -1) { + CloseHandle(reinterpret_cast(pair.second)); + } + } + handles_.clear(); +} + +WindowsHandleKeeper::~WindowsHandleKeeper() { CloseAll(); } // NOLINT +#endif + } // namespace paddle::memory::allocation diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index a24b3de4e0f54f..f7826dfdd41f16 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -171,6 +172,24 @@ class PADDLE_API MemoryMapFdSet { std::mutex mtx_; }; +#ifdef _WIN32 +// Tracks HANDLEs from CreateFileMappingA that must stay open (refcount > 0) +// to keep the named section alive for readers. These HANDLEs are closed +// during worker cleanup via MemoryMapFdSet::Clear() / _cleanup_mmap_fds. +class PADDLE_API WindowsHandleKeeper { + public: + static WindowsHandleKeeper &Instance(); // NOLINT + void Insert(const std::string &ipc_name, intptr_t fd); + void CloseAll(); + ~WindowsHandleKeeper(); + + private: + WindowsHandleKeeper() = default; + std::unordered_map handles_; + std::mutex mtx_; +}; +#endif + class MemoryMapInfo { public: explicit MemoryMapInfo(int flags, From f5f53a0a431e2c80b62784db8786b0cb5f491ef7 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 20:47:57 +0800 Subject: [PATCH 17/32] Fix bot review P1 issues: leaky WindowsHandleKeeper singleton, remove RebuildMemoryMapReaderAllocation fallback, fix ParentWatchDog error handling --- .../core/memory/allocation/mmap_allocator.cc | 19 +++++++------------ python/paddle/io/dataloader/worker.py | 7 ++++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 2698d8c9282ad9..377afce15f95a6 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -500,20 +500,11 @@ std::shared_ptr RebuildMemoryMapReaderAllocation( const std::string &ipc_name, size_t size) { #ifdef _WIN32 HANDLE hMap = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, ipc_name.c_str()); - if (!hMap) { - // Fallback: create new section (non-refcounted path only). - hMap = CreateFileMappingA(INVALID_HANDLE_VALUE, - NULL, - PAGE_READWRITE, - static_cast(size >> 32), - static_cast(size & 0xffffffffULL), - ipc_name.c_str()); - } PADDLE_ENFORCE_NE( hMap, nullptr, common::errors::Unavailable( - "CreateFileMapping/OpenFileMapping for reader %s failed, error: %lu", + "OpenFileMappingA for reader %s failed, error: %lu", ipc_name.c_str(), GetLastError())); @@ -654,8 +645,12 @@ MemoryMapAllocationPool::~MemoryMapAllocationPool() { Clear(); } // NOLINT #ifdef _WIN32 WindowsHandleKeeper &WindowsHandleKeeper::Instance() { - static WindowsHandleKeeper keeper; - return keeper; + // Leaky singleton: heap-allocated, never destructed, to avoid static + // destruction ordering conflicts with MemoryMapFdSet. Normal cleanup + // is driven by MemoryMapFdSet::Clear() / _cleanup_mmap_fds(); on + // abnormal process exit the OS reclaims the memory. + static auto *keeper = new WindowsHandleKeeper(); + return *keeper; } void WindowsHandleKeeper::Insert(const std::string &ipc_name, intptr_t fd) { diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 433f5f895589e2..20fac729921625 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -116,7 +116,12 @@ def _parent_alive_win32(self): wintypes.DWORD(self._parent_pid), ) if not handle: - return True # Can't check, assume alive + err = ctypes.windll.kernel32.GetLastError() + # ERROR_INVALID_PARAMETER (87) means PID doesn't exist. + # ERROR_ACCESS_DENIED (5) means process may exist but access denied. + if err == 5: # ERROR_ACCESS_DENIED + return True + return False exit_code = wintypes.DWORD(0) ctypes.windll.kernel32.GetExitCodeProcess( handle, ctypes.byref(exit_code) From 64dc7c1ee1d71a0ee3ecf87c6528eca677364357 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 20:55:41 +0800 Subject: [PATCH 18/32] Apply clang-format formatting --- paddle/phi/core/memory/allocation/mmap_allocator.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 377afce15f95a6..7df3d2fe6cab29 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -500,13 +500,12 @@ std::shared_ptr RebuildMemoryMapReaderAllocation( const std::string &ipc_name, size_t size) { #ifdef _WIN32 HANDLE hMap = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, ipc_name.c_str()); - PADDLE_ENFORCE_NE( - hMap, - nullptr, - common::errors::Unavailable( - "OpenFileMappingA for reader %s failed, error: %lu", - ipc_name.c_str(), - GetLastError())); + PADDLE_ENFORCE_NE(hMap, + nullptr, + common::errors::Unavailable( + "OpenFileMappingA for reader %s failed, error: %lu", + ipc_name.c_str(), + GetLastError())); void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); PADDLE_ENFORCE_NE(ptr, From a38de2e7e80d152a2368e300b686b37ad33a006a Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 21:24:57 +0800 Subject: [PATCH 19/32] Fix hMap leak on MapViewOfFile failure; keep writer HANDLE alive until ~MemoryMapWriterAllocation --- .../core/memory/allocation/mmap_allocator.cc | 53 ++++++++++++------- .../core/memory/allocation/mmap_allocator.h | 11 +++- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 7df3d2fe6cab29..528a60affcc512 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -87,12 +87,14 @@ void AllocateMemoryMap(std::string filename, filename.c_str(), GetLastError())); void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); - PADDLE_ENFORCE_NE(ptr, - nullptr, - common::errors::Unavailable( - "MapViewOfFile for reader %s failed, error: %lu", - filename.c_str(), - GetLastError())); + if (ptr == nullptr) { + DWORD err = GetLastError(); + CloseHandle(hMap); + PADDLE_THROW(common::errors::Unavailable( + "MapViewOfFile for reader %s failed, error: %lu", + filename.c_str(), + err)); + } *shared_fd = reinterpret_cast(hMap); *map_ptr_ = ptr; return; @@ -403,6 +405,9 @@ void RefcountedMemoryMapAllocation::close() { MemoryMapWriterAllocation::~MemoryMapWriterAllocation() { #ifdef _WIN32 UnmapViewOfFile(this->ptr()); + if (fd_ != -1) { + CloseHandle(reinterpret_cast(fd_)); + } #else if (munmap(this->ptr(), this->size()) == -1) { common::errors::Unavailable("could not unmap the shared memory file %s", @@ -464,14 +469,20 @@ std::shared_ptr AllocateMemoryMapWriterAllocation( ipc_name.c_str())); void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); - PADDLE_ENFORCE_NE( - ptr, - nullptr, - common::errors::Unavailable("MapViewOfFile for writer %s failed", - ipc_name.c_str())); - // Close the handle; the named section stays alive through MapViewOfFile. - CloseHandle(hMap); - return std::make_shared(ptr, size, ipc_name); + if (ptr == nullptr) { + DWORD err = GetLastError(); + CloseHandle(hMap); + PADDLE_THROW(common::errors::Unavailable( + "MapViewOfFile for writer %s failed, error: %lu", + ipc_name.c_str(), + err)); + } + // Keep the HANDLE open until ~MemoryMapWriterAllocation (where + // UnmapViewOfFile runs first, then CloseHandle). Closing it here + // would destroy the named section as soon as the writer unmaps, + // before the reader has a chance to OpenFileMappingA it. + return std::make_shared( + ptr, size, ipc_name, reinterpret_cast(hMap)); #else int flags = O_RDWR | O_CREAT; int fd = shm_open(ipc_name.c_str(), flags, 0600); @@ -508,12 +519,14 @@ std::shared_ptr RebuildMemoryMapReaderAllocation( GetLastError())); void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); - PADDLE_ENFORCE_NE(ptr, - nullptr, - common::errors::Unavailable( - "MapViewOfFile for reader %s failed, error: %lu", - ipc_name.c_str(), - GetLastError())); + if (ptr == nullptr) { + DWORD err = GetLastError(); + CloseHandle(hMap); + PADDLE_THROW(common::errors::Unavailable( + "MapViewOfFile for reader %s failed, error: %lu", + ipc_name.c_str(), + err)); + } CloseHandle(hMap); return std::make_shared(ptr, size, ipc_name); #else diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index f7826dfdd41f16..f62bcbc12d0e44 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -119,15 +119,22 @@ class PADDLE_API MemoryMapWriterAllocation : public Allocation { size_t size, std::string ipc_name) : Allocation(ptr, size, CPUPlace()), ipc_name_(std::move(ipc_name)) {} + explicit MemoryMapWriterAllocation(void *ptr, + size_t size, + std::string ipc_name, + intptr_t fd) + : Allocation(ptr, size, CPUPlace()), + ipc_name_(std::move(ipc_name)), + fd_(fd) {} inline const std::string &ipc_name() const { return ipc_name_; } - inline int shared_fd() const { return fd_; } + inline intptr_t shared_fd() const { return fd_; } ~MemoryMapWriterAllocation() override; private: std::string ipc_name_; - int fd_ = -1; + intptr_t fd_ = -1; }; class PADDLE_API MemoryMapReaderAllocation : public Allocation { From 25b7e069b569660c6ea7a2185d3b46510215a800 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 22:30:44 +0800 Subject: [PATCH 20/32] Fix collision retry IPC name mismatch: AllocateMemoryMap takes std::string* to propagate retried name --- .../core/memory/allocation/mmap_allocator.cc | 54 +++++++++---------- .../core/memory/allocation/mmap_allocator.h | 2 +- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 528a60affcc512..0ea8054a5fe4a8 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -69,22 +69,22 @@ struct CountInfo { std::atomic refcount; }; -void AllocateMemoryMap(std::string filename, +void AllocateMemoryMap(std::string *fname_ptr, intptr_t *shared_fd, int flags, size_t size, void **map_ptr_) { + std::string &fname = *fname_ptr; #ifdef _WIN32 DWORD protect = (flags & MAPPED_SHAREDMEM) ? PAGE_READWRITE : PAGE_READONLY; // For reader path (MAPPED_NOCREATE): open existing section, never create. if (flags & MAPPED_NOCREATE) { - HANDLE hMap = - OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, filename.c_str()); + HANDLE hMap = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, fname.c_str()); PADDLE_ENFORCE_NE(hMap, nullptr, common::errors::Unavailable( "OpenFileMappingA failed for %s, error: %lu", - filename.c_str(), + fname.c_str(), GetLastError())); void *ptr = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, size); if (ptr == nullptr) { @@ -92,7 +92,7 @@ void AllocateMemoryMap(std::string filename, CloseHandle(hMap); PADDLE_THROW(common::errors::Unavailable( "MapViewOfFile for reader %s failed, error: %lu", - filename.c_str(), + fname.c_str(), err)); } *shared_fd = reinterpret_cast(hMap); @@ -108,7 +108,7 @@ void AllocateMemoryMap(std::string filename, protect, static_cast(size >> 32), static_cast(size & 0xffffffffULL), - filename.c_str()); + fname.c_str()); if (hMap == NULL) { DWORD err = GetLastError(); @@ -117,24 +117,22 @@ void AllocateMemoryMap(std::string filename, "[PADDLE_MMAP] PID=%lu CreateFileMapping FAILED " "name=%s size=%zu flags=0x%x attempt=%d err=%lu\n", GetCurrentProcessId(), - filename.c_str(), + fname.c_str(), size, flags, attempt, err); fflush(stderr); PADDLE_THROW(common::errors::Unavailable( - "CreateFileMapping failed for %s, error: %lu", - filename.c_str(), - err)); + "CreateFileMapping failed for %s, error: %lu", fname.c_str(), err)); } if ((flags & MAPPED_EXCLUSIVE) && GetLastError() == ERROR_ALREADY_EXISTS) { CloseHandle(hMap); VLOG(3) << "[PADDLE_MMAP] PID=" << GetCurrentProcessId() << " name collision, retrying attempt=" << attempt - << " name=" << filename; - filename = GetIPCName(); // name collision; retry with fresh name + << " name=" << fname; + fname = GetIPCName(); // name collision; retry with fresh name continue; } @@ -147,12 +145,12 @@ void AllocateMemoryMap(std::string filename, "[PADDLE_MMAP] PID=%lu MapViewOfFile FAILED " "name=%s size=%zu err=%lu\n", GetCurrentProcessId(), - filename.c_str(), + fname.c_str(), size, err); fflush(stderr); PADDLE_THROW(common::errors::Unavailable( - "MapViewOfFile failed for %s, error: %lu", filename.c_str(), err)); + "MapViewOfFile failed for %s, error: %lu", fname.c_str(), err)); } // On Windows, always keep the HANDLE so the section stays alive @@ -166,7 +164,7 @@ void AllocateMemoryMap(std::string filename, *shared_fd = reinterpret_cast(hMap); if (flags & MAPPED_UNLINK) { - VLOG(6) << "CreateFileMapping (unlink mode): " << filename; + VLOG(6) << "CreateFileMapping (unlink mode): " << fname; } // Caller (AllocateRefcountedMemoryMapAllocation) handles Insert @@ -193,15 +191,15 @@ void AllocateMemoryMap(std::string filename, if (!(flags & MAPPED_FROMFD) && fd == -1) { if (flags & MAPPED_SHAREDMEM) { - fd = shm_open(filename.c_str(), file_flags, (mode_t)0600); + fd = shm_open(fname.c_str(), file_flags, (mode_t)0600); PADDLE_ENFORCE_NE( fd, -1, common::errors::Unavailable( "File descriptor %s open failed, unable in read-write mode", - filename.c_str())); - VLOG(6) << "shm_open: " << filename; - MemoryMapFdSet::Instance().Insert(filename); + fname.c_str())); + VLOG(6) << "shm_open: " << fname; + MemoryMapFdSet::Instance().Insert(fname); } } @@ -217,8 +215,8 @@ void AllocateMemoryMap(std::string filename, } if (flags & MAPPED_UNLINK) { - VLOG(6) << "shm_unlink: " << filename; - shm_unlink(filename.c_str()); + VLOG(6) << "shm_unlink: " << fname; + shm_unlink(fname.c_str()); } PADDLE_ENFORCE_NE(*map_ptr_, @@ -232,14 +230,14 @@ void AllocateMemoryMap(std::string filename, PADDLE_ENFORCE_NE(::close(fd), -1, common::errors::Unavailable( - "Error closing memory mapped file %s", filename)); + "Error closing memory mapped file %s", fname)); *shared_fd = -1; } #endif } std::shared_ptr -AllocateRefcountedMemoryMapAllocation(std::string filename, +AllocateRefcountedMemoryMapAllocation(std::string fname, intptr_t shared_fd, int flags, size_t size, @@ -247,18 +245,18 @@ AllocateRefcountedMemoryMapAllocation(std::string filename, intptr_t fd = shared_fd; void *base_ptr = nullptr; if (buffer_id == -1) { - AllocateMemoryMap(filename, &fd, flags, size + mmap_alignment, &base_ptr); - VLOG(4) << "Create and mmap a new shm: " << filename; + AllocateMemoryMap(&fname, &fd, flags, size + mmap_alignment, &base_ptr); + VLOG(4) << "Create and mmap a new shm: " << fname; } else { base_ptr = MemoryMapAllocationPool::Instance().GetById(buffer_id).mmap_ptr_; fd = shared_fd; - VLOG(4) << "Get a cached shm " << filename; + VLOG(4) << "Get a cached shm " << fname; } void *aligned_base_ptr = static_cast(static_cast(base_ptr) + mmap_alignment); - MemoryMapFdSet::Instance().Insert(filename); + MemoryMapFdSet::Instance().Insert(fname); return std::make_shared( - aligned_base_ptr, size, filename, fd, flags, buffer_id); + aligned_base_ptr, size, fname, fd, flags, buffer_id); } RefcountedMemoryMapAllocation::RefcountedMemoryMapAllocation( diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index f62bcbc12d0e44..0fd63354574901 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -100,7 +100,7 @@ class PADDLE_API RefcountedMemoryMapAllocation : public MemoryMapAllocation { void resetBaseptr(); }; -PADDLE_API void AllocateMemoryMap(std::string filename, +PADDLE_API void AllocateMemoryMap(std::string *filename, intptr_t *shared_fd, int flags, size_t size, From 0b82ea506fe9bdd8f5981afa9f81227e637909dd Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 23:34:20 +0800 Subject: [PATCH 21/32] Fix worker except path referencing unassigned idx on unpacking failure --- python/paddle/io/dataloader/worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 20fac729921625..1a57939a36aa53 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -443,6 +443,7 @@ def numpy2lodtensor(arr): if done_event.is_set() or iterator_drained: continue + idx = None idx, indices = data if init_exception is not None: batch = init_exception From da519c29f3fec5242043457be5fc46c857880023 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Sun, 5 Jul 2026 23:48:27 +0800 Subject: [PATCH 22/32] Move idx=None to try entry to cover all exception paths --- python/paddle/io/dataloader/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/io/dataloader/worker.py b/python/paddle/io/dataloader/worker.py index 1a57939a36aa53..c2000dd98b3a21 100644 --- a/python/paddle/io/dataloader/worker.py +++ b/python/paddle/io/dataloader/worker.py @@ -420,6 +420,7 @@ def numpy2lodtensor(arr): continue try: + idx = None if isinstance(data, _ResumeIteration): out_queue.put((data, None, None)) iterator_drained = False @@ -443,7 +444,6 @@ def numpy2lodtensor(arr): if done_event.is_set() or iterator_drained: continue - idx = None idx, indices = data if init_exception is not None: batch = init_exception From 3a10eb9df34acd838b0594c31cb54027de149a3d Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 10:18:15 +0800 Subject: [PATCH 23/32] Add Windows worker failure polling in _get_data timeout path --- python/paddle/io/dataloader/dataloader_iter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/paddle/io/dataloader/dataloader_iter.py b/python/paddle/io/dataloader/dataloader_iter.py index 7401e988928aa0..e3b42f86a966cd 100644 --- a/python/paddle/io/dataloader/dataloader_iter.py +++ b/python/paddle/io/dataloader/dataloader_iter.py @@ -725,6 +725,8 @@ def _get_data(self): continue # check failed workers + if sys.platform == 'win32': + core._throw_error_if_process_failed() failed_workers = [] for i, w in enumerate(self._workers): if self._worker_status[i] and not w.is_alive(): From 687f19d02d6ac1ba70ba986e7a085ebc6a966989 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 10:28:37 +0800 Subject: [PATCH 24/32] Ensure _exit_thread_unexpectedly() before re-raising worker failure exception --- python/paddle/io/dataloader/dataloader_iter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/paddle/io/dataloader/dataloader_iter.py b/python/paddle/io/dataloader/dataloader_iter.py index e3b42f86a966cd..356cf7de4b54d6 100644 --- a/python/paddle/io/dataloader/dataloader_iter.py +++ b/python/paddle/io/dataloader/dataloader_iter.py @@ -726,7 +726,11 @@ def _get_data(self): # check failed workers if sys.platform == 'win32': - core._throw_error_if_process_failed() + try: + core._throw_error_if_process_failed() + except Exception: + self._exit_thread_unexpectedly() + raise failed_workers = [] for i, w in enumerate(self._workers): if self._worker_status[i] and not w.is_alive(): From e09accbb79fdb41f73e4ac158d415ee1a96f31a4 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 11:41:13 +0800 Subject: [PATCH 25/32] Fix F1: shared_fd() returns intptr_t; Fix F2: remove __main__ from test --- paddle/phi/core/memory/allocation/mmap_allocator.h | 2 +- test/legacy_test/test_dataloader_multiprocess.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index 0fd63354574901..2f1b95bcde1935 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -64,7 +64,7 @@ class PADDLE_API MemoryMapAllocation : public Allocation { map_size_(size) {} inline const std::string &ipc_name() const { return ipc_name_; } - inline int shared_fd() const { return static_cast(fd_); } + inline intptr_t shared_fd() const { return fd_; } virtual void close(); diff --git a/test/legacy_test/test_dataloader_multiprocess.py b/test/legacy_test/test_dataloader_multiprocess.py index 6170992e5da473..a048d06bdd4dc3 100644 --- a/test/legacy_test/test_dataloader_multiprocess.py +++ b/test/legacy_test/test_dataloader_multiprocess.py @@ -175,7 +175,3 @@ def test_multiprocess_iterable_dataset(self): for batch in loader(): count += 1 self.assertGreater(count, 0) - - -if __name__ == '__main__': - unittest.main() From 6825f41b94d7500b74180849bb88924470eb197b Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 11:41:58 +0800 Subject: [PATCH 26/32] Revert F2: __main__ pattern is standard across Paddle tests (753/753 files) --- test/legacy_test/test_dataloader_multiprocess.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/legacy_test/test_dataloader_multiprocess.py b/test/legacy_test/test_dataloader_multiprocess.py index a048d06bdd4dc3..6170992e5da473 100644 --- a/test/legacy_test/test_dataloader_multiprocess.py +++ b/test/legacy_test/test_dataloader_multiprocess.py @@ -175,3 +175,7 @@ def test_multiprocess_iterable_dataset(self): for batch in loader(): count += 1 self.assertGreater(count, 0) + + +if __name__ == '__main__': + unittest.main() From b3d28f7effb0d81af222b3d9ba53a8d0ec35c36c Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 12:42:07 +0800 Subject: [PATCH 27/32] Fix shared_fd pybind read boundary: cast as intptr_t instead of int --- paddle/fluid/pybind/tensor.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/pybind/tensor.cc b/paddle/fluid/pybind/tensor.cc index 9cb0992ebdf11a..2edb86ff4d9cf9 100644 --- a/paddle/fluid/pybind/tensor.cc +++ b/paddle/fluid/pybind/tensor.cc @@ -1310,7 +1310,7 @@ void BindTensor(pybind11::module &m) { // NOLINT if (find_id != -1) { handle = memory::allocation::MemoryMapAllocationPool::Instance().GetById(find_id).file_name_; // NOLINT } - int shared_fd = -1; + intptr_t shared_fd = -1; auto shared_holder = memory::allocation::AllocateRefcountedMemoryMapAllocation( handle, shared_fd, flags, data_size, find_id); @@ -1361,7 +1361,7 @@ void BindTensor(pybind11::module &m) { // NOLINT // 2. Rebuild Allocation const std::string &ipc_name = t[0].cast(); - const int shared_fd = t[1].cast(); + const intptr_t shared_fd = t[1].cast(); const bool use_file_descriptor = t[6].cast(); size_t size = t[2].cast(); @@ -1602,7 +1602,7 @@ void BindTensor(pybind11::module &m) { // NOLINT .GetById(find_id) .file_name_; // NOLINT } - int shared_fd = -1; + intptr_t shared_fd = -1; auto shared_holder = memory::allocation::AllocateRefcountedMemoryMapAllocation( handle, shared_fd, flags, data_size, find_id); @@ -1646,7 +1646,7 @@ void BindTensor(pybind11::module &m) { // NOLINT DenseTensor tensor; const std::string &ipc_name = t[0].cast(); - const int shared_fd = t[1].cast(); + const intptr_t shared_fd = t[1].cast(); const bool use_file_descriptor = t[6].cast(); size_t size = t[2].cast(); From 58ddb8062eddf853066c2d629184ac254303d3e3 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 13:58:51 +0800 Subject: [PATCH 28/32] Close HANDLE immediately when reader has already opened (ref>1); only keeper when reader hasn't opened (ref==1) --- .../core/memory/allocation/mmap_allocator.cc | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 0ea8054a5fe4a8..140ce6034d1ce7 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -375,15 +375,28 @@ void RefcountedMemoryMapAllocation::close() { #endif } else { #ifdef _WIN32 - // Refcount > 0: transfer the HANDLE to WindowsHandleKeeper so it - // stays open (keeping the named section alive for readers) but is - // properly closed during worker cleanup (CloseAll) rather than - // leaked until process exit. - WindowsHandleKeeper::Instance().Insert(ipc_name_, fd_); - fd_ = -1; - closed_fd_ = true; - VLOG(6) << "UnmapViewOfFile (refcount>0): " << ipc_name_; - UnmapViewOfFile(map_ptr_); + if (info->refcount > 1) { + // Refcount > 1 means the reader has already opened this section + // (initializeRefercount was called after MapViewOfFile). The + // reader's mapping keeps the section alive, so we can safely + // CloseHandle immediately — no accumulation needed. + VLOG(6) << "close handle (reader already opened): " << ipc_name_; + CloseHandle(reinterpret_cast(fd_)); + fd_ = -1; + closed_fd_ = true; + VLOG(6) << "UnmapViewOfFile: " << ipc_name_; + UnmapViewOfFile(map_ptr_); + } else { + // Refcount == 1: reader hasn't opened yet. Transfer the HANDLE to + // WindowsHandleKeeper so the named section stays alive until the + // reader opens it. The HANDLE is closed during worker cleanup + // (MemoryMapFdSet::Clear → CloseAll). + WindowsHandleKeeper::Instance().Insert(ipc_name_, fd_); + fd_ = -1; + closed_fd_ = true; + VLOG(6) << "UnmapViewOfFile (refcount>0): " << ipc_name_; + UnmapViewOfFile(map_ptr_); + } #endif } #ifndef _WIN32 From 27dba42b028bfb67611229a27c8568f7158cabb5 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 15:40:14 +0800 Subject: [PATCH 29/32] Fix cache bypass HANDLE lifecycle; add refcount-aware WindowsHandleKeeper sweep for refcount==1 entries --- .../core/memory/allocation/mmap_allocator.cc | 50 +++++++++++++++---- .../core/memory/allocation/mmap_allocator.h | 14 +++++- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 140ce6034d1ce7..dbcebea76ff005 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -339,6 +339,11 @@ void RefcountedMemoryMapAllocation::close() { --info->refcount; if (FLAGS_use_shm_cache && buffer_id_ != -1) { +#ifdef _WIN32 + // Prevent base class MemoryMapAllocation::close() from closing the HANDLE + // (which would destroy the named section before the reader opens it). + closed_fd_ = true; +#endif return; } else { if (FLAGS_use_shm_cache && @@ -387,15 +392,14 @@ void RefcountedMemoryMapAllocation::close() { VLOG(6) << "UnmapViewOfFile: " << ipc_name_; UnmapViewOfFile(map_ptr_); } else { - // Refcount == 1: reader hasn't opened yet. Transfer the HANDLE to - // WindowsHandleKeeper so the named section stays alive until the - // reader opens it. The HANDLE is closed during worker cleanup - // (MemoryMapFdSet::Clear → CloseAll). - WindowsHandleKeeper::Instance().Insert(ipc_name_, fd_); + // Refcount == 1: reader hasn't opened yet. Transfer both HANDLE + // and map_ptr_ to WindowsHandleKeeper so the named section stays + // alive until the reader opens it. The keeper will UnmapViewOfFile + // + CloseHandle when refcount reaches 0 (via SweepClosedMappings). + WindowsHandleKeeper::Instance().Insert( + ipc_name_, fd_, map_ptr_, map_size_); fd_ = -1; closed_fd_ = true; - VLOG(6) << "UnmapViewOfFile (refcount>0): " << ipc_name_; - UnmapViewOfFile(map_ptr_); } #endif } @@ -676,16 +680,40 @@ WindowsHandleKeeper &WindowsHandleKeeper::Instance() { return *keeper; } -void WindowsHandleKeeper::Insert(const std::string &ipc_name, intptr_t fd) { +void WindowsHandleKeeper::Insert(const std::string &ipc_name, + intptr_t fd, + void *map_ptr, + size_t map_size) { + std::lock_guard lock(mtx_); + SweepClosedMappings(); + handles_[ipc_name] = {fd, map_ptr}; +} + +void WindowsHandleKeeper::SweepClosedMappings() { std::lock_guard lock(mtx_); - handles_[ipc_name] = fd; + for (auto it = handles_.begin(); it != handles_.end();) { + // CountInfo::refcount is the first field at map_ptr. A value of 0 means + // all references (including the reader's) have been released — the + // section is no longer in use and can be cleaned up. + auto *refcnt = + static_cast *>(it->second.map_ptr); + if (refcnt->load(std::memory_order_acquire) == 0) { + VLOG(6) << "WindowsHandleKeeper sweeping: " << it->first; + UnmapViewOfFile(it->second.map_ptr); + CloseHandle(reinterpret_cast(it->second.fd)); + it = handles_.erase(it); + } else { + ++it; + } + } } void WindowsHandleKeeper::CloseAll() { std::lock_guard lock(mtx_); for (auto &pair : handles_) { - if (pair.second != -1) { - CloseHandle(reinterpret_cast(pair.second)); + if (pair.second.fd != -1) { + UnmapViewOfFile(pair.second.map_ptr); + CloseHandle(reinterpret_cast(pair.second.fd)); } } handles_.clear(); diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index 2f1b95bcde1935..a382074a0a11f4 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -186,13 +186,23 @@ class PADDLE_API MemoryMapFdSet { class PADDLE_API WindowsHandleKeeper { public: static WindowsHandleKeeper &Instance(); // NOLINT - void Insert(const std::string &ipc_name, intptr_t fd); + + struct PendingMapping { + intptr_t fd; + void *map_ptr; + }; + + void Insert(const std::string &ipc_name, + intptr_t fd, + void *map_ptr, + size_t map_size); + void SweepClosedMappings(); void CloseAll(); ~WindowsHandleKeeper(); private: WindowsHandleKeeper() = default; - std::unordered_map handles_; + std::unordered_map handles_; std::mutex mtx_; }; #endif From 38e635a96ad7a4cd499ecdafd732553ad5faec2b Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 17:56:16 +0800 Subject: [PATCH 30/32] Fix self-deadlock: split SweepClosedMappings into locked helper + public wrapper --- paddle/phi/core/memory/allocation/mmap_allocator.cc | 10 +++++++--- paddle/phi/core/memory/allocation/mmap_allocator.h | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index dbcebea76ff005..95f7a72b22169a 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -685,12 +685,11 @@ void WindowsHandleKeeper::Insert(const std::string &ipc_name, void *map_ptr, size_t map_size) { std::lock_guard lock(mtx_); - SweepClosedMappings(); + SweepClosedMappingsLocked(); handles_[ipc_name] = {fd, map_ptr}; } -void WindowsHandleKeeper::SweepClosedMappings() { - std::lock_guard lock(mtx_); +void WindowsHandleKeeper::SweepClosedMappingsLocked() { for (auto it = handles_.begin(); it != handles_.end();) { // CountInfo::refcount is the first field at map_ptr. A value of 0 means // all references (including the reader's) have been released — the @@ -708,6 +707,11 @@ void WindowsHandleKeeper::SweepClosedMappings() { } } +void WindowsHandleKeeper::SweepClosedMappings() { + std::lock_guard lock(mtx_); + SweepClosedMappingsLocked(); +} + void WindowsHandleKeeper::CloseAll() { std::lock_guard lock(mtx_); for (auto &pair : handles_) { diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index a382074a0a11f4..21e74329d912b5 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -202,6 +202,8 @@ class PADDLE_API WindowsHandleKeeper { private: WindowsHandleKeeper() = default; + // Caller must hold mtx_ before calling this. + void SweepClosedMappingsLocked(); std::unordered_map handles_; std::mutex mtx_; }; From 7d9bef0815cd157dcbdb4ff24e11ff7e6a1566f3 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 18:25:31 +0800 Subject: [PATCH 31/32] Fix pool.Insert path: set closed_fd_=true on Windows to prevent base destructor from closing HANDLE --- paddle/phi/core/memory/allocation/mmap_allocator.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 95f7a72b22169a..9d3718e669dc12 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -352,6 +352,13 @@ void RefcountedMemoryMapAllocation::close() { MemoryMapAllocationPool::Instance().MaxPoolSize())) { MemoryMapAllocationPool::Instance().Insert(MemoryMapInfo( flags_, map_size_ - mmap_alignment, ipc_name_, map_ptr_)); +#ifdef _WIN32 + // Prevent base class destructor from closing the HANDLE. + // When the buffer is later retrieved, a new RefcountedMMap is + // constructed from the same map_ptr_ and increments refcount — the + // HANDLE must stay alive so the named section survives until then. + closed_fd_ = true; +#endif } else { if (info->refcount == 0) { #ifdef _WIN32 From 56566afb03bcb76a1d9cd2702136d2a3fbce1913 Mon Sep 17 00:00:00 2001 From: PlumBlossomMaid <1589524335@qq.com> Date: Mon, 6 Jul 2026 18:47:16 +0800 Subject: [PATCH 32/32] Fix CloseAll race: remove from Clear(); store fd in MemoryMapInfo for pool cleanup --- .../core/memory/allocation/mmap_allocator.cc | 22 +++++++++++++--- .../core/memory/allocation/mmap_allocator.h | 26 +++++++++++++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.cc b/paddle/phi/core/memory/allocation/mmap_allocator.cc index 9d3718e669dc12..0ee97258f1c8f9 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.cc +++ b/paddle/phi/core/memory/allocation/mmap_allocator.cc @@ -351,7 +351,15 @@ void RefcountedMemoryMapAllocation::close() { static_cast( MemoryMapAllocationPool::Instance().MaxPoolSize())) { MemoryMapAllocationPool::Instance().Insert(MemoryMapInfo( - flags_, map_size_ - mmap_alignment, ipc_name_, map_ptr_)); + flags_, + map_size_ - mmap_alignment, + ipc_name_, + map_ptr_ +#ifdef _WIN32 + , + fd_ +#endif + )); #ifdef _WIN32 // Prevent base class destructor from closing the HANDLE. // When the buffer is later retrieved, a new RefcountedMMap is @@ -606,9 +614,12 @@ void MemoryMapFdSet::Clear() { } fd_set_.clear(); #ifdef _WIN32 - // Close pending HANDLEs that were kept open for cross-process section - // lifecycle (refcount > 0 path in RefcountedMemoryMapAllocation::close). - WindowsHandleKeeper::Instance().CloseAll(); + // NOTE: Do NOT CloseAll() here. The keeper's SweepClosedMappings + // reclaims entries whose refcount has reached 0 during normal operation. + // HANDLEs still in the keeper at process exit are reclaimed by the OS. + // Calling CloseAll() here while batches are still in-flight in the + // multiprocessing queue would destroy named sections before the reader + // has a chance to OpenFileMappingA them. #endif } @@ -659,6 +670,9 @@ void MemoryMapAllocationPool::Clear() { #ifdef _WIN32 VLOG(4) << "MemoryMapAllocationPool: clear " << mmap.file_name_; UnmapViewOfFile(mmap.mmap_ptr_); + if (mmap.fd_ != -1) { + CloseHandle(reinterpret_cast(mmap.fd_)); + } #else int rlt = shm_unlink(mmap.file_name_.c_str()); if (rlt == 0) { diff --git a/paddle/phi/core/memory/allocation/mmap_allocator.h b/paddle/phi/core/memory/allocation/mmap_allocator.h index 21e74329d912b5..83db302d07b91f 100644 --- a/paddle/phi/core/memory/allocation/mmap_allocator.h +++ b/paddle/phi/core/memory/allocation/mmap_allocator.h @@ -180,9 +180,11 @@ class PADDLE_API MemoryMapFdSet { }; #ifdef _WIN32 -// Tracks HANDLEs from CreateFileMappingA that must stay open (refcount > 0) -// to keep the named section alive for readers. These HANDLEs are closed -// during worker cleanup via MemoryMapFdSet::Clear() / _cleanup_mmap_fds. +// Tracks HANDLEs and mapped views from CreateFileMappingA that must stay +// open (refcount == 1 at writer close time) to keep the named section alive +// for readers. SweepClosedMappings is called on each Insert() and reclaims +// entries whose refcount has reached 0. Remaining entries at process exit +// are cleaned up by the OS (handle closure on process termination). class PADDLE_API WindowsHandleKeeper { public: static WindowsHandleKeeper &Instance(); // NOLINT @@ -214,16 +216,30 @@ class MemoryMapInfo { explicit MemoryMapInfo(int flags, size_t data_size, std::string file_name, - void *mmap_ptr) + void *mmap_ptr +#ifdef _WIN32 + , + intptr_t fd = -1 +#endif + ) : flags_(flags), data_size_(data_size), file_name_(file_name), - mmap_ptr_(mmap_ptr) {} + mmap_ptr_(mmap_ptr) +#ifdef _WIN32 + , + fd_(fd) +#endif + { + } int flags_ = 0; size_t data_size_ = 0; std::string file_name_; void *mmap_ptr_ = nullptr; +#ifdef _WIN32 + intptr_t fd_ = -1; +#endif }; /* Note(zhangbo):