Fix Python 3.12 build issues in thrift Python (#3276)

- Add pyproject.toml with setuptools build requirement for PEP 517 compliance
- Replace distutils imports with setuptools equivalents
- Use setuptools error names directly (CompileError, ExecError, PlatformError)
- Fix macOS header collision with ntohll/htonll macros in endian.h
- Add a matrix of MacOS versions (macos-15-intel, macos-14, macos-15,
  macos-26)
- Add a matrix of non-EOL Python versions for testing
- Remove MSVC2015 from the test matrix (very old).
- Support MSVC2022, the latest in AppVeyor.
- Upgrade tornado, twisted, and zope.interface versions to the first
  that support Python 3.12.
- Try to make the test_socket, RunClientServer, and TestServer tests less flaky.

This fixes the ModuleNotFoundError: No module named 'distutils' error
when building thrift with Python 3.12+.

Author: greggdonovan

.github/workflows/build.yml

@@ -78,6 +78,42 @@ jobs:
           path: compiler/cpp/thrift
           retention-days: 3
 
+  compiler-macos:
+    strategy:
+      matrix:
+        os: &macos_versions [macos-15-intel, macos-14, macos-15, macos-26]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install dependencies
+        run: |
+          brew install automake bison flex boost libevent openssl libtool pkg-config
+          echo "$(brew --prefix bison)/bin" >> $GITHUB_PATH
+
+      - name: Run bootstrap
+        run: ./bootstrap.sh
+
+      - name: Run configure
+        run: ./configure --disable-debug --disable-tests --disable-libs
+
+      - name: Run make
+        run: make -j$(sysctl -n hw.ncpu)
+
+      - name: Run install
+        run: sudo make install
+
+      - name: Run thrift version
+        run: /usr/local/bin/thrift -version
+
+      - name: Archive built thrift compiler (macOS)
+        uses: actions/upload-artifact@v5
+        with:
+          name: thrift-compiler-${{ matrix.os }}
+          path: compiler/cpp/thrift
+          retention-days: 3
+
   lib-php:
     needs: compiler
     runs-on: ubuntu-24.04
@@ -495,8 +531,7 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python-version:
-          - "3.13" # Pin to 3.13 for now -> see THRIFT-5900
+        python-version: &python_versions ["3.10", "3.11", "3.12", "3.13", "3.14"]
       fail-fast: false
     steps:
       - uses: actions/checkout@v6
@@ -514,7 +549,7 @@ jobs:
 
       - name: Python setup
         run: |
-          python -m pip install --upgrade pip setuptools wheel flake8 tornado twisted zope.interface
+          python -m pip install --upgrade pip setuptools wheel flake8 "tornado>=6.3.0" "twisted>=24.3.0" "zope.interface>=6.1"
           python --version
           pip --version
 
@@ -553,6 +588,65 @@ jobs:
       - name: Run make check for python code
         run: make -C test/py check
 
+  lib-python-macos:
+    needs: compiler-macos
+    strategy:
+      matrix:
+        os: *macos_versions
+        python-version: *python_versions
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install dependencies
+        run: |
+          brew install automake bison flex boost libevent openssl libtool pkg-config
+          echo "$(brew --prefix bison)/bin" >> $GITHUB_PATH
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Python setup
+        run: |
+          python -m pip install --upgrade pip setuptools wheel flake8 "tornado>=6.3.0" "twisted>=24.3.0" "zope.interface>=6.1"
+          python --version
+          pip --version
+
+      - name: Run bootstrap
+        run: ./bootstrap.sh
+
+      - name: Run configure
+        run: ./configure $(echo $CONFIG_ARGS_FOR_LIBS | sed 's/without-py3/with-py3/')
+
+      - uses: actions/download-artifact@v6
+        with:
+          name: thrift-compiler-${{ matrix.os }}
+          path: compiler/cpp
+
+      - name: Run thrift-compiler
+        run: |
+          chmod a+x compiler/cpp/thrift
+          compiler/cpp/thrift -version
+
+      - name: Run make for python
+        run: make -C lib/py
+
+      - name: Run make install for python
+        run: |
+          sudo make -C lib/py install PY_PREFIX="$(python -c 'import sys; print(sys.prefix)')"
+
+      - name: Run make for python libs
+        run: make -C lib/py
+
+      - name: Run make check for python libs
+        run: make -C lib/py check
+
+      - name: Run make check for python code
+        run: make -C test/py check
+
   lib-nodejs:
     needs: compiler
     runs-on: ubuntu-24.04

appveyor.yml

@@ -33,31 +33,70 @@ branches:
 #matrix:
 #  fast_finish: true
 
+# See https://www.appveyor.com/docs/windows-images-software/ for available versions.
+
 environment:
   matrix:
-   - PROFILE: MSVC2017
+   # Python version test matrix (3.10-3.14) using MSVC2022
+   - PROFILE: MSVC2022
+     PROFILE_CLASS: MSVC
+     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+     PLATFORM: x64
+     CONFIGURATION: Release
+     BUILD_SHARED_LIBS: ON
+     BOOST_VERSION: 1.89.0
+     LIBEVENT_VERSION: 2.1.12
+     PYTHON_VERSION: "3.10"
+     QT_VERSION: 6.9.3
+     ZLIB_VERSION: 1.3.1
+
+   - PROFILE: MSVC2022
      PROFILE_CLASS: MSVC
-     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
      PLATFORM: x64
      CONFIGURATION: Release
      BUILD_SHARED_LIBS: ON
-     BOOST_VERSION: 1.67.0
-     LIBEVENT_VERSION: 2.1.8
-     PYTHON_VERSION: 3.6
-     QT_VERSION: 5.10
-     ZLIB_VERSION: 1.2.11
+     BOOST_VERSION: 1.89.0
+     LIBEVENT_VERSION: 2.1.12
+     PYTHON_VERSION: "3.11"
+     QT_VERSION: 6.9.3
+     ZLIB_VERSION: 1.3.1
 
-   - PROFILE: MSVC2015
+   - PROFILE: MSVC2022
      PROFILE_CLASS: MSVC
-     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-     PLATFORM: x86
-     CONFIGURATION: Debug
-     BUILD_SHARED_LIBS: OFF
-     BOOST_VERSION: 1.62.0
-     LIBEVENT_VERSION: 2.0.22
-     PYTHON_VERSION: 3.5
-     QT_VERSION: 5.8
-     ZLIB_VERSION: 1.2.8
+     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+     PLATFORM: x64
+     CONFIGURATION: Release
+     BUILD_SHARED_LIBS: ON
+     BOOST_VERSION: 1.89.0
+     LIBEVENT_VERSION: 2.1.12
+     PYTHON_VERSION: "3.12"
+     QT_VERSION: 6.9.3
+     ZLIB_VERSION: 1.3.1
+
+   - PROFILE: MSVC2022
+     PROFILE_CLASS: MSVC
+     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+     PLATFORM: x64
+     CONFIGURATION: Release
+     BUILD_SHARED_LIBS: ON
+     BOOST_VERSION: 1.89.0
+     LIBEVENT_VERSION: 2.1.12
+     PYTHON_VERSION: "3.13"
+     QT_VERSION: 6.9.3
+     ZLIB_VERSION: 1.3.1
+
+   - PROFILE: MSVC2022
+     PROFILE_CLASS: MSVC
+     APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+     PLATFORM: x64
+     CONFIGURATION: Release
+     BUILD_SHARED_LIBS: ON
+     BOOST_VERSION: 1.89.0
+     LIBEVENT_VERSION: 2.1.12
+     PYTHON_VERSION: "3.14"
+     QT_VERSION: 6.9.3
+     ZLIB_VERSION: 1.3.1
 
    - PROFILE: MINGW
      PROFILE_CLASS: MINGW

build/appveyor/MSVC-appveyor-full.bat

@@ -31,14 +31,7 @@ SET INSTDIR=%APPVEYOR_BUILD_FOLDER%\..\install\%PROFILE%\%PLATFORM%
 SET SRCDIR=%APPVEYOR_BUILD_FOLDER%
 
 
-IF "%PROFILE%" == "MSVC2015" (
-  IF "%PLATFORM%" == "x86" (
-    CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 || EXIT /B
-  ) ELSE (
-    CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 || EXIT /B
-    CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64 || EXIT /B
-  )
-) ELSE IF "%PROFILE%" == "MSVC2017" (
+IF "%PROFILE%" == "MSVC2017" (
   IF "%PLATFORM%" == "x86" (
     CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars32.bat" || EXIT /B
   ) ELSE (
@@ -50,6 +43,12 @@ IF "%PROFILE%" == "MSVC2015" (
   ) ELSE (
     CALL "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat" || EXIT /B
   )
+) ELSE IF "%PROFILE%" == "MSVC2022" (
+  IF "%PLATFORM%" == "x86" (
+    CALL "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars32.bat" || EXIT /B
+  ) ELSE (
+    CALL "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat" || EXIT /B
+  )
 ) ELSE (
   ECHO Unsupported PROFILE=%PROFILE% or PLATFORM=%PLATFORM%
   EXIT /B 1
@@ -59,16 +58,24 @@ IF "%PROFILE%" == "MSVC2015" (
 @ECHO ON
 
 :: compiler and generator detection
-IF /i "%PLATFORM%" == "x64" SET GENARCH= Win64
-IF "%PROFILE%" == "MSVC2015" (
-  SET GENERATOR=Visual Studio 14 2015!GENARCH!
-  SET COMPILER=vc140
-) ELSE IF "%PROFILE%" == "MSVC2017" (
+:: VS2017 uses "Generator Win64" syntax, VS2019+ use "-A x64" flag
+IF /i "%PLATFORM%" == "x64" (
+  SET GENARCH= Win64
+  SET CMAKE_ARCH_FLAG=-A x64
+) ELSE (
+  SET GENARCH=
+  SET CMAKE_ARCH_FLAG=-A Win32
+)
+IF "%PROFILE%" == "MSVC2017" (
   SET GENERATOR=Visual Studio 15 2017!GENARCH!
   SET COMPILER=vc141
+  SET CMAKE_ARCH_FLAG=
 ) ELSE IF "%PROFILE%" == "MSVC2019" (
-  SET GENERATOR=Visual Studio 16 2019!GENARCH!
+  SET GENERATOR=Visual Studio 16 2019
   SET COMPILER=vc142
+) ELSE IF "%PROFILE%" == "MSVC2022" (
+  SET GENERATOR=Visual Studio 17 2022
+  SET COMPILER=vc143
 ) ELSE (
   ECHO [error] unable to determine the CMake generator and compiler to use from MSVC profile %PROFILE%
   EXIT /B 1
@@ -96,7 +103,7 @@ IF "%PYTHON_VERSION%" == "" (
   IF /i "%PLATFORM%" == "x64" (SET PTEXT=-x64)
   SET PYTHON_ROOT=C:\Python%PYTHON_VERSION:.=%!PTEXT!
   SET PATH=!PYTHON_ROOT!\scripts;!PYTHON_ROOT!;!PATH!
-  SET CMAKE_PYTHON_OPTS=-DPython3_FIND_STRATEGY=LOCATION -DPython3_ROOT=!PYTHON_ROOT!
+  SET CMAKE_PYTHON_OPTS=-DPython3_FIND_STRATEGY=LOCATION -DPython3_ROOT=!PYTHON_ROOT! -DPython3_EXECUTABLE=!PYTHON_ROOT!\python.exe
 )
 
 IF "%CONFIGURATION%" == "Debug" (SET ZLIB_LIB_SUFFIX=d)
@@ -115,15 +122,15 @@ choco feature enable -n allowGlobalConfirmation || EXIT /B
 
 :: Things to install when NOT running in appveyor:
 IF "%APPVEYOR_BUILD_ID%" == "" (
-    cup -y chocolatey || EXIT /B
-    cinst -y curl || EXIT /B
-    cinst -y 7zip || EXIT /B
-    cinst -y python3 || EXIT /B
-    cinst -y openssl.light || EXIT /B
+    choco upgrade -y chocolatey || EXIT /B
+    choco install -y curl || EXIT /B
+    choco install -y 7zip || EXIT /B
+    choco install -y python3 || EXIT /B
+    choco install -y openssl.light || EXIT /B
 )
 
-cinst -y jdk8 || EXIT /B
-cinst -y winflexbison3 || EXIT /B
+choco install -y jdk8 || EXIT /B
+choco install -y winflexbison3 || EXIT /B
 
 :: zlib - not available through chocolatey
 CD "%APPVEYOR_SCRIPTS%" || EXIT /B
@@ -133,12 +140,17 @@ call build-zlib.bat || EXIT /B
 CD "%APPVEYOR_SCRIPTS%" || EXIT /B
 call build-libevent.bat || EXIT /B
 
-:: python packages (correct path to pip set above)
-pip.exe ^
-    install backports.ssl_match_hostname ^
-            ipaddress ^
-            tornado ^
-            twisted || EXIT /B
+:: python packages (ensure we use the configured Python)
+IF "%WITH_PYTHON%" == "ON" (
+  "!PYTHON_ROOT!\python.exe" -m ensurepip --upgrade || EXIT /B
+  "!PYTHON_ROOT!\python.exe" -m pip install --upgrade pip setuptools wheel || EXIT /B
+  "!PYTHON_ROOT!\python.exe" -m pip ^
+      install backports.ssl_match_hostname ^
+              ipaddress ^
+              tornado>=6.3.0 ^
+              twisted>=24.3.0 ^
+              zope.interface>=6.1 || EXIT /B
+)
 
 :: Adobe Flex SDK 4.6 for ActionScript
 MKDIR "C:\Adobe\Flex\SDK\4.6" || EXIT /B
@@ -171,7 +183,7 @@ CD "%BUILDDIR%" || EXIT /B
 :: DIR C:\Libraries\boost_1_60_0\lib*
 
 cmake.exe "%SRCDIR%" ^
-  -G"%GENERATOR%" ^
+  -G"%GENERATOR%" %CMAKE_ARCH_FLAG% ^
   -DBISON_EXECUTABLE="C:\ProgramData\chocolatey\lib\winflexbison3\tools\win_bison.exe" ^
   -DBOOST_ROOT="%BOOST_ROOT%" ^
   -DBOOST_LIBRARYDIR="%BOOST_LIBRARYDIR%" ^

build/appveyor/build-libevent.bat

@@ -26,6 +26,13 @@ IF "%APPVEYOR_BUILD_ID%" == "" (
 )
 7z x "%URLFILE%" -so | 7z x -si -ttar > nul || EXIT /B
 CD "libevent-%LIBEVENT_VERSION%-stable" || EXIT /B
+:: libevent's nmake config ships with EVENT__HAVE_STDINT_H commented out,
+:: but MSVC needs stdint.h for UINT32_MAX in minheap-internal.h.
+IF EXIST "WIN32-Code\nmake\event2\event-config.h" (
+    powershell -NoProfile -Command "(Get-Content 'WIN32-Code\nmake\event2\event-config.h') -replace '/\* #define EVENT__HAVE_STDINT_H 1 \*/', '#define EVENT__HAVE_STDINT_H 1' | Set-Content 'WIN32-Code\nmake\event2\event-config.h'"
+) ELSE IF EXIST "WIN32-Code\event2\event-config.h" (
+    powershell -NoProfile -Command "(Get-Content 'WIN32-Code\event2\event-config.h') -replace '/\* #define EVENT__HAVE_STDINT_H 1 \*/', '#define EVENT__HAVE_STDINT_H 1' | Set-Content 'WIN32-Code\event2\event-config.h'"
+)
 nmake -f Makefile.nmake static_libs || EXIT /B
 
 :: in libevent 2.0 there is no nmake subdirectory in WIN32-Code, but in 2.1 there is

build/cmake/BoostMacros.cmake

@@ -23,6 +23,12 @@ if(POLICY CMP0167)
   cmake_policy(SET CMP0167 OLD)
 endif()
 
+# CMake 3.27+ warns and ignores upper-case <PACKAGENAME>_ROOT variables unless
+# CMP0144 is set. We pass BOOST_ROOT on Windows builds, so enable NEW behavior.
+if(POLICY CMP0144)
+  cmake_policy(SET CMP0144 NEW)
+endif()
+
 # Force using FindBoost instead of Boost's own BoostConfig.cmake.
 # BoostConfig.cmake does not populate ${Boost_LIBRARIES} the same way,
 # which causes linking failures on Windows.

build/docker/ubuntu-focal/Dockerfile

@@ -261,10 +261,9 @@ RUN apt-get install -y --no-install-recommends \
       python3-all-dev \
       python3-pip \
       python3-setuptools \
-      python3-tornado \
-      python3-twisted \
-      python3-wheel \
-      python3-zope.interface
+      python3-wheel
+
+RUN python3 -m pip install --no-cache-dir --upgrade "tornado>=6.3.0" "twisted>=24.3.0" "zope.interface>=6.1"
 
 RUN apt-get install -y --no-install-recommends \
       `# Ruby dependencies` \