Experimental macOS (x86_64) port -- Mono + XQuartz, unlocks waterbox (DS) on macOS

.gitignore

@@ -48,3 +48,5 @@ UpgradeLog.htm
 /packages
 
 launchSettings.json
+Assets/EmuHawkMono_laststderr.txt
+Assets/EmuHawkMono_laststdout.txt

Assets/EmuHawkMonoMacOS.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+# Launcher for EmuHawk on macOS via Mono (x86_64, under Rosetta 2 on Apple Silicon).
+# Mono's System.Windows.Forms is X11-based, so an X server (XQuartz) must be running.
+# See also EmuHawkMono.sh (the Linux equivalent this is modelled on).
+cd "$(dirname "$(realpath "$0")")"
+
+# Native libs ship in ./dll; also let the loader find Homebrew-provided deps
+# (SDL2, OpenAL, Lua, zstd, libgdiplus, ...) under /usr/local, and XQuartz's X11
+# libs under /opt/X11/lib (Mono's WinForms dlopens libX11 etc. from there).
+export DYLD_LIBRARY_PATH="$PWD/dll:$PWD:/usr/local/lib:/opt/X11/lib${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}"
+export DYLD_FALLBACK_LIBRARY_PATH="$PWD/dll:$PWD:/usr/local/lib:/opt/X11/lib:/usr/lib${DYLD_FALLBACK_LIBRARY_PATH:+:$DYLD_FALLBACK_LIBRARY_PATH}"
+
+export MONO_CRASH_NOFILE=1
+export MONO_WINFORMS_XIM_STYLE=disabled # see https://bugzilla.xamarin.com/show_bug.cgi?id=28047#c9
+# Mono's default macOS WinForms backend is the Carbon driver, which is unported to 64-bit
+# ("very few parts of Windows.Forms will work properly, or at all"). Force the X11 driver,
+# which renders through XQuartz.
+export MONO_MWF_MAC_FORCE_X11=1
+
+# XQuartz: ensure DISPLAY points at a running X server.
+if [ -z "$DISPLAY" ]; then
+	export DISPLAY=:0
+fi
+if ! command -v Xquartz >/dev/null 2>&1 && [ ! -d "/Applications/Utilities/XQuartz.app" ] && [ ! -d "/opt/X11" ]; then
+	printf "%s\n" "XQuartz does not appear to be installed; Mono WinForms needs an X11 server. Install with: brew install --cask xquartz" >&2
+fi
+
+# Prefer the x86_64 Mono from Homebrew at /usr/local. Invoke it DIRECTLY (not via `arch`):
+# /usr/bin/arch is SIP-protected and strips DYLD_* from the environment, which breaks our
+# library paths. An x86_64-only binary auto-runs under Rosetta 2 when exec'd directly.
+mono_bin="mono"
+if [ -x "/usr/local/bin/mono" ]; then
+	mono_bin="/usr/local/bin/mono"
+fi
+
+# Force the GdiPlus (software) display method: XQuartz's OpenGL is Apple's legacy GLX bridge,
+# which is only GL 2.1 and crashes (CGLSetCurrentContext) under Rosetta. Without this, EmuHawk
+# aborts trying to init OpenGL. GdiPlus is full-speed here since the host only blits frames.
+set -- --gdi "$@"
+
+if (ps -A -o "command" | grep -F "EmuHawk.exe" | grep -Fvq "grep"); then
+	printf "(it seems EmuHawk is already running, NOT capturing output)\n" >&2
+	exec $mono_bin EmuHawk.exe "$@"
+fi
+o="$(mktemp -u)"
+e="$(mktemp -u)"
+mkfifo "$o" "$e"
+printf "(capturing output in %s/EmuHawkMono_last*.txt)\n" "$PWD" >&2
+tee EmuHawkMono_laststdout.txt <"$o" &
+tee EmuHawkMono_laststderr.txt <"$e" | sed "s/.*/$(tput setaf 1)&$(tput sgr0)/" >&2 &
+exec $mono_bin EmuHawk.exe "$@" >"$o" 2>"$e"

Dist/stage-macos-dylibs.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+# Stage the macOS (x86_64 / Rosetta) native-library glue into the EmuHawk output dir.
+#
+# Run this once after building (Dist/BuildRelease.sh) and before launching with
+# Assets/EmuHawkMonoMacOS.sh. It symlinks the Homebrew-provided dependencies (SDL deps,
+# OpenAL, Lua, zstd, SQLite, the X11 client stack) and creates the Linux-soname aliases
+# (libX11.so.6, etc.) that BizHawk's P/Invokes and Mono expect, all pointing at the SAME
+# Homebrew libX11 so cairo/libgdiplus/WinForms/SDL agree (see notes in EmuHawkMonoMacOS.sh).
+#
+# The dylibs BizHawk itself builds/bundles (libSDL2, libgdiplus, libcimgui, libwaterboxhost)
+# ship in Assets/dll and are copied to output/dll by the build, so they are not handled here.
+#
+# Prereqs (x86_64 Homebrew under Rosetta):
+#   brew install mono sdl2 openal-soft lua@5.4 zstd sqlite mono-libgdiplus \
+#                libx11 libxext libxrender libxcursor libxinerama libxi libxrandr \
+#                libxtst libxfixes libxscrnsaver libxau libxdmcp libxcb
+#   brew install --cask xquartz
+set -e
+
+# Target dll dir: arg 1, else ./dll relative to cwd, else output/dll under the repo.
+DLL="${1:-}"
+if [ -z "$DLL" ]; then
+	if [ -d "dll" ]; then DLL="dll"
+	else DLL="$(cd "$(dirname "$0")/.." && pwd)/output/dll"; fi
+fi
+if [ ! -d "$DLL" ]; then printf "output dll dir not found: %s\n" "$DLL" >&2; exit 1; fi
+cd "$DLL"
+
+opt() { echo "/usr/local/opt/$1"; }
+link() { # link <target-abs-path> <linkname>
+	[ -e "$1" ] && ln -sf "$1" "$2" || printf "WARN missing %s (for %s)\n" "$1" "$2" >&2
+}
+
+# Homebrew runtime deps under their canonical names
+link "$(opt lua@5.4)/lib/liblua5.4.dylib"        liblua54.dylib
+link "$(opt openal-soft)/lib/libopenal.dylib"    libopenal.dylib
+link "$(opt openal-soft)/lib/libopenal.1.dylib"  libopenal.1.dylib
+link "$(opt zstd)/lib/libzstd.1.dylib"           libzstd.1.dylib
+link "$(opt sqlite)/lib/libsqlite3.dylib"        libe_sqlite3.dylib
+link "$(opt sqlite)/lib/libsqlite3.dylib"        e_sqlite3.dylib
+
+# X11 client stack (all from Homebrew, so they share one libX11)
+link "$(opt libx11)/lib/libX11.6.dylib"          libX11.6.dylib
+link "$(opt libxext)/lib/libXext.6.dylib"        libXext.6.dylib
+link "$(opt libxrender)/lib/libXrender.1.dylib"  libXrender.1.dylib
+link "$(opt libxcursor)/lib/libXcursor.1.dylib"  libXcursor.1.dylib
+link "$(opt libxinerama)/lib/libXinerama.1.dylib" libXinerama.1.dylib
+link "$(opt libxi)/lib/libXi.6.dylib"            libXi.6.dylib
+link "$(opt libxrandr)/lib/libXrandr.2.dylib"    libXrandr.2.dylib
+link "$(opt libxtst)/lib/libXtst.6.dylib"        libXtst.6.dylib
+link "$(opt libxfixes)/lib/libXfixes.3.dylib"    libXfixes.3.dylib
+link "$(opt libxscrnsaver)/lib/libXss.1.dylib"   libXss.1.dylib
+link "$(opt libxau)/lib/libXau.6.dylib"          libXau.6.dylib
+link "$(opt libxdmcp)/lib/libXdmcp.6.dylib"      libXdmcp.6.dylib
+link "$(opt libxcb)/lib/libxcb.1.dylib"          libxcb.1.dylib
+
+# Linux-soname aliases that BizHawk's P/Invokes and Mono hardcode (XlibImports etc.)
+ln -sf libX11.6.dylib     libX11.dylib
+ln -sf libX11.6.dylib     libX11.so.6
+ln -sf libXfixes.3.dylib  libXfixes.so.3
+ln -sf libXi.6.dylib      libXi.so.6
+ln -sf libzstd.1.dylib    libzstd.so.1
+ln -sf libgdiplus.0.dylib libgdiplus.dylib
+
+printf "staged macOS dylib symlinks into %s\n" "$DLL"

README.md

@@ -172,14 +172,31 @@ As with Apple silicon Macs, not available.
 
 If you were looking to emulate iOS apps, see [#3956](https://github.com/TASEmulators/BizHawk/issues/3956).
 
-#### macOS (legacy BizHawk)
-
-EmuHawk depends on certain libraries for graphics, and these don't work on macOS. Users on macOS have three options:
-* Use another machine with Windows or Linux, or install either in an x86_64 VM (Wine is not a VM).
-* Use an older 1.x release, which was ported to macOS by @Sappharad (with replacements for the missing libraries), via Rosetta. Links and more details are in [this TASVideos forum thread](https://tasvideos.org/Forum/Topics/12659) (jump to last page for latest binaries). See [#3697](https://github.com/TASEmulators/BizHawk/issues/3697) for details.
-* For the technically-minded, download the [source](https://github.com/Sappharad/BizHawk/tree/MacUnixMonoCompat) of an older 2.x release. @Sappharad put a lot of work into it but ultimately decided to stop.
-	* ...or use the Nix expression as a starting point instead.
-	* Either way, this probably won't work on Apple silicon without a lot more effort. You'll probably want to build for x86_64 and run Mono via Rosetta. See [#4052](https://github.com/TASEmulators/BizHawk/issues/4052) re: Linux AArch64.
+#### macOS (experimental x86_64 port)
+
+> [!WARNING]
+> **This macOS port is EXPERIMENTAL and community-maintained — it is *not* officially supported by the BizHawk team.**
+> It runs the **x86_64** build under Mono + XQuartz (natively on Intel Macs, via Rosetta 2 on Apple silicon). Expect rough edges: software (GdiPlus) video only, no hardware OpenGL (so no HD/upscaled rendering or GL shader filters), and it needs a manual dependency setup.
+> **Please do not open issues about this port on the main tracker** unless you can also reproduce them on Windows/Linux — most of the dev team is on Windows and cannot support it. Discuss macOS-specific problems in the relevant macOS thread instead.
+
+Requirements: **macOS 14 (Sonoma) or newer** (this is the floor imposed by the Homebrew dependencies; the bundled native libraries themselves target macOS 11). x86_64 only — on Apple silicon everything runs through **Rosetta 2**.
+
+Setup (all the brew commands must be the **x86_64** Homebrew under `/usr/local`):
+
+1. Apple silicon only: install Rosetta 2 — `softwareupdate --install-rosetta --agree-to-license`.
+2. Install an **x86_64 Homebrew** at `/usr/local`. On Intel Macs the normal Homebrew is already x86_64. On Apple silicon, install a second one under Rosetta:
+   `arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"`
+   (prefix every command below with `arch -x86_64` and use `/usr/local/bin/brew`).
+3. Install the runtime dependencies:
+   `brew install mono mono-libgdiplus sdl2 openal-soft lua@5.4 zstd sqlite`
+   `brew install --cask xquartz` (then log out/in so the X server is registered)
+4. Get a build: either download the macOS dev build, or build from source (see [*Building*](#building)) and then run `Dist/stage-macos-dylibs.sh` to link the dependencies into `output/dll`.
+
+Run `EmuHawkMonoMacOS.sh` to start EmuHawk (it forces the X11 WinForms driver and GdiPlus video, which are required on macOS). **XQuartz must be running.** It takes the same command-line arguments as on Windows: see [*Passing command-line arguments*](#passing-command-line-arguments), e.g. `./EmuHawkMonoMacOS.sh --lua=/path/to/script.lua /path/to/rom.nds`.
+
+What works: most non-GL cores including **Game Boy/Color (Gambatte)**, **GBA (mGBA)**, and **Nintendo DS (melonDS)**, plus Lua scripting. Cores that require host OpenGL fall back to their software renderers. N64 and other GL-only paths are not expected to work.
+
+If you need an older 1.x macOS build instead, @Sappharad's Rosetta port is described in [this TASVideos forum thread](https://tasvideos.org/Forum/Topics/12659); see also [#3697](https://github.com/TASEmulators/BizHawk/issues/3697) and [#1430](https://github.com/TASEmulators/BizHawk/issues/1430).
 
 #### Nix/NixOS
 

src/BizHawk.Bizware.Graphics/OpenGL/SDL2OpenGLContext.cs

@@ -23,19 +23,23 @@ static SDL2OpenGLContext()
 		{
 			if (OSTailoredCode.IsUnixHost)
 			{
-				// make sure that Linux uses the x11 video driver
-				// we need this as mono winforms uses x11
+				// make sure that we use the x11 video driver
+				// we need this as mono winforms uses x11 (on macOS, via XQuartz)
 				// and the user could potentially try to force the wayland video driver via env vars
 				SDL_SetHintWithPriority("SDL_VIDEODRIVER", "x11", SDL_HintPriority.SDL_HINT_OVERRIDE);
-				// try to use EGL if it is available
-				// GLX is the old API, and is the more or less "deprecated" at this point, and potentially more buggy with some drivers
-				// we do need to a bit more work, in case EGL is not actually available or potentially doesn't have desktop GL support
-				SDL_SetHint(SDL_HINT_VIDEO_X11_FORCE_EGL, "1");
-				// set EGL_PLATFORM, many users have reported crashes with SDL_GL_LoadLibrary
-				// this is from EGL_PLATFORM being set to wayland by users to workaround wonky autodetection with nvidia drivers
-				// this is made worse with SDL as it uses eglGetDisplay rather than eglGetPlatformDisplay
-				// as such EGL assumes the display is a wayland display and proceeds to crash with that assumption
-				Environment.SetEnvironmentVariable("EGL_PLATFORM", "x11");
+				if (OSTailoredCode.CurrentOS != OSTailoredCode.DistinctOS.macOS)
+				{
+					// try to use EGL if it is available
+					// GLX is the old API, and is the more or less "deprecated" at this point, and potentially more buggy with some drivers
+					// we do need to a bit more work, in case EGL is not actually available or potentially doesn't have desktop GL support
+					SDL_SetHint(SDL_HINT_VIDEO_X11_FORCE_EGL, "1");
+					// set EGL_PLATFORM, many users have reported crashes with SDL_GL_LoadLibrary
+					// this is from EGL_PLATFORM being set to wayland by users to workaround wonky autodetection with nvidia drivers
+					// this is made worse with SDL as it uses eglGetDisplay rather than eglGetPlatformDisplay
+					// as such EGL assumes the display is a wayland display and proceeds to crash with that assumption
+					Environment.SetEnvironmentVariable("EGL_PLATFORM", "x11");
+					// on macOS, XQuartz provides GLX (not EGL), so we leave the above unset and let SDL use GLX
+				}
 			}
 
 			// init SDL video

src/BizHawk.Bizware.Input/KeyMouseInput/KeyMouseInputFactory.cs

@@ -9,7 +9,10 @@ internal static class KeyMouseInputFactory
 		public static IKeyMouseInput CreateKeyMouseInput() => OSTailoredCode.CurrentOS switch
 		{
 			OSTailoredCode.DistinctOS.Linux => new X11KeyMouseInput(),
-			OSTailoredCode.DistinctOS.macOS => new QuartzKeyMouseInput(),
+			// macOS EmuHawk runs under XQuartz (Mono WinForms is X11), so read the keyboard/mouse
+			// through X11 like Linux. The Quartz path (CGEventSourceKeyState) needs Accessibility
+			// permission and Mac HID keycodes, and doesn't integrate with the XQuartz window.
+			OSTailoredCode.DistinctOS.macOS => new X11KeyMouseInput(),
 			OSTailoredCode.DistinctOS.Windows => new RawKeyMouseInput(),
 			_ => throw new InvalidOperationException("Unknown OS"),
 		};

src/BizHawk.Bizware.Input/KeyMouseInput/X11KeyMouseInput.cs

@@ -1,4 +1,4 @@
-#nullable enable
+#nullable enable
 
 using System.Collections.Generic;
 using System.Runtime.InteropServices;
@@ -23,9 +23,9 @@ internal sealed class X11KeyMouseInput : IKeyMouseInput
 
 		public X11KeyMouseInput()
 		{
-			if (OSTailoredCode.CurrentOS != OSTailoredCode.DistinctOS.Linux)
+			if (OSTailoredCode.CurrentOS != OSTailoredCode.DistinctOS.Linux && OSTailoredCode.CurrentOS != OSTailoredCode.DistinctOS.macOS)
 			{
-				throw new NotSupportedException("X11 is Linux only");
+				throw new NotSupportedException("X11 is Linux/macOS only");
 			}
 
 			Display = XOpenDisplay(null);

src/BizHawk.Client.EmuHawk/GraphicsImplementations/OpenGLProvider.cs

@@ -1,4 +1,5 @@
 using BizHawk.Bizware.Graphics;
+using BizHawk.Common;
 using BizHawk.Emulation.Common;
 
 namespace BizHawk.Client.EmuHawk
@@ -9,7 +10,11 @@ namespace BizHawk.Client.EmuHawk
 	public class OpenGLProvider : IOpenGLProvider
 	{
 		public bool SupportsGLVersion(int major, int minor)
-			=> OpenGLVersion.SupportsVersion(major, minor);
+			// On macOS the only GL available (XQuartz/GLX) is the legacy Apple bridge, which is
+			// capped at GL 2.1 and aborts under Rosetta; probing it crashes. Report no host GL so
+			// cores (melonDS, N64, ...) fall back to their software renderers.
+			=> OSTailoredCode.CurrentOS != OSTailoredCode.DistinctOS.macOS
+				&& OpenGLVersion.SupportsVersion(major, minor);
 
 		public object RequestGLContext(int major, int minor, bool coreProfile)
 			=> new SDL2OpenGLContext(major, minor, coreProfile);

src/BizHawk.Common/MemoryBlock/MemoryBlockLinuxPal.cs

@@ -17,9 +17,13 @@ internal sealed class MemoryBlockLinuxPal : IMemoryBlockPal
 		/// <exception cref="InvalidOperationException">
 		/// failed to mmap
 		/// </exception>
+		// MAP_PRIVATE is 0x02 everywhere, but MAP_ANON differs: 0x20 on Linux/BSD, 0x1000 on macOS.
+		private static readonly int MapPrivateAnon
+			= 0x02 | (OSTailoredCode.CurrentOS == OSTailoredCode.DistinctOS.macOS ? 0x1000 : 0x20);
+
 		public MemoryBlockLinuxPal(ulong size)
 		{
-			var ptr = mmap(IntPtr.Zero, Z.UU(size), MemoryProtection.None, 0x22 /* MAP_PRIVATE | MAP_ANON */, -1, IntPtr.Zero);
+			var ptr = mmap(IntPtr.Zero, Z.UU(size), MemoryProtection.None, MapPrivateAnon, -1, IntPtr.Zero);
 			if (ptr == new IntPtr(-1))
 			{
 				throw new InvalidOperationException($"{nameof(mmap)}() failed with error {Marshal.GetLastWin32Error()}");

src/BizHawk.Common/OSTailoredCode.cs

@@ -12,6 +12,12 @@ public static class OSTailoredCode
 		public static readonly DistinctOS CurrentOS;
 		public static readonly bool IsUnixHost;
 
+		/// <summary>
+		/// The host OS' conventional file extension for dynamically-linked libraries, including the leading dot
+		/// (<c>".dll"</c> on Windows, <c>".dylib"</c> on macOS, <c>".so"</c> on Linux/BSD).
+		/// </summary>
+		public static readonly string DllExtension;
+
 		static OSTailoredCode()
 		{
 			if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
@@ -36,6 +42,13 @@ static OSTailoredCode()
 			}
 
 			IsUnixHost = CurrentOS != DistinctOS.Windows;
+
+			DllExtension = CurrentOS switch
+			{
+				DistinctOS.Windows => ".dll",
+				DistinctOS.macOS => ".dylib",
+				_ => ".so", // Linux/BSD
+			};
 		}
 
 		private static readonly Lazy<(WindowsVersion, Version?)?> _HostWindowsVersion = new(() =>

src/BizHawk.Emulation.Cores/Waterbox/WaterboxHost.cs

@@ -75,7 +75,9 @@ public sealed class WaterboxHost : IMonitor, IImportResolver, IStatable, IDispos
 		static WaterboxHost()
 		{
 			NativeImpl = BizInvoker.GetInvoker<WaterboxHostNative>(
-				new DynamicLibraryImportResolver(OSTailoredCode.IsUnixHost ? "libwaterboxhost.so" : "waterboxhost.dll", hasLimitedLifetime: false),
+				new DynamicLibraryImportResolver(
+					(OSTailoredCode.IsUnixHost ? "libwaterboxhost" : "waterboxhost") + OSTailoredCode.DllExtension,
+					hasLimitedLifetime: false),
 				CallingConventionAdapters.Native);
 #if !DEBUG
 			NativeImpl.wbx_set_always_evict_blocks(false);

waterbox/waterboxhost/build-release-macos.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+# Build libwaterboxhost for macOS as an x86_64 Mach-O dylib (runs under Rosetta 2 on
+# Apple Silicon). Cross-compiles from any host; only needs rustup + the x86_64-apple-darwin
+# std for a compatible nightly. See build-release.sh for the Linux/.so equivalent.
+#
+# NOTE: this builds the host. Guest-entry on macOS is not yet fully correct; see the
+# TODO[macOS] in src/context/mod.rs (the %gs / TLS scratch mechanism still needs work).
+set -e
+if [ -z "$BIZHAWKBUILD_HOME" ]; then export BIZHAWKBUILD_HOME="$(realpath "$(dirname "$0")/../..")"; fi
+cd "$(dirname "$0")"
+
+# This crate predates several nightly API changes (try_trait_v2, unsafe intrinsics), so pin
+# a known-good nightly rather than the floating channel in rust-toolchain.toml.
+TOOLCHAIN="${WBX_NIGHTLY:-nightly-2024-10-18}"
+TARGET="x86_64-apple-darwin"
+# Build for an older baseline so the dylib loads on older Macs, not just the build host.
+# (The overall floor is set by the Homebrew deps, currently macOS 14.)
+export MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-11.0}"
+
+rustup toolchain install "$TOOLCHAIN" --profile minimal
+rustup target add --toolchain "$TOOLCHAIN" "$TARGET"
+
+# cargo invokes a bare `rustc`; if another rustc (e.g. Homebrew's) precedes ~/.cargo/bin on
+# PATH it gets picked and lacks the cross std. Pin RUSTC to the toolchain's rustc explicitly.
+RUSTC_BIN="$(rustup which --toolchain "$TOOLCHAIN" rustc)"
+
+# Regenerate the interop blobs (needs nasm: `brew install nasm`). The macOS variant
+# (interop_macos.bin) handles %gs differently — see src/context/interop_macos.s.
+if command -v nasm >/dev/null 2>&1; then
+	make -C src/context
+fi
+
+RUSTC="$RUSTC_BIN" cargo "+$TOOLCHAIN" build --release --target "$TARGET"
+
+OUT="target/$TARGET/release/libwaterboxhost.dylib"
+cp "$OUT" "$BIZHAWKBUILD_HOME/Assets/dll/libwaterboxhost.dylib"
+if [ -e "$BIZHAWKBUILD_HOME/output" ]; then
+	cp "$OUT" "$BIZHAWKBUILD_HOME/output/dll/libwaterboxhost.dylib"
+fi
+printf "copied libwaterboxhost.dylib (%s) into Assets/dll\n" "$TARGET"

waterbox/waterboxhost/src/context/Makefile

@@ -1,2 +1,7 @@
-interop.bin: interop.s
-	nasm -f bin -Wall -o $@ $<
+all: interop.bin interop_macos.bin
+
+interop.bin: interop.s
+	nasm -f bin -Wall -o $@ $<
+
+interop_macos.bin: interop_macos.s
+	nasm -f bin -Wall -o $@ $<