Add macOS build for AArch64/ARM64

rdesgroppes · rdesgroppes · commit 9938ed81b8a4 · 2025-07-22T14:02:39.000+02:00
This is the payload for adding macOS "native" build support for the AArch64/ARM64 architecture to the Agent, that is by removing the need for Rosetta 2 translation at runtime. Groundwork enabling the present payload: - DataDog/integrations-core#20455 - DataDog/integrations-core#20577 - DataDog/integrations-core#20699 - DataDog/omnibus-ruby#239 - DataDog/omnibus-ruby#240 - #38498 - #39082 - DataDog/agent-release-management#364 Some earlier scouting: - DataDog/integrations-core#19193 (credits to @Pythyu) - DataDog/omnibus-software#553 (credits to @Pythyu) - #31787 (credits to @Pythyu) - DataDog/ci-platform-machine-images#321 Some follow-ups: - DataDog/datadog-agent-dev#137 - DataDog/datadog-agent-buildimages#888 - #38446 - DataDog/integrations-core#20812 - DataDog/integrations-core#20815 - DataDog/integrations-core#20645 - DataDog/ci-platform-machine-images#343 - DataDog/agent-release-management#365
diff --git a/.gitlab/.ci-linters.yml b/.gitlab/.ci-linters.yml
@@ -30,6 +30,7 @@ needs-rules:
     - lint_linux-arm64
     - lint_linux-x64
     - lint_macos_gitlab_amd64
+    - lint_macos_gitlab_arm64
     - new-e2e-eks-cleanup-on-failure
     - protobuf_test
     - publish_winget_7_x64
@@ -39,6 +40,7 @@ needs-rules:
     - tests_ebpf_arm64
     - tests_ebpf_x64
     - tests_macos_gitlab_amd64
+    - tests_macos_gitlab_arm64
     - tests_windows_secagent_x64
     - tests_windows_sysprobe_x64
     - trigger_auto_staging_release
diff --git a/.gitlab/deploy_packages/nix.yml b/.gitlab/deploy_packages/nix.yml
@@ -146,17 +146,29 @@ deploy_packages_dogstatsd_suse_rpm-x64-7:
   variables:
     PACKAGE_ARCH: x86_64
 
-deploy_packages_dmg-x64-a7:
+.deploy_packages_dmg:
   rules:
     !reference [.on_deploy]
   stage: deploy_packages
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX:$CI_IMAGE_GITLAB_AGENT_DEPLOY
-  tags: ["arch:amd64"]
-  needs: ["agent_dmg-x64-a7"]
   before_script:
     - ls $OMNIBUS_PACKAGE_DIR
   script:
-    - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-7*.dmg" $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/dmg/x86_64/
+    - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-7*.dmg" $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/dmg/$PACKAGE_ARCH/
+
+deploy_packages_dmg-x64-a7:
+  extends: .deploy_packages_dmg
+  tags: ["arch:amd64"]
+  needs: ["agent_dmg-x64-a7"]
+  variables:
+    PACKAGE_ARCH: x86_64
+
+deploy_packages_dmg-arm64-a7:
+  extends: .deploy_packages_dmg
+  tags: ["arch:arm64"]
+  needs: ["agent_dmg-arm64-a7"]
+  variables:
+    PACKAGE_ARCH: arm64
 
 # deploy dogstatsd x64, non-static binary to staging bucket
 deploy_staging_dsd:
diff --git a/.gitlab/functional_test/static_quality_gate.yml b/.gitlab/functional_test/static_quality_gate.yml
@@ -51,6 +51,8 @@ static_quality_gates:
     - windows_msi_and_bosh_zip_x64-a7
  #  - job: agent_dmg-x64-a7
  #    optional: True
+ #  - job: agent_dmg-arm64-a7
+ #    optional: True
  #
   script:
     - !reference [.login_to_docker_readonly_crane]
diff --git a/.gitlab/lint/macos.yml b/.gitlab/lint/macos.yml
@@ -6,6 +6,9 @@ include:
   stage: lint
   extends: .macos_gitlab
   needs: ["go_deps", "go_tools_deps"]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   script:
     - !reference [.retrieve_linux_go_deps]
     - !reference [.retrieve_linux_go_tools_deps]
@@ -14,13 +17,7 @@ include:
 lint_macos_gitlab_amd64:
   extends: .lint_macos_gitlab
   tags: ["macos:ventura-amd64", "specific:true"]
-  rules:
-    - !reference [.except_mergequeue]
-    - when: on_success
 
 lint_macos_gitlab_arm64:
   extends: .lint_macos_gitlab
-  rules:
-    - !reference [.on_main]
-    - !reference [.manual]
   tags: ["macos:ventura-arm64", "specific:true"]
diff --git a/.gitlab/package_build/dmg.yml b/.gitlab/package_build/dmg.yml
@@ -23,9 +23,8 @@
     export GOMODCACHE=~/gomodcache
     mkdir -p $GOMODCACHE
 
-agent_dmg-x64-a7:
+.agent_dmg:
   stage: package_build
-  tags: ["macos:ventura-amd64", "specific:true"]
   needs: ["go_mod_tidy_check"]
   rules:
      - !reference [.on_macos_gui_change]
@@ -71,3 +70,11 @@ agent_dmg-x64-a7:
     - !reference [.macos_setup_cache]
     - bash .gitlab/package_build/build_agent_dmg.sh
     - !reference [.upload_sbom_artifacts]
+
+agent_dmg-x64-a7:
+  extends: .agent_dmg
+  tags: ["macos:ventura-amd64", "specific:true"]
+
+agent_dmg-arm64-a7:
+  extends: .agent_dmg
+  tags: ["macos:ventura-arm64", "specific:true"]
diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml
@@ -21,6 +21,11 @@ include:
     - if [[ "$FAST_TESTS" == "true" ]]; then FAST_TESTS_FLAG="--only-impacted-packages"; fi
     - dda inv -- -e test --rerun-fails=2 --race --profile --cpus 12 --result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" $FAST_TESTS_FLAG --test-washer --coverage
     - dda inv -- -e invoke-unit-tests
+  after_script:
+    - !reference [.vault_login]
+    - !reference [.select_python_env_commands]
+    - !reference [.upload_junit_source]
+    - !reference [.upload_coverage]
   artifacts:
     expire_in: 2 weeks
     when: always
@@ -35,19 +40,7 @@ include:
 tests_macos_gitlab_amd64:
   extends: .tests_macos_gitlab
   tags: ["macos:ventura-amd64", "specific:true"]
-  after_script:
-    - !reference [.vault_login]
-    - !reference [.select_python_env_commands]
-    - !reference [.upload_junit_source]
-    - !reference [.upload_coverage]
 
 tests_macos_gitlab_arm64:
   extends: .tests_macos_gitlab
-  rules:
-    !reference [.manual]
   tags: ["macos:ventura-arm64", "specific:true"]
-  after_script:
-    - !reference [.vault_login]
-    - !reference [.select_python_env_commands]
-    - !reference [.upload_junit_source]
-    - !reference [.upload_coverage]
diff --git a/omnibus/config/software/datadog-agent-integrations-py3.rb b/omnibus/config/software/datadog-agent-integrations-py3.rb
@@ -86,7 +86,7 @@
     when linux_target?
       arm_target? ? "linux-aarch64" : "linux-x86_64"
     when osx_target?
-      "macos-x86_64"
+      arm_target? ? "macos-aarch64" : "macos-x86_64"
     when windows_target?
       "windows-x86_64"
   end + "_#{python_version}.txt"
diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb
@@ -256,7 +256,8 @@
     systray_build_dir = "#{project_dir}/comp/core/gui/guiimpl/systray"
     # Target OSX 10.10 (it brings significant changes to Cocoa and Foundation APIs, and older versions of OSX are EOL'ed)
     # Add @executable_path/../Frameworks to rpath to find the swift libs in the Frameworks folder.
-    command 'swiftc -O -swift-version "5" -target "x86_64-apple-macosx10.10" -Xlinker \'-rpath\' -Xlinker \'@executable_path/../Frameworks\' Sources/*.swift -o gui', cwd: systray_build_dir
+    target = arm_target? ? 'arm64-apple-macosx11.0' : 'x86_64-apple-macosx10.10'
+    command "swiftc -O -swift-version \"5\" -target \"#{target}\" -Xlinker '-rpath' -Xlinker '@executable_path/../Frameworks' Sources/*.swift -o gui", cwd: systray_build_dir
     copy "#{systray_build_dir}/gui", "#{app_temp_dir}/MacOS/"
     copy "#{systray_build_dir}/agent.png", "#{app_temp_dir}/MacOS/"
   end
diff --git a/omnibus/config/software/openssl3.rb b/omnibus/config/software/openssl3.rb
@@ -60,7 +60,7 @@
   configure_args = []
   if mac_os_x?
     configure_cmd = "./Configure"
-    configure_args << "darwin64-x86_64-cc"
+    configure_args << "darwin64-#{arm_target? ? "arm64" : "x86_64"}-cc"
   elsif windows?
     configure_cmd = "perl.exe ./Configure"
     configure_args << (windows_arch_i386? ? "mingw" : "mingw64")
diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb
@@ -32,7 +32,7 @@
 
     if mac_os_x?
       python_configure_options.push("--enable-ipv6",
-                            "--with-universal-archs=intel",
+                            "--with-universal-archs=#{arm_target? ? "universal2" : "intel"}",
                             "--enable-shared")
     elsif linux_target?
       python_configure_options.push("--enable-shared",
diff --git a/pkg/process/checks/host_info_test.go b/pkg/process/checks/host_info_test.go
@@ -100,7 +100,7 @@ func TestGetHostnameFromCmd(t *testing.T) {
 }
 
 func TestResolveHostname(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && (runtime.GOARCH == "amd64" || runtime.GOARCH == "arm64") {
 		t.Skip("TestResolveHostname is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	osHostname, err := os.Hostname()
diff --git a/pkg/serverless/metrics/metric_test.go b/pkg/serverless/metrics/metric_test.go
@@ -39,7 +39,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestStartDoesNotBlock(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && (runtime.GOARCH == "amd64" || runtime.GOARCH == "arm64") {
 		t.Skip("TestStartDoesNotBlock is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	mockConfig := configmock.New(t)
@@ -118,7 +118,7 @@ func TestStartWithProxy(t *testing.T) {
 }
 
 func TestRaceFlushVersusAddSample(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && (runtime.GOARCH == "amd64" || runtime.GOARCH == "arm64") {
 		t.Skip("TestRaceFlushVersusAddSample is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	metricAgent := &ServerlessMetricAgent{
diff --git a/pkg/serverless/proxy/proxy_test.go b/pkg/serverless/proxy/proxy_test.go
@@ -69,7 +69,7 @@ func (tp *testProcessorResponseError) GetExecutionInfo() *invocationlifecycle.Ex
 }
 
 func TestProxyResponseValid(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && (runtime.GOARCH == "amd64" || runtime.GOARCH == "arm64") {
 		t.Skip("TestProxyResponseValid is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	// fake the runtime API running on 5001
diff --git a/releasenotes/notes/add-macos-build-for-aarch64-767531f392e77140.yaml b/releasenotes/notes/add-macos-build-for-aarch64-767531f392e77140.yaml
@@ -0,0 +1,11 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+enhancements:
+  - |
+    Add macOS build for AArch64/ARM64.
diff --git a/tasks/omnibus.py b/tasks/omnibus.py
@@ -110,7 +110,7 @@ def get_omnibus_env(
 
     if sys.platform == 'darwin':
         # Target MacOS 10.12
-        env['MACOSX_DEPLOYMENT_TARGET'] = '10.12'
+        env['MACOSX_DEPLOYMENT_TARGET'] = '11.0' if os.uname().machine == "arm64" else '10.12'
 
     if skip_sign:
         env['SKIP_SIGN_MAC'] = 'true'

Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,8 @@ static_quality_gates:`
`51`	`51`	`- windows_msi_and_bosh_zip_x64-a7`
`52`	`52`	`# - job: agent_dmg-x64-a7`
`53`	`53`	`# optional: True`
	`54`	`+ # - job: agent_dmg-arm64-a7`
	`55`	`+ # optional: True`
`54`	`56`	`#`
`55`	`57`	`script:`
`56`	`58`	`- !reference [.login_to_docker_readonly_crane]`