[ABLD-28] Backport upstream macOS multiarch support (#240)

While looking at the macOS installation script, I realized no architecture was passed to the `dd-agent` [query string](https://github.com/DataDog/datadog-agent/blob/711ec69670e90a5066d5672a6ac8a9783871a37b/cmd/agent/install_mac_os.sh#L78):

```bash
curl "https://s3.amazonaws.com/dd-agent?prefix=datadog-agent-${major_minor}."
```

There was no architecture disambiguation so far because our PKGs and DMGs had been primarily targeting x86_64/AMD64, and relying on Rosetta 2 translation for AArch64/ARM64.

The present change is therefore to backport (`git cherry-pick`) upstream efforts on the matter, i.e.:
1. chef/omnibus@2b1b503
2. chef/omnibus@6173b5f

[Without](https://gitlab.ddbuild.io/DataDog/datadog-agent/-/jobs/1031599309#L1387)/[with](https://gitlab.ddbuild.io/DataDog/datadog-agent/-/jobs/1032219779#L2302) an architecture qualifier:
```diff
-datadog-agent-7.70.0-devel.git.75.d018f55.pipeline.70720451-1.dmg
+datadog-agent-7.70.0-devel.git.79.1816262.pipeline.70752546-1.x86_64.dmg
```
(examples from development pipelines)

Squashed here:

* Add support for macOS 11 and multiple architectures

* Provide the correct platform version (11 vs 11.x) for macOS.
* Add the architecture (x86_64, arm64) to the file name

Cherry-picked-from: chef/omnibus@2b1b503
Co-authored-by: Régis Desgroppes <regis.desgroppes@datadoghq.com>

* Add hostArchitectures to macOS Distribution file

Unless hostArchitectures is set, macOS 11 will assume that the package
is x86_64 and require Rosetta 2.

* https://scriptingosx.com/2020/12/platform-support-in-macos-installer-packages-pkg/
* https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html#//apple_ref/doc/uid/TP40005370-CH100-SW7

Cherry-picked-from: chef/omnibus@6173b5f
Co-authored-by: Régis Desgroppes <regis.desgroppes@datadoghq.com>

Author: rdesgroppes

lib/omnibus/metadata.rb

@@ -170,9 +170,13 @@ def truncate_platform_version(platform_version, platform)
         when "centos", "debian", "el", "fedora", "freebsd", "omnios", "pidora", "raspbian", "rhel", "sles", "suse", "smartos", "nexus", "ios_xr"
           # Only want MAJOR (e.g. Debian 7, OmniOS r151006, SmartOS 20120809T221258Z)
           platform_version.split(".").first
-        when "aix", "gentoo", "mac_os_x", "openbsd", "slackware", "solaris2", "opensuse", "ubuntu", "macos"
-          # Only want MAJOR.MINOR (e.g. Mac OS X 10.9, Ubuntu 12.04)
+        when "aix", "gentoo", "openbsd", "slackware", "solaris2", "opensuse", "ubuntu"
+          # Only want MAJOR.MINOR (e.g. Ubuntu 12.04)
           platform_version.split(".")[0..1].join(".")
+        when "mac_os_x", "darwin", "macos"
+          # If running macOS >= 11, use only MAJOR version. Otherwise, use MAJOR.MINOR
+          pv_bits = platform_version.split(".")
+          pv_bits[0].to_i >= 11 ? pv_bits[0] : pv_bits[0..1].join(".")
         when "arch"
           # Arch Linux does not have a platform_version ohai attribute, it is rolling release (lsb_release -r)
           "rolling"

lib/omnibus/packagers/pkg.rb

@@ -146,7 +146,7 @@ def install_location(val = NULL)
 
     # @see Base#package_name
     def package_name
-      "#{safe_base_package_name}-#{safe_version}-#{safe_build_iteration}.pkg"
+      "#{safe_base_package_name}-#{safe_version}-#{safe_build_iteration}.#{safe_architecture}.pkg"
     end
 
     #
@@ -242,6 +242,7 @@ def write_distribution_file
                         identifier: safe_identifier,
                         version: safe_version,
                         component_pkg: component_pkg,
+                        host_architecture: safe_architecture,
                       })
     end
 
@@ -276,6 +277,15 @@ def component_pkg
       "#{safe_base_package_name}-core.pkg"
     end
 
+    #
+    # Return the architecture
+    #
+    # @return [String]
+    #
+    def safe_architecture
+      @safe_architecture ||= Ohai["kernel"]["machine"]
+    end
+
     #
     # Return the PKG-ready base package name, removing any invalid characters.
     #

resources/pkg/distribution.xml.erb

@@ -7,7 +7,7 @@
 
   <!-- Generated by productbuild - - synthesize -->
   <pkg-ref id="<%= identifier %>"/>
-  <options customize="never" require-scripts="false"/>
+  <options customize="never" require-scripts="false" hostArchitectures="<%= host_architecture %>" />
   <choices-outline>
     <line choice="default">
       <line choice="<%= identifier %>"/>

spec/unit/compressors/dmg_spec.rb

@@ -114,7 +114,7 @@ module Omnibus
               --detach-retries=5 \\
               --settings="#{subject.resource_path('settings.py')}" \\
               -Dbackground="#{subject.resource_path('background.png')}" \\
-              -Dpkg="#{package_dir}/project-1.2.3-2.pkg" \\
+              -Dpkg="#{package_dir}/project-1.2.3-2.x86_64.pkg" \\
               -Dpkg_position="535, 50" \\
               -Dvolume_icon="#{staging_dir}/tmp.icns" \\
               -Dwindow_bounds="100, 100, 750, 600" \\
@@ -136,7 +136,7 @@ module Omnibus
         expect(subject).to receive(:shellout!)
           .with <<-EOH.gsub(/^ {12}/, "")
             hdiutil verify \\
-              "#{package_dir}/project-1.2.3-2.dmg" \\
+              "#{package_dir}/project-1.2.3-2.x86_64.dmg" \\
               -puppetstrings
           EOH
 
@@ -162,10 +162,10 @@ module Omnibus
             DeRez -only icns "#{icon}" > tmp.rsrc
 
             # Append the icon reosurce to the DMG
-            Rez -append tmp.rsrc -o "#{package_dir}/project-1.2.3-2.dmg"
+            Rez -append tmp.rsrc -o "#{package_dir}/project-1.2.3-2.x86_64.dmg"
 
             # Source the icon
-            SetFile -a C "#{package_dir}/project-1.2.3-2.dmg"
+            SetFile -a C "#{package_dir}/project-1.2.3-2.x86_64.dmg"
           EOH
 
         subject.set_dmg_icon
@@ -174,11 +174,11 @@ module Omnibus
 
     describe '#package_name' do
       it 'reflects the packager\'s unmodified package_name' do
-        expect(subject.package_name).to eq("project-1.2.3-2.dmg")
+        expect(subject.package_name).to eq("project-1.2.3-2.x86_64.dmg")
       end
 
       it 'reflects the packager\'s modified package_name' do
-        package_basename = "projectsub-1.2.3-3"
+        package_basename = "projectsub-1.2.3-3.x86_64"
         allow(project.packagers_for_system[0]).to receive(:package_name)
           .and_return("#{package_basename}.pkg")
 

spec/unit/compressors/tgz_spec.rb

@@ -38,25 +38,25 @@ module Omnibus
 
     describe '#package_name' do
       it "returns the name of the packager" do
-        expect(subject.package_name).to eq("project-1.2.3-2.pkg.tar.gz")
+        expect(subject.package_name).to eq("project-1.2.3-2.x86_64.pkg.tar.gz")
       end
     end
 
     describe '#write_tgz' do
       before do
-        File.open("#{staging_dir}/project-1.2.3-2.pkg", "wb") do |f|
+        File.open("#{staging_dir}/project-1.2.3-2.x86_64.pkg", "wb") do |f|
           f.write " " * 1_000_000
         end
       end
 
       it "generates the file" do
         subject.write_tgz
-        expect("#{staging_dir}/project-1.2.3-2.pkg.tar.gz").to be_a_file
+        expect("#{staging_dir}/project-1.2.3-2.x86_64.pkg.tar.gz").to be_a_file
       end
 
       it "has the correct content" do
         subject.write_tgz
-        file = File.open("#{staging_dir}/project-1.2.3-2.pkg.tar.gz", "rb")
+        file = File.open("#{staging_dir}/project-1.2.3-2.x86_64.pkg.tar.gz", "rb")
         contents = file.read
         file.close
 

spec/unit/metadata_spec.rb

@@ -220,6 +220,8 @@ module Omnibus
       it_behaves_like "a version manipulator", "gentoo", "2004.3", "2004.3"
       it_behaves_like "a version manipulator", "ios_xr", "6.0.0.14I", "6"
       it_behaves_like "a version manipulator", "mac_os_x", "10.9.1", "10.9"
+      it_behaves_like "a version manipulator", "mac_os_x", "10.15.7", "10.15"
+      it_behaves_like "a version manipulator", "mac_os_x", "11.2.1", "11"
       it_behaves_like "a version manipulator", "nexus", "5.0", "5"
       it_behaves_like "a version manipulator", "omnios", "r151010", "r151010"
       it_behaves_like "a version manipulator", "openbsd", "5.4.4", "5.4"

spec/unit/packagers/pkg_spec.rb

@@ -47,8 +47,8 @@ module Omnibus
     end
 
     describe '#package_name' do
-      it "includes the name, version, and build iteration" do
-        expect(subject.package_name).to eq("project-full-name-1.2.3-2.pkg")
+      it "includes the name, version, build iteration, and architecture" do
+        expect(subject.package_name).to eq("project-full-name-1.2.3-2.x86_64.pkg")
       end
     end
 
@@ -137,8 +137,24 @@ module Omnibus
 
         expect(contents).to include('<pkg-ref id="com.getchef.project-full-name"/>')
         expect(contents).to include('<line choice="com.getchef.project-full-name"/>')
+        expect(contents).to include('hostArchitectures="x86_64"')
         expect(contents).to include("project-full-name-core.pkg")
       end
+
+      context "for arm64 builds" do
+        before do
+          stub_ohai(platform: "mac_os_x", version: "11.0") do |data|
+            data["kernel"]["machine"] = "arm64"
+          end
+        end
+
+        it "sets the hostArchitectures to include arm64" do
+          subject.write_distribution_file
+          contents = File.read("#{staging_dir}/Distribution")
+
+          expect(contents).to include('hostArchitectures="arm64"')
+        end
+      end
     end
 
     describe '#build_product_pkg' do
@@ -148,7 +164,7 @@ module Omnibus
             productbuild \\
               --distribution "#{staging_dir}/Distribution" \\
               --resources "#{staging_dir}/Resources" \\
-              "#{package_dir}/project-full-name-1.2.3-2.pkg"
+              "#{package_dir}/project-full-name-1.2.3-2.x86_64.pkg"
           EOH
 
           subject.build_product_pkg
@@ -166,7 +182,7 @@ module Omnibus
               --distribution "#{staging_dir}/Distribution" \\
               --resources "#{staging_dir}/Resources" \\
               --sign "My Special Identity" \\
-              "#{package_dir}/project-full-name-1.2.3-2.pkg"
+              "#{package_dir}/project-full-name-1.2.3-2.x86_64.pkg"
             EOH
           subject.build_product_pkg
         end