Skip to content

Create portable toolchain when installing SDKs for macOS/iOS builds #662

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hach-que wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Create portable toolchain when installing SDKs for macOS/iOS builds #662

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 79 additions & 6 deletions UET/Redpoint.Uet.SdkManagement/Sdk/MacSdkSetup.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@
{
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Redpoint.Hashing;
using Redpoint.PackageManagement;
using Redpoint.ProcessExecution;
using Redpoint.ProgressMonitor;
using Redpoint.Uet.SdkManagement.Sdk.VersionNumbers;
using System.Globalization;
using System.Runtime.Versioning;
using System.Text;

public class MacSdkSetup : ISdkSetup
{
Expand Down Expand Up @@ -55,7 +58,18 @@ public async Task<string> ComputeSdkPackageId(string unrealEnginePath, Cancellat
{
if (OperatingSystem.IsMacOS())
{
var versionNumber = await _versionNumberResolver.For<IMacVersionNumbers>(unrealEnginePath).GetXcodeVersion(unrealEnginePath).ConfigureAwait(false);
var resolver = _versionNumberResolver.For<IMacVersionNumbers>(unrealEnginePath);
var xcodeVersion = await resolver.GetXcodeVersion(unrealEnginePath).ConfigureAwait(false);
var portableToolchainInfo = await resolver.GetPortableToolchainVersion(unrealEnginePath, xcodeVersion).ConfigureAwait(false);
var versionNumber = xcodeVersion;
if (portableToolchainInfo != null)
{
versionNumber += '-' + Hash.XxHash64(
portableToolchainInfo.Value.ClangSubdir + "-" +
portableToolchainInfo.Value.OSSHeadersSubdir + "-" +
portableToolchainInfo.Value.PortableToolchainVersion,
Encoding.UTF8).Hash.ToString(CultureInfo.InvariantCulture);
}
return $"{versionNumber}-iOS";
}
else if (OperatingSystem.IsWindows())
Expand All @@ -70,7 +84,7 @@ public async Task<string> ComputeSdkPackageId(string unrealEnginePath, Cancellat
}

[SupportedOSPlatform("macos")]
public async Task InstallXcode(string xcodeVersion, string sdkPackagePath, CancellationToken cancellationToken)
public async Task InstallXcode(string xcodeVersion, MacPortableToolchainInfo? portableToolchainInfo, string sdkPackagePath, CancellationToken cancellationToken)
{
// Check that the required environment variables have been set.
var appleXcodeStoragePath = Environment.GetEnvironmentVariable("UET_APPLE_XCODE_STORAGE_PATH");
Expand Down Expand Up @@ -231,14 +245,54 @@ await monitor.MonitorAsync(
{
throw new SdkSetupPackageGenerationFailedException("Xcode was unable to install iOS platform support.");
}

// Create AutoSDK directory.
if (portableToolchainInfo.HasValue)
{
var clangBase = Path.GetDirectoryName(Path.GetDirectoryName(portableToolchainInfo.Value.ClangSubdir)!)!;

var toolchainPath = Path.Combine(sdkPackagePath, "AutoSDK", portableToolchainInfo.Value.PortableToolchainVersion);
Directory.CreateDirectory(toolchainPath);
Directory.CreateDirectory(Path.Combine(toolchainPath, clangBase));
Directory.CreateDirectory(Path.Combine(toolchainPath, portableToolchainInfo.Value.OSSHeadersSubdir, "usr"));

var links = new Dictionary<string, string>
{
{
Path.Combine(toolchainPath, portableToolchainInfo.Value.OSSHeadersSubdir, "usr", "include"),
Path.Combine(sdkPackagePath, "Xcode.app", "Contents", "Developer", "Platforms", "MacOSX.platform", "Developer", "SDKs", "MacOSX.sdk", "usr", "include")
},
{
Path.Combine(toolchainPath, clangBase, "include"),
Path.Combine(sdkPackagePath, "Xcode.app", "Contents", "Developer", "Platforms", "MacOSX.platform", "Developer", "SDKs", "MacOSX.sdk", "usr", "include")
},
{
Path.Combine(toolchainPath, clangBase, "lib"),
Path.Combine(sdkPackagePath, "Xcode.app", "Contents", "Developer", "Toolchains", "XcodeDefault.xctoolchain", "usr", "lib")
},
{
Path.Combine(toolchainPath, clangBase, "bin"),
Path.Combine(sdkPackagePath, "Xcode.app", "Contents", "Developer", "Toolchains", "XcodeDefault.xctoolchain", "usr", "bin")
},
};

foreach (var kv in links)
{
Directory.CreateSymbolicLink(
kv.Key,
Path.GetRelativePath(kv.Key, kv.Value).Substring(3));
}
}
}

public async Task GenerateSdkPackage(string unrealEnginePath, string sdkPackagePath, CancellationToken cancellationToken)
{
if (OperatingSystem.IsMacOS())
{
var xcodeVersion = await _versionNumberResolver.For<IMacVersionNumbers>(unrealEnginePath).GetXcodeVersion(unrealEnginePath).ConfigureAwait(false);
await InstallXcode(xcodeVersion, sdkPackagePath, cancellationToken);
var versionResolver = _versionNumberResolver.For<IMacVersionNumbers>(unrealEnginePath);
var xcodeVersion = await versionResolver.GetXcodeVersion(unrealEnginePath).ConfigureAwait(false);
var portableToolchainVersion = await versionResolver.GetPortableToolchainVersion(unrealEnginePath, xcodeVersion).ConfigureAwait(false);
await InstallXcode(xcodeVersion, portableToolchainVersion, sdkPackagePath, cancellationToken);
}
else
{
Expand All @@ -248,7 +302,26 @@ public async Task GenerateSdkPackage(string unrealEnginePath, string sdkPackageP

public Task<AutoSdkMapping[]> GetAutoSdkMappingsForSdkPackage(string sdkPackagePath, CancellationToken cancellationToken)
{
return Task.FromResult(Array.Empty<AutoSdkMapping>());
if (OperatingSystem.IsMacOS())
{
return Task.FromResult(new[]
{
new AutoSdkMapping
{
RelativePathInsideAutoSdkPath = "Apple",
RelativePathInsideSdkPackagePath = "AutoSDK",
},
new AutoSdkMapping
{
RelativePathInsideAutoSdkPath = "Xcode.app",
RelativePathInsideSdkPackagePath = "Xcode.app",
},
});
}
else
{
return Task.FromResult(Array.Empty<AutoSdkMapping>());
}
}

public async Task<EnvironmentForSdkUsage> GetRuntimeEnvironmentForSdkPackage(string sdkPackagePath, CancellationToken cancellationToken)
Expand Down Expand Up @@ -277,7 +350,7 @@ await _processExecutor.ExecuteAsync(
// Emit the environment variable required to use Xcode from the package directory.
var envs = new Dictionary<string, string>
{
{ "DEVELOPER_DIR", Path.Combine(sdkPackagePath, "Xcode.app") }
{ "DEVELOPER_DIR", Path.Combine(sdkPackagePath, "Xcode.app") },
};
var currentPath = Environment.GetEnvironmentVariable("PATH");
if (currentPath != null)
Expand Down
5 changes: 5 additions & 0 deletions UET/Redpoint.Uet.SdkManagement/Sdk/VersionNumbers/EmbeddedMacVersionNumbers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ public async Task<string> GetXcodeVersion(string unrealEnginePath)
return await ParseXcodeVersion(applePlatformSdk).ConfigureAwait(false);
}

public Task<MacPortableToolchainInfo?> GetPortableToolchainVersion(string unrealEnginePath, string xcodeVersion)
{
return Task.FromResult<MacPortableToolchainInfo?>(null);
}

public async Task<string> GetITunesVersion(string unrealEnginePath)
{
return string.Empty;
Expand Down
10 changes: 10 additions & 0 deletions UET/Redpoint.Uet.SdkManagement/Sdk/VersionNumbers/IMacVersionNumbers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,21 @@
{
using System.Runtime.Versioning;

public readonly record struct MacPortableToolchainInfo
{
public required readonly string PortableToolchainVersion { get; init; }
public required readonly string ClangSubdir { get; init; }
public required readonly string OSSHeadersSubdir { get; init; }
}

internal interface IMacVersionNumbers : IVersionNumbers
{
[SupportedOSPlatform("macos")]
Task<string> GetXcodeVersion(string unrealEnginePath);

[SupportedOSPlatform("macos")]
Task<MacPortableToolchainInfo?> GetPortableToolchainVersion(string unrealEnginePath, string xcodeVersion);

[SupportedOSPlatform("windows")]
Task<string> GetITunesVersion(string unrealEnginePath);
}
Expand Down
85 changes: 85 additions & 0 deletions UET/Redpoint.Uet.SdkManagement/Sdk/VersionNumbers/JsonMacVersionNumbers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,91 @@ await File.ReadAllTextAsync(engineBuildVersionPath),
throw new InvalidOperationException("Unable to read Apple SDK versions from either 'MaxVersion' or 'MainVersion' in Apple_SDK.json!");
}

[SupportedOSPlatform("macos")]
public async Task<MacPortableToolchainInfo?> GetPortableToolchainVersion(string unrealEnginePath, string xcodeVersion)
{
var jsonPath = Path.Combine(
unrealEnginePath,
"Engine",
"Config",
"Apple",
"AppleOpenSource_SDK.json");
if (!File.Exists(jsonPath))
{
return null;
}

if (!Version.TryParse(xcodeVersion, out var xcodeParsedVersion) || xcodeParsedVersion == null)
{
throw new InvalidOperationException($"Unable to parse {xcodeVersion} to generic platform version!");
}

var json = await File.ReadAllTextAsync(jsonPath).ConfigureAwait(false);
var dictionary = JsonSerializer.Deserialize(
json,
new JsonConfigJsonSerializerContext(new JsonSerializerOptions
{
AllowTrailingCommas = true,
ReadCommentHandling = JsonCommentHandling.Skip,
}).DictionaryStringJsonElement);
if (dictionary == null)
{
throw new InvalidOperationException("Unable to read Apple open source SDK versions from Apple_SDK.json!");
}

string? clangSubdir = null;
string? ossHeadersSubdir = null;

if (dictionary.TryGetValue("ClangSubdir", out var clangSubdirElement) &&
clangSubdirElement.ValueKind == JsonValueKind.String)
{
clangSubdir = clangSubdirElement.GetString();
}
if (dictionary.TryGetValue("OSSHeadersSubdir", out var ossHeadersSubdirElement) &&
ossHeadersSubdirElement.ValueKind == JsonValueKind.String)
{
ossHeadersSubdir = ossHeadersSubdirElement.GetString();
}

string? matchingVersion = null;
if (dictionary.TryGetValue("XcodeToToolchainVersion", out var versionMappings) &&
versionMappings.ValueKind == JsonValueKind.Array)
{
var jsonArray = versionMappings.EnumerateArray().ToArray();
for (int mappingIndex = jsonArray.Length - 1; mappingIndex >= 0; mappingIndex--)
{
var jsonElement = jsonArray[mappingIndex];
if (jsonElement.ValueKind == JsonValueKind.String)
{
var split = jsonElement.GetString()!.Split('-');
if (split.Length == 2)
{
if (Version.TryParse(split[0], out var mappingVersion) && xcodeParsedVersion >= mappingVersion)
{
_logger.LogInformation($"Selected Xcode version '{xcodeParsedVersion}' is within bounds '0'-'{mappingVersion}', with toolchain version '{split[1]}'.");
matchingVersion = split[1];
break;
}
}
}
}
}

if (!string.IsNullOrWhiteSpace(clangSubdir) &&
!string.IsNullOrWhiteSpace(ossHeadersSubdir) &&
!string.IsNullOrWhiteSpace(matchingVersion))
{
return new MacPortableToolchainInfo
{
ClangSubdir = clangSubdir,
OSSHeadersSubdir = ossHeadersSubdir,
PortableToolchainVersion = matchingVersion,
};
}

return null;
}

[SupportedOSPlatform("windows")]
public async Task<string> GetITunesVersion(string unrealEnginePath)
{
Expand Down
23 changes: 15 additions & 8 deletions UET/uet/Commands/Generate/GenerateCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,17 +426,24 @@ public async Task<int> ExecuteAsync(ICommandInvocationContext context)
}
else if (OperatingSystem.IsMacOS())
{
var solutionPath = Path.Combine(workingDirectory, outputFilenameWithoutExtension + ".xcworkspace");
if (File.Exists(solutionPath))
var opened = false;
foreach (var candidateFilename in new[] { outputFilenameWithoutExtension, outputFilenameWithoutExtension + " (Mac)" })
{
_logger.LogInformation($"Opening generated project file due to --open|-o being passed: {solutionPath}");
Process.Start(new ProcessStartInfo
var solutionPath = Path.Combine(workingDirectory, candidateFilename + ".xcworkspace");
_logger.LogInformation($"Checking for project: {solutionPath}");
if (File.Exists(solutionPath) || Directory.Exists(solutionPath))
{
UseShellExecute = true,
FileName = solutionPath,
});
_logger.LogInformation($"Opening generated project file due to --open|-o being passed: {solutionPath}");
Process.Start(new ProcessStartInfo
{
UseShellExecute = true,
FileName = solutionPath,
});
opened = true;
break;
}
}
else
if (!opened)
{
_logger.LogWarning("Unable to automatically open generated project file because it doesn't exist. This can happen if you've configured Unreal Engine to generate project files for something other than Xcode.");
}
Expand Down
33 changes: 19 additions & 14 deletions UET/uet/Commands/InstallSdks/InstallSdksCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ public InstallSdksCommandInstance(

public async Task<int> ExecuteAsync(ICommandInvocationContext context)
{
if (!OperatingSystem.IsWindows())
if (OperatingSystem.IsLinux())
{
_logger.LogError("This command is not currently supported on non-Windows platforms.");
_logger.LogError("This command is not currently supported on Linux.");
return 1;
}

Expand All @@ -103,7 +103,9 @@ public async Task<int> ExecuteAsync(ICommandInvocationContext context)
_logger.LogInformation($" - {platform}");
}

var packagePath = UetPaths.UetDefaultWindowsSdkStoragePath;
var packagePath = OperatingSystem.IsWindows()
? UetPaths.UetDefaultWindowsSdkStoragePath
: UetPaths.UetDefaultMacSdkStoragePath;
Directory.CreateDirectory(packagePath);

var envVars = await _localSdkManager.SetupEnvironmentForSdkSetups(
Expand All @@ -112,21 +114,24 @@ public async Task<int> ExecuteAsync(ICommandInvocationContext context)
sdkSetups.ToHashSet(),
context.GetCancellationToken()).ConfigureAwait(false);

if (!context.ParseResult.GetValueForOption(_options.SkipPermissionUpdate))
if (OperatingSystem.IsWindows())
{
_logger.LogInformation("Updating permissions on SDK directories so all users have read/write access...");
await _worldPermissionApplier.GrantEveryonePermissionAsync(packagePath, context.GetCancellationToken()).ConfigureAwait(false);
}
if (!context.ParseResult.GetValueForOption(_options.SkipPermissionUpdate))
{
_logger.LogInformation("Updating permissions on SDK directories so all users have read/write access...");
await _worldPermissionApplier.GrantEveryonePermissionAsync(packagePath, context.GetCancellationToken()).ConfigureAwait(false);
}

_logger.LogInformation("Setting environment variables to user scope...");
foreach (var kv in envVars)
{
_logger.LogInformation($" {kv.Key} = {kv.Value}");
if (Environment.GetEnvironmentVariable(kv.Key, EnvironmentVariableTarget.User) != kv.Value)
_logger.LogInformation("Setting environment variables to user scope...");
foreach (var kv in envVars)
{
Environment.SetEnvironmentVariable(kv.Key, kv.Value, EnvironmentVariableTarget.User);
_logger.LogInformation($" {kv.Key} = {kv.Value}");
if (Environment.GetEnvironmentVariable(kv.Key, EnvironmentVariableTarget.User) != kv.Value)
{
Environment.SetEnvironmentVariable(kv.Key, kv.Value, EnvironmentVariableTarget.User);
}
context.GetCancellationToken().ThrowIfCancellationRequested();
}
context.GetCancellationToken().ThrowIfCancellationRequested();
}

return 0;
Expand Down
1 change: 1 addition & 0 deletions UET/uet/Commands/Internal/InstallXcode/InstallXcodeCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ public async Task<int> ExecuteAsync(ICommandInvocationContext context)

await macSdkSetup.InstallXcode(
version,
null,
packageWorkingPath,
context.GetCancellationToken());
await File.WriteAllTextAsync(Path.Combine(packageWorkingPath, "sdk-ready"), "ready", context.GetCancellationToken()).ConfigureAwait(false);
Expand Down
Loading