Skip to content

[toolchain] Upload published toolchains with --upload #38688

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cdesouza-chromium wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[toolchain] Upload published toolchains with --upload #38688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 10 additions & 38 deletions tools/cr/toolchain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from __future__ import annotations

from dataclasses import dataclass
import hashlib
import os
import re
import textwrap
Expand Down Expand Up @@ -593,21 +592,6 @@ def _commit_title(objects: list[dict]) -> str:
return (f'Rust/WASM toolchain ({match["rust"][:12]}-{match["sub"]}, '
f'{match["clang"]}, sub {match["sub"]})')

@staticmethod
def _fetch_archive_info(url: str) -> tuple[str, int]:
"""Download `url` and return its `(sha256sum, size_bytes)`.

`setdep` values are read straight off the published archive.
"""
digest = hashlib.sha256()
size = 0
with requests.get(url, stream=True, timeout=60) as response:
response.raise_for_status()
for chunk in response.iter_content(chunk_size=1 << 20):
digest.update(chunk)
size += len(chunk)
return digest.hexdigest(), size

# `brave_subrevision` is required here (unlike the base's `**kwargs`
# catch-all) since this toolchain has no side index to auto-discover it
# from; see the base `repin`'s docstring.
Expand All @@ -623,9 +607,9 @@ def repin(self,
this Chromium tag's Rust+Clang revision (e.g. `1` for a fresh
Chromium-version bump, or whatever `gen-rust-toolchain
--brave-subrevision` last built) -- there is no side index to
auto-discover it from; every object name and its overlay base are
derived deterministically and fetched straight from the bucket, like
`tools/clang/scripts/sync_deps.py`'s `GetDepsObjectInfo`.
auto-discover it from; every platform's object is read straight from
its sibling index (see
`build_rust_toolchain.rust_toolchain_extra_dep`).
"""
self._require_no_staged_files()

Expand All @@ -634,25 +618,13 @@ def repin(self,
commit=ref)
upstream_stem = self._upstream_stem(revision_text)

objects = []
platform_conditions = build_rust_toolchain.SUPPORTED_PLATFORM_CONDITIONS
for platform_prefix in platform_conditions:
object_name = (f'{platform_prefix}-{upstream_stem}-'
f'{brave_subrevision}.tar.xz')
url = f'{build_rust_toolchain.TOOLCHAIN_BUCKET_URL}/{object_name}'
try:
sha256sum, size_bytes = self._fetch_archive_info(url)
except requests.RequestException as e:
raise BadOutcomeException(
f'Could not fetch published toolchain {url}: {e}') from e
host_os = build_rust_toolchain.PLATFORM_PREFIX_TO_CHROMIUM_HOST_OS[
platform_prefix]
objects.append({
'object_name': object_name,
'sha256sum': sha256sum,
'size_bytes': size_bytes,
'overlayed_on': f'{host_os}/{upstream_stem}.tar.xz',
})
try:
extra_dep = build_rust_toolchain.rust_toolchain_extra_dep(
upstream_stem, brave_subrevision)
except RuntimeError as e:
raise BadOutcomeException(str(e)) from e
objects = extra_dep[
build_rust_toolchain.RUST_TOOLCHAIN_DEP_PATH]['objects']

installer = self.spec.installer
path = repository.brave.root / installer
Expand Down
50 changes: 34 additions & 16 deletions tools/cr/toolchain_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,21 +308,41 @@ class RustRepinTest(_FakeRepoTest):
BRAVE_SUBREVISION = 1

@staticmethod
def _fake_archive_info(url: str) -> tuple[str, int]:
"""A deterministic stand-in for a real download: derives a fake
`(sha256sum, size_bytes)` from the object name in `url`, so each
platform's fetched values are distinguishable without any network
access."""
name = url.rsplit('/', 1)[-1]
return f'sha-{name}', len(name)
def _fake_extra_dep(upstream_stem: str, brave_subrevision: int) -> dict:
"""A deterministic stand-in for
`build_rust_toolchain.rust_toolchain_extra_dep`: derives a fake
`sha256sum`/`size_bytes` per platform from its object name, so each
platform's values are distinguishable without any network access."""
build_rust_toolchain = toolchain.build_rust_toolchain
objects = []
for platform_prefix, condition in (
build_rust_toolchain.SUPPORTED_PLATFORM_CONDITIONS.items()):
object_name = (f'{platform_prefix}-{upstream_stem}-'
f'{brave_subrevision}.tar.xz')
host_os = build_rust_toolchain.PLATFORM_PREFIX_TO_CHROMIUM_HOST_OS[
platform_prefix]
objects.append({
'object_name': object_name,
'sha256sum': f'sha-{object_name}',
'size_bytes': len(object_name),
'overlayed_on': f'{host_os}/{upstream_stem}.tar.xz',
'condition': condition,
})
return {
build_rust_toolchain.RUST_TOOLCHAIN_DEP_PATH: {
'bucket': f'{build_rust_toolchain.TOOLCHAIN_BUCKET_URL}/',
'condition': build_rust_toolchain.RUST_TOOLCHAIN_DEP_CONDITION,
'objects': objects,
}
}

def setUp(self):
super().setUp()
self.rust = toolchain.RustToolchain()
self._seed_installer()
patcher = patch.object(toolchain.RustToolchain,
'_fetch_archive_info',
side_effect=self._fake_archive_info)
patcher = patch.object(toolchain.build_rust_toolchain,
'rust_toolchain_extra_dep',
side_effect=self._fake_extra_dep)
self.fetch = patcher.start()
self.addCleanup(patcher.stop)

Expand Down Expand Up @@ -376,11 +396,9 @@ def test_updates_and_commits_with_auto_culprit(self):
self.assertIn(f'Linux_x64/{self.UPSTREAM_STEM}.tar.xz', text)
self.assertIn(f'Win/{self.UPSTREAM_STEM}.tar.xz', text)
self.assertNotIn('old-upstream.tar.xz', text)
# Every platform is fetched straight from the bucket, no side index.
self.assertEqual(self.fetch.call_count, 4)
self.fetch.assert_any_call(
f'{toolchain.build_rust_toolchain.TOOLCHAIN_BUCKET_URL}/'
f'{linux_name}')
# Every platform's object is read from its sibling index in one call.
self.fetch.assert_called_once_with(self.UPSTREAM_STEM,

@netzenbot netzenbot Aug 1, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: the old assert_any_call(f'{TOOLCHAIN_BUCKET_URL}/{linux_name}') was the only coverage of the per-platform object-name/URL derivation. That derivation now lives in rust_toolchain_extra_dep (plus the new -{brave_subrevision}.tar.xz suffix and toolchain_index_name), which has no test file, and _fake_extra_dep re-implements the naming rather than exercising it — so the check isn't relocated, just dropped. Consider a small test for rust_toolchain_extra_dep/toolchain_index_name in tools/cr/toolchains/. (✅ Relocate Test Checks When Refactoring)

self.BRAVE_SUBREVISION)
# Everything setdep does not touch survives byte for byte.
self.assertIn("'src/other-dep'", text)
self.assertIn('OTHER_CONSTANT = 1', text)
Expand Down Expand Up @@ -427,7 +445,7 @@ def test_staged_files_block_the_update(self):

def test_fetch_failure_raises(self):
self._seed_rust_revision_bump()
self.fetch.side_effect = toolchain.requests.RequestException('boom')
self.fetch.side_effect = RuntimeError('boom')
with self.assertRaises(toolchain.BadOutcomeException):
self._repin(culprit=None)

Expand Down
28 changes: 4 additions & 24 deletions tools/cr/toolchains/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,27 +16,7 @@ build:
used to build Brave without having a Visual Studio install, or to cross-
compile Brave.

### Reproducibility

Reproducibility of the rust toolchain is important, and the builder we have has
the necessary constrols to make sure we have enough information to generate the
same toolchain again if necessary.

Another aspect of reproducibility for the builder is to make sure we are able to
verify that a toolchain used is distinct, and that there is no chance for an
archived toolchain to be replaced. For this reason, the builder also maintains
and index for a given toolchain signature. This archive index can be used by the
client to query what toolchains we have, and to pin any given toolchain with a
hashsum value during checkout.

The index aggregates every build for the same platform and upstream Rust+Clang
combination, and is the canonical source for "what toolchains are available."
Each entry stores all the relevant information for all the necessary elements
that led to the creation of the toolchain.

`--brave-subrevision=<int>` exists as the lever for minting a distinct new
toolchain at the same upstream Rust+Clang revisions. The script encodes it as
the final section of the tarball filename, so bumping it produces a fresh URL.
This control is important for cases where there are changes on our end for how
the toolchain should be generated, and a new distinct archive is necessary for a
toolchain already in use.
The three builders above share their publishing plumbing (probing whether a
toolchain is already published, writing a sibling YAML index, and uploading the
archive + index pair) via `toolchain_publish.py`, so it stays consistent instead
of being carried as three near-identical copies.
Loading
Loading