Skip to content

Multiple package repos in config.yaml #294

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
msimberg wants to merge 18 commits into
base: main
Choose a base branch
from
Open

Multiple package repos in config.yaml #294

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
6f966f2
Initial version of multiple package repos in recipe
msimberg May 15, 2026
7dd9d65
Separate alps-cluster-config and recipe package repos
msimberg May 15, 2026
f9e0ee5
Put package repos under repos in build root
msimberg May 15, 2026
fe85aac
Fix custom repos in repos.yaml
msimberg May 15, 2026
cfb0389
Clean up package repo config
msimberg May 18, 2026
c7bbb88
Small cleanup
msimberg May 18, 2026
db34c24
A bit of cleanup
msimberg May 18, 2026
1538303
Add multi-package-repo test
msimberg May 18, 2026
d649156
Use parameterized test fixtures
msimberg May 18, 2026
92ab8e6
Add test-envvars.sh to CI
msimberg May 19, 2026
f8204d9
Merge remote-tracking branch 'origin/main' into multiple-package-repos
msimberg May 19, 2026
d38d855
Formatting
msimberg May 20, 2026
3cb3aff
Run test-envvars.sh with uv venv
msimberg May 20, 2026
7d7710d
Move repo.yaml to common template for recipe and alps repos
msimberg May 29, 2026
2d06546
Rename some pkg to pkg_repo for clarity
msimberg May 29, 2026
dfdaa62
Consistency renaming
msimberg May 29, 2026
68d97ee
Add default for commit when multiple package repos are given in confi…
msimberg May 29, 2026
960f1aa
Format files
msimberg May 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/main.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,8 @@ jobs:
run: uv sync --group dev
- name: Run Unit Tests
run: uv run pytest
- name: Run Envvars Test
working-directory: unittests
run: |
sudo apt-get install -y jq
uv run bash ./test-envvars.sh
147 changes: 98 additions & 49 deletions stackinator/builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,26 +191,15 @@ def generate(self, recipe):

spack_git_commit_result = self._git_clone("spack", spack_repo, spack_commit, spack_path)

# Clone the spack-packages repository and check out commit if one was given
spack_packages = spack["packages"]
spack_packages_repo = spack_packages["repo"]
spack_packages_commit = spack_packages["commit"]
spack_packages_path = self.path / "spack-packages"

spack_packages_git_commit_result = self._git_clone(
"spack-packages",
spack_packages_repo,
spack_packages_commit,
spack_packages_path,
)
packages_meta = self._resolve_packages(spack["packages"])
for pkg in packages_meta:
pkg["commit"] = self._git_clone(pkg["name"], pkg["url"], pkg["ref"], pkg["path"])

spack_meta = {
"url": spack_repo,
"ref": spack_commit,
"commit": spack_git_commit_result,
"packages_url": spack_packages_repo,
"packages_ref": spack_packages_commit,
"packages_commit": spack_packages_git_commit_result,
"packages": packages_meta,
}

# load the jinja templating environment
Expand Down Expand Up @@ -331,16 +320,12 @@ def generate(self, recipe):
# 2. cluster-config/repos.yaml
# - if the repos.yaml file exists it will contain a list of relative paths
# to search for package
# 1. builtin repo
# 1. package repos from config.yaml in the order specified (typically
# only spack-packages builtin repo)

# Build a list of repos with packages to install.
# Build a list of repos with packages to install from system config and recipe.
repos = []

# check for a repo in the recipe
if recipe.spack_repo is not None:
self._logger.debug(f"adding recipe spack package repo: {recipe.spack_repo}")
repos.append(recipe.spack_repo)

# look for repos.yaml file in the system configuration
repo_yaml = recipe.system_config_path / "repos.yaml"
if repo_yaml.exists() and repo_yaml.is_file():
Expand All @@ -361,7 +346,7 @@ def generate(self, recipe):
self._logger.error(f"{repo_path} from {repo_yaml} is not a spack package repository")
raise RuntimeError("invalid system-provided package repository")

self._logger.debug(f"full list of spack package repo: {repos}")
self._logger.debug(f"full list of system spack package repos: {repos}")

# Delete the store/repo path, if it already exists.
# Do this so that incremental builds (though not officially supported) won't break if a repo is updated.
Expand All @@ -378,7 +363,7 @@ def generate(self, recipe):
self._logger.debug(f"created the repo packages path {pkg_dst}")

# create the repository step 2: create the repo.yaml file that
# configures alps and builtin repos
# configures the alps repo
with (repo_dst / "repo.yaml").open("w") as f:
f.write(
"""\
Expand All @@ -388,43 +373,84 @@ def generate(self, recipe):
"""
)

# If the recipe provides a package repo, install it as a separate
# "recipe" repo in the store with highest precedence.
has_recipe_repo = recipe.spack_repo is not None
if has_recipe_repo:
recipe_dst = repos_path / "recipe"
self._logger.debug(f"creating the recipe spack repo in {recipe_dst}")
if recipe_dst.exists():
self._logger.debug(f"{recipe_dst} exists ... deleting")
shutil.rmtree(recipe_dst)

recipe_pkg_dst = recipe_dst / "packages"
recipe_pkg_dst.mkdir(mode=0o755, parents=True)

with (recipe_dst / "repo.yaml").open("w") as f:
f.write(
"""\
repo:
namespace: recipe
api: v2.0
"""
Comment thread
msimberg marked this conversation as resolved.
Outdated
)

packages_path = recipe.spack_repo / "packages"
for pkg_path in packages_path.iterdir():
dst = recipe_pkg_dst / pkg_path.name
if pkg_path.is_dir():
self._logger.debug(f" installing recipe package {pkg_path} to {recipe_pkg_dst}")
install(pkg_path, dst)

# create the repository step 2: create the repos.yaml file in build_path/config
repos_yaml_template = jinja_env.get_template("repos.yaml")
with (config_path / "repos.yaml").open("w") as f:
repo_path = recipe.mount / "repos" / "spack_repo" / "alps"
builtin_repo_path = recipe.mount / "repos" / "spack_repo" / "builtin"
recipe_repo_path = recipe.mount / "repos" / "spack_repo" / "recipe"
package_repos = [
{
"name": pkg["name"],
"path": (recipe.mount / "repos" / "spack_repo" / pkg["name"]).as_posix(),
}
for pkg in spack_meta["packages"]
Copy link
Copy Markdown
Contributor

@albestro albestro May 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just a renaming suggestion for the variable

Suggested change
"name": pkg["name"],
"path": (recipe.mount / "repos" / "spack_repo" / pkg["name"]).as_posix(),
}
for pkg in spack_meta["packages"]
"name": repo["name"],
"path": (recipe.mount / "repos" / "spack_repo" / repo["name"]).as_posix(),
}
for repo in spack_meta["packages"]

also in other similar loops (both before and after this one), I'd suggest to use repo or pkg_repo or repo_meta...pkg sounds like it's targeting a single package

Copy link
Copy Markdown
Collaborator Author

@msimberg msimberg May 29, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did a few renamings hopefully for the better. I tried to avoid renaming things outside of the diff of this PR.

]
f.write(
repos_yaml_template.render(
repo_path=repo_path.as_posix(),
builtin_repo_path=builtin_repo_path.as_posix(),
package_repos=package_repos,
recipe_repo_path=recipe_repo_path.as_posix(),
has_recipe_repo=has_recipe_repo,
verbose=False,
)
)
f.write("\n")

# Iterate over the source repositories copying their contents to the consolidated repo in the uenv.
# Do overwrite packages that have been copied from an earlier source repo, enforcing a descending
# order of precidence.
if len(repos) > 0:
for repo_src in repos:
self._logger.debug(f"installing repo {repo_src}")
packages_path = repo_src / "packages"
for pkg_path in packages_path.iterdir():
dst = pkg_dst / pkg_path.name
if pkg_path.is_dir() and not dst.exists():
self._logger.debug(f" installing package {pkg_path} to {pkg_dst}")
install(pkg_path, dst)
elif dst.exists():
self._logger.debug(f" NOT installing package {pkg_path}")

# Copy the builtin repo to store, delete if it already exists.
spack_packages_builtin_path = spack_packages_path / "repos" / "spack_repo" / "builtin"
spack_packages_store_path = store_path / "repos" / "spack_repo" / "builtin"
self._logger.debug(f"copying builtin repo from {spack_packages_builtin_path} to {spack_packages_store_path}")
if spack_packages_store_path.exists():
self._logger.debug(f"{spack_packages_store_path} exists ... deleting")
shutil.rmtree(spack_packages_store_path)
install(spack_packages_builtin_path, spack_packages_store_path)
# Iterate over the alps and recipe repositories copying their contents
# to the final repo locations. Because of the order of repos in the
# repos.yaml config file, recipe packages have precedence.
for repo_src in repos:
self._logger.debug(f"installing repo {repo_src}")
packages_path = repo_src / "packages"
for pkg_path in packages_path.iterdir():
dst = pkg_dst / pkg_path.name
if pkg_path.is_dir() and not dst.exists():
self._logger.debug(f" installing package {pkg_path} to {pkg_dst}")
install(pkg_path, dst)
elif dst.exists():
self._logger.debug(f" NOT installing package {pkg_path}")

# Copy all package repos defined in config.yaml to their final repo
# locations.
for idx, pkg_meta in enumerate(spack_meta["packages"]):
clone_path = pkg_meta["path"]
name = pkg_meta["name"]
src_path = clone_path / pkg_meta["repo_path"]
dst_path = store_path / "repos" / "spack_repo" / name
self._logger.debug(f"copying repo '{name}' from {src_path} to {dst_path}")
if dst_path.exists():
self._logger.debug(f"{dst_path} exists ... deleting")
shutil.rmtree(dst_path)
install(src_path, dst_path)

# Generate the makefile and spack.yaml files that describe the compilers
compiler_files = recipe.compiler_files
Expand Down Expand Up @@ -525,6 +551,29 @@ def generate(self, recipe):
)
f.write("\n")

def _resolve_packages(self, packages):
base = self.path / "repos"
if isinstance(packages.get("repo"), str):
return [
{
"name": "builtin",
"url": packages["repo"],
"ref": packages.get("commit"),
"path": base / "builtin",
"repo_path": "repos/spack_repo/builtin",
}
]
return [
{
"name": name,
"url": val["repo"],
"ref": val.get("commit"),
"path": base / name,
"repo_path": val.get("path", f"repos/spack_repo/{name}"),
}
for name, val in packages.items()
]

def _git_clone(self, name, repo, commit, path):
if not (path / ".git").is_dir():
self._logger.info(f"{name}: clone repository {repo} to {path}")
Expand Down
41 changes: 25 additions & 16 deletions stackinator/etc/envvars.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,12 +621,27 @@ def meta_impl(args):

if args.spack is not None:
spack_url, spack_ref, spack_commit = args.spack.split(",")
spack_packages_url = None
spack_packages_ref = None
spack_packages_commit = None
if args.spack_packages is not None:
spack_packages_url, spack_packages_ref, spack_packages_commit = args.spack_packages.split(",")
spack_path = f"{args.mount}/config".replace("//", "/")
scalar_vars = {
"UENV_SPACK_CONFIG_PATH": spack_path,
"UENV_SPACK_URL": spack_url,
"UENV_SPACK_REF": spack_ref,
"UENV_SPACK_COMMIT": spack_commit,
}
if args.spack_package_repo:
repo_names = []
for entry in args.spack_package_repo:
name, url, ref, commit = entry.split(",")
repo_names.append(name)
name_upper = name.upper().replace("-", "_")
scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_URL"] = url
scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_REF"] = ref
scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_COMMIT"] = commit
if name == "builtin":
scalar_vars["UENV_SPACK_PACKAGES_URL"] = url
scalar_vars["UENV_SPACK_PACKAGES_REF"] = ref
scalar_vars["UENV_SPACK_PACKAGES_COMMIT"] = commit
scalar_vars["UENV_PACKAGE_REPOS"] = ",".join(repo_names)
meta["views"]["spack"] = {
"activate": "/dev/null",
"description": "configure spack upstream",
Expand All @@ -636,15 +651,7 @@ def meta_impl(args):
"type": "augment",
"values": {
"list": {},
"scalar": {
"UENV_SPACK_CONFIG_PATH": spack_path,
"UENV_SPACK_URL": spack_url,
"UENV_SPACK_REF": spack_ref,
"UENV_SPACK_COMMIT": spack_commit,
"UENV_SPACK_PACKAGES_URL": spack_packages_url,
"UENV_SPACK_PACKAGES_REF": spack_packages_ref,
"UENV_SPACK_PACKAGES_COMMIT": spack_packages_commit,
},
"scalar": scalar_vars,
},
},
}
Expand Down Expand Up @@ -686,9 +693,11 @@ def meta_impl(args):
default=None,
)
uenv_parser.add_argument(
"--spack-packages",
help='configure spack-packages repository metadata. Format is "spack_url,git_ref,git_commit"',
"--spack-package-repo",
help="configure spack package repository metadata. "
'Format is "name,spack_url,git_ref,git_commit". Can be repeated.',
type=str,
action="append",
default=None,
)

Expand Down
52 changes: 39 additions & 13 deletions stackinator/schema/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,47 @@
"default": null
},
"packages": {
"type": "object",
"additionalProperties": false,
"required": ["repo"],
"properties" : {
"repo": {
"type": "string"
"oneOf": [
{
"type": "object",
"additionalProperties": false,
"required": ["repo"],
"properties": {
"repo": {
"type": "string"
},
"commit": {
"oneOf": [
{"type" : "string"},
{"type" : "null"}
]
}
Comment on lines +40 to +45
Copy link
Copy Markdown
Contributor

@albestro albestro May 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at the test, this is managed at python level with dict.get which returns None if not set, instead of "enforcing" it with the schema.

If it was on purpose, it's ok for me. Just noting this because it might have happened "by accident" and it is not what has been done elsewhere, where default values are used

Suggested change
"commit": {
"oneOf": [
{"type" : "string"},
{"type" : "null"}
]
}
"commit": {
"oneOf": [
{"type" : "string"},
{"type" : "null"}
],
default: null
}

In case, it applies also to the next more "advanced" schema.

Copy link
Copy Markdown
Collaborator Author

@msimberg msimberg May 29, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Semi-intentional, but you might have ideas on how to improve this since I know you've worked more with the schema+validator. The current validator seems to set defaults unconditionally for subobjects in oneOf. It'd e.g. take

...
  packages:
    myrepo:
      repo: https://...

and make it into

...
  packages:
    commit: null
    myrepo:
      repo: https://...

which doesn't pass validation anymore, because it should be one of those two.

It seems I can set a default for commit in the branch where there's a list of repos so I added that default now. But for packages:commit I can't add back the old default null without changing the validator.

But that raises a good question: should we even allow commit to be empty? I see now there's a fallback path for it when cloning repos and when commit isn't given it'll just use whatever is checked out by default after cloning a repo. I think this is probably not something we ever want. I couldn't see any recipes in alps-uenv that would set repo but not set the commit.

}
},
"commit": {
"oneOf": [
{"type" : "string"},
{"type" : "null"}
],
"default": null
{
"type": "object",
"minProperties": 1,
"additionalProperties": {
"type": "object",
"additionalProperties": false,
"required": ["repo"],
"properties": {
"repo": {
"type": "string"
},
"commit": {
"oneOf": [
{"type" : "string"},
{"type" : "null"}
]
},
"path": {
"type": "string"
}
}
}
}
}
]
}
}
},
Expand Down
2 changes: 1 addition & 1 deletion stackinator/templates/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ modules-done: environments generate-config

env-meta: generate-config environments{% if modules %} modules-done{% endif %}

$(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' --spack-packages='{{ spack_meta.packages_url }},{{ spack_meta.packages_ref }},{{ spack_meta.packages_commit }}' $(STORE)
$(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}'{% for pkg in spack_meta.packages %} --spack-package-repo='{{ pkg.name }},{{ pkg.url }},{{ pkg.ref }},{{ pkg.commit }}'{% endfor %} $(STORE)
touch env-meta

post-install: env-meta
Expand Down
7 changes: 6 additions & 1 deletion stackinator/templates/repos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
repos:
{% if has_recipe_repo %}
recipe: {{ recipe_repo_path }}
{% endif %}
alps: {{ repo_path }}
builtin: {{ builtin_repo_path }}
{% for repo in package_repos %}
{{ repo.name }}: {{ repo.path }}
{% endfor %}
2 changes: 2 additions & 0 deletions unittests/data/arbor-uenv/meta/env.json.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@
"arbor": {
"activate": "@@mount@@/env/arbor/activate.sh",
"description": "",
"recipe_variables": {"scalar": {}, "list": {}},
"root": "@@mount@@/env/arbor"
},
"develop": {
"activate": "@@mount@@/env/develop/activate.sh",
"description": "",
"recipe_variables": {"scalar": {}, "list": {}},
"root": "@@mount@@/env/develop"
}
}
Expand Down
2 changes: 2 additions & 0 deletions unittests/recipes/with-multi-repos/compilers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
gcc:
version: "11"
13 changes: 13 additions & 0 deletions unittests/recipes/with-multi-repos/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
name: with-multi-repos
store: '/user-environment'
spack:
repo: https://github.com/spack/spack.git
commit: v21.0
packages:
my-packages:
repo: https://github.com/example/spack-packages.git
commit: v1.0
path: custom/path/to/packages
other-packages:
repo: https://github.com/example/other-packages.git
version: 2
Loading
Loading