Multiple package repos in config.yaml

.github/workflows/main.yaml

@@ -23,6 +23,11 @@ jobs:
         done
         exit $errors
     - name: Install Dependencies
-      run: uv sync --group dev
+      run: |
+        uv sync --group dev
+        sudo apt-get install -y jq
     - name: Run Unit Tests
       run: uv run pytest
+    - name: Run Envvars Test
+      working-directory: unittests
+      run: uv run bash ./test-envvars.sh

stackinator/builder.py

@@ -13,6 +13,12 @@
 
 from . import VERSION, cache, root_logger, spack_util
 
+_REPO_YAML = """\
+repo:
+  namespace: {namespace}
+  api: v2.0
+"""
+
 
 def install(src, dst, *, ignore=None, symlinks=False):
     """Call shutil.copytree or shutil.copy2. copy2 is used if `src` is not a directory.
@@ -191,26 +197,16 @@ def generate(self, recipe):
 
         spack_git_commit_result = self._git_clone("spack", spack_repo, spack_commit, spack_path)
 
-        # Clone the spack-packages repository and check out commit if one was given
-        spack_packages = spack["packages"]
-        spack_packages_repo = spack_packages["repo"]
-        spack_packages_commit = spack_packages["commit"]
-        spack_packages_path = self.path / "spack-packages"
-
-        spack_packages_git_commit_result = self._git_clone(
-            "spack-packages",
-            spack_packages_repo,
-            spack_packages_commit,
-            spack_packages_path,
-        )
+        package_repos = recipe.spack_package_repos
+        for pkg_repo in package_repos:
+            pkg_repo["path"] = self.path / "repos" / pkg_repo["name"]
+            pkg_repo["commit"] = self._git_clone(pkg_repo["name"], pkg_repo["url"], pkg_repo["ref"], pkg_repo["path"])
 
         spack_meta = {
             "url": spack_repo,
             "ref": spack_commit,
             "commit": spack_git_commit_result,
-            "packages_url": spack_packages_repo,
-            "packages_ref": spack_packages_commit,
-            "packages_commit": spack_packages_git_commit_result,
+            "packages": package_repos,
         }
 
         # load the jinja templating environment
@@ -331,16 +327,12 @@ def generate(self, recipe):
         # 2. cluster-config/repos.yaml
         #   - if the repos.yaml file exists it will contain a list of relative paths
         #     to search for package
-        # 1. builtin repo
+        # 1. package repos from config.yaml in the order specified (typically
+        #    only spack-packages builtin repo)
 
-        # Build a list of repos with packages to install.
+        # Build a list of repos with packages to install from system config and recipe.
         repos = []
 
-        # check for a repo in the recipe
-        if recipe.spack_repo is not None:
-            self._logger.debug(f"adding recipe spack package repo: {recipe.spack_repo}")
-            repos.append(recipe.spack_repo)
-
         # look for repos.yaml file in the system configuration
         repo_yaml = recipe.system_config_path / "repos.yaml"
         if repo_yaml.exists() and repo_yaml.is_file():
@@ -361,7 +353,7 @@ def generate(self, recipe):
                     self._logger.error(f"{repo_path} from {repo_yaml} is not a spack package repository")
                     raise RuntimeError("invalid system-provided package repository")
 
-        self._logger.debug(f"full list of spack package repo: {repos}")
+        self._logger.debug(f"full list of system spack package repos: {repos}")
 
         # Delete the store/repo path, if it already exists.
         # Do this so that incremental builds (though not officially supported) won't break if a repo is updated.
@@ -378,53 +370,82 @@ def generate(self, recipe):
         self._logger.debug(f"created the repo packages path {pkg_dst}")
 
         # create the repository step 2: create the repo.yaml file that
-        # configures alps and builtin repos
+        # configures the alps repo
         with (repo_dst / "repo.yaml").open("w") as f:
-            f.write(
-                """\
-repo:
-  namespace: alps
-  api: v2.0
-"""
-            )
+            f.write(_REPO_YAML.format(namespace="alps"))
+
+        # If the recipe provides a package repo, install it as a separate
+        # "recipe" repo in the store with highest precedence.
+        has_recipe_repo = recipe.spack_repo is not None
+        if has_recipe_repo:
+            recipe_dst = repos_path / "recipe"
+            self._logger.debug(f"creating the recipe spack repo in {recipe_dst}")
+            if recipe_dst.exists():
+                self._logger.debug(f"{recipe_dst} exists ... deleting")
+                shutil.rmtree(recipe_dst)
+
+            recipe_pkg_dst = recipe_dst / "packages"
+            recipe_pkg_dst.mkdir(mode=0o755, parents=True)
+
+            with (recipe_dst / "repo.yaml").open("w") as f:
+                f.write(_REPO_YAML.format(namespace="recipe"))
+
+            packages_path = recipe.spack_repo / "packages"
+            for pkg_path in packages_path.iterdir():
+                dst = recipe_pkg_dst / pkg_path.name
+                if pkg_path.is_dir():
+                    self._logger.debug(f"  installing recipe package {pkg_path} to {recipe_pkg_dst}")
+                    install(pkg_path, dst)
 
         # create the repository step 2: create the repos.yaml file in build_path/config
         repos_yaml_template = jinja_env.get_template("repos.yaml")
         with (config_path / "repos.yaml").open("w") as f:
             repo_path = recipe.mount / "repos" / "spack_repo" / "alps"
-            builtin_repo_path = recipe.mount / "repos" / "spack_repo" / "builtin"
+            recipe_repo_path = recipe.mount / "repos" / "spack_repo" / "recipe"
+            package_repos = [
+                {
+                    "name": pkg_repo["name"],
+                    "path": (recipe.mount / "repos" / "spack_repo" / pkg_repo["name"]).as_posix(),
+                }
+                for pkg_repo in spack_meta["packages"]
+            ]
             f.write(
                 repos_yaml_template.render(
                     repo_path=repo_path.as_posix(),
-                    builtin_repo_path=builtin_repo_path.as_posix(),
+                    package_repos=package_repos,
+                    recipe_repo_path=recipe_repo_path.as_posix(),
+                    has_recipe_repo=has_recipe_repo,
                     verbose=False,
                 )
             )
             f.write("\n")
 
-        # Iterate over the source repositories copying their contents to the consolidated repo in the uenv.
-        # Do overwrite packages that have been copied from an earlier source repo, enforcing a descending
-        # order of precidence.
-        if len(repos) > 0:
-            for repo_src in repos:
-                self._logger.debug(f"installing repo {repo_src}")
-                packages_path = repo_src / "packages"
-                for pkg_path in packages_path.iterdir():
-                    dst = pkg_dst / pkg_path.name
-                    if pkg_path.is_dir() and not dst.exists():
-                        self._logger.debug(f"  installing package {pkg_path} to {pkg_dst}")
-                        install(pkg_path, dst)
-                    elif dst.exists():
-                        self._logger.debug(f"  NOT installing package {pkg_path}")
-
-        # Copy the builtin repo to store, delete if it already exists.
-        spack_packages_builtin_path = spack_packages_path / "repos" / "spack_repo" / "builtin"
-        spack_packages_store_path = store_path / "repos" / "spack_repo" / "builtin"
-        self._logger.debug(f"copying builtin repo from {spack_packages_builtin_path} to {spack_packages_store_path}")
-        if spack_packages_store_path.exists():
-            self._logger.debug(f"{spack_packages_store_path} exists ... deleting")
-            shutil.rmtree(spack_packages_store_path)
-        install(spack_packages_builtin_path, spack_packages_store_path)
+        # Iterate over the alps and recipe repositories copying their contents
+        # to the final repo locations. Because of the order of repos in the
+        # repos.yaml config file, recipe packages have precedence.
+        for repo_src in repos:
+            self._logger.debug(f"installing repo {repo_src}")
+            packages_path = repo_src / "packages"
+            for pkg_path in packages_path.iterdir():
+                dst = pkg_dst / pkg_path.name
+                if pkg_path.is_dir() and not dst.exists():
+                    self._logger.debug(f"  installing package {pkg_path} to {pkg_dst}")
+                    install(pkg_path, dst)
+                elif dst.exists():
+                    self._logger.debug(f"  NOT installing package {pkg_path}")
+
+        # Copy all package repos defined in config.yaml to their final repo
+        # locations.
+        for pkg_repo in spack_meta["packages"]:
+            clone_path = pkg_repo["path"]
+            name = pkg_repo["name"]
+            src_path = clone_path / pkg_repo["repo_path"]
+            dst_path = store_path / "repos" / "spack_repo" / name
+            self._logger.debug(f"copying repo '{name}' from {src_path} to {dst_path}")
+            if dst_path.exists():
+                self._logger.debug(f"{dst_path} exists ... deleting")
+                shutil.rmtree(dst_path)
+            install(src_path, dst_path)
 
         # Generate the makefile and spack.yaml files that describe the compilers
         compiler_files = recipe.compiler_files

stackinator/etc/envvars.py

@@ -621,12 +621,27 @@ def meta_impl(args):
 
     if args.spack is not None:
         spack_url, spack_ref, spack_commit = args.spack.split(",")
-        spack_packages_url = None
-        spack_packages_ref = None
-        spack_packages_commit = None
-        if args.spack_packages is not None:
-            spack_packages_url, spack_packages_ref, spack_packages_commit = args.spack_packages.split(",")
         spack_path = f"{args.mount}/config".replace("//", "/")
+        scalar_vars = {
+            "UENV_SPACK_CONFIG_PATH": spack_path,
+            "UENV_SPACK_URL": spack_url,
+            "UENV_SPACK_REF": spack_ref,
+            "UENV_SPACK_COMMIT": spack_commit,
+        }
+        if args.spack_package_repo:
+            repo_names = []
+            for entry in args.spack_package_repo:
+                name, url, ref, commit = entry.split(",")
+                repo_names.append(name)
+                name_upper = name.upper().replace("-", "_")
+                scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_URL"] = url
+                scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_REF"] = ref
+                scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_COMMIT"] = commit
+                if name == "builtin":
+                    scalar_vars["UENV_SPACK_PACKAGES_URL"] = url
+                    scalar_vars["UENV_SPACK_PACKAGES_REF"] = ref
+                    scalar_vars["UENV_SPACK_PACKAGES_COMMIT"] = commit
+            scalar_vars["UENV_PACKAGE_REPOS"] = ",".join(repo_names)
         meta["views"]["spack"] = {
             "activate": "/dev/null",
             "description": "configure spack upstream",
@@ -636,15 +651,7 @@ def meta_impl(args):
                 "type": "augment",
                 "values": {
                     "list": {},
-                    "scalar": {
-                        "UENV_SPACK_CONFIG_PATH": spack_path,
-                        "UENV_SPACK_URL": spack_url,
-                        "UENV_SPACK_REF": spack_ref,
-                        "UENV_SPACK_COMMIT": spack_commit,
-                        "UENV_SPACK_PACKAGES_URL": spack_packages_url,
-                        "UENV_SPACK_PACKAGES_REF": spack_packages_ref,
-                        "UENV_SPACK_PACKAGES_COMMIT": spack_packages_commit,
-                    },
+                    "scalar": scalar_vars,
                 },
             },
         }
@@ -686,9 +693,11 @@ def meta_impl(args):
         default=None,
     )
     uenv_parser.add_argument(
-        "--spack-packages",
-        help='configure spack-packages repository metadata. Format is "spack_url,git_ref,git_commit"',
+        "--spack-package-repo",
+        help="configure spack package repository metadata. "
+        'Format is "name,spack_url,git_ref,git_commit". Can be repeated.',
         type=str,
+        action="append",
         default=None,
     )
 

stackinator/recipe.py

@@ -213,6 +213,39 @@ def spack_repo(self):
             return repo_path
         return None
 
+    _RESERVED_REPO_NAMES = {"alps", "recipe"}
+
+    @property
+    def spack_package_repos(self):
+        packages = self.config["spack"]["packages"]
+        if isinstance(packages.get("repo"), str):
+            return [
+                {
+                    "name": "builtin",
+                    "url": packages["repo"],
+                    "ref": packages.get("commit"),
+                    "repo_path": packages.get("path", "repos/spack_repo/builtin"),
+                }
+            ]
+        repos = [
+            {
+                "name": name,
+                "url": val["repo"],
+                "ref": val.get("commit"),
+                "repo_path": val.get("path", f"repos/spack_repo/{name}"),
+            }
+            for name, val in packages.items()
+        ]
+        for repo in repos:
+            name = repo["name"]
+            if name in self._RESERVED_REPO_NAMES:
+                raise RuntimeError(
+                    f"The package repo name '{name}' is reserved for stackinator internal use. "
+                    f"Reserved names are: {self._RESERVED_REPO_NAMES}. "
+                    "Choose a different name in config.yaml:spack:packages."
+                )
+        return repos
+
     # Returns:
     #   Path: of the recipe extra path if it exists
     #   None: if there is no user-provided extra path in the recipe

stackinator/schema/config.json

@@ -4,6 +4,24 @@
     "type" : "object",
     "additionalProperties": false,
     "required": ["name", "spack"],
+    "defs": {
+        "repo_def": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["repo", "commit"],
+            "properties": {
+                "repo": {
+                    "type": "string"
+                },
+                "commit": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                }
+            }
+        }
+    },
     "properties" : {
         "name" : {
             "type": "string"
@@ -28,21 +46,21 @@
                     "default": null
                 },
                 "packages": {
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": ["repo"],
-                    "properties" : {
-                        "repo": {
-                            "type": "string"
+                    "oneOf": [
+                        {
+                            "$ref": "#/defs/repo_def"
                         },
-                        "commit": {
-                            "oneOf": [
-                                {"type" : "string"},
-                                {"type" : "null"}
-                            ],
-                            "default": null
+                        {
+                            "type": "object",
+                            "minProperties": 1,
+                            "patternProperties": {
+                                "^\\w[\\w-]*$": {
+                                    "$ref": "#/defs/repo_def"
+                                }
+                            },
+                            "additionalProperties": false
                         }
-                    }
+                    ]
                 }
             }
         },

stackinator/templates/Makefile

@@ -59,7 +59,7 @@ modules-done: environments generate-config
 
 env-meta: generate-config environments{% if modules %} modules-done{% endif %}
 
-	$(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' --spack-packages='{{ spack_meta.packages_url }},{{ spack_meta.packages_ref }},{{ spack_meta.packages_commit }}' $(STORE)
+	$(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}'{% for pkg_repo in spack_meta.packages %} --spack-package-repo='{{ pkg_repo.name }},{{ pkg_repo.url }},{{ pkg_repo.ref }},{{ pkg_repo.commit }}'{% endfor %} $(STORE)
 	touch env-meta
 
 post-install: env-meta

stackinator/templates/repos.yaml

@@ -1,3 +1,8 @@
 repos:
+{% if has_recipe_repo %}
+  recipe: {{ recipe_repo_path }}
+{% endif %}
   alps: {{ repo_path }}
-  builtin: {{ builtin_repo_path }}
+{% for repo in package_repos %}
+  {{ repo.name }}: {{ repo.path }}
+{% endfor %}

unittests/data/arbor-uenv/meta/env.json.in

@@ -9,11 +9,13 @@
     "arbor": {
       "activate": "@@mount@@/env/arbor/activate.sh",
       "description": "",
+      "recipe_variables": {"scalar": {}, "list": {}},
       "root": "@@mount@@/env/arbor"
     },
     "develop": {
       "activate": "@@mount@@/env/develop/activate.sh",
       "description": "",
+      "recipe_variables": {"scalar": {}, "list": {}},
       "root": "@@mount@@/env/develop"
     }
   }