Feature: Add DockerImage classes for docker.io, nvcr.io, and quay.io

docs/getting_started/user-guide.rst

@@ -1351,7 +1351,7 @@ As of version 0.0.52, you can request on demand updates of container.yaml recipe
 where an update means we ping the registry or resource for the module and find
 updated tags. An update generally means that:
 
- - We start with the 50 latest tags of the container, as determined by `crane.ggcr.dev <https://crane.ggcr.dev/ls/quay.io/biocontainers/samtools>`_
+ - We start with the latest tags of the container, as determined by `crane.ggcr.dev <https://crane.ggcr.dev/ls/quay.io/biocontainers/samtools>`_ or the REST API associated with a container registry (e.g., `https://hub.docker.com/v2/`)
  - We filter according to any recipe ``filters`` in the container.yaml
  - Given a convention of including a hash, we try to remove it and generate a loose version
  - Any versions (including latest) that cannot be sorted based on some semblance to a version are filtered out
@@ -1420,11 +1420,44 @@ If you are using an earlier release than 0.0.58 you can accomplish the same as f
         shpc update ${name} --dry-run
       done
 
+As of version 0.1.34, there's support to clear all existing tags in the
+`container.yaml` file and replace with the new tags. This is useful if you need
+to recover your `container.yaml` from a bad state, or if you want to change the
+filtering scheme and regenerate the whole tag collection. **Check the output with
+`--dry-run` first, this option is destructive!**
+
+.. code-block:: console
+
+    $ shpc update --purge --dry-run
+
+As of version 0.1.34, you can specify the maximum number of new tags to add
+from the command line. Without the flag, the default is up to 5 new tags. This is
+particularly helpful for generating new `container.yaml` files from stubs, or
+when used in combination with `--purge`.
+
+.. code-block:: console
+
+    $ shpc update --max-tags=10
 
 Let us know if there are other features you'd like for update! For specific recipes
 it could be that a different method of choosing or sorting tags (beyond the defaults mentioned above
 and filter) is needed.
 
+
+Updating from Nvidia GPU Cloud (NGC)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As of version 0.1.34, there is support to update from the NGC API, which provides the
+most accurate information for `nvcr.io` containers. To use NGC, install SHPC with
+the `ngc` or `all` option, and export the environment variable `SHPC_NGC_API_KEY`
+prior to running `shpc update`. See the `NGC portal https://org.ngc.nvidia.com/account/api-key`_
+for how to get an NGC API key.
+
+.. code-block:: console
+
+    $ export SHPC_NGC_API_KEY="your_ngc_api_key_here"
+    $ shpc update nvcr.io/nvidia/pytorch
+
 .. _getting_started-commands-sync-registry:
 
 

setup.py

@@ -69,6 +69,7 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"):
     INSTALL_REQUIRES = get_reqs(lookup)
     TESTS_REQUIRES = get_reqs(lookup, "TESTS_REQUIRES")
     INSTALL_REQUIRES_ALL = get_reqs(lookup, "INSTALL_REQUIRES_ALL")
+    NGC_REQUIRES = get_reqs(lookup, "NGC_REQUIRES")
 
     setup(
         name=NAME,
@@ -89,6 +90,7 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"):
         install_requires=INSTALL_REQUIRES,
         tests_require=TESTS_REQUIRES,
         extras_require={
+            "ngc": [NGC_REQUIRES],
             "all": [INSTALL_REQUIRES_ALL],
         },
         classifiers=[

shpc/client/__init__.py

@@ -340,6 +340,18 @@ def get_parser():
         help="ignore container.yaml filters, run an update with this specific set",
         dest="filters",
     )
+    update.add_argument(
+        "--purge",
+        help="purge existing tags and only add new ones found in the registry",
+        default=False,
+        action="store_true",
+    )
+    update.add_argument(
+        "--max-tags",
+        help="set a maximum number of tags to add after filtering. Defaults to 5.",
+        type=int,
+        default=5,
+    )
 
     # sync-registry gets latest files and non-existing containers from upstream shpc
     sync = subparsers.add_parser(

shpc/client/update.py

@@ -11,4 +11,10 @@ def main(args, parser, extra, subparser):
     shpc.utils.ensure_no_extra(extra)
 
     cli = get_client(quiet=args.quiet, settings_file=args.settings_file)
-    cli.update(args.module_name, dryrun=args.dryrun, filters=args.filters)
+    cli.update(
+        args.module_name,
+        dryrun=args.dryrun,
+        filters=args.filters,
+        purge=args.purge,
+        max_tags=args.max_tags,
+    )

shpc/main/client.py

@@ -117,7 +117,7 @@ def _load_container(self, name):
         config.set_tag(tag)
         return config
 
-    def update(self, name=None, dryrun=False, filters=None):
+    def update(self, name=None, dryrun=False, filters=None, purge=None, max_tags=None):
         """
         Given a module name (or None for all modules) upgrade the registry.
         """
@@ -129,7 +129,9 @@ def update(self, name=None, dryrun=False, filters=None):
 
         for module_name in modules:
             config = self._load_container(module_name)
-            config.update(dryrun=dryrun, filters=filters)
+            config.update(
+                dryrun=dryrun, filters=filters, purge=purge, max_tags=max_tags
+            )
 
     def test(
         self,

shpc/main/container/config.py

@@ -176,15 +176,17 @@ def check_overrides(self):
                 return False
         return True
 
-    def update(self, dryrun=False, filters=None):
+    def update(self, dryrun=False, filters=None, purge=None, max_tags=None):
         """
         Update a container.yaml, meaning the tags and latest.
         """
         updated = None
         if self.docker or self.oras:
             previous_tags = self.get("tags", {})
             previous_latest = self.get("latest", {})
-            updated = update.update_config_tags(self, filters=filters)
+            updated = update.update_config_tags(
+                self, filters=filters, purge=purge, max_length=max_tags
+            )
 
             # print the container name and latest tag:
             print(add_prefix(underline(self.docker or self.oras)))

shpc/main/container/update/__init__.py

@@ -2,16 +2,18 @@
 __copyright__ = "Copyright 2021-2025, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
+import os
+
 from shpc.logger import logger
 
 from .diff import print_diff
-from .docker import DockerImage
+from .docker import DockerHubImage, DockerImage, NGCImage, QuayDockerImage
 from .versions import filter_versions
 
 assert print_diff
 
 
-def update_config_tags(config, filters=None):
+def update_config_tags(config, filters=None, purge=None, max_length=None):
     """
     Given a container config, update the latest tags
     """
@@ -20,13 +22,20 @@ def update_config_tags(config, filters=None):
         uri = config.docker or config.oras
 
         logger.info("Looking for updated digests for %s" % uri)
-        latest_tags = get_latest_tags(uri)
+
+        image = _get_image_type(uri, config)
+        latest_tags = get_latest_tags(uri, config, image=image)
 
         # Notice this API call truncates at 50
-        versions = filter_versions(latest_tags, filters=filters or config.filter)
+        versions = filter_versions(
+            latest_tags, filters=filters or config.filter, max_length=max_length
+        )
 
         # Get list of current tags, and update with new versions
-        current_tags = dict(config.get("tags", {}))
+        if purge:
+            current_tags = dict({})
+        else:
+            current_tags = dict(config.get("tags", {}))
 
         # Now do the same practice as before - try to derive what is latest
         sorted_tags = filter_versions(
@@ -59,7 +68,7 @@ def update_config_tags(config, filters=None):
         tags = list(current_tags.keys())
         for tag in tags:
             try:
-                digest = get_container_tag(uri, tag)
+                digest = get_container_tag(uri, config, tag, image=image)
             except Exception:
                 digest = {tag: current_tags[tag]}
 
@@ -108,22 +117,39 @@ def get_earliest_tag(sorted_tags):
     return earliest_tag
 
 
-def get_container_tag(container_name, tag=None):
+def _get_image_type(container_name, config):
+    container_prefix = container_name.split("/")[0]
+
+    if container_prefix == "docker.io" or "." not in container_prefix:
+        return DockerHubImage(container_name)
+    elif container_prefix == "quay.io":
+        return QuayDockerImage(container_name)
+    elif (container_prefix == "nvcr.io") and ("SHPC_NGC_API_KEY" in os.environ):
+        return NGCImage(container_name)
+    else:
+        return DockerImage(container_name)
+
+
+def get_container_tag(container_name, config, tag=None, image=None):
     """
     Given a container name, get the latest list of tags and digests.
     This can be extended when we have a container updater.
     """
-    image = DockerImage(container_name)
+
+    if image is None:
+        image = _get_image_type(container_name, config)
 
     # Get a specific tag
     tag = tag or "latest"
     digest = image.digest(tag)
     return {tag: digest}
 
 
-def get_latest_tags(container_name, tag=None):
+def get_latest_tags(container_name, config, tag=None, image=None):
     """
     Given a container name, get the latest tags.
     """
-    image = DockerImage(container_name)
+
+    if image is None:
+        image = _get_image_type(container_name, config)
     return image.tags()