Add CLI to control caching (#69)

* Collect cache statistics

and make it easier to allow using the FileCache with a directory
structure that represents a project.

* Add '--no-cache' CLI option

and show in output whether the cache was hit or not for each file.

* Make sure docstub generated stubs are ignored

when walking a source package.

* Add the `clean` subcommand

* Add tests for caching-related CLI

* Validate cache directory before cleaning it

* Preserve click command decorator for stub files

* Fix and annotate verbose parameter

* Use "\f" to hide "Parameter" docstring section of clean command

Author: lagru

docs/command_line.md

@@ -15,7 +15,8 @@ Options:
   -h, --help  Show this message and exit.
 
 Commands:
-  run  Generate Python stub files.
+  clean  Clean the cache.
+  run    Generate Python stub files.
 ```
 
 <!--- end cli-docstub --->
@@ -52,8 +53,30 @@ Options:
   --allow-errors INT  Allow this many or fewer errors. If docstub reports
                       more, exit with error code '1'. This is useful to adopt
                       docstub gradually.  [default: 0; x>=0]
+  --no-cache          Ignore pre-existing cache and don't create a new one.
   -v, --verbose       Print more details (repeatable).
   -h, --help          Show this message and exit.
 ```
 
 <!--- end cli-docstub-run --->
+
+
+## Command `docstub clean`
+
+<!--- The following block is checked by the test suite --->
+<!--- begin cli-docstub-clean --->
+
+```plain
+Usage: docstub clean [OPTIONS]
+
+  Clean the cache.
+
+  Looks for a cache directory relative to the current working directory. If
+  one exists, remove it.
+
+Options:
+  -v, --verbose  Print more details (repeatable).
+  -h, --help     Show this message and exit.
+```
+
+<!--- end cli-docstub-clean --->

src/docstub/_cache.py

@@ -5,10 +5,21 @@
 logger = logging.getLogger(__name__)
 
 
+CACHE_DIR_NAME = ".docstub_cache"
+
+
 CACHEDIR_TAG_CONTENT = """\
-Signature: 8a477f597d28d172789f06886806bc55\
-# This file is a cache directory tag automatically created by docstub.\n"
-# For information about cache directory tags see https://bford.info/cachedir/\n"
+Signature: 8a477f597d28d172789f06886806bc55
+# Mark this directory as a cache [1], created by docstub [2]
+# [1] https://bford.info/cachedir/
+# [2] https://github.com/scientific-python/docstub
+"""
+
+
+GITHUB_IGNORE_CONTENT = """\
+# Make git ignore this cache directory, created by docstub [1]
+# [1] https://github.com/scientific-python/docstub
+*
 """
 
 
@@ -43,22 +54,42 @@ def create_cache(path):
     """
     path.mkdir(parents=True, exist_ok=True)
     cachdir_tag_path = path / "CACHEDIR.TAG"
-    cachdir_tag_content = (
-        "Signature: 8a477f597d28d172789f06886806bc55\n"
-        "# This file is a cache directory tag automatically created by docstub.\n"
-        "# For information about cache directory tags see https://bford.info/cachedir/\n"
-    )
+
     if not cachdir_tag_path.is_file():
         with open(cachdir_tag_path, "w") as fp:
-            fp.write(cachdir_tag_content)
+            fp.write(CACHEDIR_TAG_CONTENT)
 
     gitignore_path = path / ".gitignore"
-    gitignore_content = (
-        "# This file is a cache directory automatically created by docstub.\n" "*\n"
-    )
     if not gitignore_path.is_file():
         with open(gitignore_path, "w") as fp:
-            fp.write(gitignore_content)
+            fp.write(GITHUB_IGNORE_CONTENT)
+
+
+def validate_cache(path):
+    """Make sure the given path is a cache created by docstub.
+
+    Parameters
+    ----------
+    path : Path
+
+    Raises
+    ------
+    FileNotFoundError
+    """
+    if not path.is_dir():
+        raise FileNotFoundError(f"expected '{path}' to be a valid directory")
+
+    if not path.name == CACHE_DIR_NAME:
+        raise FileNotFoundError(
+            f"expected directory '{path}' be named '{CACHE_DIR_NAME}'"
+        )
+
+    cachdir_tag_path = path / "CACHEDIR.TAG"
+    if not cachdir_tag_path.is_file():
+        raise FileNotFoundError(f"expected '{path}' to contain a 'CACHEDIR.TAG' file")
+    gitignore_path = path / ".gitignore"
+    if not gitignore_path.is_file():
+        raise FileNotFoundError(f"expected '{path}' to contain a '.gitignore' file")
 
 
 class FuncSerializer[T](Protocol):
@@ -90,9 +121,24 @@ class FileCache:
     This class can cache results of a function to the disk. A unique key is
     generated from the arguments to the function, and the result is cached
     inside a file named after this key.
+
+    Attributes
+    ----------
+    func : Callable
+        The function whose output shall be cached.
+    serializer : FuncSerializer
+        An interface that matches the given `func`. It must implement the
+        `FuncSerializer` protocol.
+    sub_dir : str
+        A unique name to structure multiple / parallel caches inside `cache_dir`.
+    cache_hits, cache_misses : int
+        Records how many times this object returned results from a cache (hits)
+        or by computing it (misses).
+    cached_last_call : bool or None
+        Whether the last call was cached. ``None`` if not called yet.
     """
 
-    def __init__(self, *, func, serializer, cache_dir, name):
+    def __init__(self, *, func, serializer, cache_dir, sub_dir=None):
         """
         Parameters
         ----------
@@ -103,27 +149,45 @@ def __init__(self, *, func, serializer, cache_dir, name):
             `FuncSerializer` protocol.
         cache_dir : Path
             The directory of the cache.
-        name : str
-            A unique name to separate parallel caches inside `cache_dir`.
+        sub_dir : str
+            A unique name to structure multiple / parallel caches inside `cache_dir`.
         """
         self.func = func
         self.serializer = serializer
         self._cache_dir = cache_dir
-        self.name = name
+        self.sub_dir = sub_dir
+
+        self.cache_hits = 0
+        self.cache_misses = 0
+        self.cached_last_call = None
 
     @cached_property
-    def named_cache_dir(self):
-        """Path to the named subdirectory inside the cache.
+    def cache_dir(self):
+        """Return and create cache dir on first use - also check its size.
+
+        Returns
+        -------
+        cache_dir : Path
+        """
+        create_cache(self._cache_dir)
+
+        if _directory_size(self._cache_dir) > 512 * 1024**2:
+            logger.warning("cache size at %r exceeds 512 MiB", self._cache_dir)
+
+        return self._cache_dir
+
+    @property
+    def cache_sub_dir(self):
+        """Create and return path to a specific subdirectory inside the cache.
 
         Warns when cache size exceeds 512 MiB.
         """
-        cache_dir = self._cache_dir
-        create_cache(cache_dir)
-        if _directory_size(cache_dir) > 512 * 1024**2:
-            logger.warning("cache size at %r exceeds 512 MiB", cache_dir)
-        _named_cache_dir = cache_dir / self.name
-        _named_cache_dir.mkdir(parents=True, exist_ok=True)
-        return _named_cache_dir
+        named_dir = self.cache_dir
+        if self.sub_dir:
+            named_dir /= self.sub_dir
+        named_dir.mkdir(parents=True, exist_ok=True)
+
+        return named_dir
 
     def __call__(self, *args, **kwargs):
         """Call the wrapped `func` and cache each result in a file.
@@ -138,14 +202,25 @@ def __call__(self, *args, **kwargs):
         data : Any
         """
         key = self.serializer.hash_args(*args, **kwargs)
-        entry_path = self.named_cache_dir / f"{key}{self.serializer.suffix}"
+        entry_path = self.cache_sub_dir / f"{key}{self.serializer.suffix}"
+
         if entry_path.is_file():
+            # `data` is already cached
             with entry_path.open("rb") as fp:
                 raw = fp.read()
             data = self.serializer.deserialize(raw)
+
+            self.cached_last_call = True
+            self.cache_hits += 1
+
         else:
+            # `data` isn't cached, write cache
             data = self.func(*args, **kwargs)
             raw = self.serializer.serialize(data)
             with entry_path.open("xb") as fp:
                 fp.write(raw)
+
+            self.cached_last_call = False
+            self.cache_misses += 1
+
         return data