ci: fix tests and readthedocs

.build_rtd_docs/requirements.rtd.txt

@@ -8,7 +8,4 @@ ipykernel
 rtds_action
 myst_parser
 sphinx_rtd_theme
-
-
-
-
+pytest

.github/workflows/ci.yml

@@ -99,12 +99,6 @@ jobs:
           repository: MODFLOW-USGS/modflow6-testmodels
           path: modflow6-testmodels
 
-      - name: Checkout modflow6-examples
-        uses: actions/checkout@v3
-        with:
-          repository: MODFLOW-USGS/modflow6-examples
-          path: modflow6-examples
-
       - name: Setup GNU Fortran ${{ env.GCC_V }}
         uses: awvwgk/setup-fortran@main
         with:
@@ -118,24 +112,6 @@ jobs:
           cache-downloads: true
           cache-env: true
 
-      - name: Cache modflow6 examples
-        id: cache-examples
-        uses: actions/cache@v3
-        with:
-          path: modflow6-examples/examples
-          key: modflow6-examples-${{ hashFiles('modflow6-examples/scripts/**') }}
-
-      - name: Install extra Python packages
-        if: steps.cache-examples.outputs.cache-hit != 'true'
-        working-directory: modflow6-examples/etc
-        run: |
-          pip install -r requirements.pip.txt
-
-      - name: Build example models
-        if: steps.cache-examples.outputs.cache-hit != 'true'
-        working-directory: modflow6-examples/etc
-        run: python ci_build_files.py
-
       - name: Build modflow6
         working-directory: modflow6
         run: |
@@ -157,9 +133,9 @@ jobs:
         working-directory: modflow6/autotest
         run: |
           if [ "${{ github.ref_name }}" == "master" ]; then
-            pytest -v -n auto --durations 0 -m "not developmode"
+            pytest -v -n auto --durations 0 -m "not large and not developmode"
           else
-            pytest -v -n auto --durations 0
+            pytest -v -n auto --durations 0 -m "not large"
           fi
 
       - name: Test scripts
@@ -231,9 +207,9 @@ jobs:
         working-directory: modflow6/autotest
         run: |
           if [ "${{ github.ref_name }}" == "master" ]; then
-            pytest -v -n auto --durations 0 -m "not developmode"
+            pytest -v -n auto --durations 0 -m "not large and not developmode"
           else
-            pytest -v -n auto --durations 0
+            pytest -v -n auto --durations 0 -m "not large"
           fi
 
   test_ifort:
@@ -262,12 +238,6 @@ jobs:
           repository: MODFLOW-USGS/modflow6-testmodels
           path: modflow6-testmodels
 
-      - name: Checkout modflow6-examples
-        uses: actions/checkout@v3
-        with:
-          repository: MODFLOW-USGS/modflow6-examples
-          path: modflow6-examples
-
       - name: Setup Micromamba
         uses: mamba-org/provision-with-micromamba@main
         with:
@@ -286,23 +256,6 @@ jobs:
           $mamba_bin = "C:\Users\runneradmin\micromamba-root\envs\modflow6\Scripts"
           echo $mamba_bin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
-      - name: Cache modflow6 examples
-        id: cache-examples
-        uses: actions/cache@v3
-        with:
-          path: modflow6-examples/examples
-          key: modflow6-examples-${{ hashFiles('modflow6-examples/scripts/**') }}
-
-      - name: Install extra Python packages
-        if: steps.cache-examples.outputs.cache-hit != 'true'
-        working-directory: modflow6-examples/etc
-        run: pip install -r requirements.pip.txt
-
-      - name: Build example models
-        if: steps.cache-examples.outputs.cache-hit != 'true'
-        working-directory: modflow6-examples/etc
-        run: python ci_build_files.py
-
       - name: Update version files
         working-directory: modflow6/distribution
         run: python update_version.py
@@ -348,9 +301,9 @@ jobs:
         working-directory: modflow6/autotest
         run: |
           if [ "${{ github.ref_name }}" == "master" ]; then
-            pytest -v -n auto --durations 0 -m "not developmode"
+            pytest -v -n auto --durations 0 -m "not large and not developmode"
           else
-            pytest -v -n auto --durations 0
+            pytest -v -n auto --durations 0 -m "not large"
           fi
 
       - name: Test programs (Windows)
@@ -359,9 +312,9 @@ jobs:
         shell: pwsh
         run: |
           if ( "${{ github.ref_name }}" -eq "master" ) {
-            pytest -v -n auto --durations 0 -m "not developmode"
+            pytest -v -n auto --durations 0 -m "not large and not developmode"
           } else {
-            pytest -v -n auto --durations 0
+            pytest -v -n auto --durations 0 -m "not large"
           }
 
       - name: Test scripts

DEVELOPER.md

@@ -1,17 +1,39 @@
-# Building and Testing MODFLOW 6
+# Developing MODFLOW 6
 
 This document describes how to set up your development environment to build and test MODFLOW 6.
-It also explains the basic mechanics of using `git`. Details on how to contribute your code to the repository are found in the separate document [CONTRIBUTING.md](CONTRIBUTING.md)
-
-* [Prerequisite Software](#prerequisite-software)
-* [Getting the Sources](#getting-the-sources)
-* [Building](#building)
-* [Running Tests Locally](#running-tests-locally)
-
-See the [contribution guidelines](https://github.com/MODFLOW-USGS/modflow6/blob/develop/CONTRIBUTING.md)
-if you'd like to contribute to MODFLOW 6.
-
-## Prerequisite Software
+It also explains the basic mechanics of using `git`. Details on how to contribute your code to the repository are found in the separate document [CONTRIBUTING.md](CONTRIBUTING.md).
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
+- [Requirements](#requirements)
+  - [Git](#git)
+  - [gfortran (version 4.9 to 10)](#gfortran-version-49-to-10)
+    - [Linux](#linux)
+    - [macOS](#macos)
+    - [Windows](#windows)
+  - [Python](#python)
+  - [ifort (optional)](#ifort-optional)
+    - [Windows](#windows-1)
+  - [Doxygen & LaTeX (optional)](#doxygen--latex-optional)
+  - [fprettify](#fprettify)
+- [Installation](#installation)
+- [Building](#building)
+  - [Meson](#meson)
+  - [Visual Studio](#visual-studio)
+  - [Pymake](#pymake)
+  - [Make](#make)
+- [Testing](#testing)
+  - [External model repos](#external-model-repos)
+    - [Installing external repos](#installing-external-repos)
+      - [Test models](#test-models)
+      - [Example models](#example-models)
+    - [Running external model tests](#running-external-model-tests)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Requirements
 
 Before you can build and test MODFLOW 6, you must install and configure the
 following products on your development machine.
@@ -79,7 +101,7 @@ These programs can be installed from various sources, including by conda, macpor
 
 [fprettify](https://github.com/pseewald/fprettify) can be used to format Fortran source code and in combination with the [MODFLOW 6 fprettify configuration](https://github.com/MODFLOW-USGS/modflow6/blob/develop/distribution/.fprettify.yaml) establishes a contribution standard for properly formatted MODFLOW 6 Fortran source. This tool can be installed with `pip` or `conda` and used from the command line or integrated with a [VSCode](https://github.com/MODFLOW-USGS/modflow6/blob/develop/.vscode/README.md) or Visual Studio development environment. See [contribution guidelines](https://github.com/MODFLOW-USGS/modflow6/blob/develop/CONTRIBUTING.md) for additional information.
 
-## Getting the Sources
+## Installation
 
 Fork and clone the MODFLOW 6 repository:
 
@@ -151,15 +173,14 @@ The README also explains how to build MODFLOW 6 with it.
 We also provide make files which can be used to build MODFLOW 6 with [GNU Make](https://www.gnu.org/software/make/).
 For the build instructions we refer to the [GNU Make Manual](https://www.gnu.org/software/make/manual/).
 
-
-## Running Tests
+## Testing
 
 Tests should pass locally before a PR is opened on Github. All the tests are executed by the CI system and a pull request can only be merged with passing tests.
 
-Tests must be run from the `autotest` folder:
+Tests must be run from the `autotest` folder.
 
 ```shell
-cd modflow6/autotest
+cd autotest
 ```
 
 FloPy plugins must first be updated:
@@ -204,7 +225,7 @@ While many tests create models programmatically, the full suite tests MODFLOW 6
 
 #### Installing external repos
 
-By default, the tests expect these repositories side-by-side with (i.e. in the same parent directory as) the `modflow6` repository. If the repos are somewhere else, you can set the `REPOS_PATH` environment variable to point to their parent directory.
+By default, the tests expect these repositories side-by-side with (i.e. in the same parent directory as) the `modflow6` repository. If the repos are somewhere else, you can set the `REPOS_PATH` environment variable to point to their parent directory. If external model repositories are not found, tests requiring them will be skipped.
 
 **Note:** a convenient way to persist environment variables needed for tests is to store them in a `.env` file in the `autotest` folder. Each variable should be defined on a separate line, with format `KEY=VALUE`. The `pytest-dotenv` plugin will then automatically load any variables found in this file into the test process' environment.
 

autotest/conftest.py

@@ -7,18 +7,18 @@
 project_root_path = Path(__file__).parent.parent
 
 
-def should_compare(test: str, comparisons: dict, executables: Executables) -> bool:
+def should_compare(
+    test: str, comparisons: dict, executables: Executables
+) -> bool:
     if test in comparisons.keys():
-        version = Executables.get_version(path=executables.mf6)
-        print(f"MODFLOW 6 development version='{version}'")
-        version = Executables.get_version(path=executables.mf6_regression)
-        print(f"MODFLOW 6 regression version='{version}'")
-        if version in comparisons[test]:
+        dev_ver = Executables.get_version(path=executables.mf6).split(' ')[0]
+        reg_ver = Executables.get_version(path=executables.mf6_regression).split(' ')[0]
+        print(f"MODFLOW 6 development version: {dev_ver}")
+        print(f"MODFLOW 6 regression version: {reg_ver}")
+        excluded = list(comparisons[test])
+        if reg_ver in excluded:
             print(
-                f"Test {test} does not run with versions {comparisons[test]}"
-            )
-            print(
-                f"Skipping regression test of sim {test} because the version is {version}"
+                f"Regression version {reg_ver} not supported for test {test}, skipping comparison"
             )
             return False
     return True
@@ -45,5 +45,5 @@ def pytest_addoption(parser):
         "--original-regression",
         action="store_true",
         default=False,
-        help="TODO"
+        help="TODO",
     )

autotest/pytest.ini

@@ -1,3 +1,7 @@
 [pytest]
 markers =
+    slow: tests taking more than a few seconds to complete
+    repo: tests using models loaded from an external repository
+    large: tests using large models (examples and largetestmodels)
+    regression: tests comparing results from different versions
     developmode: tests that should only run with IDEVELOPMODE = 1

autotest/test_z01_testmodels_mf6.py

@@ -1,12 +1,12 @@
 import pytest
-
-
 from conftest import should_compare
 from simulation import Simulation
 
-
-excluded = ["alt_model"]
-comparisons = {
+excluded_models = [
+    "alt_model",
+    "test205_gwtbuy-henrytidal"
+]
+excluded_comparisons = {
     "test001e_noUZF_3lay": ("6.2.1",),
     "test005_advgw_tidal": ("6.2.1",),
     "test017_Crinkle": ("6.2.1",),
@@ -29,22 +29,25 @@
 }
 
 
+@pytest.mark.repo
+@pytest.mark.regression
 def test_model(function_tmpdir, test_model_mf6, targets, original_regression):
     exdir = test_model_mf6.parent
     name = exdir.name
 
-    if name in excluded:
+    if name in excluded_models:
         pytest.skip(f"Excluding mf6 model: {name}")
 
     sim = Simulation(
         name=name,
+        exe_dict=targets.as_dict(),
         mf6_regression=not original_regression,
         cmp_verbose=False,
-        make_comparison=should_compare(exdir, comparisons, targets),
-        simpath=str(exdir)
+        make_comparison=should_compare(name, excluded_comparisons, targets),
+        simpath=str(exdir),
     )
 
-    src = sim.simpath
+    src = exdir
     dst = str(function_tmpdir)
 
     # Run the MODFLOW 6 simulation and compare to results generated using

autotest/test_z02_testmodels_mf5to6.py

@@ -2,35 +2,41 @@
 
 import flopy
 import pytest
-
+from common_regression import get_namefiles, model_setup
 from conftest import should_compare
-from common_regression import model_setup, get_namefiles
 from simulation import Simulation
 
-
 sfmt = "{:25s} - {}"
-excluded = ["alt_model"]
-comparisons = {
+excluded_models = [
+    "alt_model",
+    "mf2005"
+]
+excluded_comparisons = {
     "testPr2": ("6.2.1",),
     "testUzfLakSfr": ("6.2.1",),
     "testUzfLakSfr_laketable": ("6.2.1",),
     "testWetDry": ("6.2.1",),
 }
 
 
-def test_model(function_tmpdir, test_model_mf5to6, targets, original_regression):
+@pytest.mark.repo
+@pytest.mark.regression
+def test_model(
+    function_tmpdir, test_model_mf5to6, targets, original_regression
+):
     exdir = test_model_mf5to6.parent
     name = exdir.name
 
-    if name in excluded:
+    if name in excluded_models:
         pytest.skip(f"Excluding mf5to6 model: {name}")
 
     sim = Simulation(
-        exdir.name,
+        name=exdir.name,
+        exe_dict=targets.as_dict(),
         mf6_regression=not original_regression,
         cmp_verbose=False,
-        make_comparison=should_compare(name, comparisons, targets),
-        simpath=str(exdir)
+        make_comparison=should_compare(name, excluded_comparisons, targets),
+        simpath=str(exdir),
     )
 
     src = sim.simpath
@@ -100,4 +106,3 @@ def test_model(function_tmpdir, test_model_mf5to6, targets, original_regression)
     # appropriate MODFLOW-2005, MODFLOW-NWT, MODFLOW-USG, or MODFLOW-LGR run.
     sim.run()
     sim.compare()
-    sim.teardown()