diff --git a/CMakeLists.txt b/CMakeLists.txt index 463e37056..100f776ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,10 @@ if(HAVE_PYTHON_ZARR_INTERFACE OR HAVE_PYTHON_FDB_INTERFACE) find_package(Python 3.11 COMPONENTS Interpreter Development REQUIRED) endif() +ecbuild_add_option( FEATURE PYTHON_EXTENDED_FDB_TOOL_TESTS + DESCRIPTION "Build extendend fdb tool tests" + CONDITION HAVE_PYTHON_FDB_INTERFACE + DEFAULT OFF) ### build the tools diff --git a/pytest.toml b/pytest.toml new file mode 100644 index 000000000..88d2bf3b1 --- /dev/null +++ b/pytest.toml @@ -0,0 +1,22 @@ +[pytest] +addopts = ["-p no:doctest"] + +markers = [ + "files: Test the files setup of FDB tool.", + "simple: Test the simple setup of FDB tool.", + "yaml: Test the yaml config setup of FDB tool.", + "yaml_mars_disks: Test the yaml mars disks config setup of FDB tool.", + "yaml_tools: Test the yaml tools config setup of FDB tool.", + "json: Test the json config setup of FDB tool.", + "json_tools: Test the json tools config setup of FDB tool.", + "info: Test the corresponding FDB tool.", + "hide: Test the corresponding FDB tool.", + "list: Test the corresponding FDB tool.", + "read: Test the corresponding FDB tool.", + "root: Test the corresponding FDB tool.", + "wipe: Test the corresponding FDB tool.", + "purge: Test the corresponding FDB tool.", + "write: Test the corresponding FDB tool.", + "overlay: Test the corresponding FDB tool.", + "grib2fdb: Test the corresponding FDB tool." +] diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f7390ffb5..f8415ac92 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,4 +25,5 @@ endif() if (HAVE_PYTHON_FDB_INTERFACE) add_subdirectory( pyfdb ) + add_subdirectory( fdb_e2e ) endif() diff --git a/tests/fdb_e2e/CMakeLists.txt b/tests/fdb_e2e/CMakeLists.txt new file mode 100644 index 000000000..740ccf597 --- /dev/null +++ b/tests/fdb_e2e/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory( tool_tests ) diff --git a/tests/fdb_e2e/conftest.py b/tests/fdb_e2e/conftest.py new file mode 100644 index 000000000..28fcb6595 --- /dev/null +++ b/tests/fdb_e2e/conftest.py @@ -0,0 +1,826 @@ +# list (C) Copyright 2011- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +import json +import logging +import os +import pathlib +import shutil +from collections.abc import Generator + +import git +import pytest +import yaml +from findlibs import Path + +from pyfdb.pyfdb import FDB + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + + +@pytest.fixture +def test_logger(): + return logger + + +@pytest.fixture(scope="function") +def data_path() -> pathlib.Path: + """ + Provides path to test data + """ + path = pathlib.Path(__file__).resolve().parent / "data" + assert path.exists() + return path + + +@pytest.fixture(scope="function") +def test_data_path() -> pathlib.Path: + """ + Provides path to test data + """ + + git_repo = git.Repo(__file__, search_parent_directories=True) + git_root = pathlib.Path(git_repo.git.rev_parse("--show-toplevel")) + path = git_root / "tests" / "fdb" / "e2e" / "data" + assert path.exists() + return path + + +@pytest.fixture(scope="function") +def get_git_root() -> Path: + git_repo = git.Repo(__file__, search_parent_directories=True) + git_root = git_repo.git.rev_parse("--show-toplevel") + return Path(git_root) + + +@pytest.fixture(scope="session") +def session_tmp(tmp_path_factory) -> Generator[pathlib.Path, None, None]: + tmp_dir = tmp_path_factory.mktemp("session_data") + yield tmp_dir + + +@pytest.fixture(scope="function") +def function_tmp(tmp_path_factory) -> Generator[pathlib.Path, None, None]: + tmp_function_dir = tmp_path_factory.mktemp("test_data_", numbered=True) + yield tmp_function_dir + + +@pytest.fixture(scope="function", autouse=False) +def empty_fdb_setup(data_path, function_tmp) -> pathlib.Path: + """ + Creates a FDB setup in this tests temp directory. + This setup can be shared between tests + """ + schema_path_src = data_path / "schema" + assert schema_path_src.is_file() + + fdb_home_dir = function_tmp + fdb_config_dir = fdb_home_dir / "etc" / "fdb" + schema_path = fdb_config_dir / "schema" + shutil.copy(schema_path_src, schema_path) + + fdb5_root = function_tmp / "root" + fdb5_root.mkdir() + fdb_config = { + "type": "local", + "engine": "toc", + "schema": str(schema_path), + "spaces": [ + { + "handler": "Default", + "roots": [ + {"path": str(fdb5_root)}, + ], + } + ], + } + fdb_config_str = yaml.dump(fdb_config) + fdb_config_path = fdb_config_dir / "fdb_config.yaml" + fdb_config_path.write_text(fdb_config_str) + return fdb_home_dir + + +def simple_fdb_setup(data_path, function_tmp, subtocs: bool, expver_handler: bool) -> tuple[pathlib.Path, pathlib.Path]: + """ + Creates a FDB setup in this tests temp directory. + This setup can be shared between tests + + This is a setup which creates a fdb config file. The schema file need to + be placed in etc/fdb subfolder but the config can be located in the root. + """ + + fdb_home_dir = function_tmp + + # Copy schema to config folder + schema_path_src = data_path / "schema" + assert schema_path_src.is_file() + + schema_dir = fdb_home_dir / "etc" / "fdb" + schema_dir.mkdir(parents=True) + + schema_path = fdb_home_dir / "etc" / "fdb" / "schema" + shutil.copy(schema_path_src, schema_path) + + invalid_root_path = fdb_home_dir / "invalid-root" + + root_path = fdb_home_dir / "root" + root_path.mkdir() + + fdb_config = { + "type": "local", + "engine": "toc", + "schema": str(schema_path), + "spaces": [ + { + "regex": "rd:?.*", + "handler": "expver" if expver_handler is True else "Default", + "roots": [ + {"path": str(invalid_root_path)}, + {"path": str(root_path)}, + ], + } + ], + } + + fdb_config_dir = fdb_home_dir # Config file in home dir, env contains the FDB5_CONFIG_PATH + + fdb_config_str = yaml.dump(fdb_config) + fdb_config_path = fdb_config_dir / "config.yaml" + fdb_config_path.write_text(fdb_config_str) + + fdb_user_config = {"useSubToc": subtocs} + fdb_user_config_str = yaml.dump(fdb_user_config) + fdb_user_config_path = fdb_config_dir / "user_config.yaml" + fdb_user_config_path.write_text(fdb_user_config_str) + + return fdb_config_path, fdb_user_config_path + + +@pytest.fixture(scope="function", autouse=False) +def simple_fdb_setup_no_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = simple_fdb_setup(data_path, function_tmp, subtocs=False, expver_handler=False) + + os.environ["FDB_HOME"] = str(function_tmp) + os.environ["FDB5_CONFIG_FILE"] = str(function_tmp / "config.yaml") + + fdb = FDB() + + del os.environ["FDB_HOME"] + del os.environ["FDB5_CONFIG_FILE"] + + # Check that the temporary fdb is configured and picked up + assert str(function_tmp) in fdb.config()[0]["schema"] + all(str(function_tmp) in root["path"] for root in fdb.config()[0]["spaces"][0]["roots"]) + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def simple_env_no_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + env["FDB5_CONFIG_FILE"] = str(function_tmp / "config.yaml") + + return env + + +@pytest.fixture(scope="function", autouse=False) +def simple_fdb_setup_no_subtocs_expver_handler(data_path, function_tmp) -> FDB: + _, _ = simple_fdb_setup(data_path, function_tmp, subtocs=False, expver_handler=True) + + os.environ["FDB_HOME"] = str(function_tmp) + os.environ["FDB5_CONFIG_FILE"] = str(function_tmp / "config.yaml") + + fdb = FDB() + + del os.environ["FDB_HOME"] + del os.environ["FDB5_CONFIG_FILE"] + + # Check that the temporary fdb is configured and picked up + assert str(function_tmp) in fdb.config()[0]["schema"] + all(str(function_tmp) in root["path"] for root in fdb.config()[0]["spaces"][0]["roots"]) + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def simple_env_no_subtocs_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + env["FDB_EXPVER_FILE"] = f"{str(function_tmp)}/expver.map" + env["FDB5_CONFIG_FILE"] = str(function_tmp / "config.yaml") + + file: Path = Path(function_tmp / "expver.map") + file.touch() + + assert file.exists() + + return env + + +@pytest.fixture(scope="function", autouse=False) +def simple_fdb_setup_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = simple_fdb_setup(data_path, function_tmp, subtocs=True, expver_handler=False) + + os.environ["FDB_HOME"] = str(function_tmp) + os.environ["FDB5_CONFIG_FILE"] = str(function_tmp / "config.yaml") + + fdb = FDB() + + del os.environ["FDB_HOME"] + del os.environ["FDB5_CONFIG_FILE"] + + # Check that the temporary fdb is configured and picked up + assert str(function_tmp) in fdb.config()[0]["schema"] + all(str(function_tmp) in root["path"] for root in fdb.config()[0]["spaces"][0]["roots"]) + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def simple_env_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + env["FDB5_SUB_TOCS"] = "1" + env["FDB5_CONFIG_FILE"] = str(function_tmp / "config.yaml") + + return env + + +@pytest.fixture(scope="function", autouse=False) +def simple_fdb_setup_subtocs_expver_handler(data_path, function_tmp) -> FDB: + _, _ = simple_fdb_setup(data_path, function_tmp, subtocs=True, expver_handler=True) + + os.environ["FDB_HOME"] = str(function_tmp) + os.environ["FDB5_CONFIG_FILE"] = str(function_tmp / "config.yaml") + + fdb = FDB() + + del os.environ["FDB_HOME"] + del os.environ["FDB5_CONFIG_FILE"] + + # Check that the temporary fdb is configured and picked up + assert str(function_tmp) in fdb.config()[0]["schema"] + all(str(function_tmp) in root["path"] for root in fdb.config()[0]["spaces"][0]["roots"]) + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def simple_env_subtocs_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + env["FDB_EXPVER_FILE"] = f"{str(function_tmp)}/expver.map" + env["FDB5_SUB_TOCS"] = "1" + env["FDB5_CONFIG_FILE"] = str(function_tmp / "config.yaml") + + file: Path = Path(function_tmp / "expver.map") + file.touch() + + assert file.exists() + + return env + + +def files_fdb_setup( + data_path, function_tmp, subtocs: bool, expver_handler: bool +) -> tuple[pathlib.Path | None, pathlib.Path | None]: + """ + Creates a FDB setup in this tests temp directory. + This setup can be shared between tests + + The main intention is to check whether the setup of the FDB is possible without supplying + a config file. If no config file is given we need to create the corresponding files in the + fdb_config_dir. + """ + + fdb_home_dir = function_tmp + fdb_config_dir = fdb_home_dir / "etc" / "fdb" + fdb_config_dir.mkdir(parents=True) + + # Copy schema to config folder + schema_path_src = data_path / "schema" + assert schema_path_src.is_file() + schema_path = fdb_config_dir / "schema" + shutil.copy(schema_path_src, schema_path) + + # invalid_root_path = fdb_home_dir / "invalid-root" + + root_path = fdb_home_dir / "root" + root_path.mkdir() + + fdb_spaces_path = fdb_config_dir / "spaces" + if expver_handler is True: + fdb_spaces_path.write_text("rd:?.* rd expver") + else: + fdb_spaces_path.write_text("rd:?.* rd Default") + + fdb_roots_path: Path = fdb_config_dir / "roots" + + with fdb_roots_path.open("w+") as file_fdb_roots: + file_fdb_roots.write(f"{str(fdb_home_dir)}/root rd yes yes" + os.linesep) + if expver_handler: + fdb_roots_path.write_text(f"{str(fdb_home_dir)}/invalid-root rd yes yes") + else: + fdb_roots_path.write_text(f"{str(fdb_home_dir)}/invalid-root rd no no") + + return None, None + + +@pytest.fixture(scope="function", autouse=False) +def files_fdb_setup_no_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = files_fdb_setup(data_path, function_tmp, subtocs=False, expver_handler=False) + + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there is no config found + assert config == {} + assert user_config == {} + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def files_env_no_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + + return env + + +@pytest.fixture(scope="function", autouse=False) +def files_fdb_setup_no_subtocs_expver_handler(data_path, function_tmp) -> FDB: + _, _ = files_fdb_setup(data_path, function_tmp, subtocs=False, expver_handler=True) + + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there is no config found + assert config == {} + assert user_config == {} + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def files_env_no_subtocs_expver_handler(function_tmp) -> dict[str, str]: + fdb_schema_dir = function_tmp / "etc" / "fdb" + + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + env["FDB5_ROOT"] = str(function_tmp / "root") + env["FDB_EXPVER_FILE"] = f"{str(fdb_schema_dir / 'expver.map')}" + + file: Path = Path(fdb_schema_dir / "expver.map") + file.touch() + + assert file.exists() + + return env + + +@pytest.fixture(scope="function", autouse=False) +def files_fdb_setup_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = files_fdb_setup(data_path, function_tmp, subtocs=True, expver_handler=False) + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there is no config found + assert config == {} + assert user_config == {} + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def files_env_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + env["FDB5_SUB_TOCS"] = "1" + + return env + + +@pytest.fixture(scope="function", autouse=False) +def files_fdb_setup_subtocs_expver_handler(data_path, function_tmp) -> FDB: + _, _ = files_fdb_setup(data_path, function_tmp, subtocs=True, expver_handler=True) + + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there is no config found + assert config == {} + assert user_config == {} + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def files_env_subtocs_expver_handler(function_tmp) -> dict[str, str]: + fdb_schema_dir = function_tmp / "etc" / "fdb" + + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + env["FDB5_ROOT"] = str(function_tmp / "root") + env["FDB5_SUB_TOCS"] = "1" + env["FDB_EXPVER_FILE"] = f"{str(fdb_schema_dir / 'expver.map')}" + + file: Path = Path(fdb_schema_dir / "expver.map") + file.touch() + + assert file.exists() + + return env + + +def config_yaml_fdb_setup( + data_path, function_tmp, subtocs: bool, expver_handler: bool +) -> tuple[pathlib.Path, pathlib.Path | None]: + """ + Creates a FDB setup in this tests temp directory. + This setup can be shared between tests + + All the needed config files are located in etc/fdb subfolder + of the temporary work directory. + + The only env variable which is set should be FDB_HOME, so no + FDB5_CONFIG_PATH. + """ + + fdb_home_dir = function_tmp + fdb_config_dir = fdb_home_dir / "etc" / "fdb" + fdb_config_dir.mkdir(parents=True) + + # Copy schema to config folder + schema_path_src = data_path / "schema" + assert schema_path_src.is_file() + schema_path = fdb_config_dir / "schema" + shutil.copy(schema_path_src, schema_path) + + invalid_root_path = fdb_home_dir / "invalid-root" + + root_path = fdb_home_dir / "root" + root_path.mkdir() + + fdb_config = { + "type": "local", + "engine": "toc", + "schema": str(schema_path), + "spaces": [ + { + "regex": "rd:?.*", + "handler": "expver" if expver_handler is True else "Default", + "roots": [ + {"path": str(invalid_root_path)}, + {"path": str(root_path)}, + ], + } + ], + } + + fdb_config_str = yaml.dump(fdb_config) + fdb_config_path = fdb_config_dir / "config.yaml" + fdb_config_path.write_text(fdb_config_str) + + return fdb_config_path, None + + +@pytest.fixture(scope="function", autouse=False) +def config_yaml_env_no_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + + return env + + +@pytest.fixture(scope="function", autouse=False) +def config_yaml_fdb_setup_no_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = config_yaml_fdb_setup(data_path, function_tmp, subtocs=False, expver_handler=False) + + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there a config found, the one in etc/fdb + assert config != {} + assert user_config == {} + + return fdb + + +def config_json_fdb_setup( + data_path, function_tmp, subtocs: bool, expver_handler: bool +) -> tuple[pathlib.Path, pathlib.Path | None]: + """ + Creates a FDB setup in this tests temp directory. + This setup can be shared between tests + + All the needed config files are located in etc/fdb subfolder + of the temporary work directory. + + The only env variable which is set should be FDB_HOME, so no + FDB5_CONFIG_PATH. + """ + + fdb_home_dir = function_tmp + fdb_config_dir = fdb_home_dir / "etc" / "fdb" + fdb_config_dir.mkdir(parents=True) + + # Copy schema to config folder + schema_path_src = data_path / "schema" + assert schema_path_src.is_file() + schema_path = fdb_config_dir / "schema" + shutil.copy(schema_path_src, schema_path) + + invalid_root_path = fdb_home_dir / "invalid-root" + + root_path = fdb_home_dir / "root" + root_path.mkdir() + + fdb_config = { + "type": "local", + "engine": "toc", + "schema": str(schema_path), + "spaces": [ + { + "regex": "rd:?.*", + "handler": "expver" if expver_handler is True else "Default", + "roots": [ + {"path": str(invalid_root_path)}, + {"path": str(root_path)}, + ], + } + ], + } + + fdb_config_str = json.dumps(fdb_config) + fdb_config_path = fdb_config_dir / "config.json" + fdb_config_path.write_text(fdb_config_str) + + return fdb_config_path, None + + +@pytest.fixture(scope="function", autouse=False) +def config_json_env_no_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + + return env + + +@pytest.fixture(scope="function", autouse=False) +def config_json_fdb_setup_no_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = config_json_fdb_setup(data_path, function_tmp, subtocs=False, expver_handler=False) + + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there a config found, the one in etc/fdb + assert config != {} + assert user_config == {} + + return fdb + + +def config_mars_disks_fdb_setup( + data_path, function_tmp, subtocs: bool, expver_handler: bool +) -> tuple[pathlib.Path, pathlib.Path | None]: + """ + Creates a FDB setup in this tests temp directory. + This setup can be shared between tests + + All the needed config files are located in etc/fdb subfolder + of the temporary work directory. + + The only env variable which is set should be FDB_HOME, so no + FDB5_CONFIG_PATH. + """ + + fdb_home_dir = function_tmp + fdb_config_dir = fdb_home_dir / "etc" / "fdb" + fdb_config_dir.mkdir(parents=True) + + # Copy schema to config folder + schema_path_src = data_path / "schema" + assert schema_path_src.is_file() + schema_path = fdb_config_dir / "schema" + shutil.copy(schema_path_src, schema_path) + + fdb_config = { + "type": "local", + "engine": "toc", + "schema": str(schema_path), + "spaces": [ + { + "marsDisks": True, + "name": "marsFdb", + "regex": "r.*", + "handler": "WeightedRandom", + } + ], + } + + fdb_config_str = yaml.dump(fdb_config) + fdb_config_path = fdb_config_dir / "config.yaml" + fdb_config_path.write_text(fdb_config_str) + + mars_disks_path = fdb_home_dir / "etc" / "disks" + mars_disks_path.mkdir(parents=True) + + (mars_disks_path / "fdb").write_text(f"{str(fdb_home_dir)}/root") + + root_path = fdb_home_dir / "root" + root_path.mkdir() + + return fdb_config_path, None + + +@pytest.fixture(scope="function", autouse=False) +def config_yaml_mars_disks_env_no_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + + return env + + +@pytest.fixture(scope="function", autouse=False) +def config_yaml_mars_disks_fdb_setup_no_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = config_mars_disks_fdb_setup(data_path, function_tmp, subtocs=False, expver_handler=False) + + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there a config found, the one in etc/fdb + assert config != {} + assert user_config == {} + + return fdb + + +def config_tools_fdb_setup( + data_path, function_tmp, dump_json: bool = False +) -> tuple[pathlib.Path, pathlib.Path | None]: + """ + Creates a FDB setup in this tests temp directory. + This setup can be shared between tests + + All the needed config files are located in etc/fdb subfolder + of the temporary work directory. + + The only env variable which is set should be FDB_HOME, so no + FDB5_CONFIG_PATH. + + Write the config as yaml per default and json if the corresponding flag is supplied + """ + + fdb_home_dir = function_tmp + fdb_config_dir = fdb_home_dir / "etc" / "fdb" + fdb_config_dir.mkdir(parents=True) + + # Copy schema to config folder + schema_path_src = data_path / "schema" + assert schema_path_src.is_file() + schema_path = fdb_config_dir / "schema" + shutil.copy(schema_path_src, schema_path) + + invalid_root_path = fdb_home_dir / "invalid-root" + + root_path = fdb_home_dir / "root" + root_path.mkdir() + + fdb_config = { + "type": "local", + "engine": "toc", + "schema": str(schema_path), + "spaces": [ + { + "regex": "rd:?.*", + "handler": "Default", + "roots": [ + {"path": str(invalid_root_path)}, + {"path": str(root_path)}, + ], + } + ], + } + + tool_names = [ + "base_config", + "fdb-root", + "fdb-info", + "fdb-write", + "fdb-purge", + "fdb-wipe", + "fdb-list", + "fdb-read", + "fdb-hide", + "fdb-overlay", + "fdb-write-legacy", + "fdb-where", + "fdb-adopt", + "grib2fdb5", + ] + + for tool_name in tool_names: + if dump_json is False: + fdb_config_str = yaml.dump(fdb_config) + fdb_config_path = fdb_config_dir / f"{tool_name}.yaml" + fdb_config_path.write_text(fdb_config_str) + else: + fdb_config_str = json.dumps(fdb_config) + fdb_config_path = fdb_config_dir / f"{tool_name}.json" + fdb_config_path.write_text(fdb_config_str) + + fdb_config_path = fdb_config_dir / "base_config.yaml" + + return fdb_config_path, None + + +@pytest.fixture(scope="function", autouse=False) +def config_yaml_tools_env_no_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + + return env + + +@pytest.fixture(scope="function", autouse=False) +def config_yaml_tools_fdb_setup_no_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = config_tools_fdb_setup(data_path, function_tmp, dump_json=False) + + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there a config found, the one in etc/fdb + assert config == {} + assert user_config == {} + + return fdb + + +@pytest.fixture(scope="function", autouse=False) +def config_json_tools_env_no_subtocs_no_expver_handler(function_tmp) -> dict[str, str]: + env = {} + env["PATH"] = os.environ["PATH"] + env["FDB_HOME"] = str(function_tmp) + + return env + + +@pytest.fixture(scope="function", autouse=False) +def config_json_tools_fdb_setup_no_subtocs_no_expver_handler(data_path, function_tmp) -> FDB: + _, _ = config_tools_fdb_setup(data_path, function_tmp, dump_json=True) + + os.environ["FDB_HOME"] = str(function_tmp) + fdb = FDB() + del os.environ["FDB_HOME"] + + config, user_config = fdb.config() + + # Check that there a config found, the one in etc/fdb + assert config == {} + assert user_config == {} + + return fdb diff --git a/tests/fdb_e2e/data/oper.grib b/tests/fdb_e2e/data/oper.grib new file mode 100644 index 000000000..8a08820af Binary files /dev/null and b/tests/fdb_e2e/data/oper.grib differ diff --git a/tests/fdb_e2e/data/quantile.grib b/tests/fdb_e2e/data/quantile.grib new file mode 100644 index 000000000..c3b70d596 Binary files /dev/null and b/tests/fdb_e2e/data/quantile.grib differ diff --git a/tests/fdb_e2e/data/schema b/tests/fdb_e2e/data/schema new file mode 100644 index 000000000..ec87e7b92 --- /dev/null +++ b/tests/fdb_e2e/data/schema @@ -0,0 +1,558 @@ +# * Format of the rules is: + +# [a1, a2, a3 ...[b1, b2, b3... [c1, c2, c3...]]] + +# - The first level (a) defines which attributes are used to name the top level directory +# - The second level (b) defines which attributes are used to name the data files +# - The third level (c) defines which attributes are used as index keys + +# * Rules can be grouped + +# [a1, a2, a3 ... +# [b1, b2, b3... [c1, c2, c3...]] +# [B1, B2, B3... [C1, C2, C3...]] +# ] + +# * A list of values can be given for an attribute +# [ ..., stream=enfo/efov, ... ] +# This will be used when matching rules. + +# * Attributes can be typed +# Globally, at the begining of this file: + +# refdate: Date; + +# or in the context of a rule: +# [type=cl, ... [date:ClimateMonth, ...]] + +# Typing attributes is done when the user's requests or the GRIB values need to be modified before directories, files and indexes are created. For example, ClimateMonth will transform 2010-04-01 to 'may' internally. + +# * Attributes can be optional +# [ step, levelist?, param ] +# They will be replaced internally by an empty value. It is also posiible to provide a default subtitution value: e.g. [domain?g] will consider the domain to be 'g' if missing. + +# * Attributes can be removed: +# [grid-] +# This is useful to remove attributes present in the GRIB that should not be ignored + +# * Rules are matched: + +# - If the attributes are present in the GRIB/Request, or marked optional or ignored +# - If a list of possible value is provided, one of them must match, for example +# [ class, expver, stream=enfo/efov, date, time, domain ] +# will match either stream=enfo or stream=efov, all other attributes will be matched if they exist in the GRIB or user's request + +# * On archive: +# - Attributes are extracted from the GRIB (namespace 'mars'), possibly modified by the attribute type +# - Only the first rule is used, so order is important +# - All GRIB attributes must be used by the rules, otherwise an error is raised + +# * On retrieve: +# - Attributes are extracted from the user's request, possibly modified by the attribute type (e.g. for handling of U/V) +# - All the matching rules are considered +# - Only attributes listed in the rules are used to extract values from the user's request + + +# Default types + +param: Param; +step: Step; +date: Date; +hdate: Date; +refdate: Date; +latitude: Double; +longitude: Double; +levelist: Double; +grid: Grid; +expver: Expver; + +time: Time; +fcmonth: Integer; + +number: Integer; +frequency: Integer; +direction: Integer; +channel: Integer; + +instrument: Integer; +ident: Integer; + +diagnostic: Integer; +iteration: Integer; +system: Integer; +method: Integer; + +# ??????? + +# reference: Integer; +# fcperiod: Integer; + +# opttime: Integer; +# leadtime: Integer; + +# quantile: ?????? +# range: ?????? + +# band: Integer; + + +######################################################## +# These rules must be first, otherwise fields of These +# classes will be index with the default rule for oper +[ class=ti/s2, expver, stream, date, time, model + [ origin, type, levtype, hdate? + [ step, number?, levelist?, param ]] +] + +[ class=ms, expver, stream, date, time, country=de + [ domain, type, levtype, dbase, rki, rty, ty + [ step, levelist?, param ]] +] + +[ class=ms, expver, stream, date, time, country=it + [ domain, type, levtype, model, bcmodel, icmodel:First3 + [ step, levelist?, param ] + ] +] + +[ class=el, expver, stream, date, time, domain + [ origin, type, levtype + [ step, levelist?, param ]] +] + +######################################################## +# The are the rules matching most of the fields +# oper/dcda +[ class, expver, stream=oper/dcda/scda, date, time, domain + + [ type=im/sim + [ step?, ident, instrument, channel ]] + + [ type=ssd + [ step, param, ident, instrument, channel ]] + + [ type=4i, levtype + [ step, iteration, levelist, param ]] + + [ type=me, levtype + [ step, number, levelist?, param ]] + + [ type=ef, levtype + [ step, levelist?, param, channel? ]] + + [ type, levtype + [ step, levelist?, param ]] + +] + +# dcwv/scwv/wave +[ class, expver, stream=dcwv/scwv/wave, date, time, domain + [ type, levtype + [ step, param, frequency?, direction? ]]] + +# enfo +[ class, expver, stream=enfo/efov, date, time, domain + + [ type, levtype=dp, product?, section? + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type=tu, levtype, reference + [ step, number, levelist?, param ]] + + [ type, levtype + [ step, quantile?, number?, levelist?, param ]] + +] + +# waef/weov +[ class, expver, stream=waef/weov, date, time, domain + [ type, levtype + [ step, number?, param, frequency?, direction? ]] +] + +######################################################## +# enda +[ class, expver, stream=enda, date, time, domain + + [ type=ef/em/es/ses, levtype + [ step, number?, levelist?, param, channel? ]] + + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + + + [ type, levtype + [ step, number?, levelist?, param ]] +] + +# ewda +[ class, expver, stream=ewda, date, time, domain + [ type, levtype + [ step, number, param, frequency?, direction? ]] +] + + +######################################################## +# elda +[ class, expver, stream=elda, date, time, domain + + [ type, levtype, anoffset + [ step, number?, levelist?, iteration?, param, channel? ]] +] + +# ewda +[ class, expver, stream=ewla, date, time, domain + [ type, levtype, anoffset + [ step, number?, param, frequency?, direction? ]] +] + +######################################################## +# elda +[ class, expver, stream=lwda, date, time, domain + + [ type=ssd, anoffset + [ step, param, ident, instrument, channel ]] + + [type=me, levtype, anoffset + [ number, step, levelist?, param]] + + [ type=4i, levtype, anoffset + [ step, iteration, levelist, param ]] + + [ type, levtype, anoffset + [ step, levelist?, param]] +] + +# ewda +[ class, expver, stream=lwwv, date, time, domain + [ type, levtype, anoffset + [ step, param, frequency?, direction? ]] +] +######################################################## +# amap +[ class, expver, stream=amap, date, time, domain + [ type, levtype, origin + [ step, levelist?, param ]]] + +# maed +[ class, expver, stream=maed, date, time, domain + [ type, levtype, origin + [ step, levelist?, param ]]] + +# mawv +[ class, expver, stream=mawv, date, time, domain + [ type, levtype, origin + [ step, param, frequency?, direction? ]]] + +# cher +[ class, expver, stream=cher, date, time, domain + [ type, levtype + [ step, levelist, param ]]] + + +# efhc +[ class, expver, stream=efhc, refdate, time, domain + [ type, levtype, date + [ step, number?, levelist?, param ]]] + +# efho +[ class, expver, stream=efho, date, time, domain + [ type, levtype, hdate + [ step, number?, levelist?, param ]]] + + +# efhs +[ class, expver, stream=efhs, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]]] + +# wehs +[ class, expver, stream=wehs, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]]] + +# kwbc +[ class, expver, stream=kwbc, date, time, domain + [ type, levtype + [ step, number?, levelist?, param ]]] + +# ehmm +[ class, expver, stream=ehmm, date, time, domain + [ type, levtype, hdate + [ fcmonth, levelist?, param ]]] + + +# ammc/cwao/edzw/egrr/lfpw/rjtd/toga +[ class, expver, stream=ammc/cwao/edzw/egrr/lfpw/rjtd/toga/fgge, date, time, domain + [ type, levtype + [ step, levelist?, param ]]] + +######################################################################## + +# enfh +[ class, expver, stream=enfh, date, time, domain + + [ type, levtype=dp, hdate, product?, section? + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, hdate + [ step, number?, levelist?, param ]] +] + +# enwh +[ class, expver, stream=enwh, date, time, domain + [ type, levtype, hdate + [ step, number?, param, frequency?, direction? ]] +] + +######################################################################## +# sens +[ class, expver, stream=sens, date, time, domain + [ type, levtype + [ step, diagnostic, iteration, levelist?, param ]]] + +######################################################################## +# esmm +[ class, expver, stream=esmm, date, time, domain + [ type, levtype + [ fcmonth, levelist?, param ]]] +# ewhc +[ class, expver, stream=ewhc, refdate, time, domain + [ type, levtype, date + [ step, number?, param, frequency?, direction? ]]] + +######################################################################## +# ewho +[ class, expver, stream=ewho, date, time, domain + [ type, levtype, hdate + [ step, number?, param, frequency?, direction? ]]] + +# mfam +[ class, expver, stream=mfam, date, time, domain + + [ type=pb/pd, levtype, origin, system?, method + [ fcperiod, quantile, levelist?, param ]] + + [ type, levtype, origin, system?, method + [ fcperiod, number?, levelist?, param ]] + +] + +# mfhm +[ class, expver, stream=mfhm, refdate, time, domain + [ type, levtype, origin, system?, method, date? + [ fcperiod, number?, levelist?, param ]]] +# mfhw +[ class, expver, stream=mfhw, refdate, time, domain + [ type, levtype, origin, system?, method, date + [ step, number?, param ]]] +# mfwm +[ class, expver, stream=mfwm, date, time, domain + [ type, levtype, origin, system?, method + [ fcperiod, number, param ]]] +# mhwm +[ class, expver, stream=mhwm, refdate, time, domain + [ type, levtype, origin, system?, method, date + [ fcperiod, number, param ]]] + +# mmsf +[ class, expver, stream=mmsf, date, time, domain + + [ type, levtype=dp, origin, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, origin, system?, method + [ step, number, levelist?, param ]] +] + +# mnfc +[ class, expver, stream=mnfc, date, time, domain + + [ type, levtype=dp, origin, product, section, system?, method + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, origin, system?, method + [ step, number?, levelist?, param ]] +] + +# mnfh +[ class, expver, stream=mnfh, refdate, time, domain + [ type, levtype=dp, origin, product, section, system?, method, date + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + [ type, levtype, origin, system?, method, date? + [ step, number?, levelist?, param ]] +] + +# mnfm +[ class, expver, stream=mnfm, date, time, domain + [ type, levtype, origin, system?, method + [ fcperiod, number?, levelist?, param ]]] + +# mnfw +[ class, expver, stream=mnfw, date, time, domain + [ type, levtype, origin, system?, method + [ step, number?, param ]]] + +# mnth +[ class, expver, stream=mnth, domain + [ type=cl, levtype + [ date: ClimateMonthly, time, levelist?, param ]] + [ type, levtype + [ date: Month, time, step?, levelist?, param ]]] + +# mofc +[ class, expver, stream=mofc, date, time, domain + [ type, levtype=dp, product, section, system?, method + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + [ type, levtype, system?, method + [ step, number?, levelist?, param ]] +] + +# mofm +[ class, expver, stream=mofm, date, time, domain + [ type, levtype, system?, method + [ fcperiod, number, levelist?, param ]]] + +# mmsa/msmm +[ class, expver, stream=mmsa, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number?, levelist?, param ]]] + +[ class, expver, stream=msmm, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number?, levelist?, param ]]] + +# ocea +[ class, expver, stream=ocea, date, time, domain + [ type, levtype, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] +] + +#=# seas +[ class, expver, stream=seas, date, time, domain + + [ type, levtype=dp, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, system?, method + [ step, number, levelist?, param ]] +] + +# sfmm/smma +[ class, expver, stream=sfmm/smma, date, time, domain + [ type, levtype, system?, method + [ fcmonth, number?, levelist?, param ]]] + +# supd +[ class=od, expver, stream=supd, date, time, domain + [ type, levtype, origin?, grid + [ step, levelist?, param ]]] + +# For era +[ class, expver, stream=supd, date, time, domain + [ type, levtype, grid- # The minus sign is here to consume 'grid', but don't index it + [ step, levelist?, param ]]] + +# swmm +[ class, expver, stream=swmm, date, time, domain + [ type, levtype, system?, method + [ fcmonth, number, param ]]] + +# wamf +[ class, expver, stream=wamf, date, time, domain + [ type, levtype, system?, method + [ step, number?, param ]]] +# wamo +[ class, expver, stream=wamo, domain + [ type=cl, levtype + [ date: ClimateMonthly, time, param ]] + [ type, levtype + [ date: Month, time, step?, param ]]] + +# wasf +[ class, expver, stream=wasf, date, time, domain + [ type, levtype, system?, method + [ step, number, param ]]] +# wmfm +[ class, expver, stream=wmfm, date, time, domain + [ type, levtype, system?, method + [ fcperiod, number, param ]]] +#= +[ class, expver, stream=moda/msdc/mdfa/msda, domain + [ type, levtype + [ date: Month, time?, step?, levelist?, param ]]] + #[ step, leadtime?, opttime?, number?, levelist?, param ]]] + +# seap +[ class, expver, stream=seap, date, time, domain + [ type=sv/svar, levtype, origin, method? + [ step, leadtime, opttime, number, levelist?, param ]] + + [ type=ef, levtype, origin + [ step, levelist?, param, channel? ]] + + [ type, levtype, origin + [ step, levelist?, param ]] + + ] + +[ class, expver, stream=mmaf, date, time, domain + [ type, levtype, origin, system?, method + [ step, number, levelist?, param ]] +] + +[ class, expver, stream=mmam, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number, levelist?, param ]] +] + + +[ class, expver, stream=dacl, domain + [ type=pb, levtype + [ date: ClimateDaily, time, step, quantile, levelist?, param ]] + [ type, levtype + [ date: ClimateDaily, time, step, levelist?, param ]] + +] + +[ class, expver, stream=dacw, domain + [ type=pb, levtype + [ date: ClimateDaily, time, step, quantile, param ]] + [ type, levtype + [ date: ClimateDaily, time, step, param ]] + +] + +[ class, expver, stream=edmm/ewmm, date, time, domain + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + [ type, levtype + [ step, number, levelist?, param ]] +] + +[ class, expver, stream=edmo/ewmo, date, domain + [ type, levtype + [ number, levelist?, param ]] +] + +# stream gfas +[ class=mc, expver, stream=gfas, date, time, domain + [ type=ga, levtype + [ step, param ]] + + [ type=gsd + [ param, ident, instrument ]] + +] + +# class is e2 +[ class, expver, stream=espd, date, time, domain + [ type, levtype, origin, grid + [ step, number, levelist?, param ]]] + +[ class=cs, expver, stream, date:Default, time, domain + [ type, levtype + [ step, levelist?, param ]]] + + + diff --git a/tests/fdb_e2e/data/steprange.grib b/tests/fdb_e2e/data/steprange.grib new file mode 100644 index 000000000..92fd43a92 Binary files /dev/null and b/tests/fdb_e2e/data/steprange.grib differ diff --git a/tests/fdb_e2e/data/template.grib b/tests/fdb_e2e/data/template.grib new file mode 100644 index 000000000..76804d01d Binary files /dev/null and b/tests/fdb_e2e/data/template.grib differ diff --git a/tests/fdb_e2e/data/x138-300.grib b/tests/fdb_e2e/data/x138-300.grib new file mode 100644 index 000000000..569f5c84c Binary files /dev/null and b/tests/fdb_e2e/data/x138-300.grib differ diff --git a/tests/fdb_e2e/python_api/list/test_list.py b/tests/fdb_e2e/python_api/list/test_list.py new file mode 100644 index 000000000..c06c3d2b2 --- /dev/null +++ b/tests/fdb_e2e/python_api/list/test_list.py @@ -0,0 +1,462 @@ +import datetime +from pathlib import Path + +import pytest +from python_api.util.tools import Tool + +from pyfdb import FDB +from pyfdb.pyfdb_iterator import ListElement + + +def generate_test_files_expver_step(source_file_path, function_tmp): + # Data setup + target_files: list[Path] = [] + + for step in range(0, 3): + for expver in ["xxxx", "xxxy"]: + target_file_path = function_tmp / f"data.{expver}.{step}.grib" + Tool.modify_metadata_source_file( + [("class", "rd"), ("expver", expver), ("type", "fc"), ("step", str(step))], + source_file=source_file_path, + target_file=target_file_path, + ) + target_files.append(target_file_path) + + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "2")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "2")], + ], + [ + "data.xxxx.0.grib", + "data.xxxy.0.grib", + "data.xxxx.1.grib", + "data.xxxy.1.grib", + "data.xxxx.2.grib", + "data.xxxy.2.grib", + ], + ) + + return target_files + + +def generate_test_files_key_value( + source_file_path, + function_tmp, + key_value_list: list[list[tuple[str, str | int]]], + output_file_names: list[str], +): + """Creates copies of the GRIB file at source_file_path in function_tmp. For each + element of the key_value_list all given key-value-pairs are modified in the resulting GRIB + file and the corresponding entry from output_file_names is used as a name. + + This means: + + setup_fdb_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "2")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "2")], + ], + [ + "data.xxxx.0.grib", + "data.xxxy.0.grib", + "data.xxxx.1.grib", + "data.xxxy.1.grib", + "data.xxxx.2.grib", + "data.xxxy.2.grib", + ], + ) + + Is creating 6 files with the names of output_file_names. For each file the GRIB metadata + is modified as given in key_value_list. + + + Args: + source_file_path (Path): Path to the template GRIB file + function_tmp (Path): Path to the output folder + key_value_list: List of key-value pair lists for modifying the GRIB metadata + output_file_names: Names of the output files + + Returns: + target_files - A list of paths to the created files + """ + assert len(key_value_list) == len(output_file_names) + + # Data setup + target_files: list[Path] = [] + + for i in range(len(key_value_list)): + cur_key_value_list = key_value_list[i] + output_file_name = output_file_names[i] + target_file_path = function_tmp / f"data.{output_file_name}.grib" + Tool.modify_metadata_source_file( + cur_key_value_list, + source_file=source_file_path, + target_file=target_file_path, + ) + target_files.append(target_file_path) + + return target_files + + +@pytest.mark.parametrize("no_files_saved", [1, 2, 3, 4, 5, 6]) +def test_list_all(simple_fdb_setup_no_subtocs_no_expver_handler, data_path, function_tmp, no_files_saved): + # Data setup + source_file_path = data_path / "oper.grib" + + target_files: list[Path] = [] + + for step in range(0, 3): + for expver in ["xxxx", "xxxy"]: + target_file_path = function_tmp / f"data.{expver}.{step}.grib" + Tool.modify_metadata_source_file( + [("class", "rd"), ("expver", expver), ("type", "fc"), ("step", str(step))], + source_file=source_file_path, + target_file=target_file_path, + ) + target_files.append(target_file_path) + + with FDB(simple_fdb_setup_no_subtocs_no_expver_handler) as fdb: + for i in range(no_files_saved): + fdb.archive(target_files[i].read_bytes()) + + list_elements = list(fdb.list({})) + assert len(list_elements) == 24 * no_files_saved + + list_elements = list(fdb.list({"class": "rd", "expver": "xxxx", "step": "0"})) + assert len(list_elements) == 24 + + list_elements = list(fdb.list({"class": "rd", "expver": "xxxy", "step": "0"})) + assert len(list_elements) == (24 if no_files_saved >= 2 else 0) + + list_elements = list(fdb.list({"class": "rd", "expver": "xxxx", "step": "1"})) + assert len(list_elements) == (24 if no_files_saved >= 3 else 0) + + list_elements = list(fdb.list({"class": "rd", "expver": "xxxy", "step": "1"})) + assert len(list_elements) == (24 if no_files_saved >= 4 else 0) + + list_elements = list(fdb.list({"class": "rd", "expver": "xxxx", "step": "2"})) + assert len(list_elements) == (24 if no_files_saved >= 5 else 0) + + list_elements = list(fdb.list({"class": "rd", "expver": "xxxy", "step": "2"})) + assert len(list_elements) == (24 if no_files_saved >= 6 else 0) + + +def test_list_all_including_masked(simple_fdb_setup_no_subtocs_no_expver_handler, data_path, function_tmp): + fdb = FDB(simple_fdb_setup_no_subtocs_no_expver_handler) + + source_file_path = data_path / "oper.grib" + target_files = generate_test_files_expver_step(source_file_path, function_tmp) + + # Second run masks the elements + for i in range(2): + with fdb: + for target_file in target_files: + fdb.archive(target_file.read_bytes()) + + # Check total amount of elements in the FDB + list_elements = list(fdb.list({}, include_masked=True)) + assert len(list_elements) == 144 * (i + 1) + + for expver in ["xxxx", "xxxy"]: + for step in range(0, 3): + # Check that for every of the given test files there are exact 24 entries + # In the second loop, those are masked and should return twice as many elements + list_elements = list(fdb.list({"class": "rd", "expver": expver, "step": step}, include_masked=True)) + assert len(list_elements) == 24 * (i + 1) + + # Also check that without the masked elements we only retain 24 elements + list_elements = list(fdb.list({"class": "rd", "expver": expver, "step": step})) + assert len(list_elements) == 24 + + +def test_list_location(simple_fdb_setup_no_subtocs_no_expver_handler, data_path, function_tmp): + fdb = FDB(simple_fdb_setup_no_subtocs_no_expver_handler) + + source_file_path = data_path / "oper.grib" + target_files = generate_test_files_expver_step(source_file_path, function_tmp) + + # Second run masks the elements + with fdb: + fdb.archive(target_files[0].read_bytes()) + + # Check total amount of elements in the FDB + list_elements = list(fdb.list({})) + assert len(list_elements) == 24 + + # Check total amount of elements at time 0000 + list_elements: list[ListElement] = list(fdb.list({"class": "rd", "expver": "xxxx", "time": "0000"})) + + locations = [] + + for el in list_elements: + print(f"URI: {el.uri}, Offset: {el.offset()}, Length: {el.length()}") + assert el.uri + locations.append(el.uri) + + assert len(locations) == 12, "Wrong amount of archived data for expver 'xxxx' and time '0000' should be 12" + + for el in list_elements: + print(f"{el}") + + +def test_list_masking(simple_fdb_setup_no_subtocs_no_expver_handler, data_path, function_tmp): + fdb = FDB(simple_fdb_setup_no_subtocs_no_expver_handler) + + yesterday = f"{(datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y%m%d')}" + + source_file_path = data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [ + ("class", "rd"), + ("expver", "xxxx"), + ("date", int(yesterday)), + ], + [ + ("class", "rd"), + ("expver", "xxxx"), + ("date", 20170101), + ], + ], + ["xxxx.d1", "xxxx.d2"], + ) + + for _ in range(2): + with fdb: + for target_file in target_files: + fdb.archive(target_file.read_bytes()) + + # Check total amount of elements in the FDB + list_elements = list(fdb.list({})) + assert len(list_elements) == 48 + + # Check total amount of elements in the FDB with masked included + list_elements = list(fdb.list({}, include_masked=True)) + assert len(list_elements) == 96 + + # Check total amount of elements in the FDB with masked included + selection = { + "class": "rd", + "expver": "xxxx", + "date": ["-1", "20170101"], + "stream": "oper", + "type": "an", + "levtype": "pl", + "param": ["155", "138"], + "levelist": ["300", "400", "500", "700", "850", "1000"], + } + list_elements = list(fdb.list(selection)) + assert len(list_elements) == 48, "Entries should be masked per default" + + list_elements = list(fdb.list(selection, include_masked=True)) + assert len(list_elements) == 96, "Masked entries should be shown" + + selection_non_existing_param_level = { + "class": "rd", + "expver": "xxxx", + "date": ["-1", "20170101"], + "stream": "oper", + "type": "an", + "levtype": "pl", + "param": ["130", "138"], + "levelist": ["300", "123", "1000"], + } + + list_elements = list(fdb.list(selection_non_existing_param_level)) + assert len(list_elements) == 8, "Entries should be masked per default" + + list_elements = list(fdb.list(selection_non_existing_param_level, include_masked=True)) + assert len(list_elements) == 16, "Masked entries should be shown" + + +# TODO(TKR): Should we include a test case for the minimum keys or is this not necessary due to the tools not being covered + + +def test_list_ranges_include_masked(simple_fdb_setup_no_subtocs_no_expver_handler, data_path, function_tmp): + fdb = FDB(simple_fdb_setup_no_subtocs_no_expver_handler) + + yesterday = f"{(datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y%m%d')}" + + source_file_path = data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [ + ("class", "rd"), + ("expver", "xxxx"), + ("date", int(yesterday)), + ], + [ + ("class", "rd"), + ("expver", "xxxx"), + ("date", 20170101), + ], + [ + ("class", "rd"), + ("expver", "xxxx"), + ("date", 20180101), + ], + ], + ["xxxx.d1", "xxxx.d2", "xxxx.d3"], + ) + + with fdb: + for target_file in target_files: + fdb.archive(target_file.read_bytes()) + + all_entries = list(fdb.list({}, include_masked=True)) + assert len(all_entries) == 72 + + all_entries = list(fdb.list({})) + assert len(all_entries) == 72 + + +request_entries_tuples = [ + ({"class": "rd", "expver": "xxxx", "date": "-1"}, 24), + ({"class": "rd", "expver": "xxxx", "date": "-1/20170101"}, 48), + ({"class": "rd", "expver": "xxxx", "date": ["-1", "20170101"]}, 48), + ({"class": "rd", "expver": "xxxx", "date": "-1/20170101/20180101"}, 72), + ({"class": "rd", "expver": "xxxx", "date": ["-1", "20170101", "20180101"]}, 72), + ( + { + "class": "rd", + "expver": "xxxx", + "date": ["-1", "20170101", "20180101"], + "stream": "oper", + "type": "an", + "levtype": "pl", + "param": 60, + }, + 0, + ), + ( + { + "class": "rd", + "expver": "xxxx", + "date": "20060101", + "stream": "oper", + "type": "an", + "levtype": "pl", + "param": 155, + }, + 0, + ), + ( + { + "class": "rd", + "expver": "xxxx", + "date": ["-1", "20170101", "20180101", "20060101"], + "stream": "oper", + "type": "an", + "levtype": "pl", + "param": 155, + }, + 36, + ), + ( + { + "class": "rd", + "expver": "xxxx", + "date": ["-1", "20170101", "20180101", "20060101"], # date 20060101 is not available + "stream": "oper", + "type": "an", + "levtype": "pl", + "param": [155, 138], + }, + 72, + ), + ( + { + "class": "rd", + "expver": "xxxx", + "date": ["-1", "20170101", "20180101", "20060101"], # date 20060101 is not available + "stream": "oper", + "type": "an", + "levtype": "pl", + "param": [155, 60, 138], # param 60 not available + }, + 72, + ), + ( + { + "class": "rd", + "expver": "xxxx", + "date": ["-1", "20170101", "20180101", "20060101"], # date 20060101 is not available + "stream": "oper", + "type": "an", + "levtype": "pl", + "levelist": "300/400/500/700/850/1000", + "param": [155, 60, 138], # param 60 not available + }, + 72, + ), + ( + { + "class": "rd", + "expver": "xxxx", + "date": ["-1", "20170101", "20180101", "20060101"], # date 20060101 is not available + "stream": "oper", + "type": "an", + "levtype": "pl", + "levelist": "300/123/1000", + "param": [155, 60, 138], # param 60 not available + }, + 24, + ), +] + + +@pytest.mark.parametrize("selection, expected_entries", request_entries_tuples) +def test_list_ranges( + simple_fdb_setup_no_subtocs_no_expver_handler, data_path, function_tmp, selection, expected_entries +): + fdb = FDB(simple_fdb_setup_no_subtocs_no_expver_handler) + + yesterday = f"{(datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y%m%d')}" + + source_file_path = data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [ + ("class", "rd"), + ("expver", "xxxx"), + ("date", int(yesterday)), + ], + [ + ("class", "rd"), + ("expver", "xxxx"), + ("date", 20170101), + ], + [ + ("class", "rd"), + ("expver", "xxxx"), + ("date", 20180101), + ], + ], + ["xxxx.d1", "xxxx.d2", "xxxx.d3"], + ) + + with fdb: + for target_file in target_files: + fdb.archive(target_file.read_bytes()) + + all_entries = list(fdb.list(selection)) + assert len(all_entries) == expected_entries diff --git a/tests/fdb_e2e/python_api/util/__init__.py b/tests/fdb_e2e/python_api/util/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fdb_e2e/python_api/util/dispatcher.py b/tests/fdb_e2e/python_api/util/dispatcher.py new file mode 100644 index 000000000..d1146b3e2 --- /dev/null +++ b/tests/fdb_e2e/python_api/util/dispatcher.py @@ -0,0 +1,36 @@ +import logging +import os +import shlex +import subprocess +from pathlib import Path + + +def run_request(cmd: str, workdir: Path, env) -> int: + env = {**os.environ.copy(), **env} + lex = shlex.shlex(cmd) + lex.whitespace_split = True + + lexed_command = list(lex) + + log_file_name = workdir / "test_log.txt" + + with subprocess.Popen( + lexed_command, + env=env, + cwd=workdir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + process.stdin.flush() + process.stdin.close() + + while True: + line = process.stdout.readline() + if not line: + break + logging.debug(line.decode("utf-8").rstrip()) + with log_file_name.open("a+") as log_file: + log_file.write(line.decode("utf-8")) + + return process.returncode diff --git a/tests/fdb_e2e/python_api/util/tools.py b/tests/fdb_e2e/python_api/util/tools.py new file mode 100644 index 000000000..6b7e8a34d --- /dev/null +++ b/tests/fdb_e2e/python_api/util/tools.py @@ -0,0 +1,116 @@ +import datetime +from pathlib import Path + +import eccodes as ec + + +class Tool: + # e.g grib_set -s class=rd,expver=xxxx,type=fc,step=0,date={{ DATE|LONG_DATE }} "{{ SOURCE_DATA }}" data.xxxx.0.grib + @classmethod + def modify_metadata_source_file( + cls, + args: list[tuple[str, str | int]], + source_file: Path, + target_file: Path, + ): + assert source_file.is_file() + template_grib_fd = source_file.open("rb") + + while True: + gid = ec.codes_grib_new_from_file(template_grib_fd) + + if gid is None: + break + + count_data_points = int(ec.codes_get(gid, "numberOfDataPoints")) + count_values = int(ec.codes_get(gid, "numberOfValues")) + + try: + count_missing = int(ec.codes_get(gid, "numberOfMissing")) + assert count_missing == 0 + except ec.KeyValueNotFoundError: + pass + + # This only supports messages without missing datapoints + assert count_data_points == count_values + + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + ec.codes_set(gid, "date", int(yesterday)) + + for k, v in args: + if isinstance(v, int): + ec.codes_set(gid, k, v) + else: + ec.codes_set_string(gid, k, v) + + with target_file.open("a+b") as out: + ec.codes_write(gid, out) + + ec.codes_release(gid) + + template_grib_fd.close() + + +def generate_test_files_key_value( + source_file_path, + function_tmp, + key_value_list: list[list[tuple[str, str | int]]], + output_file_names: list[str], +): + """Creates copies of the GRIB file at source_file_path in function_tmp. For each + element of the key_value_list all given key-value-pairs are modified in the resulting GRIB + file and the corresponding entry from output_file_names is used as a name. + + This means: + + setup_fdb_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "2")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "2")], + ], + [ + "data.xxxx.0.grib", + "data.xxxy.0.grib", + "data.xxxx.1.grib", + "data.xxxy.1.grib", + "data.xxxx.2.grib", + "data.xxxy.2.grib", + ], + ) + + Is creating 6 files with the names of output_file_names. For each file the GRIB metadata + is modified as given in key_value_list. + + + Args: + source_file_path (Path): Path to the template GRIB file + function_tmp (Path): Path to the output folder + key_value_list: List of key-value pair lists for modifying the GRIB metadata + output_file_names: Names of the output files + + Returns: + target_files - A list of paths to the created files + """ + assert len(key_value_list) == len(output_file_names) + + # Data setup + target_files: list[Path] = [] + + for i in range(len(key_value_list)): + cur_key_value_list = key_value_list[i] + output_file_name = output_file_names[i] + target_file_path = function_tmp / f"data.{output_file_name}.grib" + Tool.modify_metadata_source_file( + cur_key_value_list, + source_file=source_file_path, + target_file=target_file_path, + ) + target_files.append(target_file_path) + + return target_files diff --git a/tests/fdb_e2e/tool_tests/CMakeLists.txt b/tests/fdb_e2e/tool_tests/CMakeLists.txt new file mode 100644 index 000000000..0653ff904 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/CMakeLists.txt @@ -0,0 +1,43 @@ +# Set the labels to the simple and yaml config subset for the 'default' tests +if(HAVE_PYTHON_EXTENDED_FDB_TOOL_TESTS) + set(labels "") +else() + set(labels "-m (simple or yaml)") +endif() + +message("Labels for the execution of e2e tests: ${labels}") + +function(add_py_test test) + get_filename_component(stem ${test} NAME_WLE) + add_test( + NAME fdb_tool_tests_${stem} + COMMAND ${Python_EXECUTABLE} + -m pytest ${labels} + --basetemp ${CMAKE_CURRENT_BINARY_DIR}/pytest-tmp + -vv -s ${CMAKE_CURRENT_SOURCE_DIR}/${test} + ) + set(_env + "PYTHONPATH=${CMAKE_BINARY_DIR}/pyfdb-python-package-staging:$ENV{PYTHONPATH}" + #prevent findlibs in pyfdb to pickup any globally installed fdb + "PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH}" + "FDB5_DIR=${CMAKE_BINARY_DIR}" + #prevent findlibs in eccodes-python to pickup any globally installed eccodes + "ECCODES_PYTHON_USE_FINDLIBS=1" + "ECCODES_DIR=$/.." + ) + set_tests_properties(fdb_tool_tests_${stem} + PROPERTIES + RESOURCE_LOCK pytest + ENVIRONMENT "${_env}" + LABELS fdb_tool_tests + ) +endfunction() + +set(test_files + test_fdb_no_subtocs.py + test_fdb_subtocs.py +) + +foreach(test_file ${test_files}) + add_py_test(${test_file}) +endforeach() diff --git a/tests/fdb_e2e/tool_tests/Remarks.md b/tests/fdb_e2e/tool_tests/Remarks.md new file mode 100644 index 000000000..b4269cec9 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/Remarks.md @@ -0,0 +1,52 @@ +Introduced changes +================== +- Added domain=g to fdb-overlay and fdb-hide commands + +FIXED +===== +FAILED test_simple_fdb_no_subtoc_no_expver_handler.py::test_read[/Users/tkremer/Code/stack_2/fdb/tests/fdb_e2e/tool_tests/no_subtoc/read/steprange.sh] - subprocess.CalledProcessError: Command '/Users/tkremer/Code/stack_2/fdb/tests/fdb_e2e/tool_tests/no_subtoc/read/steprange.sh' returned non-zero exit status 1. +FAILED test_simple_fdb_no_subtoc_no_expver_handler.py::test_overlay[/Users/tkremer/Code/stack_2/fdb/tests/fdb_e2e/tool_tests/no_subtoc/overlay/wipe.sh] - subprocess.CalledProcessError: Command '/Users/tkremer/Code/stack_2/fdb/tests/fdb_e2e/tool_tests/no_subtoc/overlay/wipe.sh' returned non-zero exit status 1. +- Added xfail for now till FDB-652 is fixed +FAILED test_simple_fdb_no_subtoc_no_expver_handler.py::test_wipe[/Users/tkremer/Code/stack_2/fdb/tests/fdb_e2e/tool_tests/no_subtoc/wipe/porcelain.sh] - subprocess.CalledProcessError: Command '/Users/tkremer/Code/stack_2/fdb/tests/fdb_e2e/tool_tests/no_subtoc/wipe/porcelain.sh' returned non-zero exit status 255. +- Output changed + +CHANGED TESTS +============= +tests/fdb_e2e/tool_tests/subtocs/wipe/duplicate_data.sh + +- Dropped config_yaml.sh setup as there is virtually no difference to the simple_setup + +TODO +==== +- assert all in conftest during setup of fdb +- delete PATH modification + +# for config_script, cleanup_script, expver, subtocs +("files_config", None, True, False), +("files_config", None, False, False), +("files_config", None, True, True), +("files_config", None, False, True), +("simple_config", None, True, False), +("simple_config", None, False, False), +("simple_config", None, True, True), +("simple_config", None, False, True), +( + "config_yaml", + None, + False, + False, +), # Just checking that the correct config is used from default +( + "config_json", + None, + False, + False, +), # locations. Don't worry about testing all variants for this - +("config_yaml_mars_disks", None, False, False), +( + "tool_json", + None, + False, + False, +), # that can be tested in the above tests +("tool_yaml", None, False, False), diff --git a/tests/fdb_e2e/tool_tests/conftest.py b/tests/fdb_e2e/tool_tests/conftest.py new file mode 100644 index 000000000..810f429d8 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/conftest.py @@ -0,0 +1,71 @@ +import pathlib +from pathlib import Path + +import git +import pytest + + +def get_git_root(path) -> Path: + git_repo = git.Repo(path, search_parent_directories=True) + git_root = git_repo.git.rev_parse("--show-toplevel") + return Path(git_root) + + +def search_no_subtoc_test_sh_files(case: str): + return sorted((get_git_root(__file__) / "tests" / "fdb_e2e" / "tool_tests" / "no_subtocs" / case).glob("*.sh")) + + +def search_subtoc_test_sh_files(case: str): + return sorted((get_git_root(__file__) / "tests" / "fdb_e2e" / "tool_tests" / "subtocs" / case).glob("*.sh")) + + +@pytest.fixture(scope="function") +def test_data_path() -> pathlib.Path: + """ + Provides path to test data + """ + + git_repo = git.Repo(__file__, search_parent_directories=True) + git_root = pathlib.Path(git_repo.git.rev_parse("--show-toplevel")) + path = git_root / "tests" / "fdb_e2e" / "data" + assert path.exists() + return path + + +def pytest_generate_tests(metafunc): + for case in ["info", "hide", "grib2fdb5", "list", "purge", "read", "root", "wipe", "write"]: + if f"{case}_no_subtoc_script" in metafunc.fixturenames: + # Collect your elements here + elements = search_no_subtoc_test_sh_files(case) + metafunc.parametrize(f"{case}_no_subtoc_script", elements, ids=[str(e.name) for e in elements]) + + if f"{case}_subtoc_script" in metafunc.fixturenames: + elements = search_subtoc_test_sh_files(case) + metafunc.parametrize(f"{case}_subtoc_script", elements, ids=[str(e.name) for e in elements]) + + # TODO(TKR): Handle overlay/wipe.sh separately due to FDB-652 + # Once the issue is fixed the lines below can be delete and "overlay" can be added to the case list + # above + if "overlay_no_subtoc_script" in metafunc.fixturenames: + # Collect your elements here + elements = search_no_subtoc_test_sh_files("overlay") + + marked_elements = [] + for el in elements: + if "wipe.sh" in str(el): + marked_elements.append(pytest.param(el, marks=[pytest.mark.xfail])) + else: + marked_elements.append(el) + + metafunc.parametrize("overlay_no_subtoc_script", marked_elements, ids=[str(e.name) for e in elements]) + + if "overlay_subtoc_script" in metafunc.fixturenames: + elements = search_subtoc_test_sh_files("overlay") + marked_elements = [] + for el in elements: + if "wipe.sh" in str(el): + marked_elements.append(pytest.param(el.name, marks=[pytest.mark.xfail])) + else: + marked_elements.append(el) + + metafunc.parametrize("overlay_subtoc_script", marked_elements, ids=[str(e.name) for e in elements]) diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/check.sh b/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/check.sh new file mode 100755 index 000000000..29a0849a3 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/check.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +# Check that the verification keys do what they say they should + +grib_ls -m data.xxxx.grib + + +for invalid_check in "-e xxxy" "-c uk" "-s enfo" "-T fc" +do + + set +e + grib2fdb5 $invalid_check -f data.xxxx.grib + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded with invalid check: $invalid_check" && exit 1 +done + + +for valid_check in "-e xxxx" "-c rd" "-s oper" "-T an" +do + + # Sanity check + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out| tr -d '[:space:]')" != "0" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "No data should be purged with insufficient keys" && exit -1 + + grib2fdb5 $valid_check -f data.xxxx.grib + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out| tr -d '[:space:]')" != "24" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + # TODO(TKR) + # [[ "$(find -name '*.index' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}4{% else %}2{% endif %}" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + + # Ensure there is no data left for the next loop + fdb-wipe --minimum-keys="class,expver" class=rd,expver=xxxx --doit +done + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/ignored_options.sh b/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/ignored_options.sh new file mode 100755 index 000000000..960fda309 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/ignored_options.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +# This is the same as simple.sh, but it sets some options that should have no effect + +grib_ls -m data.xxxx.0.grib + +# Write simple data + +grib2fdb5 -d -1 -f data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +# TODO(TKR) [[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}4{% else %}2{% endif %}" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +# TODO(TKR) [[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}2{% else %}0{% endif %}" ]] && echo "Subtocs incorrect" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# Duplicating data adds, but not to the counts + +grib2fdb5 -d -1 -f data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +# TODO(TKR) [[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}8{% else %}4{% endif %}" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 +# TODO(TKR) [[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}4{% else %}0{% endif %}" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +grib2fdb5 -d -1 -f data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +# TODO(TKR)[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}12{% else %}6{% endif %}" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +# TODO(TKR) [[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}6{% else %}0{% endif %}" ]] && echo "Subtocs incorrect" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# But we can also add to another DB + +grib2fdb5 -d -1 -f data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +# TODO(TKR) [[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}16{% else %}8{% endif %}" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +# TODO(TKR) [[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}8{% else %}0{% endif %}" ]] && echo "Subtocs incorrect" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +grib2fdb5 -d -1 -f data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +# TODO(TKR) [[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}20{% else %}10{% endif %}" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 +# TODO(TKR) [[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "{% if USE_SUBTOCS %}10{% else %}0{% endif %}" ]] && echo "Subtocs incorrect" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/simple.sh b/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/simple.sh new file mode 100755 index 000000000..bac44af43 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/grib2fdb5/simple.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +grib_ls -m data.xxxx.0.grib + +# Write simple data + +grib2fdb5 -f data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# Duplicating data adds, but not to the counts + +grib2fdb5 -f data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +grib2fdb5 -f data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# But we can also add to another DB + +grib2fdb5 -f data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +grib2fdb5 -f data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/hide/list.sh b/tests/fdb_e2e/tool_tests/no_subtocs/hide/list.sh new file mode 100755 index 000000000..780273b60 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/hide/list.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Set up the data for the test +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.0.grib + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +fdb-write data.xxxx.0.grib +fdb-write data.xxxx.1.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +echo $(wc -l < out) +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo " Expecting elements. Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# Without --doit, nothing happens + + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Nothing should change without --doit" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Nothing should change without --doit" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Nothing should change without --doit" && exit -1 + +# Hide some of the data + +fdb-hide class=rd,expver=xxxx,stream=oper,domain=g,date="$YESTERDAY",time=1200 --doit + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Unexpected visible data" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | grep time=0000 | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | grep time=0000 | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | grep time=1200 | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | grep time=1200 | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + +# And we can write data (even to the same DB) after hiding + +fdb-write data.xxxx.0.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/hide/read.sh b/tests/fdb_e2e/tool_tests/no_subtocs/hide/read.sh new file mode 100755 index 000000000..862423ab1 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/hide/read.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +YESTERDAY="$1" + +grib_ls -m data.xxxx.0.grib + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +fdb-write data.xxxx.0.grib +fdb-write data.xxxx.1.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# Without --doit, nothing happens + +fdb-hide class=rd,expver=xxxx,stream=oper,domain=g,date=$YESTERDAY,time=1200 + +fdb-read --extract data.xxxx.1.grib out.1.no-doit.grib +grib_compare out.1.no-doit.grib data.xxxx.1.grib + +# Hide some of the data + +fdb-hide class=rd,expver=xxxx,stream=oper,domain=g,date=$YESTERDAY,time=1200 --doit + +fdb-read --extract data.xxxx.1.grib out.1.hidden.grib +set +e +grib_compare out.1.hidden.grib data.xxxx.1.grib +rc=$? +set -e +[[ $rc = 0 ]] && echo "Retrieved data should not match!" && exit -1 + +grib_copy -w time=0000 data.xxxx.1.grib data.xxxx.1.time0000.grib +grib_compare out.1.hidden.grib data.xxxx.1.time0000.grib + +# And we can write data (even to the same DB) after hiding + +fdb-write data.xxxx.1.grib + +fdb-read --extract data.xxxx.1.grib out.1.rewritten.grib +grib_compare out.1.rewritten.grib data.xxxx.1.grib diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/hide/wipe.sh b/tests/fdb_e2e/tool_tests/no_subtocs/hide/wipe.sh new file mode 100755 index 000000000..42a2c33cd --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/hide/wipe.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +YESTERDAY="$1" + +grib_ls -m data.xxxx.0.grib + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +fdb-write data.xxxx.0.grib +fdb-write data.xxxx.1.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# Hide some of the data +fdb-hide class=rd,expver=xxxx,stream=oper,domain=g,date=$YESTERDAY,time=1200 --doit + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "24" ]] && echo "Unexpected visible data" && exit -1 + +# When we do fdb-wipe, we also wipe the hidden data + +[[ "$(find . -name toc | wc -l| tr -d '[:space:]')" != "2" ]] && "Incorrect data present" && exit -1 +[[ "$(find . -name "*.data" | wc -l| tr -d '[:space:]')" != "4" ]] && "Incorrect data present" && exit -1 +[[ "$(find . -name "*.index" | wc -l| tr -d '[:space:]')" != "4" ]] && "Incorrect data present" && exit -1 + +fdb-wipe class=rd,expver=xxxx --minimum-keys="" --doit + +[[ "$(find . -name toc | wc -l| tr -d '[:space:]')" != "0" ]] && echo "All data should have been wiped" && exit -1 +[[ "$(find . -name "*.data" | wc -l| tr -d '[:space:]')" != "0" ]] && echo "All data should have been wiped" && exit -1 +[[ "$(find . -name "*.index" | wc -l| tr -d '[:space:]')" != "0" ]] && echo "All data should have been wiped" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/info/version.sh b/tests/fdb_e2e/tool_tests/no_subtocs/info/version.sh new file mode 100755 index 000000000..efbf2bb9a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/info/version.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Ensure that the database corresponding to a request does not exist + +ver=$(fdb-info --version) + +[[ ! "$ver" =~ ^5.[0-9]+.[0-9]+$ ]] && echo "expected version number" && exit 1 + +echo $(fdb-info --version) + +if ! (fdb-info --all | grep -q "Version: $ver") +then + echo "Expected version in --all output" + exit 1 +fi + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/list/all.sh b/tests/fdb_e2e/tool_tests/no_subtocs/list/all.sh new file mode 100755 index 000000000..6932a80ce --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/list/all.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Ensure that listing finds the correct data, but excludes duplicates. +set -euxo pipefail + +grib_ls -m data.xxxx.0.grib + +# Set up some regexes for later testing + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_x2="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}" +regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_y2="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}" + +# We loop over all these tests twice. We should get the same listing results both times +# as the default behaviour is to only show the data that would be retrieved (i.e. that +# that matches a MARS request. + +for i in 1 2; +do + + +fdb-write data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then + [[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then [[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then + [[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + + +fdb-write data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then + [[ "$(wc -l < out | tr -d '[:space:]' | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxx.2.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then + [[ "$(wc -l < out | tr -d '[:space:]')" != "120" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxy.2.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "144" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +# End loop with duplicate writes + +done + +exit 0 + diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/list/all_full.sh b/tests/fdb_e2e/tool_tests/no_subtocs/list/all_full.sh new file mode 100755 index 000000000..c0d7054bd --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/list/all_full.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Ensure that listing finds the correct data, but excludes duplicates. +# This is the same data as in the all test, but with a full enumeration +set -euxo pipefail + +grib_ls -m data.xxxx.0.grib + +# Set up some regexes for later testing + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +# We loop over all these tests twice. We should get the same listing results both times +# as the default behaviour is to only show the data that would be retrieved (i.e. that +# that matches a MARS request. + +for i in 0 1; +do + + +fdb-write data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "$((24 + (i*96)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "$((48 + (i*96)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "$((72 + (i*96)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "$((96 + (i*96)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 + + +# End loop with duplicate writes + +done + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/list/locations.sh b/tests/fdb_e2e/tool_tests/no_subtocs/list/locations.sh new file mode 100755 index 000000000..e4c0e495f --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/list/locations.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Ensure that listing finds the correct data, but excludes duplicates. +set -euxo pipefail + +# Check that we can obtain the location of the data +grib_ls -m data.xxxx.grib +fdb-write data.xxxx.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# Specify the location flag + +fdb-list class=rd,expver=xxxx,time=0000 --porcelain --location | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect data found" && exit -1 +[[ "$(grep "root/rd:xxxx" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Locations not reported" && exit -1 + +# And without the location flag, none of the location info is included + +fdb-list class=rd,expver=xxxx,time=0000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect data found" && exit -1 +[[ "$(grep "root/rd:xxxx" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Locations incorrectly reported" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/list/masking.sh b/tests/fdb_e2e/tool_tests/no_subtocs/list/masking.sh new file mode 100755 index 000000000..8fec4cbd8 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/list/masking.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +grib_ls -m data.xxxx.d1.grib + +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Entries should be masked without --full" && exit -1 +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "All entries should be visible with --full" && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "8" ]] && echo "Entries should be masked without full" && exit -1 +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "16" ]] && echo "All entries should be visible with --full" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/list/minimum_keys.sh b/tests/fdb_e2e/tool_tests/no_subtocs/list/minimum_keys.sh new file mode 100755 index 000000000..6f81f9088 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/list/minimum_keys.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Ensure that listing finds the correct data, but excludes duplicates. +set -euxo pipefail + +# Ensure that we can wipe specified databases, but only when fully specified + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +# Check that if any of the keys are missing, then nothing happens + +for invalid_key in class=rd \ + expver=xxxx +do + set +e + fdb-list $invalid_key + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 +done + +# Check that supplying the key makes it work + +fdb-list class=rd,expver=xxxx --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Should work when we specify the keys" && exit -1 + +# Check that we can change the minimum keys + +for invalid_key in class=rd,expver=xxxx \ + class=rd,time=1200 \ + expver=xxxx,time=1200 +do + set +e + fdb-list --minimum-keys=class,expver,time $invalid_key + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 +done + +# And that the purge works now this is supplied + +fdb-list --minimum-keys=class,expver,time class=rd,expver=xxxx,time=1200 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Should work when we specify the keys" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/list/porcelain.sh b/tests/fdb_e2e/tool_tests/no_subtocs/list/porcelain.sh new file mode 100755 index 000000000..d0cf6914d --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/list/porcelain.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# If we don't specify --porcelain, then the expanded request is prepended to the output request. +# Measure the difference! + +grib_ls -m data.xxxx.grib +fdb-write data.xxxx.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# With normal output + +fdb-list class=rd,expver=xxxx | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^list,$' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "MARS request incorrectly reported" && exit -1 +[[ "$(grep '^[ ].*=.*' out | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]class=rd' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "MARS request incorrectly reported" && exit -1 +[[ "$(grep '^[ ]expver=xxxx' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "MARS request incorrectly reported" && exit -1 + + +# With porcelain output + +fdb-list class=rd,expver=xxxx --porcelain | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries reported" && exit -1 + +# Test with expansion of date and param without porcelain + +fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^list,$' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^[ ].*=.*' out | wc -l | tr -d '[:space:]')" != "7" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]class=rd' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]expver=xxxx' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]param=130' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E '^[ ]date=[0-9]{8}' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 + +# Test with expansion of date and param with porcelain + +fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t --porcelain | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries reported" && exit -1 + +# Test with expansion of date and param without porcelain + +fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^list,$' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^[ ].*=.*' out | wc -l | tr -d '[:space:]')" != "7" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]class=rd' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]expver=xxxx' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]param=138' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E '^[ ]date=[0-9]{8}' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 + +# Test with expansion of date and param with porcelain + +fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 --porcelain | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries reported" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/list/ranges.sh b/tests/fdb_e2e/tool_tests/no_subtocs/list/ranges.sh new file mode 100755 index 000000000..848a6563c --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/list/ranges.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euxo pipefail +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +grib_ls -m data.xxxx.d1.grib + +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-write data.xxxx.d3.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list class=rd,expver=xxxx --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103/20060101,stream=oper,type=an,levtype=pl,param=60 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=155 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/overlay/force.sh b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/force.sh new file mode 100755 index 000000000..d00d10527 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/force.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an existing DB, with existing Data + +fdb-write data.xxxy.3.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an overlay to xxxy -- fails without --doit + +set +e +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 +rc=$? +set -e +[[ $rc = 0 ]] && echo "Overlay should not succeed against existing DB" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 + +# And actually create the overlay + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --force + +# The overlaid data should now be visible + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "60" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=0" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=0" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/overlay/hidden.sh b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/hidden.sh new file mode 100755 index 000000000..bc7676103 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/hidden.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an existing DB, with existing Data + +fdb-write data.xxxy.3.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Hide the existing DB + +fdb-hide class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=1200 --doit +fdb-hide class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --doit + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data should be hidden" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be hidden" && exit -1 + +# Create an overlay to xxxy -- fails without --doit + +set +e +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 +rc=$? +set -e +[[ $rc = 0 ]] && echo "Overlay should not succeed against existing DB" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data shouldn't change after failing overlay" && exit -1 + +# And actually create the overlay + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --force + +# The overlaid data should now be visible + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=0" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=0" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/overlay/list.sh b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/list.sh new file mode 100755 index 000000000..3e6a5649a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/list.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an overlay to xxxy + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/overlay/permitted_keys.sh b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/permitted_keys.sh new file mode 100755 index 000000000..5f97114e1 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/permitted_keys.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Changed expver forbidden + +set +e +fdb-overlay --variable-keys="class" \ + class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 +rc=$? +set -e +[[ $rc = 0 ]] && echo "fdb-overlay should not succeed with invalid keys" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No additional data added with non-permitted keys" && exit -1 + +# Changed class forbidden + +set +e +fdb-overlay --variable-keys="expver" \ + class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=od,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 +rc=$? +set -e +[[ $rc = 0 ]] && echo "fdb-overlay should not succeed with invalid keys" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No additional data added with non-permitted keys" && exit -1 + +# And now with all the permitted keys + +fdb-overlay --variable-keys="class,expver" \ + class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/overlay/read.sh b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/read.sh new file mode 100755 index 000000000..3c01deac2 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/read.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create a comparison GRIB + +grib_copy -w time=0000 data.xxxx.grib data.xxxx.0000.grib +grib_set -s expver=xxxy data.xxxx.0000.grib data.xxxy.0000.grib +grib_set -s expver=xxxy data.xxxx.grib data.xxxy.grib + +# Create an overlay to xxxy + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-read --extract data.xxxy.0000.grib out.grib +grib_compare out.grib data.xxxy.0000.grib + +# Show that this is only the data that has been overlaid + +fdb-read --extract data.xxxy.grib out2.grib +set +e +grib_compare out2.grib data.xxxy.grib +rc=$? +set -e +[[ $rc = 0 ]] && echo "Retrieved data should be incomplete" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/overlay/remove.sh b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/remove.sh new file mode 100755 index 000000000..6d655ebce --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/remove.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an overlay to xxxy + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --remove + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/overlay/wipe.sh b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/wipe.sh new file mode 100755 index 000000000..775207f08 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/overlay/wipe.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an overlay to xxxy + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Wipe the DB. Should fail (no wipeable data found) + +set +e +fdb-wipe class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --doit +rc=$? +set -e + +[[ $rc = 0 ]] && echo "Should be nothing to wipe with overlayed data" && exit 1 + +# Write directly to the target + +fdb-write data.xxxy.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "60" ]] && echo "Incorrect number of entries written" && exit -1 + +# We can wipe the data that we added, but don't impact the written data + +fdb-wipe class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=1200/0000 --doit + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/purge/minimum_keys.sh b/tests/fdb_e2e/tool_tests/no_subtocs/purge/minimum_keys.sh new file mode 100755 index 000000000..c3adfa541 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/purge/minimum_keys.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can purge specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 + +# Check that if any of the keys are missing, then nothing happens + +for invalid_key in class=rd,expver=xxxx,stream=oper,date=-1 \ + class=rd,expver=xxxx,stream=oper,time=1200 \ + class=rd,expver=xxxx,date=-1,time=1200 \ + class=rd,stream=oper,date=-1,time=1200 \ + expver=xxxx,stream=oper,date=-1,time=1200 +do + + set +e + fdb-purge $invalid_key --doit + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + +done + +# Check that supplying the key makes it work + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Data should be purged when fully specified" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data should be purged when fully specified" && exit -1 + +# Re-duplicate the data + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 + +# Check that we can change the minimum keys + +for invalid_key in class=rd,time=1200 \ + class=rd,expver=xxxx \ + expver=xxxx,time=1200 +do + + set +e + fdb-purge --minimum-keys=class,expver,time $invalid_key --doit + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + +done + +# And that the purge works now this is supplied + +fdb-purge --minimum-keys=class,expver,time class=rd,expver=xxxx,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Data should be purged when fully specified" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/purge/overspecified.sh b/tests/fdb_e2e/tool_tests/no_subtocs/purge/overspecified.sh new file mode 100755 index 000000000..c583e7886 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/purge/overspecified.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# If we specify such that we select only within a given index, we should never delete data. + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly duplicated" && exit -1 + +# TODO: We may in future support this type of purging. If so, remove the type=an +# test, and consider the tests in partial.sh + +for over in "type=an" "levelist=300" +do + +set +e +fdb-purge --minimum-keys=class,expver class=rd,expver=xxxx,$over --doit +rc=$? +set -e +[[ $rc = 0 ]] && echo "Purging with overspecified key should not have succeeded" && exit 1 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Nothing should change when overspecified" && exit -1 + +done + +fdb-purge --minimum-keys=class,expver class=rd,expver=xxxx --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Full purge should occur when not overspecified" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/purge/porcelain.sh b/tests/fdb_e2e/tool_tests/no_subtocs/purge/porcelain.sh new file mode 100755 index 000000000..09392efee --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/purge/porcelain.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 | tee purge_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "FDB modified without --doit" && exit -1 +(( "$(wc -l < purge_out | tr -d '[:space:]')" < "10" )) && echo "normal purge output incomplete" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --porcelain | tee purge_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(wc -l < purge_out | tr -d '[:space:]')" != "2" ]] && echo "Unexpected file list for removal" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --porcelain --doit | tee purge_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(wc -l < purge_out | tr -d '[:space:]')" != "2" ]] && echo "Unexpected file list for removal" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/purge/simple.sh b/tests/fdb_e2e/tool_tests/no_subtocs/purge/simple.sh new file mode 100755 index 000000000..ea62b59d3 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/purge/simple.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euxo pipefail +# Ensure that we can purge specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "FDB modified without --doit" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "60" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "5" ]] && echo "Incorrect data files created" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "5" ]] && echo "Incorrect indexes created" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=0000 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not fully deduplicated" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data not fully deduplicated" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data not fully deduplicated" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/read/extract.sh b/tests/fdb_e2e/tool_tests/no_subtocs/read/extract.sh new file mode 100755 index 000000000..2d842478a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/read/extract.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.source.xxxy.grib + +fdb-write data.source.xxxx.grib +fdb-write data.source.xxxy.grib +fdb-write data.source.xxxz.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxz" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Select only part of the grib + +grib_copy -w time=1200 data.source.xxxy.grib data.12.xxxy.grib + +# And attempt a read + +fdb-read --extract data.12.xxxy.grib output.grib + +# Check that we have returned the same data as was in the source GRIB + +grib_compare output.grib data.12.xxxy.grib + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/read/quantile.sh b/tests/fdb_e2e/tool_tests/no_subtocs/read/quantile.sh new file mode 100755 index 000000000..c2766b299 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/read/quantile.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.xxxy.grib + +fdb-write data.xxxx.grib +fdb-write data.xxxy.grib +fdb-write data.xxxz.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "3" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "3" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxz" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create a request for reading (only selects +fdb-read req out.grib +grib_compare out.grib data.xxxy.grib + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/read/schema_change.sh b/tests/fdb_e2e/tool_tests/no_subtocs/read/schema_change.sh new file mode 100755 index 000000000..afc7b04cc --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/read/schema_change.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euxo pipefail + +TODAY=$1 +YESTERDAY=$2 +BEFORE_YESTERDAY=$3 +BEFORE_BEFORE_YESTERDAY=$4 + +# Change the schema +# i) Ensure that appropriate copies of the schemas are taken +# ii) Ensure that we can successfully write +# iii) Ensure that we can successfully read + +grib_ls -m data.xxxx.0.grib + +# Rather than using the default schema +# TODO(TKR) The copied schema from /home/fdbdev/etc/fdb/schema is read only and therefore +# we are getting permission denied issues here + +cat > ./etc/fdb/schema < find.1 +[[ "$(wc -l < find.1 | tr -d '[:space:]')" != "1" ]] && echo "Incorrect schema count" && exit -1 +cmp $(cat find.1) ./etc/fdb/schema + +# Change the schema. Change it in such a way that it actually makes a difference for this data! + +cat > ./etc/fdb/schema < find.2 +[[ "$(wc -l < find.2 | tr -d '[:space:]')" != "1" ]] && echo "Incorrect schema count" && exit -1 +cmp $(cat find.2) ./etc/fdb/schema + +# The DBs should have differing schemas + +set +e +cmp $(cat find.1) $(cat find.2) +rc=$? +set -e +[[ $rc = 0 ]] && echo "Stored schemas should differ" && exit -1 + +# Test that we have the data we are expecting + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$TODAY" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$YESTERDAY" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$BEFORE_YESTERDAY" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$BEFORE_BEFORE_YESTERDAY" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$TODAY" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$TODAY" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Check that we can read data from before the change + +# cat > req.xxxx.-3 << EOF +# retrieve,{{ SAMPLE_REQUEST | FLATTEN_MARS_REQUEST({"class": "rd", "expver": "xxxx", "date": (DATE-(3*DAY))|LONG_DATE, "domain": "g"}) }} +# EOF +fdb-read req.xxxx.-3 out.xxxx.-3.grib +grib_compare out.xxxx.-3.grib data.xxxx.-3.grib + +# Check that we can read data from after the change + +# cat > req.xxxx.0 << EOF +# retrieve,{{ SAMPLE_REQUEST | FLATTEN_MARS_REQUEST({"class": "rd", "expver": "xxxx", "date": DATE|LONG_DATE, "domain": "g"}) }} +# EOF +fdb-read req.xxxx.0 out.xxxx.0.grib +grib_compare out.xxxx.0.grib data.xxxx.0.grib + +# Check that we can read + +# cat > req.combined << EOF +# retrieve,{{ SAMPLE_REQUEST | FLATTEN_MARS_REQUEST({"class": "rd", "expver": "xxxx", "date": [(DATE-(3*DAY))|LONG_DATE, (DATE-(2*DAY))|LONG_DATE, (DATE-DAY)|LONG_DATE,DATE|LONG_DATE], "domain": "g"}) }} +# EOF +cat data.xxxx.{-{3,2,1},0}.grib > data.xxxx.combined.grib +fdb-read req.xxxx.combined out.xxxx.combined.grib +grib_compare out.xxxx.combined.grib data.xxxx.combined.grib + +exit 0 + diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/read/simple.sh b/tests/fdb_e2e/tool_tests/no_subtocs/read/simple.sh new file mode 100755 index 000000000..86d5e69ee --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/read/simple.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.source.xxxy.grib + +fdb-write data.source.xxxx.grib +fdb-write data.source.xxxy.grib +fdb-write data.source.xxxz.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxz" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + + +# Create a subselected GRIB for comparison purposes + +grib_copy -w time=1200 data.source.xxxy.grib data.12.xxxy.grib + +# Create a request for reading (only selects + +fdb-read req.xxxy.simple out.grib +grib_compare out.grib data.12.xxxy.grib + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/read/steprange.sh b/tests/fdb_e2e/tool_tests/no_subtocs/read/steprange.sh new file mode 100755 index 000000000..84a02358b --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/read/steprange.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.steprange.xxxy.grib + +fdb-write data.steprange.xxxx.grib +fdb-write data.steprange.xxxy.grib +fdb-write data.steprange.xxxz.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "3" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "3" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxz" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create a request for reading (only selects + +fdb-read req.xxxy.steprange out.grib +grib_compare out.grib data.steprange.xxxy.grib + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/root/create.sh b/tests/fdb_e2e/tool_tests/no_subtocs/root/create.sh new file mode 100755 index 000000000..d39d14f33 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/root/create.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that the database corresponding to a request does not exist + +SIMPLE_REQUEST=$(cat req.xxxy.simple) + +fdb-where --ignore-errors=false $SIMPLE_REQUEST && echo "Database already exists" && exit-1 + +# By default, we don't create a database just by asking where it is + +fdb-root $SIMPLE_REQUEST +fdb-where --ignore-errors=false $SIMPLE_REQUEST && echo "Database incorrectly created" && exit-1 + +# We can create a database if needed! + +fdb-root --create $SIMPLE_REQUEST +fdb-where --ignore-errors=false $SIMPLE_REQUEST + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/damaged.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/damaged.sh new file mode 100755 index 000000000..48c9976f9 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/damaged.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euxo pipefail +# Ensure that we can wipe specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect files present" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect files present" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect files present" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect files present" && exit -1 + +rm -rf $(find . -name '*.data') +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Failed to nuke data files" && exit -1 + +# Check that the wipe command still works! + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.index.extra' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/duplicate_data.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/duplicate_data.sh new file mode 100755 index 000000000..1b5cd2a3f --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/duplicate_data.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "FDB modified without --doit" && exit -1 + + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-write data.xxxx.grib +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/extra.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/extra.sh new file mode 100755 index 000000000..6af111072 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/extra.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euxo pipefail +# +# Ensure that we can wipe specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect files present" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect files present" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect files present" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect files present" && exit -1 + +# Create some additional files + +for f in $(find . -name '*.index') +do + touch $f.extra +done + +[[ "$(find . -name '*.index.extra' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Extra files created incorrectly" && exit -1 + + +set +e +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +rc=$? +set -e + +[[ $rc = 0 ]] && echo "fdb-wipe should not succeed with unexpected data present" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(find . -name '*.index.extra' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Nothing should be removed with extra data" && exit -1 + + +# And forcibly remove everything + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit --unsafe-wipe-all + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.index.extra' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/minimum_keys.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/minimum_keys.sh new file mode 100755 index 000000000..446b172f8 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/minimum_keys.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases, but only when fully specified + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly duplicated" && exit -1 + +# Check that if any of the keys are missing, then nothing happens + +for invalid_key in class=rd,expver=xxxx,stream=oper,date=-1 \ + class=rd,expver=xxxx,stream=oper,time=1200 \ + class=rd,expver=xxxx,date=-1,time=1200 \ + class=rd,stream=oper,date=-1,time=1200 \ + expver=xxxx,stream=oper,date=-1,time=1200 +do + + set +e + fdb-wipe $invalid_key --doit + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + +done + +# Check that supplying the key makes it work + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Data should be purged when fully specified" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 + +# Re-trite the data + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 + +# Check that we can change the minimum keys + +for invalid_key in class=rd,time=1200 \ + class=rd,expver=xxxx \ + expver=xxxx,time=1200 +do + + set +e + fdb-wipe --minimum-keys=class,expver,time $invalid_key --doit + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + +done + +# And that the purge works now this is supplied + +fdb-wipe --minimum-keys=class,expver,time class=rd,expver=xxxx,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/overspecified.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/overspecified.sh new file mode 100755 index 000000000..bc9eea86c --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/overspecified.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# If we specify such that we select only within a given index, we should never delete data. + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +set +e +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,levelist=300 --doit +rc=$? +set -e +[[ $rc = 0 ]] && echo "Should not have succeeded with overspecified key." && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Nothing should change when overspecified" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Full purge should occur when not overspecified" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/partial.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/partial.sh new file mode 100755 index 000000000..64ae1eeee --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/partial.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Do a partial wipe that deletes all of the data + +grib_ls -m data.xxxx.an.grib + +fdb-write data.xxxx.an.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,type=an +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No change should be made without --doit" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,type=an --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find root -name 'schema' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/partial_complete.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/partial_complete.sh new file mode 100755 index 000000000..f945c27fe --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/partial_complete.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can purge a subset of the data within the DB, which +# is indexed according to different second level metadata + +grib_ls -m data.xxxx.an.grib +grib_ls -m data.xxxx.fc.grib + +fdb-write data.xxxx.an.grib +fdb-write data.xxxx.fc.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly written" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,type=fc +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "No change should be made without --doit" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,type=fc --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep type=fc out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep type=an out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect amount of data removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/porcelain.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/porcelain.sh new file mode 100755 index 000000000..0822dceb3 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/porcelain.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 | tee wipe_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 +(( "$(wc -l < wipe_out | tr -d '[:space:]')" < "10" )) && echo "normal wipe output incomplete" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --porcelain | tee wipe_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(wc -l < wipe_out | tr -d '[:space:]')" != "11" ]] && echo "Unexpected file list for removal" && exit -1 # n.b. includes directory + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit --porcelain | tee wipe_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(wc -l < wipe_out | tr -d '[:space:]')" != "24" ]] && echo "Unexpected file list for removal" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/ranges.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/ranges.sh new file mode 100755 index 000000000..c4aba84c7 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/ranges.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +grib_ls -m data.xxxx.d1.grib + +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-write data.xxxx.d3.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1/20180103,time=1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "FDB modified without --doit" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1/20180103,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep date=20170101 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep date=20180103 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=20170101,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep date=20170101 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep date=20180103 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-write data.xxxx.d3.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1/20170101/20180103,time=0000/1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "FDB modified without --doit" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1/20170101/20180103,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "All data should have been deleted" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/wipe/simple.sh b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/simple.sh new file mode 100755 index 000000000..5fdb6fd81 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/wipe/simple.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euxo pipefail +# +# Ensure that we can wipe specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect data files created" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect indexes created" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/write/exclude_filter.sh b/tests/fdb_e2e/tool_tests/no_subtocs/write/exclude_filter.sh new file mode 100755 index 000000000..acc44b039 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/write/exclude_filter.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# test the EXCLUDE FILTER + +grib_ls -m data.xxxx.grib + +fdb-write --exclude-filter=levelist=800/to/900/by/50 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "20" ]] && exit -1 +[[ "$(grep levelist=850 out | wc -l | tr -d '[:space:]')" != "0" ]] && exit -1 + +fdb-wipe --doit --minimum-keys=class,expver class=rd,expver=xxxx +fdb-write --exclude-filter=time=0000 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && exit -1 + +fdb-wipe --doit --minimum-keys=class,expver class=rd,expver=xxxx +fdb-write --exclude-filter=time=0000/1200,levelist=500/to/1500/by/100 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "6" ]] && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "6" ]] && exit -1 + +exit 0 + diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/write/include_filter.sh b/tests/fdb_e2e/tool_tests/no_subtocs/write/include_filter.sh new file mode 100755 index 000000000..9c812c458 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/write/include_filter.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# test the INCLUDE FILTER + +grib_ls -m data.xxxx.grib + +fdb-write --include-filter=time=0000 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep -v time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && exit -1 + +# If we filter on a non-existent experiment, we should get no data +fdb-wipe --doit --minimum-keys=class,expver class=rd,expver=xxxx +fdb-write --include-filter=expver=yyyy data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && exit -1 + +fdb-write --include-filter=param=vo data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "6" ]] && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "6" ]] && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/write/intersection_filter.sh b/tests/fdb_e2e/tool_tests/no_subtocs/write/intersection_filter.sh new file mode 100755 index 000000000..f442ad9f8 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/write/intersection_filter.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# test the INCLUDE FILTER + +grib_ls -m data.xxxx.grib + +fdb-write --exclude-filter=time=1200 --include-filter=time=1200 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data written when all should be excluded" && exit -1 + +fdb-write --exclude-filter=time=1200/0000,levelist=500/to/1500/by/100 --include-filter=time=1200 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "6" ]] && echo "Incorrect number of elements written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Incorrect amount of data written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/no_subtocs/write/simple.sh b/tests/fdb_e2e/tool_tests/no_subtocs/write/simple.sh new file mode 100755 index 000000000..5c5b6a5f4 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/no_subtocs/write/simple.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.xxxx.0.grib + +# Write simple data + +fdb-write data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# Duplicating data adds, but not to the counts + +fdb-write data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +fdb-write data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# But we can also add to another DB + +fdb-write data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +fdb-write data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Subtocs incorrect" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/check.sh b/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/check.sh new file mode 100755 index 000000000..2b0634c0b --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/check.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +# Check that the verification keys do what they say they should + +grib_ls -m data.xxxx.grib + + +for invalid_check in "-e xxxy" "-c uk" "-s enfo" "-T fc" +do + + set +e + grib2fdb5 $invalid_check -f data.xxxx.grib + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded with invalid check: $invalid_check" && exit 1 +done + + +for valid_check in "-e xxxx" "-c rd" "-s oper" "-T an" +do + + # Sanity check + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out| tr -d '[:space:]')" != "0" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "0" ]] && echo "No data should be purged with insufficient keys" && exit -1 + + grib2fdb5 $valid_check -f data.xxxx.grib + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out| tr -d '[:space:]')" != "24" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + + # Ensure there is no data left for the next loop + fdb-wipe --minimum-keys="class,expver" class=rd,expver=xxxx --doit +done + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/ignored_options.sh b/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/ignored_options.sh new file mode 100755 index 000000000..1560d440c --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/ignored_options.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +# This is the same as simple.sh, but it sets some options that should have no effect + +grib_ls -m data.xxxx.0.grib + +# Write simple data + +grib2fdb5 -d -1 -f data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Subtocs incorrect" && exit -1 + +# Duplicating data adds, but not to the counts + +grib2fdb5 -d -1 -f data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +grib2fdb5 -d -1 -f data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "12" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "6" ]] && echo "Subtocs incorrect" && exit -1 + +# But we can also add to another DB + +grib2fdb5 -d -1 -f data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "16" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "8" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +grib2fdb5 -d -1 -f data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l| tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l| tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l| tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l| tr -d '[:space:]')" != "20" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l| tr -d '[:space:]')" != "10" ]] && echo "Subtocs incorrect" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/simple.sh b/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/simple.sh new file mode 100755 index 000000000..4112d773e --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/grib2fdb5/simple.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +grib_ls -m data.xxxx.0.grib + +# Write simple data + +grib2fdb5 -f data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Subtocs incorrect" && exit -1 + +# Duplicating data adds, but not to the counts + +grib2fdb5 -f data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +grib2fdb5 -f data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Subtocs incorrect" && exit -1 + +# But we can also add to another DB + +grib2fdb5 -f data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "16" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +grib2fdb5 -f data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "20" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "10" ]] && echo "Subtocs incorrect" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/hide/list.sh b/tests/fdb_e2e/tool_tests/subtocs/hide/list.sh new file mode 100755 index 000000000..780273b60 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/hide/list.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Set up the data for the test +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.0.grib + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +fdb-write data.xxxx.0.grib +fdb-write data.xxxx.1.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +echo $(wc -l < out) +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo " Expecting elements. Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# Without --doit, nothing happens + + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Nothing should change without --doit" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Nothing should change without --doit" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Nothing should change without --doit" && exit -1 + +# Hide some of the data + +fdb-hide class=rd,expver=xxxx,stream=oper,domain=g,date="$YESTERDAY",time=1200 --doit + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Unexpected visible data" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | grep time=0000 | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | grep time=0000 | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | grep time=1200 | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | grep time=1200 | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + +# And we can write data (even to the same DB) after hiding + +fdb-write data.xxxx.0.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/hide/read.sh b/tests/fdb_e2e/tool_tests/subtocs/hide/read.sh new file mode 100755 index 000000000..862423ab1 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/hide/read.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +YESTERDAY="$1" + +grib_ls -m data.xxxx.0.grib + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +fdb-write data.xxxx.0.grib +fdb-write data.xxxx.1.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# Without --doit, nothing happens + +fdb-hide class=rd,expver=xxxx,stream=oper,domain=g,date=$YESTERDAY,time=1200 + +fdb-read --extract data.xxxx.1.grib out.1.no-doit.grib +grib_compare out.1.no-doit.grib data.xxxx.1.grib + +# Hide some of the data + +fdb-hide class=rd,expver=xxxx,stream=oper,domain=g,date=$YESTERDAY,time=1200 --doit + +fdb-read --extract data.xxxx.1.grib out.1.hidden.grib +set +e +grib_compare out.1.hidden.grib data.xxxx.1.grib +rc=$? +set -e +[[ $rc = 0 ]] && echo "Retrieved data should not match!" && exit -1 + +grib_copy -w time=0000 data.xxxx.1.grib data.xxxx.1.time0000.grib +grib_compare out.1.hidden.grib data.xxxx.1.time0000.grib + +# And we can write data (even to the same DB) after hiding + +fdb-write data.xxxx.1.grib + +fdb-read --extract data.xxxx.1.grib out.1.rewritten.grib +grib_compare out.1.rewritten.grib data.xxxx.1.grib diff --git a/tests/fdb_e2e/tool_tests/subtocs/hide/wipe.sh b/tests/fdb_e2e/tool_tests/subtocs/hide/wipe.sh new file mode 100755 index 000000000..198c6256a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/hide/wipe.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Set up the data for the test + +set -euxo pipefail + +YESTERDAY="$1" + +grib_ls -m data.xxxx.0.grib + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +fdb-write data.xxxx.0.grib +fdb-write data.xxxx.1.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l| tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# Hide some of the data +fdb-hide class=rd,expver=xxxx,stream=oper,domain=g,date=$YESTERDAY,time=1200 --doit + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out| tr -d '[:space:]')" != "24" ]] && echo "Unexpected visible data" && exit -1 + +# When we do fdb-wipe, we also wipe the hidden data + +[[ "$(find . -name toc | wc -l| tr -d '[:space:]')" != "2" ]] && echo "Incorrect data present" && exit -1 +[[ "$(find . -name "toc.*" | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Incorrect data present" && exit -1 +[[ "$(find . -name "*.data" | wc -l| tr -d '[:space:]')" != "4" ]] && echo "Incorrect data present" && exit -1 +[[ "$(find . -name "*.index" | wc -l| tr -d '[:space:]')" != "8" ]] && echo "Incorrect data present" && exit -1 + +fdb-wipe class=rd,expver=xxxx --minimum-keys="" --doit + +[[ "$(find . -name toc | wc -l| tr -d '[:space:]')" != "0" ]] && echo "All data should have been wiped" && exit -1 +[[ "$(find . -name "*.data" | wc -l| tr -d '[:space:]')" != "0" ]] && echo "All data should have been wiped" && exit -1 +[[ "$(find . -name "*.index" | wc -l| tr -d '[:space:]')" != "0" ]] && echo "All data should have been wiped" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/info/version.sh b/tests/fdb_e2e/tool_tests/subtocs/info/version.sh new file mode 100755 index 000000000..efbf2bb9a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/info/version.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Ensure that the database corresponding to a request does not exist + +ver=$(fdb-info --version) + +[[ ! "$ver" =~ ^5.[0-9]+.[0-9]+$ ]] && echo "expected version number" && exit 1 + +echo $(fdb-info --version) + +if ! (fdb-info --all | grep -q "Version: $ver") +then + echo "Expected version in --all output" + exit 1 +fi + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/list/all.sh b/tests/fdb_e2e/tool_tests/subtocs/list/all.sh new file mode 100755 index 000000000..6932a80ce --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/list/all.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Ensure that listing finds the correct data, but excludes duplicates. +set -euxo pipefail + +grib_ls -m data.xxxx.0.grib + +# Set up some regexes for later testing + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_x2="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}" +regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_y2="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}" + +# We loop over all these tests twice. We should get the same listing results both times +# as the default behaviour is to only show the data that would be retrieved (i.e. that +# that matches a MARS request. + +for i in 1 2; +do + + +fdb-write data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then + [[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then [[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then + [[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + + +fdb-write data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then + [[ "$(wc -l < out | tr -d '[:space:]' | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxx.2.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +if [[ "$i" = "1" ]]; then + [[ "$(wc -l < out | tr -d '[:space:]')" != "120" ]] && echo "Incorrect number of entries written" && exit -1 + [[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +fi +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxy.2.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "144" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x2" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y2" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + + +# End loop with duplicate writes + +done + +exit 0 + diff --git a/tests/fdb_e2e/tool_tests/subtocs/list/all_full.sh b/tests/fdb_e2e/tool_tests/subtocs/list/all_full.sh new file mode 100755 index 000000000..c0d7054bd --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/list/all_full.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Ensure that listing finds the correct data, but excludes duplicates. +# This is the same data as in the all test, but with a full enumeration +set -euxo pipefail + +grib_ls -m data.xxxx.0.grib + +# Set up some regexes for later testing + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +# We loop over all these tests twice. We should get the same listing results both times +# as the default behaviour is to only show the data that would be retrieved (i.e. that +# that matches a MARS request. + +for i in 0 1; +do + + +fdb-write data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "$((24 + (i*96)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "$((48 + (i*96)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "$((72 + (i*96)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "$((0+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 + + +fdb-write data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "$((96 + (i*96)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_x1" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y0" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E "$regex_y1" out | wc -l | tr -d '[:space:]')" != "$((24+(24*i)))" ]] && echo "Incorrect number of entries written" && exit -1 + + +# End loop with duplicate writes + +done + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/list/locations.sh b/tests/fdb_e2e/tool_tests/subtocs/list/locations.sh new file mode 100755 index 000000000..e4c0e495f --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/list/locations.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Ensure that listing finds the correct data, but excludes duplicates. +set -euxo pipefail + +# Check that we can obtain the location of the data +grib_ls -m data.xxxx.grib +fdb-write data.xxxx.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# Specify the location flag + +fdb-list class=rd,expver=xxxx,time=0000 --porcelain --location | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect data found" && exit -1 +[[ "$(grep "root/rd:xxxx" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Locations not reported" && exit -1 + +# And without the location flag, none of the location info is included + +fdb-list class=rd,expver=xxxx,time=0000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect data found" && exit -1 +[[ "$(grep "root/rd:xxxx" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Locations incorrectly reported" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/list/masking.sh b/tests/fdb_e2e/tool_tests/subtocs/list/masking.sh new file mode 100755 index 000000000..8fec4cbd8 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/list/masking.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +grib_ls -m data.xxxx.d1.grib + +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Entries should be masked without --full" && exit -1 +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "All entries should be visible with --full" && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "8" ]] && echo "Entries should be masked without full" && exit -1 +fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "16" ]] && echo "All entries should be visible with --full" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/list/minimum_keys.sh b/tests/fdb_e2e/tool_tests/subtocs/list/minimum_keys.sh new file mode 100755 index 000000000..6f81f9088 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/list/minimum_keys.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Ensure that listing finds the correct data, but excludes duplicates. +set -euxo pipefail + +# Ensure that we can wipe specified databases, but only when fully specified + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +# Check that if any of the keys are missing, then nothing happens + +for invalid_key in class=rd \ + expver=xxxx +do + set +e + fdb-list $invalid_key + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 +done + +# Check that supplying the key makes it work + +fdb-list class=rd,expver=xxxx --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Should work when we specify the keys" && exit -1 + +# Check that we can change the minimum keys + +for invalid_key in class=rd,expver=xxxx \ + class=rd,time=1200 \ + expver=xxxx,time=1200 +do + set +e + fdb-list --minimum-keys=class,expver,time $invalid_key + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 +done + +# And that the purge works now this is supplied + +fdb-list --minimum-keys=class,expver,time class=rd,expver=xxxx,time=1200 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Should work when we specify the keys" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/list/porcelain.sh b/tests/fdb_e2e/tool_tests/subtocs/list/porcelain.sh new file mode 100755 index 000000000..d0cf6914d --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/list/porcelain.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# If we don't specify --porcelain, then the expanded request is prepended to the output request. +# Measure the difference! + +grib_ls -m data.xxxx.grib +fdb-write data.xxxx.grib + +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +# With normal output + +fdb-list class=rd,expver=xxxx | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^list,$' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "MARS request incorrectly reported" && exit -1 +[[ "$(grep '^[ ].*=.*' out | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]class=rd' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "MARS request incorrectly reported" && exit -1 +[[ "$(grep '^[ ]expver=xxxx' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "MARS request incorrectly reported" && exit -1 + + +# With porcelain output + +fdb-list class=rd,expver=xxxx --porcelain | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries reported" && exit -1 + +# Test with expansion of date and param without porcelain + +fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^list,$' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^[ ].*=.*' out | wc -l | tr -d '[:space:]')" != "7" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]class=rd' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]expver=xxxx' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]param=130' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E '^[ ]date=[0-9]{8}' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 + +# Test with expansion of date and param with porcelain + +fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t --porcelain | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries reported" && exit -1 + +# Test with expansion of date and param without porcelain + +fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^list,$' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(grep '^[ ].*=.*' out | wc -l | tr -d '[:space:]')" != "7" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]class=rd' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]expver=xxxx' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep '^[ ]param=138' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep -E '^[ ]date=[0-9]{8}' out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 + +# Test with expansion of date and param with porcelain + +fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 --porcelain | tee out +[[ "$(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries reported" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries reported" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/list/ranges.sh b/tests/fdb_e2e/tool_tests/subtocs/list/ranges.sh new file mode 100755 index 000000000..848a6563c --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/list/ranges.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euxo pipefail +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +grib_ls -m data.xxxx.d1.grib + +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-write data.xxxx.d3.grib +fdb-list --all --minimum-keys="" --porcelain --full | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list class=rd,expver=xxxx --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103/20060101,stream=oper,type=an,levtype=pl,param=60 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=155 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && exit -1 + +fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/overlay/force.sh b/tests/fdb_e2e/tool_tests/subtocs/overlay/force.sh new file mode 100755 index 000000000..d00d10527 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/overlay/force.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an existing DB, with existing Data + +fdb-write data.xxxy.3.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an overlay to xxxy -- fails without --doit + +set +e +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 +rc=$? +set -e +[[ $rc = 0 ]] && echo "Overlay should not succeed against existing DB" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 + +# And actually create the overlay + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --force + +# The overlaid data should now be visible + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "60" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=0" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=0" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/overlay/hidden.sh b/tests/fdb_e2e/tool_tests/subtocs/overlay/hidden.sh new file mode 100755 index 000000000..bc7676103 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/overlay/hidden.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an existing DB, with existing Data + +fdb-write data.xxxy.3.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=3" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Hide the existing DB + +fdb-hide class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=1200 --doit +fdb-hide class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --doit + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data should be hidden" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be hidden" && exit -1 + +# Create an overlay to xxxy -- fails without --doit + +set +e +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 +rc=$? +set -e +[[ $rc = 0 ]] && echo "Overlay should not succeed against existing DB" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "step=0" | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data shouldn't change after failing overlay" && exit -1 + +# And actually create the overlay + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --force + +# The overlaid data should now be visible + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | grep "step=0" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data shouldn't change after failing overlay" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | grep "step=0" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data shouldn't change after failing overlay" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/overlay/list.sh b/tests/fdb_e2e/tool_tests/subtocs/overlay/list.sh new file mode 100755 index 000000000..3e6a5649a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/overlay/list.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an overlay to xxxy + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/overlay/permitted_keys.sh b/tests/fdb_e2e/tool_tests/subtocs/overlay/permitted_keys.sh new file mode 100755 index 000000000..5f97114e1 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/overlay/permitted_keys.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Changed expver forbidden + +set +e +fdb-overlay --variable-keys="class" \ + class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 +rc=$? +set -e +[[ $rc = 0 ]] && echo "fdb-overlay should not succeed with invalid keys" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No additional data added with non-permitted keys" && exit -1 + +# Changed class forbidden + +set +e +fdb-overlay --variable-keys="expver" \ + class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=od,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 +rc=$? +set -e +[[ $rc = 0 ]] && echo "fdb-overlay should not succeed with invalid keys" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No additional data added with non-permitted keys" && exit -1 + +# And now with all the permitted keys + +fdb-overlay --variable-keys="class,expver" \ + class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/overlay/read.sh b/tests/fdb_e2e/tool_tests/subtocs/overlay/read.sh new file mode 100755 index 000000000..3c01deac2 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/overlay/read.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create a comparison GRIB + +grib_copy -w time=0000 data.xxxx.grib data.xxxx.0000.grib +grib_set -s expver=xxxy data.xxxx.0000.grib data.xxxy.0000.grib +grib_set -s expver=xxxy data.xxxx.grib data.xxxy.grib + +# Create an overlay to xxxy + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-read --extract data.xxxy.0000.grib out.grib +grib_compare out.grib data.xxxy.0000.grib + +# Show that this is only the data that has been overlaid + +fdb-read --extract data.xxxy.grib out2.grib +set +e +grib_compare out2.grib data.xxxy.grib +rc=$? +set -e +[[ $rc = 0 ]] && echo "Retrieved data should be incomplete" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/overlay/remove.sh b/tests/fdb_e2e/tool_tests/subtocs/overlay/remove.sh new file mode 100755 index 000000000..6d655ebce --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/overlay/remove.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an overlay to xxxy + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --remove + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/overlay/wipe.sh b/tests/fdb_e2e/tool_tests/subtocs/overlay/wipe.sh new file mode 100755 index 000000000..775207f08 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/overlay/wipe.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euxo pipefail + +YESTERDAY=$1 + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create an overlay to xxxy + +fdb-overlay class=rd,expver=xxxx,stream=oper,date=$YESTERDAY,domain=g,time=0000 \ + class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Wipe the DB. Should fail (no wipeable data found) + +set +e +fdb-wipe class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=0000 --doit +rc=$? +set -e + +[[ $rc = 0 ]] && echo "Should be nothing to wipe with overlayed data" && exit 1 + +# Write directly to the target + +fdb-write data.xxxy.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "60" ]] && echo "Incorrect number of entries written" && exit -1 + +# We can wipe the data that we added, but don't impact the written data + +fdb-wipe class=rd,expver=xxxy,stream=oper,date=$YESTERDAY,domain=g,time=1200/0000 --doit + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect number of entries written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/purge/minimum_keys.sh b/tests/fdb_e2e/tool_tests/subtocs/purge/minimum_keys.sh new file mode 100755 index 000000000..f2f2291ba --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/purge/minimum_keys.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can purge specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly duplicated" && exit -1 + +# Check that if any of the keys are missing, then nothing happens + +for invalid_key in class=rd,expver=xxxx,stream=oper,date=-1 \ + class=rd,expver=xxxx,stream=oper,time=1200 \ + class=rd,expver=xxxx,date=-1,time=1200 \ + class=rd,stream=oper,date=-1,time=1200 \ + expver=xxxx,stream=oper,date=-1,time=1200 +do + + set +e + fdb-purge $invalid_key --doit + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "No data should be purged with insufficient keys" && exit -1 + +done + +# Check that supplying the key makes it work + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "5" ]] && echo "Data should be purged when fully specified" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data should be purged when fully specified" && exit -1 + +# Re-duplicate the data + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Data not correctly duplicated" && exit -1 + +# Check that we can change the minimum keys + +for invalid_key in class=rd,time=1200 \ + class=rd,expver=xxxx \ + expver=xxxx,time=1200 +do + + set +e + fdb-purge --minimum-keys=class,expver,time $invalid_key --doit + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "No data should be purged with insufficient keys" && exit -1 + +done + +# And that the purge works now this is supplied + +fdb-purge --minimum-keys=class,expver,time class=rd,expver=xxxx,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data should be purged when fully specified" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/purge/overspecified.sh b/tests/fdb_e2e/tool_tests/subtocs/purge/overspecified.sh new file mode 100755 index 000000000..c583e7886 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/purge/overspecified.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# If we specify such that we select only within a given index, we should never delete data. + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly duplicated" && exit -1 + +# TODO: We may in future support this type of purging. If so, remove the type=an +# test, and consider the tests in partial.sh + +for over in "type=an" "levelist=300" +do + +set +e +fdb-purge --minimum-keys=class,expver class=rd,expver=xxxx,$over --doit +rc=$? +set -e +[[ $rc = 0 ]] && echo "Purging with overspecified key should not have succeeded" && exit 1 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Nothing should change when overspecified" && exit -1 + +done + +fdb-purge --minimum-keys=class,expver class=rd,expver=xxxx --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Full purge should occur when not overspecified" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/purge/porcelain.sh b/tests/fdb_e2e/tool_tests/subtocs/purge/porcelain.sh new file mode 100755 index 000000000..373a123b7 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/purge/porcelain.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly duplicated" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 | tee purge_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "FDB modified without --doit" && exit -1 +(( "$(wc -l < purge_out | tr -d '[:space:]')" < "10" )) && echo "normal purge output incomplete" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --porcelain | tee purge_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(wc -l < purge_out | tr -d '[:space:]')" != "6" ]] && echo "Unexpected file list for removal" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --porcelain --doit | tee purge_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(wc -l < purge_out | tr -d '[:space:]')" != "6" ]] && echo "Unexpected file list for removal" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "5" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/purge/simple.sh b/tests/fdb_e2e/tool_tests/subtocs/purge/simple.sh new file mode 100755 index 000000000..0dc1b3c4a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/purge/simple.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euxo pipefail +# Ensure that we can purge specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly duplicated" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "FDB modified without --doit" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "5" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "60" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "5" ]] && echo "Incorrect data files created" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "9" ]] && echo "Incorrect indexes created" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=0000 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-purge class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not fully deduplicated" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data not fully deduplicated" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data not fully deduplicated" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/read/extract.sh b/tests/fdb_e2e/tool_tests/subtocs/read/extract.sh new file mode 100755 index 000000000..2d842478a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/read/extract.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.source.xxxy.grib + +fdb-write data.source.xxxx.grib +fdb-write data.source.xxxy.grib +fdb-write data.source.xxxz.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxz" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Select only part of the grib + +grib_copy -w time=1200 data.source.xxxy.grib data.12.xxxy.grib + +# And attempt a read + +fdb-read --extract data.12.xxxy.grib output.grib + +# Check that we have returned the same data as was in the source GRIB + +grib_compare output.grib data.12.xxxy.grib + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/read/quantile.sh b/tests/fdb_e2e/tool_tests/subtocs/read/quantile.sh new file mode 100755 index 000000000..c2766b299 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/read/quantile.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.xxxy.grib + +fdb-write data.xxxx.grib +fdb-write data.xxxy.grib +fdb-write data.xxxz.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "3" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "3" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxz" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create a request for reading (only selects +fdb-read req out.grib +grib_compare out.grib data.xxxy.grib + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/read/schema_change.sh b/tests/fdb_e2e/tool_tests/subtocs/read/schema_change.sh new file mode 100755 index 000000000..afc7b04cc --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/read/schema_change.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euxo pipefail + +TODAY=$1 +YESTERDAY=$2 +BEFORE_YESTERDAY=$3 +BEFORE_BEFORE_YESTERDAY=$4 + +# Change the schema +# i) Ensure that appropriate copies of the schemas are taken +# ii) Ensure that we can successfully write +# iii) Ensure that we can successfully read + +grib_ls -m data.xxxx.0.grib + +# Rather than using the default schema +# TODO(TKR) The copied schema from /home/fdbdev/etc/fdb/schema is read only and therefore +# we are getting permission denied issues here + +cat > ./etc/fdb/schema < find.1 +[[ "$(wc -l < find.1 | tr -d '[:space:]')" != "1" ]] && echo "Incorrect schema count" && exit -1 +cmp $(cat find.1) ./etc/fdb/schema + +# Change the schema. Change it in such a way that it actually makes a difference for this data! + +cat > ./etc/fdb/schema < find.2 +[[ "$(wc -l < find.2 | tr -d '[:space:]')" != "1" ]] && echo "Incorrect schema count" && exit -1 +cmp $(cat find.2) ./etc/fdb/schema + +# The DBs should have differing schemas + +set +e +cmp $(cat find.1) $(cat find.2) +rc=$? +set -e +[[ $rc = 0 ]] && echo "Stored schemas should differ" && exit -1 + +# Test that we have the data we are expecting + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$TODAY" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$YESTERDAY" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$BEFORE_YESTERDAY" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$BEFORE_BEFORE_YESTERDAY" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$TODAY" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "date=$TODAY" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + +# Check that we can read data from before the change + +# cat > req.xxxx.-3 << EOF +# retrieve,{{ SAMPLE_REQUEST | FLATTEN_MARS_REQUEST({"class": "rd", "expver": "xxxx", "date": (DATE-(3*DAY))|LONG_DATE, "domain": "g"}) }} +# EOF +fdb-read req.xxxx.-3 out.xxxx.-3.grib +grib_compare out.xxxx.-3.grib data.xxxx.-3.grib + +# Check that we can read data from after the change + +# cat > req.xxxx.0 << EOF +# retrieve,{{ SAMPLE_REQUEST | FLATTEN_MARS_REQUEST({"class": "rd", "expver": "xxxx", "date": DATE|LONG_DATE, "domain": "g"}) }} +# EOF +fdb-read req.xxxx.0 out.xxxx.0.grib +grib_compare out.xxxx.0.grib data.xxxx.0.grib + +# Check that we can read + +# cat > req.combined << EOF +# retrieve,{{ SAMPLE_REQUEST | FLATTEN_MARS_REQUEST({"class": "rd", "expver": "xxxx", "date": [(DATE-(3*DAY))|LONG_DATE, (DATE-(2*DAY))|LONG_DATE, (DATE-DAY)|LONG_DATE,DATE|LONG_DATE], "domain": "g"}) }} +# EOF +cat data.xxxx.{-{3,2,1},0}.grib > data.xxxx.combined.grib +fdb-read req.xxxx.combined out.xxxx.combined.grib +grib_compare out.xxxx.combined.grib data.xxxx.combined.grib + +exit 0 + diff --git a/tests/fdb_e2e/tool_tests/subtocs/read/simple.sh b/tests/fdb_e2e/tool_tests/subtocs/read/simple.sh new file mode 100755 index 000000000..86d5e69ee --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/read/simple.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.source.xxxy.grib + +fdb-write data.source.xxxx.grib +fdb-write data.source.xxxy.grib +fdb-write data.source.xxxz.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxz" out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=1200" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | grep "time=0000" | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 + + +# Create a subselected GRIB for comparison purposes + +grib_copy -w time=1200 data.source.xxxy.grib data.12.xxxy.grib + +# Create a request for reading (only selects + +fdb-read req.xxxy.simple out.grib +grib_compare out.grib data.12.xxxy.grib + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/read/steprange.sh b/tests/fdb_e2e/tool_tests/subtocs/read/steprange.sh new file mode 100755 index 000000000..84a02358b --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/read/steprange.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.steprange.xxxy.grib + +fdb-write data.steprange.xxxx.grib +fdb-write data.steprange.xxxy.grib +fdb-write data.steprange.xxxz.grib + +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "3" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "3" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxx" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxy" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep "expver=xxxz" out | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect number of entries written" && exit -1 + +# Create a request for reading (only selects + +fdb-read req.xxxy.steprange out.grib +grib_compare out.grib data.steprange.xxxy.grib + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/root/create.sh b/tests/fdb_e2e/tool_tests/subtocs/root/create.sh new file mode 100755 index 000000000..19895e671 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/root/create.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that the database corresponding to a request does not exist + +SIMPLE_REQUEST=$(cat req.xxxy.simple) + +fdb-where --ignore-errors=false $SIMPLE_REQUEST && echo "Database already exists" && exit -1 + +# By default, we don't create a database just by asking where it is + +fdb-root $SIMPLE_REQUEST +fdb-where --ignore-errors=false $SIMPLE_REQUEST && echo "Database incorrectly created" && exit -1 + +# We can create a database if needed! + +fdb-root --create $SIMPLE_REQUEST +fdb-where --ignore-errors=false $SIMPLE_REQUEST + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/damaged.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/damaged.sh new file mode 100755 index 000000000..824f8722a --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/damaged.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euxo pipefail +# Ensure that we can wipe specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect files present" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect files present" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect files present" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Incorrect files present" && exit -1 + +rm -rf $(find . -name '*.data') +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Failed to nuke data files" && exit -1 + +# Check that the wipe command still works! + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.index.extra' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/duplicate_data.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/duplicate_data.sh new file mode 100755 index 000000000..4f10feaaa --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/duplicate_data.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "FDB modified without --doit" && exit -1 + + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-write data.xxxx.grib +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/extra.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/extra.sh new file mode 100755 index 000000000..62c481a92 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/extra.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euxo pipefail +# +# Ensure that we can wipe specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect files present" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect files present" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect files present" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Incorrect files present" && exit -1 + +# Create some additional files + +for f in $(find . -name '*.index') +do + touch $f.extra +done + +[[ "$(find . -name '*.index.extra' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Extra files created incorrectly" && exit -1 + + +set +e +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +rc=$? +set -e + +[[ $rc = 0 ]] && echo "fdb-wipe should not succeed with unexpected data present" && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Nothing should be removed with extra data" && exit -1 +[[ "$(find . -name '*.index.extra' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Nothing should be removed with extra data" && exit -1 + + +# And forcibly remove everything + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit --unsafe-wipe-all + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.index.extra' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/minimum_keys.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/minimum_keys.sh new file mode 100755 index 000000000..63d9fb63e --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/minimum_keys.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases, but only when fully specified + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly duplicated" && exit -1 + +# Check that if any of the keys are missing, then nothing happens + +for invalid_key in class=rd,expver=xxxx,stream=oper,date=-1 \ + class=rd,expver=xxxx,stream=oper,time=1200 \ + class=rd,expver=xxxx,date=-1,time=1200 \ + class=rd,stream=oper,date=-1,time=1200 \ + expver=xxxx,stream=oper,date=-1,time=1200 +do + + set +e + fdb-wipe $invalid_key --doit + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + +done + +# Check that supplying the key makes it work + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data should be purged when fully specified" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 + +# Re-trite the data + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 + +# Check that we can change the minimum keys + +for invalid_key in class=rd,time=1200 \ + class=rd,expver=xxxx \ + expver=xxxx,time=1200 +do + + set +e + fdb-wipe --minimum-keys=class,expver,time $invalid_key --doit + rc=$? + set -e + + [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1 + + fdb-list --all --full --minimum-keys="" --porcelain | tee out + [[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "No data should be purged with insufficient keys" && exit -1 + [[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "No data should be purged with insufficient keys" && exit -1 + +done + +# And that the purge works now this is supplied + +fdb-wipe --minimum-keys=class,expver,time class=rd,expver=xxxx,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data should be purged when fully specified" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/overspecified.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/overspecified.sh new file mode 100755 index 000000000..bc9eea86c --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/overspecified.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# If we specify such that we select only within a given index, we should never delete data. + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +set +e +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,levelist=300 --doit +rc=$? +set -e +[[ $rc = 0 ]] && echo "Should not have succeeded with overspecified key." && exit 1 + +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Nothing should change when overspecified" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Full purge should occur when not overspecified" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/partial.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/partial.sh new file mode 100755 index 000000000..64ae1eeee --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/partial.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Do a partial wipe that deletes all of the data + +grib_ls -m data.xxxx.an.grib + +fdb-write data.xxxx.an.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Data not correctly written" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,type=an +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "No change should be made without --doit" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,type=an --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find root -name 'schema' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/partial_complete.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/partial_complete.sh new file mode 100755 index 000000000..f945c27fe --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/partial_complete.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can purge a subset of the data within the DB, which +# is indexed according to different second level metadata + +grib_ls -m data.xxxx.an.grib +grib_ls -m data.xxxx.fc.grib + +fdb-write data.xxxx.an.grib +fdb-write data.xxxx.fc.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Data not correctly written" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,type=fc +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "No change should be made without --doit" && exit -1 + +fdb-wipe --minimum-keys=class,expver class=rd,expver=xxxx,type=fc --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep type=fc out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep type=an out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect amount of data removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/porcelain.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/porcelain.sh new file mode 100755 index 000000000..a97550fa5 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/porcelain.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 | tee wipe_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 +(( "$(wc -l < wipe_out | tr -d '[:space:]')" < "10" )) && echo "normal wipe output incomplete" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --porcelain | tee wipe_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 +[[ "$(wc -l < wipe_out | tr -d '[:space:]')" != "13" ]] && echo "Unexpected file list for removal" && exit -1 # n.b. includes directory + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit --porcelain | tee wipe_out +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(wc -l < wipe_out | tr -d '[:space:]')" != "28" ]] && echo "Unexpected file list for removal" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/ranges.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/ranges.sh new file mode 100755 index 000000000..60baa868f --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/ranges.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +grib_ls -m data.xxxx.d1.grib + +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-write data.xxxx.d3.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1/20180103,time=1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "FDB modified without --doit" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1/20180103,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep date=20170101 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep date=20180103 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=20170101,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep date=20170101 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep date=20180103 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-write data.xxxx.d1.grib +fdb-write data.xxxx.d2.grib +fdb-write data.xxxx.d3.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1/20170101/20180103,time=0000/1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "FDB modified without --doit" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1/20170101/20180103,time=0000/1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "All data should have been deleted" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/wipe/simple.sh b/tests/fdb_e2e/tool_tests/subtocs/wipe/simple.sh new file mode 100755 index 000000000..a7fa4d4a6 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/wipe/simple.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euxo pipefail +# +# Ensure that we can wipe specified databases + +grib_ls -m data.xxxx.grib + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "FDB modified without --doit" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-write data.xxxx.grib +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "36" ]] && echo "Data not correctly duplicated" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect amount of data removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "3" ]] && echo "Incorrect data files created" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Incorrect indexes created" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=0000 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data incorrectly removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "1" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Incorrect indexes removed" && exit -1 + +fdb-wipe class=rd,expver=xxxx,stream=oper,date=-1,time=1200 --doit +fdb-list --all --full --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Data not fully removed" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect data files removed" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect indexes removed" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/write/exclude_filter.sh b/tests/fdb_e2e/tool_tests/subtocs/write/exclude_filter.sh new file mode 100755 index 000000000..acc44b039 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/write/exclude_filter.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# test the EXCLUDE FILTER + +grib_ls -m data.xxxx.grib + +fdb-write --exclude-filter=levelist=800/to/900/by/50 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "20" ]] && exit -1 +[[ "$(grep levelist=850 out | wc -l | tr -d '[:space:]')" != "0" ]] && exit -1 + +fdb-wipe --doit --minimum-keys=class,expver class=rd,expver=xxxx +fdb-write --exclude-filter=time=0000 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && exit -1 + +fdb-wipe --doit --minimum-keys=class,expver class=rd,expver=xxxx +fdb-write --exclude-filter=time=0000/1200,levelist=500/to/1500/by/100 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "6" ]] && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "6" ]] && exit -1 + +exit 0 + diff --git a/tests/fdb_e2e/tool_tests/subtocs/write/include_filter.sh b/tests/fdb_e2e/tool_tests/subtocs/write/include_filter.sh new file mode 100755 index 000000000..9c812c458 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/write/include_filter.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# test the INCLUDE FILTER + +grib_ls -m data.xxxx.grib + +fdb-write --include-filter=time=0000 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep -v time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && exit -1 + +# If we filter on a non-existent experiment, we should get no data +fdb-wipe --doit --minimum-keys=class,expver class=rd,expver=xxxx +fdb-write --include-filter=expver=yyyy data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && exit -1 + +fdb-write --include-filter=param=vo data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "12" ]] && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "6" ]] && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "6" ]] && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/write/intersection_filter.sh b/tests/fdb_e2e/tool_tests/subtocs/write/intersection_filter.sh new file mode 100755 index 000000000..f442ad9f8 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/write/intersection_filter.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# test the INCLUDE FILTER + +grib_ls -m data.xxxx.grib + +fdb-write --exclude-filter=time=1200 --include-filter=time=1200 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "0" ]] && echo "Data written when all should be excluded" && exit -1 + +fdb-write --exclude-filter=time=1200/0000,levelist=500/to/1500/by/100 --include-filter=time=1200 data.xxxx.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "6" ]] && echo "Incorrect number of elements written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Incorrect amount of data written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "0" ]] && echo "Incorrect amount of data written" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/subtocs/write/simple.sh b/tests/fdb_e2e/tool_tests/subtocs/write/simple.sh new file mode 100755 index 000000000..6ed411e73 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/subtocs/write/simple.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euxo pipefail + +grib_ls -m data.xxxx.0.grib + +# Write simple data + +fdb-write data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Subtocs incorrect" && exit -1 + +# Duplicating data adds, but not to the counts + +fdb-write data.xxxx.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +fdb-write data.xxxx.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "24" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "12" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "2" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "6" ]] && echo "Subtocs incorrect" && exit -1 + +# But we can also add to another DB + +fdb-write data.xxxy.0.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "72" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "36" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "16" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "8" ]] && echo "Subtocs incorrect" && exit -1 + +# Adding data in the same DB doesn't add more tocs + +fdb-write data.xxxy.1.grib +fdb-list --all --minimum-keys="" --porcelain | tee out +[[ "$(wc -l < out | tr -d '[:space:]')" != "96" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=1200 out | wc -l | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(grep time=0000 out | wc -l | tr -d '[:space:]')" != "48" ]] && echo "Incorrect number of entries written" && exit -1 +[[ "$(find . -name '*.data' | wc -l | tr -d '[:space:]')" != "10" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name '*.index' | wc -l | tr -d '[:space:]')" != "20" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc' | wc -l | tr -d '[:space:]')" != "4" ]] && echo "Data not correctly written" && exit -1 +[[ "$(find . -name 'toc.*' | wc -l | tr -d '[:space:]')" != "10" ]] && echo "Subtocs incorrect" && exit -1 + +exit 0 diff --git a/tests/fdb_e2e/tool_tests/test_fdb_no_subtocs.py b/tests/fdb_e2e/tool_tests/test_fdb_no_subtocs.py new file mode 100644 index 000000000..89c2b8a15 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/test_fdb_no_subtocs.py @@ -0,0 +1,408 @@ +import datetime + +import pytest +from python_api.util.tools import generate_test_files_key_value +from util.run_sh import run_script + +no_subtoc_combinations = [ + pytest.param( + "simple_fdb_setup_no_subtocs_no_expver_handler", + "simple_env_no_subtocs_no_expver_handler", + marks=pytest.mark.simple, + ), + pytest.param( + "simple_fdb_setup_no_subtocs_expver_handler", "simple_env_no_subtocs_expver_handler", marks=pytest.mark.simple + ), + pytest.param( + "files_fdb_setup_no_subtocs_no_expver_handler", + "files_env_no_subtocs_no_expver_handler", + marks=pytest.mark.files, + ), + pytest.param( + "files_fdb_setup_no_subtocs_expver_handler", "files_env_no_subtocs_expver_handler", marks=pytest.mark.files + ), + pytest.param( + "config_yaml_fdb_setup_no_subtocs_no_expver_handler", + "config_yaml_env_no_subtocs_no_expver_handler", + marks=pytest.mark.yaml, + ), + pytest.param( + "config_json_fdb_setup_no_subtocs_no_expver_handler", + "config_json_env_no_subtocs_no_expver_handler", + marks=pytest.mark.json, + ), + pytest.param( + "config_yaml_mars_disks_fdb_setup_no_subtocs_no_expver_handler", + "config_yaml_mars_disks_env_no_subtocs_no_expver_handler", + marks=pytest.mark.yaml_mars_disks, + ), + pytest.param( + "config_yaml_tools_fdb_setup_no_subtocs_no_expver_handler", + "config_yaml_tools_env_no_subtocs_no_expver_handler", + marks=pytest.mark.yaml_tools, + ), + pytest.param( + "config_json_tools_fdb_setup_no_subtocs_no_expver_handler", + "config_json_tools_env_no_subtocs_no_expver_handler", + marks=pytest.mark.json_tools, + ), +] + + +@pytest.mark.info +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_info(fdb_setup, env_setup, function_tmp, info_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + run_script(script=info_no_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.hide +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_hide(fdb_setup, env_setup, function_tmp, test_data_path, hide_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + + source_file_path = test_data_path / "oper.grib" + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0"), ("date", int(yesterday))], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1"), ("date", int(yesterday))], + ], + ["xxxx.0", "xxxx.1"], + ) + + run_script(script=hide_no_subtoc_script, args=[yesterday], cwd=function_tmp, env=env) + + +@pytest.mark.grib2fdb +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_grib2fdb(fdb_setup, env_setup, function_tmp, test_data_path, grib2fdb5_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + source_file_path = test_data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "1")], + ], + ["xxxx", "xxxx.0", "xxxx.1", "xxxy.0", "xxxy.1"], + ) + + assert len(target_files) == 5 + + # We are skipping the archive as the tests are expecting an empty FDB + print(target_files) + run_script(script=grib2fdb5_no_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.list +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_list(fdb_setup, env_setup, function_tmp, test_data_path, list_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + source_file_path = test_data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "2")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "2")], + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("date", 20170101)], + [("class", "rd"), ("expver", "xxxx"), ("date", 20180103)], + ], + ["xxxx", "xxxx.0", "xxxx.1", "xxxx.2", "xxxy.0", "xxxy.1", "xxxy.2", "xxxx.d1", "xxxx.d2", "xxxx.d3"], + ) + + assert len(target_files) == 10 + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=list_no_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.overlay +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_overlay(fdb_setup, env_setup, function_tmp, test_data_path, overlay_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + source_file_path = test_data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxy")], + [("class", "rd"), ("expver", "xxxy"), ("step", "3")], + ], + ["xxxx", "xxxy", "xxxy.3"], + ) + + assert len(target_files) == 3 + + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=overlay_no_subtoc_script, args=[yesterday], cwd=function_tmp, env=env) + + +@pytest.mark.purge +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_purge(fdb_setup, env_setup, function_tmp, test_data_path, purge_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + source_file_path = test_data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + ], + ["xxxx"], + ) + + assert len(target_files) == 1 + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=purge_no_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.read +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_read(fdb_setup, env_setup, function_tmp, test_data_path, read_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + ### Setting up needed quantile data + source_file_path = test_data_path / "quantile.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxy")], + [("class", "rd"), ("expver", "xxxz")], + ], + ["xxxx", "xxxy", "xxxz"], + ) + + quantile_request = ( + "retrieve," + "class=rd," + "type=cd," + "stream=efhs," + "expver=xxxy," + "levtype=sfc," + "param=228," + "domain=g," + "date=-1," + "time=0000," + "step=60-132," + "quantile=34:100," + "target=quantile_request.grb" + ) + + with (function_tmp / "req").open("w+") as req: + req.write(quantile_request) + + assert len(target_files) == 3 + + ### Setting up oper data + + today = (datetime.date.today()).strftime("%Y%m%d") + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + before_yesterday = (datetime.date.today() - datetime.timedelta(days=2)).strftime("%Y%m%d") + before_before_yesterday = (datetime.date.today() - datetime.timedelta(days=3)).strftime("%Y%m%d") + + source_file_path = test_data_path / "oper.grib" + target_files.extend( + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx"), ("date", int(today))], + [("class", "rd"), ("expver", "xxxx"), ("date", int(yesterday))], + [("class", "rd"), ("expver", "xxxx"), ("date", int(before_yesterday))], + [("class", "rd"), ("expver", "xxxx"), ("date", int(before_before_yesterday))], + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxy")], + [("class", "rd"), ("expver", "xxxz")], + ], + ["xxxx.0", "xxxx.-1", "xxxx.-2", "xxxx.-3", "source.xxxx", "source.xxxy", "source.xxxz"], + ) + ) + + assert len(target_files) == 10 + + for date_diff in [0, -3]: + request = ( + "retrieve,class=rd,expver=xxxx,type=an,stream=oper,levtype=pl,levelist=1000/850/700/500/400/300," + "domain=g," + f"date={(datetime.date.today() + datetime.timedelta(days=date_diff)).strftime('%Y%m%d')}," + "time=0000/1200," + "step=0,param=138/155" + ) + + with (function_tmp / f"req.xxxx.{date_diff}").open("w+") as req: + req.write(request) + + request = ( + "retrieve,class=rd,expver=xxxx,type=an,stream=oper,levtype=pl,levelist=1000/850/700/500/400/300," + "domain=g," + f"date={before_before_yesterday}/{before_yesterday}/{yesterday}/{today}," + "time=0000/1200," + "step=0,param=138/155" + ) + + with (function_tmp / "req.xxxx.combined").open("w+") as req: + req.write(request) + + request = ( + "retrieve,class=rd,expver=xxxy,type=an,stream=oper,levtype=pl,levelist=1000/850/700/500/400/300," + "domain=g," + f"date={yesterday}," + "time=1200," + "step=0,param=138/155" + ) + + with (function_tmp / "req.xxxy.simple").open("w+") as req: + req.write(request) + + ## Steprange data + + source_file_path = test_data_path / "steprange.grib" + target_files.extend( + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxy")], + [("class", "rd"), ("expver", "xxxz")], + ], + ["steprange.xxxx", "steprange.xxxy", "steprange.xxxz"], + ) + ) + request = ( + "retrieve," + "class= rd," + "type= ep," + "stream= enfo," + "expver= xxxy," + "levtype= sfc," + "param= 131070," + "domain=g," + f"date={yesterday}," + "time= 1200," + "step= 0-24" + ) + + with (function_tmp / "req.xxxy.steprange").open("w+") as req: + req.write(request) + + # We are skipping the archive as the tests are expecting an empty FDB + run_script( + script=read_no_subtoc_script, + args=[today, yesterday, before_yesterday, before_before_yesterday], + cwd=function_tmp, + env=env, + ) + + +@pytest.mark.root +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_root(fdb_setup, env_setup, function_tmp, root_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + + fdb_request = ( + "class=rd,expver=xxxy,type=an,stream=oper,levtype=pl,levelist=1000/850/700/500/400/300," + "domain=g," + f"date={yesterday}," + "time=1200," + "step=0,param=138/155" + ) + + with (function_tmp / "req.xxxy.simple").open("w+") as req: + req.write(fdb_request) + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=root_no_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.wipe +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_wipe(fdb_setup, env_setup, function_tmp, test_data_path, wipe_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + target_files = [] + + source_file_path = test_data_path / "oper.grib" + target_files.extend( + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("type", "an")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc")], + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("date", 20170101)], + [("class", "rd"), ("expver", "xxxx"), ("date", 20180103)], + ], + ["xxxx", "xxxx.an", "xxxx.fc", "xxxx.d1", "xxxx.d2", "xxxx.d3"], + ) + ) + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=wipe_no_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.write +@pytest.mark.parametrize("fdb_setup, env_setup", no_subtoc_combinations) +def test_write(fdb_setup, env_setup, function_tmp, test_data_path, write_no_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + target_files = [] + + source_file_path = test_data_path / "oper.grib" + target_files.extend( + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", 0)], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", 1)], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", 0)], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", 1)], + ], + ["xxxx", "xxxx.0", "xxxx.1", "xxxy.0", "xxxy.1"], + ) + ) + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=write_no_subtoc_script, args=None, cwd=function_tmp, env=env) diff --git a/tests/fdb_e2e/tool_tests/test_fdb_subtocs.py b/tests/fdb_e2e/tool_tests/test_fdb_subtocs.py new file mode 100644 index 000000000..6d5bcd997 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/test_fdb_subtocs.py @@ -0,0 +1,378 @@ +import datetime + +import pytest +from python_api.util.tools import generate_test_files_key_value +from util.run_sh import run_script + +subtoc_combinations = [ + pytest.param( + "simple_fdb_setup_subtocs_no_expver_handler", "simple_env_subtocs_no_expver_handler", marks=pytest.mark.simple + ), + pytest.param( + "simple_fdb_setup_subtocs_expver_handler", "simple_env_subtocs_expver_handler", marks=pytest.mark.simple + ), + pytest.param( + "files_fdb_setup_subtocs_no_expver_handler", "files_env_subtocs_no_expver_handler", marks=pytest.mark.files + ), + pytest.param("files_fdb_setup_subtocs_expver_handler", "files_env_subtocs_expver_handler", marks=pytest.mark.files), +] + + +@pytest.mark.info +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_info_subtoc(fdb_setup, env_setup, function_tmp, info_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + run_script(script=info_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.hide +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_hide(fdb_setup, env_setup, function_tmp, test_data_path, hide_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + + source_file_path = test_data_path / "oper.grib" + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0"), ("date", int(yesterday))], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1"), ("date", int(yesterday))], + ], + ["xxxx.0", "xxxx.1"], + ) + + run_script(script=hide_subtoc_script, args=[yesterday], cwd=function_tmp, env=env) + + +@pytest.mark.grib2fdb +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_grib2fdb(fdb_setup, env_setup, function_tmp, test_data_path, grib2fdb5_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + source_file_path = test_data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "1")], + ], + ["xxxx", "xxxx.0", "xxxx.1", "xxxy.0", "xxxy.1"], + ) + + assert len(target_files) == 5 + + # We are skipping the archive as the tests are expecting an empty FDB + print(target_files) + run_script(script=grib2fdb5_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.list +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_list(fdb_setup, env_setup, function_tmp, test_data_path, list_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + source_file_path = test_data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", "2")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "0")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "1")], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", "2")], + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("date", 20170101)], + [("class", "rd"), ("expver", "xxxx"), ("date", 20180103)], + ], + ["xxxx", "xxxx.0", "xxxx.1", "xxxx.2", "xxxy.0", "xxxy.1", "xxxy.2", "xxxx.d1", "xxxx.d2", "xxxx.d3"], + ) + + assert len(target_files) == 10 + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=list_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.overlay +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_overlay(fdb_setup, env_setup, function_tmp, test_data_path, overlay_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + source_file_path = test_data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxy")], + [("class", "rd"), ("expver", "xxxy"), ("step", "3")], + ], + ["xxxx", "xxxy", "xxxy.3"], + ) + + assert len(target_files) == 3 + + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=overlay_subtoc_script, args=[yesterday], cwd=function_tmp, env=env) + + +@pytest.mark.purge +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_purge(fdb_setup, env_setup, function_tmp, test_data_path, purge_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + source_file_path = test_data_path / "oper.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + ], + ["xxxx"], + ) + + assert len(target_files) == 1 + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=purge_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.read +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_read(fdb_setup, env_setup, function_tmp, test_data_path, read_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + ### Setting up needed quantile data + source_file_path = test_data_path / "quantile.grib" + target_files = generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxy")], + [("class", "rd"), ("expver", "xxxz")], + ], + ["xxxx", "xxxy", "xxxz"], + ) + + quantile_request = ( + "retrieve," + "class=rd," + "type=cd," + "stream=efhs," + "expver=xxxy," + "levtype=sfc," + "param=228," + "domain=g," + "date=-1," + "time=0000," + "step=60-132," + "quantile=34:100," + "target=quantile_request.grb" + ) + + with (function_tmp / "req").open("w+") as req: + req.write(quantile_request) + + assert len(target_files) == 3 + + ### Setting up oper data + + today = (datetime.date.today()).strftime("%Y%m%d") + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + before_yesterday = (datetime.date.today() - datetime.timedelta(days=2)).strftime("%Y%m%d") + before_before_yesterday = (datetime.date.today() - datetime.timedelta(days=3)).strftime("%Y%m%d") + + source_file_path = test_data_path / "oper.grib" + target_files.extend( + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx"), ("date", int(today))], + [("class", "rd"), ("expver", "xxxx"), ("date", int(yesterday))], + [("class", "rd"), ("expver", "xxxx"), ("date", int(before_yesterday))], + [("class", "rd"), ("expver", "xxxx"), ("date", int(before_before_yesterday))], + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxy")], + [("class", "rd"), ("expver", "xxxz")], + ], + ["xxxx.0", "xxxx.-1", "xxxx.-2", "xxxx.-3", "source.xxxx", "source.xxxy", "source.xxxz"], + ) + ) + + assert len(target_files) == 10 + + for date_diff in [0, -3]: + request = ( + "retrieve,class=rd,expver=xxxx,type=an,stream=oper,levtype=pl,levelist=1000/850/700/500/400/300," + "domain=g," + f"date={(datetime.date.today() + datetime.timedelta(days=date_diff)).strftime('%Y%m%d')}," + "time=0000/1200," + "step=0,param=138/155" + ) + + with (function_tmp / f"req.xxxx.{date_diff}").open("w+") as req: + req.write(request) + + request = ( + "retrieve,class=rd,expver=xxxx,type=an,stream=oper,levtype=pl,levelist=1000/850/700/500/400/300," + "domain=g," + f"date={before_before_yesterday}/{before_yesterday}/{yesterday}/{today}," + "time=0000/1200," + "step=0,param=138/155" + ) + + with (function_tmp / "req.xxxx.combined").open("w+") as req: + req.write(request) + + request = ( + "retrieve,class=rd,expver=xxxy,type=an,stream=oper,levtype=pl,levelist=1000/850/700/500/400/300," + "domain=g," + f"date={yesterday}," + "time=1200," + "step=0,param=138/155" + ) + + with (function_tmp / "req.xxxy.simple").open("w+") as req: + req.write(request) + + ## Steprange data + + source_file_path = test_data_path / "steprange.grib" + target_files.extend( + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxy")], + [("class", "rd"), ("expver", "xxxz")], + ], + ["steprange.xxxx", "steprange.xxxy", "steprange.xxxz"], + ) + ) + request = ( + "retrieve," + "class= rd," + "type= ep," + "stream= enfo," + "expver= xxxy," + "levtype= sfc," + "param= 131070," + "domain=g," + f"date={yesterday}," + "time= 1200," + "step= 0-24," + "target= 'steprange_request.grb'" + ) + + with (function_tmp / "req.xxxy.steprange").open("w+") as req: + req.write(request) + + # We are skipping the archive as the tests are expecting an empty FDB + run_script( + script=read_subtoc_script, + args=[today, yesterday, before_yesterday, before_before_yesterday], + cwd=function_tmp, + env=env, + ) + + +@pytest.mark.root +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_root(fdb_setup, env_setup, function_tmp, root_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y%m%d") + + fdb_request = ( + "class=rd,expver=xxxy,type=an,stream=oper,levtype=pl,levelist=1000/850/700/500/400/300," + "domain=g," + f"date={yesterday}," + "time=1200," + "step=0,param=138/155" + ) + + with (function_tmp / "req.xxxy.simple").open("w+") as req: + req.write(fdb_request) + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=root_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.wipe +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_wipe(fdb_setup, env_setup, function_tmp, test_data_path, wipe_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + target_files = [] + + source_file_path = test_data_path / "oper.grib" + target_files.extend( + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("type", "an")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc")], + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("date", 20170101)], + [("class", "rd"), ("expver", "xxxx"), ("date", 20180103)], + ], + ["xxxx", "xxxx.an", "xxxx.fc", "xxxx.d1", "xxxx.d2", "xxxx.d3"], + ) + ) + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=wipe_subtoc_script, args=None, cwd=function_tmp, env=env) + + +@pytest.mark.write +@pytest.mark.parametrize("fdb_setup, env_setup", subtoc_combinations) +def test_write(fdb_setup, env_setup, function_tmp, test_data_path, write_subtoc_script, request): + _ = request.getfixturevalue(fdb_setup) + env = request.getfixturevalue(env_setup) + + target_files = [] + + source_file_path = test_data_path / "oper.grib" + target_files.extend( + generate_test_files_key_value( + source_file_path, + function_tmp, + [ + [("class", "rd"), ("expver", "xxxx")], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", 0)], + [("class", "rd"), ("expver", "xxxx"), ("type", "fc"), ("step", 1)], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", 0)], + [("class", "rd"), ("expver", "xxxy"), ("type", "fc"), ("step", 1)], + ], + ["xxxx", "xxxx.0", "xxxx.1", "xxxy.0", "xxxy.1"], + ) + ) + + # We are skipping the archive as the tests are expecting an empty FDB + run_script(script=write_subtoc_script, args=None, cwd=function_tmp, env=env) diff --git a/tests/fdb_e2e/tool_tests/util/run_sh.py b/tests/fdb_e2e/tool_tests/util/run_sh.py new file mode 100644 index 000000000..d7e44ed40 --- /dev/null +++ b/tests/fdb_e2e/tool_tests/util/run_sh.py @@ -0,0 +1,39 @@ +import logging +import subprocess +from pathlib import Path +from typing import Optional + +# def run_script(script: Path, cwd, env): +# result = subprocess.run([str(script)], cwd=cwd, env=env, capture_output=True, text=True) +# if result.returncode != 0: +# raise AssertionError(f"Shell script failed: {script.name}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}") + + +def run_script(script: Path, args: Optional[list[str]], cwd: str, env: dict[str, str]): + command = [str(script)] + + if args is not None: + command.extend(args) + + logging.debug(f"Calling {command}") + + logging.debug(env) + + process = subprocess.Popen( + command, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, text=True + ) + + stdout = "" + + for line in process.stdout: + if not line: + break + logging.info(line.strip()) + stdout += line + + process.wait() + + if process.returncode != 0: + raise subprocess.CalledProcessError(returncode=process.returncode, cmd=str(script)) + + return stdout