diff --git a/CMakeLists.txt b/CMakeLists.txt index 463e37056..b090e1a54 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,10 +6,15 @@ project( fdb5 LANGUAGES C CXX ) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# set(CMAKE_COMPILE_WARNING_AS_ERROR ON) + # add_compile_options(-fsanitize=address) # add_compile_options(-fsanitize=address,undefined -fno-omit-frame-pointer) # add_link_options(-fsanitize=address) + +# add_compile_options(-fsanitize=thread) +# add_compile_options(-fsanitize=thread,undefined -fno-omit-frame-pointer) +# add_link_options(-fsanitize=thread) + # set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare") # set(CMAKE_CXX_FLAGS "-Wno-unused-parameter -Wno-unused-variable -Wno-reorder -Wno-sign-compare -Wvla-cxx-extension") diff --git a/src/chunked_data_view/ListIterator.cc b/src/chunked_data_view/ListIterator.cc index 606ca59e9..f7c1d9209 100644 --- a/src/chunked_data_view/ListIterator.cc +++ b/src/chunked_data_view/ListIterator.cc @@ -27,7 +27,7 @@ std::optional>> ListIte auto has_next = listIterator_.next(elem); if (has_next) { - return std::make_tuple(elem.combinedKey(), std::unique_ptr(elem.location().dataHandle())); + return std::make_tuple(elem.combinedKey(), std::unique_ptr(elem.location().dataHandle(""))); } return std::nullopt; diff --git a/src/chunked_data_view/include/chunked_data_view/ListIterator.h b/src/chunked_data_view/include/chunked_data_view/ListIterator.h index 33e43f21a..1001f8663 100644 --- a/src/chunked_data_view/include/chunked_data_view/ListIterator.h +++ b/src/chunked_data_view/include/chunked_data_view/ListIterator.h @@ -37,6 +37,7 @@ class ListIteratorWrapperImpl : public ListIteratorInterface { public: explicit ListIteratorWrapperImpl(fdb5::ListIterator listIterator) : listIterator_(std::move(listIterator)) {}; + std::optional>> next() override; }; diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 1447ea10d..bed4f3523 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -21,6 +21,8 @@ #include #include +#include + #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/DataHandle.h" @@ -39,7 +41,6 @@ #include "fdb5/database/FieldLocation.h" #include "fdb5/database/Key.h" #include "fdb5/database/WipeCoordinator.h" -#include "fdb5/database/WipeState.h" #include "fdb5/io/FieldHandle.h" #include "fdb5/io/HandleGatherer.h" #include "fdb5/message/MessageDecoder.h" @@ -66,24 +67,24 @@ FDB::FDB(FDB&&) = default; FDB& FDB::operator=(FDB&&) = default; -void FDB::archive(eckit::message::Message msg) { +void FDB::archive(eckit::message::Message msg, const std::string& tracingID) { fdb5::Key key = MessageDecoder::messageToKey(msg); - archive(key, msg.data(), msg.length()); + archive(key, msg.data(), msg.length(), tracingID); } -void FDB::archive(eckit::DataHandle& handle) { +void FDB::archive(eckit::DataHandle& handle, const std::string& tracingID) { eckit::message::Message msg; eckit::message::Reader reader(handle); while ((msg = reader.next())) { - archive(msg); + archive(msg, tracingID); } } -void FDB::archive(const void* data, size_t length) { +void FDB::archive(const void* data, size_t length, const std::string& tracingID) { eckit::MemoryHandle handle(data, length); archive(handle); } -void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& handle) { +void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& handle, const std::string& tracingID) { eckit::message::Message msg; eckit::message::Reader reader(handle); @@ -99,7 +100,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h LOG_DEBUG_LIB(LibFdb5) << ss.str(); throw eckit::UserError(ss.str(), Here()); } - archive(key, msg.data(), msg.length()); + archive(key, msg.data(), msg.length(), tracingID); } if (cube.countVacant()) { std::ostringstream ss; @@ -114,7 +115,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h } } -void FDB::archive(const Key& key, const void* data, size_t length) { +void FDB::archive(const Key& key, const void* data, size_t length, const std::string& tracingID) { eckit::Timer timer; timer.start(); @@ -134,15 +135,15 @@ void FDB::archive(const Key& key, const void* data, size_t length) { keyInternal.unset("stepunits"); } - internal_->archive(keyInternal, data, length); + internal_->archive(keyInternal, data, length, tracingID); dirty_ = true; timer.stop(); stats_.addArchive(length, timer); } -void FDB::reindex(const Key& key, const FieldLocation& location) { - internal_->reindex(key, location); +void FDB::reindex(const Key& key, const FieldLocation& location, const std::string& tracingID) { + internal_->reindex(key, location, tracingID); dirty_ = true; } @@ -162,23 +163,23 @@ bool FDB::sorted(const metkit::mars::MarsRequest& request) { return sorted; } -eckit::DataHandle* FDB::read(const eckit::URI& uri) { +eckit::DataHandle* FDB::read(const eckit::URI& uri, const std::string& tracingID) { auto location = std::unique_ptr(FieldLocationFactory::instance().build(uri.scheme(), uri)); - return location->dataHandle(); + return location->dataHandle(tracingID); } -eckit::DataHandle* FDB::read(const std::vector& uris, bool sorted) { +eckit::DataHandle* FDB::read(const std::vector& uris, const std::string& tracingID, bool sorted) { HandleGatherer result(sorted); for (const eckit::URI& uri : uris) { auto location = std::unique_ptr(FieldLocationFactory::instance().build(uri.scheme(), uri)); - result.add(location->dataHandle()); + result.add(location->dataHandle(tracingID)); } return result.dataHandle(); } -eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { +eckit::DataHandle* FDB::read(ListIterator& it, const std::string& tracingID, bool sorted) { eckit::Timer timer; timer.start(); @@ -216,53 +217,55 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { for (std::size_t i = 0; i < cube.size(); i++) { ListElement element; if (cube.find(i, element)) { - result.add(element.location().dataHandle()); + result.add(element.location().dataHandle(tracingID)); } } } } else { while (it.next(el)) { - result.add(el.location().dataHandle()); + result.add(el.location().dataHandle(tracingID)); } } return result.dataHandle(); } -eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request) { +eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request, const std::string& tracingID) { static bool seekable = eckit::Resource("fdbSeekableDataHandle;$FDB_SEEKABLE_DATA_HANDLE", false); - ListIterator it = inspect(request); - return seekable ? new FieldHandle(it) : read(it, sorted(request)); + ListIterator it = inspect(request, tracingID); + return seekable ? new FieldHandle(it, tracingID) : read(it, sorted(request), tracingID); } -ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) { - return internal_->inspect(request); +ListIterator FDB::inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) { + return internal_->inspect(request, tracingID); } -ListIterator FDB::list(const FDBToolRequest& request, const ListMode mode, const int level) { - return {internal_->list(request, level), mode}; +ListIterator FDB::list(const FDBToolRequest& request, const ListMode mode, const std::string& tracingID, + const int level) { + return {internal_->list(request, tracingID, level), mode}; } ListIterator FDB::list(const FDBToolRequest& request, const bool deduplicate, const int level) { - return list(request, deduplicate ? ListMode::Deduplicate : ListMode::Full, level); + return list(request, deduplicate ? ListMode::Deduplicate : ListMode::Full, generateTracingID("list"), level); } DumpIterator FDB::dump(const FDBToolRequest& request, bool simple) { - return internal_->dump(request, simple); + return internal_->dump(request, generateTracingID(), simple); } -StatusIterator FDB::status(const FDBToolRequest& request) { - return internal_->status(request); +StatusIterator FDB::status(const FDBToolRequest& request, const std::string& tracingID) { + return internal_->status(request, tracingID); } -WipeIterator FDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) { +WipeIterator FDB::wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit, bool porcelain, + bool unsafeWipeAll) { auto internal = internal_->shared(); - auto async = [internal, request, doit, porcelain, unsafeWipeAll](eckit::Queue& queue) { + auto async = [internal, request, tracingID, doit, porcelain, unsafeWipeAll](eckit::Queue& queue) { // Visit the catalogues to determine what they would wipe - WipeStateIterator it = internal->wipe(request, doit, porcelain, unsafeWipeAll); + WipeStateIterator it = internal->wipe(request, tracingID, doit, porcelain, unsafeWipeAll); // Coordinate the wipe across catalogues and stores WipeCoordinator coordinator{internal->config()}; @@ -279,20 +282,21 @@ WipeIterator FDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, return WipeIterator(new APIAsyncIterator(internal_->shared(), async)); } -PurgeIterator FDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) { - return internal_->purge(request, doit, porcelain); +PurgeIterator FDB::purge(const FDBToolRequest& request, const std::string& tracingID, bool doit, bool porcelain) { + return internal_->purge(request, tracingID, doit, porcelain); } -StatsIterator FDB::stats(const FDBToolRequest& request) { - return internal_->stats(request); +StatsIterator FDB::stats(const FDBToolRequest& request, const std::string& tracingID) { + return internal_->stats(request, tracingID); } -ControlIterator FDB::control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers) { - return internal_->control(request, action, identifiers); +ControlIterator FDB::control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers, + const std::string& tracingID) { + return internal_->control(request, tracingID, action, identifiers); } -MoveIterator FDB::move(const FDBToolRequest& request, const eckit::URI& dest) { - return internal_->move(request, dest); +MoveIterator FDB::move(const FDBToolRequest& request, const eckit::URI& dest, const std::string& tracingID) { + return internal_->move(request, tracingID, dest); } FDBStats FDB::stats() const { @@ -311,12 +315,12 @@ void FDB::print(std::ostream& s) const { s << *internal_; } -void FDB::flush() { +void FDB::flush(const std::string& tracingID) { if (dirty_) { eckit::Timer timer; timer.start(); - internal_->flush(); + internal_->flush(tracingID); dirty_ = false; timer.stop(); @@ -324,18 +328,18 @@ void FDB::flush() { } } -IndexAxis FDB::axes(const FDBToolRequest& request, int level) { +IndexAxis FDB::axes(const FDBToolRequest& request, const std::string& tracingID, int level) { IndexAxis axes; AxesElement elem; - auto it = axesIterator(request, level); + auto it = axesIterator(request, tracingID, level); while (it.next(elem)) { axes.merge(elem.axes()); } return axes; } -AxesIterator FDB::axesIterator(const FDBToolRequest& request, int level) { - return internal_->axesIterator(request, level); +AxesIterator FDB::axesIterator(const FDBToolRequest& request, const std::string& tracingID, int level) { + return internal_->axesIterator(request, tracingID, level); } bool FDB::dirty() const { @@ -354,6 +358,14 @@ void FDB::registerFlushCallback(FlushCallback callback) { internal_->registerFlushCallback(callback); } +std::string FDB::generateTracingID(const std::string& prefix) { + uuid_t uuid; + uuid_generate(uuid); + char uuidStr[37]; + uuid_unparse(uuid, uuidStr); + return (prefix.empty() ? "" : prefix + "-") + std::string(uuidStr); +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index bd109c5af..42fda7972 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -72,6 +72,8 @@ class FDB { public: // methods + static std::string generateTracingID(const std::string& prefix = ""); + FDB(const Config& config = Config().expandConfig()); ~FDB(); @@ -88,14 +90,16 @@ class FDB { /// Due to the message being self describing no key needs to be supplied. /// Any callback set with registerArchiveCallback will be invoked. /// @param handle eckit::message::Message to data to archive - void archive(eckit::message::Message msg); + /// @param tracingID Optional tracing ID for the archive operation + void archive(eckit::message::Message msg, const std::string& tracingID = generateTracingID("archive")); /// Archives a stream of one or more messages. /// /// Reads messages from the eckit::DatAaHandle and calls archive() on the corresponding messages. /// Any callback set with registerArchiveCallback will be invoked on each message. /// @param handle eckit::DataHandle reference data to archive - void archive(eckit::DataHandle& handle); + /// @param tracingID Optional tracing ID for the archive operation + void archive(eckit::DataHandle& handle, const std::string& tracingID = generateTracingID("archive")); /// Archive binary data to a FDB. /// @@ -103,16 +107,19 @@ class FDB { /// Any callback set with registerArchiveCallback will be invoked on each message. /// @param data Pointer to the binary data to archive /// @param length Size of the data to archive with the given - void archive(const void* data, size_t length); + /// @param tracingID Optional tracing ID for the archive operation + void archive(const void* data, size_t length, const std::string& tracingID = generateTracingID("archive")); /// Archives data from Datahandle and ensures all keys exactly match the provided MarsRequest. /// /// Any callback set with registerArchiveCallback will be invoked on each message. /// @param request a mars request /// @param handle a data handle pointing to the data + /// @param tracingID Optional tracing ID for the archive operation /// @throws eckit::UserError if there are more keys in the MarsRequest then in the messages. /// @throws eckit::UserError if message key not present in MarsRequest. - void archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& handle); + void archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& handle, + const std::string& tracingID = generateTracingID("archive")); /// Archive a binary blob into FDB. /// @@ -122,54 +129,77 @@ class FDB { /// @param key Key used for indexing and archiving the data /// @param data Pointer to the binary blob to archive /// @param length Size in bytes of the binary blob to archive - void archive(const Key& key, const void* data, size_t length); + /// @param tracingID Optional tracing ID for the archive operation + void archive(const Key& key, const void* data, size_t length, + const std::string& tracingID = generateTracingID("archive")); /// Generate an new index entry for an existing field location. /// /// Can be used to reindex existing data into a new catalogue (see fdb-reindex tool). /// @param key Key used to index the data. /// @param location Location of existing data in an FDB store. - void reindex(const Key& key, const FieldLocation& location); + /// @param tracingID Optional tracing ID for the reindex operation + void reindex(const Key& key, const FieldLocation& location, + const std::string& tracingID = generateTracingID("reindex")); /// Flush all buffers and closes all data handles into a consistent DB state /// @note always safe to call - void flush(); + /// @param tracingID Optional tracing ID for the flush operation + void flush(const std::string& tracingID = generateTracingID("flush")); // TODO(simondsmart): Review this. This is a bit odd. The purpose of a URI is that it directly describes the // data locations - and as such shouldn't need the FDB object to do the conversion into DataHandle? /// Read binary data from an URI. /// @param uri eckit uri to the data source + /// @param tracingID Optional tracing ID for the read operation /// @return DataHandle for reading the requested data from - eckit::DataHandle* read(const eckit::URI& uri); + eckit::DataHandle* read(const eckit::URI& uri, const std::string& tracingID = generateTracingID("read")); /// Read binary data from an list of URI. /// @param vector of uris eckit uris to the data source /// @param inStorageOrder if set data will be returned in the order it is stored. If unset data will be returned in /// the order it was requested. /// @return DataHandle for reading the requested data - eckit::DataHandle* read(const std::vector& uris, bool inStorageOrder = false); + eckit::DataHandle* read(const std::vector& uris, bool inStorageOrder = false) { + return read(uris, generateTracingID("read"), inStorageOrder); + } + eckit::DataHandle* read(const std::vector& uris, const std::string& tracingID, + bool inStorageOrder = false); /// Read binary from a ListIterator. /// @param uris a list iterator which resembles a set of fields which should be read /// @param inStorageOrder if set data will be returned in the order it is stored. If unset data will be returned in /// the order it was requested. /// @return DataHandle for reading the requested data from - eckit::DataHandle* read(ListIterator& it, bool inStorageOrder = false); + eckit::DataHandle* read(ListIterator& it, bool inStorageOrder = false, + const std::string& tracingID = generateTracingID("read")) { + return read(it, tracingID, inStorageOrder); + } + eckit::DataHandle* read(ListIterator& it, const std::string& tracingID, bool inStorageOrder = false); /// Retrieve data which is specified by a MARS request. /// @param request MarsRequest which describes the data which should be retrieved + /// @param tracingID Optional tracing ID for the retrieve operation /// @return DataHandle for reading the requested data from - eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request); + eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request, + const std::string& tracingID = generateTracingID("retrieve")); - // TODO(kkratz): Provide doc! - ListIterator inspect(const metkit::mars::MarsRequest& request); + /// Retrieve locations of data specified by a MARS request. + /// @param request MarsRequest which describes the data which should be retrieved + /// @param tracingID Optional tracing ID for the retrieve operation + /// @return ListIterator for iterating over the set of found items + ListIterator inspect(const metkit::mars::MarsRequest& request, + const std::string& tracingID = generateTracingID("inspect")); /// List data present at the archive and which can be retrieved. /// @param request FDBToolRequest stating which data should be queried /// @param mode select how duplicates should be handled in the returned iterator /// @param level maximum level the visitor should respect /// @return ListIterator for iterating over the set of found items - ListIterator list(const FDBToolRequest& request, ListMode mode, int level = 3); + ListIterator list(const FDBToolRequest& request, ListMode mode, int level = 3) { + return list(request, mode, generateTracingID("list"), level); + } + ListIterator list(const FDBToolRequest& request, ListMode mode, const std::string& tracingID, int level = 3); /// Backwards-compatible overload using the previous deduplicate flag. ListIterator list(const FDBToolRequest& request, bool deduplicate = false, int level = 3); @@ -187,7 +217,7 @@ class FDB { DumpIterator dump(const FDBToolRequest& request, bool simple = false); // TODO(kkratz): Provide doc! - StatusIterator status(const FDBToolRequest& request); + StatusIterator status(const FDBToolRequest& request, const std::string& tracingID = generateTracingID("status")); /// Wipe data from the database. /// @@ -201,7 +231,11 @@ class FDB { /// @param unsafeWipeAll flag for omitting all security checks and force a wipe /// @return WipeIterator for iterating over the set of wiped items WipeIterator wipe(const FDBToolRequest& request, bool doit = false, bool porcelain = false, - bool unsafeWipeAll = false); + bool unsafeWipeAll = false) { + return wipe(request, generateTracingID(), doit, porcelain, unsafeWipeAll); + } + WipeIterator wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit = false, + bool porcelain = false, bool unsafeWipeAll = false); /// Move content of one FDB database. /// @@ -210,7 +244,8 @@ class FDB { /// @param request a fdb tool request for the data which should be move /// @param dest destination uri to which the data should be moved /// @return MoveIterator for iterating over the set of found items - MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest); + MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest, + const std::string& tracingID = generateTracingID("move")); /// Remove duplicate data from the database. /// @@ -223,32 +258,43 @@ class FDB { /// @param doit bool if true the purge is triggered, otherwise a dry-run is executed /// @param porcelain bool for printing only those files which are deleted /// @return PurgeIterator for iterating over the set of found items - PurgeIterator purge(const FDBToolRequest& request, bool doit = false, bool porcelain = false); + PurgeIterator purge(const FDBToolRequest& request, bool doit = false, bool porcelain = false) { + return purge(request, generateTracingID("purge"), doit, porcelain); + } + PurgeIterator purge(const FDBToolRequest& request, const std::string& tracingID, bool doit = false, + bool porcelain = false); /// Prints information about FDB databases, aggregating the /// information over all the databases visited into a final summary. /// @param request FDB tool request for which the stats should be shown /// @return StatsIterator for iterating over the set of found items - StatsIterator stats(const FDBToolRequest& request); + StatsIterator stats(const FDBToolRequest& request, const std::string& tracingID = generateTracingID("stats")); // TODO(kkratz): Provide doc! /// @param request FDB tool request /// @param action control action /// @param identifiers identifiers /// @return ControlIterator for iterating over the set of found items - ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers); + ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers, + const std::string& tracingID = generateTracingID("control")); // TODO(kkratz): Provide doc! /// @param request FDB tool request /// @param level maximum level the axis visitor should respect /// @return IndexAxis - IndexAxis axes(const FDBToolRequest& request, int level = 3); + IndexAxis axes(const FDBToolRequest& request, int level = 3) { + return axes(request, generateTracingID("axes"), level); + } + IndexAxis axes(const FDBToolRequest& request, const std::string& tracingID, int level = 3); // TODO(kkratz): Provide doc! /// @param request FDB tool request /// @param level maximum level the axis visitor should respect /// @return AxisIterator - AxesIterator axesIterator(const FDBToolRequest& request, int level = 3); + AxesIterator axesIterator(const FDBToolRequest& request, int level = 3) { + return axesIterator(request, generateTracingID("axesIterator"), level); + } + AxesIterator axesIterator(const FDBToolRequest& request, const std::string& tracingID, int level = 3); /// Check whether a specific control identifier is enabled /// @param controlIdentifier a given control identifier diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h index 0920cf446..500206faf 100644 --- a/src/fdb5/api/FDBFactory.h +++ b/src/fdb5/api/FDBFactory.h @@ -31,6 +31,7 @@ #include "fdb5/api/helpers/StatsIterator.h" #include "fdb5/api/helpers/StatusIterator.h" #include "fdb5/config/Config.h" +#include "fdb5/database/Key.h" #include "fdb5/database/WipeState.h" namespace eckit::message { @@ -72,32 +73,34 @@ class FDBBase : public std::enable_shared_from_this, public CallbackReg // -------------- Primary API functions ---------------------------- - virtual void archive(const Key& key, const void* data, size_t length) = 0; + virtual void archive(const Key& key, const void* data, size_t length, const std::string& tracingID) = 0; - virtual void reindex(const Key& key, const FieldLocation& location) { NOTIMP; } + virtual void reindex(const Key& key, const FieldLocation& location, const std::string& tracingID) { NOTIMP; } - virtual void flush() = 0; + virtual void flush(const std::string& tracingID) = 0; - virtual ListIterator inspect(const metkit::mars::MarsRequest& request) = 0; + virtual ListIterator inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) = 0; - virtual ListIterator list(const FDBToolRequest& request, int level) = 0; + virtual ListIterator list(const FDBToolRequest& request, const std::string& tracingID, int level) = 0; - virtual DumpIterator dump(const FDBToolRequest& request, bool simple) = 0; + virtual DumpIterator dump(const FDBToolRequest& request, const std::string& tracingID, bool simple) = 0; - virtual StatusIterator status(const FDBToolRequest& request) = 0; + virtual StatusIterator status(const FDBToolRequest& request, const std::string& tracingID) = 0; - virtual WipeStateIterator wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) = 0; + virtual WipeStateIterator wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit, + bool porcelain, bool unsafeWipeAll) = 0; - virtual PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) = 0; + virtual PurgeIterator purge(const FDBToolRequest& request, const std::string& tracingID, bool doit, + bool porcelain) = 0; - virtual StatsIterator stats(const FDBToolRequest& request) = 0; + virtual StatsIterator stats(const FDBToolRequest& request, const std::string& tracingID) = 0; - virtual ControlIterator control(const FDBToolRequest& request, ControlAction action, - ControlIdentifiers identifier) = 0; + virtual ControlIterator control(const FDBToolRequest& request, const std::string& tracingID, ControlAction action, + ControlIdentifiers identifiers) = 0; - virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0; + virtual MoveIterator move(const FDBToolRequest& request, const std::string& tracingID, const eckit::URI& dest) = 0; - virtual AxesIterator axesIterator(const FDBToolRequest& request, int axes) = 0; + virtual AxesIterator axesIterator(const FDBToolRequest& request, const std::string& tracingID, int axes) = 0; // -------------- API management ---------------------------- diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc index 791f5d9ce..7f122e7bb 100644 --- a/src/fdb5/api/LocalFDB.cc +++ b/src/fdb5/api/LocalFDB.cc @@ -44,104 +44,108 @@ using namespace eckit; namespace fdb5 { -void LocalFDB::archive(const Key& key, const void* data, size_t length) { +void LocalFDB::archive(const Key& key, const void* data, size_t length, const std::string& tracingID) { if (!archiver_) { - LOG_DEBUG_LIB(LibFdb5) << *this << ": Constructing new archiver" << std::endl; - archiver_.reset(new Archiver(config_, archiveCallback_)); + LOG_DEBUG_LIB(LibFdb5) << *this << ": " << "tracingID: " << tracingID << " - Constructing new archiver" + << std::endl; + archiver_.reset(new Archiver(tracingID, config_, archiveCallback_)); } - archiver_->archive(key, data, length); + archiver_->archive(key, data, length, tracingID); } -void LocalFDB::reindex(const Key& key, const FieldLocation& location) { +void LocalFDB::reindex(const Key& key, const FieldLocation& location, const std::string& tracingID) { if (!reindexer_) { LOG_DEBUG_LIB(LibFdb5) << *this << ": Constructing new reindexer" << std::endl; - reindexer_.reset(new Reindexer(config_)); + reindexer_.reset(new Reindexer(tracingID, config_)); } - reindexer_->reindex(key, location); + reindexer_->reindex(key, location, tracingID); } -ListIterator LocalFDB::inspect(const metkit::mars::MarsRequest& request) { +ListIterator LocalFDB::inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) { if (!inspector_) { LOG_DEBUG_LIB(LibFdb5) << *this << ": Constructing new retriever" << std::endl; inspector_.reset(new Inspector(config_)); } - return inspector_->inspect(request); + return inspector_->inspect(request, tracingID); } template -APIIterator LocalFDB::queryInternal(const FDBToolRequest& request, Ts... args) { +APIIterator LocalFDB::queryInternal(const FDBToolRequest& request, + const std::string& tracingID, Ts... args) { using ValueType = typename VisitorType::ValueType; using QueryIterator = APIIterator; using AsyncIterator = APIAsyncIterator; - auto async_worker = [this, request, args...](Queue& queue) { + auto async_worker = [this, request, tracingID, args...](Queue& queue) { EntryVisitMechanism mechanism(config_); - VisitorType visitor(queue, request.request(), args...); + VisitorType visitor(queue, request.request(), tracingID, args...); mechanism.visit(request, visitor); }; return QueryIterator(new AsyncIterator(shared_from_this(), async_worker)); } -ListIterator LocalFDB::list(const FDBToolRequest& request, const int level) { +ListIterator LocalFDB::list(const FDBToolRequest& request, const std::string& tracingID, int level) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::list() : " << request << std::endl; - return queryInternal(request, level); + return queryInternal(request, tracingID, level); } -DumpIterator LocalFDB::dump(const FDBToolRequest& request, bool simple) { +DumpIterator LocalFDB::dump(const FDBToolRequest& request, const std::string& tracingID, bool simple) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::dump() : " << request << std::endl; - return queryInternal(request, simple); + return queryInternal(request, tracingID, simple); } -StatusIterator LocalFDB::status(const FDBToolRequest& request) { +StatusIterator LocalFDB::status(const FDBToolRequest& request, const std::string& tracingID) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::status() : " << request << std::endl; - return queryInternal(request); + return queryInternal(request, tracingID); } -WipeStateIterator LocalFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) { +WipeStateIterator LocalFDB::wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit, bool porcelain, + bool unsafeWipeAll) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::wipe() : " << request << std::endl; - return queryInternal(request, doit); + return queryInternal(request, tracingID, doit); } -MoveIterator LocalFDB::move(const FDBToolRequest& request, const eckit::URI& dest) { +MoveIterator LocalFDB::move(const FDBToolRequest& request, const std::string& tracingID, const eckit::URI& dest) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::move() : " << request << std::endl; - return queryInternal(request, dest); + return queryInternal(request, tracingID, dest); } -PurgeIterator LocalFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) { +PurgeIterator LocalFDB::purge(const FDBToolRequest& request, const std::string& tracingID, bool doit, bool porcelain) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::purge() : " << request << std::endl; - return queryInternal(request, doit, porcelain); + return queryInternal(request, tracingID, doit, porcelain); } -StatsIterator LocalFDB::stats(const FDBToolRequest& request) { +StatsIterator LocalFDB::stats(const FDBToolRequest& request, const std::string& tracingID) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::stats() : " << request << std::endl; - return queryInternal(request); + return queryInternal(request, tracingID); } -ControlIterator LocalFDB::control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers) { +ControlIterator LocalFDB::control(const FDBToolRequest& request, const std::string& tracingID, ControlAction action, + ControlIdentifiers identifiers) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::control() : " << request << std::endl; - return queryInternal(request, action, identifiers); + return queryInternal(request, tracingID, action, identifiers); } -AxesIterator LocalFDB::axesIterator(const FDBToolRequest& request, int level) { +AxesIterator LocalFDB::axesIterator(const FDBToolRequest& request, const std::string& tracingID, int level) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axesIterator() : " << request << std::endl; - return queryInternal(request, level); + return queryInternal(request, tracingID, level); } -void LocalFDB::flush() { +void LocalFDB::flush(const std::string& tracingID) { ASSERT(!(archiver_ && reindexer_)); if (archiver_) { - archiver_->flush(); + archiver_->flush(tracingID); flushCallback_(); } else if (reindexer_) { - reindexer_->flush(); + reindexer_->flush(tracingID); flushCallback_(); } } diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h index 72c1fe75e..c19d78f08 100644 --- a/src/fdb5/api/LocalFDB.h +++ b/src/fdb5/api/LocalFDB.h @@ -37,37 +37,40 @@ class LocalFDB : public FDBBase { using FDBBase::FDBBase; - void archive(const Key& key, const void* data, size_t length) override; + void archive(const Key& key, const void* data, size_t length, const std::string& tracingID) override; - void reindex(const Key& key, const FieldLocation& location) override; + void reindex(const Key& key, const FieldLocation& location, const std::string& tracingID) override; - ListIterator inspect(const metkit::mars::MarsRequest& request) override; + ListIterator inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) override; - ListIterator list(const FDBToolRequest& request, int level) override; + ListIterator list(const FDBToolRequest& request, const std::string& tracingID, int level) override; - DumpIterator dump(const FDBToolRequest& request, bool simple) override; + DumpIterator dump(const FDBToolRequest& request, const std::string& tracingID, bool simple) override; - StatusIterator status(const FDBToolRequest& request) override; + StatusIterator status(const FDBToolRequest& request, const std::string& tracingID) override; - WipeStateIterator wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) override; + WipeStateIterator wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit, bool porcelain, + bool unsafeWipeAll) override; - PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) override; + PurgeIterator purge(const FDBToolRequest& request, const std::string& tracingID, bool doit, + bool porcelain) override; - StatsIterator stats(const FDBToolRequest& request) override; + StatsIterator stats(const FDBToolRequest& request, const std::string& tracingID) override; - ControlIterator control(const FDBToolRequest& request, ControlAction action, + ControlIterator control(const FDBToolRequest& request, const std::string& tracingID, ControlAction action, ControlIdentifiers identifiers) override; - MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override; + MoveIterator move(const FDBToolRequest& request, const std::string& tracingID, const eckit::URI& dest) override; - AxesIterator axesIterator(const FDBToolRequest& request, int axes) override; + AxesIterator axesIterator(const FDBToolRequest& request, const std::string& tracingID, int axes) override; - void flush() override; + void flush(const std::string& tracingID) override; protected: // methods template - APIIterator queryInternal(const FDBToolRequest& request, Ts... args); + APIIterator queryInternal(const FDBToolRequest& request, + const std::string& tracingID, Ts... args); private: // methods diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc index e4a499031..74c09c6ae 100644 --- a/src/fdb5/api/RemoteFDB.cc +++ b/src/fdb5/api/RemoteFDB.cc @@ -25,108 +25,108 @@ using namespace eckit::literals; namespace { -template +using namespace fdb5; +template struct BaseAPIHelper { typedef T ValueType; static size_t bufferSize() { return 1_MiB; } static size_t queueSize() { return 100; } - static fdb5::remote::Message message() { return msgID; } + static Message message() { return msgID; } - void encodeExtra(eckit::Stream& s) const {} - static ValueType valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) { return ValueType(s); } + void encodeExtra(Stream& s) const {} + static ValueType valueFromStream(Stream& s, RemoteFDB* fdb, const std::string& tracingID) { return ValueType(s); } }; -using StatsHelper = BaseAPIHelper; +using StatsHelper = BaseAPIHelper; -struct ListHelper : BaseAPIHelper { +struct ListHelper : BaseAPIHelper { ListHelper(const int depth) : depth_(depth) {} - static fdb5::ListElement valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) { - fdb5::ListElement elem(s); - std::shared_ptr remoteLocation; + static ListElement valueFromStream(Stream& s, RemoteFDB* fdb, const std::string& tracingID) { + ListElement elem(s); + std::shared_ptr remoteLocation; if (elem.hasLocation()) { - if (fdb5::LibFdb5::instance().debug()) { - eckit::Log::debug() << "ListHelper::valueFromStream - original location: "; - elem.location().dump(eckit::Log::debug()); - eckit::Log::debug() << std::endl; + if (LibFdb5::instance().debug()) { + Log::debug() << "ListHelper::valueFromStream - original location: "; + elem.location().dump(Log::debug()); + Log::debug() << std::endl; } // TODO move the endpoint replacement to the server side () if (elem.location().uri().scheme() == "fdb") { - eckit::net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()}; + net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()}; - remoteLocation = - fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), - static_cast(elem.location())) - .make_shared(); + remoteLocation = RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), + static_cast(elem.location())) + .make_shared(); } else { - remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(); + remoteLocation = RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(); } } - return fdb5::ListElement(elem.keys(), remoteLocation, elem.timestamp()); + return ListElement(elem.keys(), remoteLocation, elem.timestamp()); } - void encodeExtra(eckit::Stream& s) const { s << depth_; } + void encodeExtra(Stream& s) const { s << depth_; } private: int depth_{3}; }; -struct AxesHelper : BaseAPIHelper { +struct AxesHelper : BaseAPIHelper { AxesHelper(int level) : level_(level) {} - void encodeExtra(eckit::Stream& s) const { s << level_; } + void encodeExtra(Stream& s) const { s << level_; } private: int level_; }; -struct InspectHelper : BaseAPIHelper { +struct InspectHelper : BaseAPIHelper { - static fdb5::ListElement valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) { - fdb5::ListElement elem(s); + static ListElement valueFromStream(Stream& s, RemoteFDB* fdb, const std::string& tracingID) { + ListElement elem(s); - if (fdb5::LibFdb5::instance().debug()) { - eckit::Log::debug() << "InspectHelper::valueFromStream - original location: "; - elem.location().dump(eckit::Log::debug()); - eckit::Log::debug() << std::endl; + if (LibFdb5::instance().debug()) { + Log::debug() << "InspectHelper::valueFromStream - original location: "; + elem.location().dump(Log::debug()); + Log::debug() << std::endl; } if (elem.location().uri().scheme() == "fdb") { - eckit::net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()}; + net::Endpoint fieldLocationEndpoint{elem.location().uri().host(), elem.location().uri().port()}; - std::shared_ptr remoteLocation = - fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), - static_cast(elem.location())) + std::shared_ptr remoteLocation = + RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), + static_cast(elem.location())) .make_shared(); - return fdb5::ListElement(elem.keys(), remoteLocation, elem.timestamp()); + return ListElement(elem.keys(), remoteLocation, elem.timestamp()); } - std::shared_ptr remoteLocation = - fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(); - return fdb5::ListElement(elem.keys(), remoteLocation, elem.timestamp()); + std::shared_ptr remoteLocation = + RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(); + return ListElement(elem.keys(), remoteLocation, elem.timestamp()); } }; -struct WipeHelper : BaseAPIHelper { +struct WipeHelper : BaseAPIHelper { WipeHelper(bool doit, bool porcelain, bool unsafeWipeAll) : doit_(doit), unsafeWipeAll_(unsafeWipeAll) {} - void encodeExtra(eckit::Stream& s) const { + void encodeExtra(Stream& s) const { s << doit_; s << unsafeWipeAll_; } - static fdb5::CatalogueWipeState valueFromStream(eckit::Stream& s, fdb5::RemoteFDB* fdb) { - return fdb5::CatalogueWipeState(s); + static CatalogueWipeState valueFromStream(Stream& s, RemoteFDB* fdb, const std::string& tracingID) { + return CatalogueWipeState(s); } private: @@ -139,13 +139,13 @@ struct WipeHelper : BaseAPIHelper " << s.second << std::endl; } - throw eckit::SeriousBug(ss.str()); + throw SeriousBug(ss.str()); } return it->second; } -RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name) : - LocalFDB(config, name), Client(config) { +RemoteFDB::RemoteFDB(const Configuration& config, const std::string& name) : LocalFDB(config, name), Client(config) { - eckit::Buffer buf = controlWriteReadResponse(remote::Message::Stores, generateRequestID()); - eckit::MemoryStream s(buf); + Buffer buf = controlWriteReadResponse(Message::Stores, generateRequestID()); + MemoryStream s(buf); size_t numStores; s >> numStores; ASSERT(numStores > 0); - std::unordered_set localFields; + std::unordered_set localFields; std::vector stores; std::vector fieldLocationEndpoints; @@ -179,22 +178,22 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name s >> store; size_t numAliases; s >> numAliases; - std::vector aliases; + std::vector aliases; if (numAliases == 0) { - eckit::net::Endpoint storeEndpoint{store}; + net::Endpoint storeEndpoint{store}; storesReadMapping_[storeEndpoint] = storeEndpoint; LOG_DEBUG_LIB(LibFdb5) << "store endpoint: " << storeEndpoint << " default data location endpoint: " << storeEndpoint << std::endl; } else { for (size_t j = 0; j < numAliases; j++) { - eckit::net::Endpoint alias(s); + net::Endpoint alias(s); if (store.empty()) { storesLocalFields_.push_back(alias); localFields.emplace(alias); } else { - eckit::net::Endpoint fieldLocationEndpoint{store}; + net::Endpoint fieldLocationEndpoint{store}; storesReadMapping_[fieldLocationEndpoint] = alias; } LOG_DEBUG_LIB(LibFdb5) << "store endpoint: " << alias << " default data location endpoint: " << store @@ -214,21 +213,18 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name fieldLocationEndpoints.push_back(""); } - eckit::Buffer buf2 = controlWriteReadResponse(remote::Message::Schema, generateRequestID()); - eckit::MemoryStream s2(buf2); + Buffer buf2 = controlWriteReadResponse(Message::Schema, generateRequestID()); + MemoryStream s2(buf2); - fdb5::Schema* schema = eckit::Reanimator::reanimate(s2); + Schema* schema = Reanimator::reanimate(s2); config_.set("stores", stores); config_.set("fieldLocationEndpoints", fieldLocationEndpoints); config_.overrideSchema(static_cast(controlEndpoint()) + "/schema", schema); +} - /// @note: We must instantiate the ReadLimiter before any RemoteStores due to their static initialisation. - /// @todo: this may change in future. - static size_t memoryLimit = eckit::Resource( - "$FDB_READ_LIMIT;fdbReadLimit", - config_.userConfig().getUnsigned("limits.read", size_t(1) * 1024 * 1024 * 1024)); // 1GiB - ReadLimiter::init(memoryLimit); +RemoteFDB::~RemoteFDB() { + deregister(); } // ----------------------------------------------------------------------------------------------------- @@ -241,7 +237,7 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name template -auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) +auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& request, const std::string& tracingID) -> APIIterator { using ValueType = typename HelperClass::ValueType; @@ -255,6 +251,7 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& // will result in return messages uint32_t id = generateRequestID(); + std::lock_guard lock(messageMutex_); auto entry = messageQueues_.emplace(id, std::make_shared(HelperClass::queueSize())); ASSERT(entry.second); std::shared_ptr messageQueue(entry.first->second); @@ -264,6 +261,9 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& Buffer encodeBuffer(HelperClass::bufferSize()); MemoryStream s(encodeBuffer); s << request; + if (tracingEnabled()) { + s << tracingID; + } helper.encodeExtra(s); controlWriteCheckResponse(HelperClass::message(), id, true, encodeBuffer, s.position()); @@ -274,39 +274,40 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& return IteratorType( // n.b. Don't worry about catching exceptions in lambda, as // this is handled in the AsyncIterator. - new AsyncIterator(shared_from_this(), [messageQueue, remoteFDB](eckit::Queue& queue) { - eckit::Buffer msg{0}; + new AsyncIterator(shared_from_this(), [messageQueue, remoteFDB, tracingID](Queue& queue) { + Buffer msg{0}; while (true) { if (messageQueue->pop(msg) == -1) { break; } else { MemoryStream s(msg); - queue.emplace(HelperClass::valueFromStream(s, remoteFDB)); + queue.emplace(HelperClass::valueFromStream(s, remoteFDB, tracingID)); } } // messageQueue goes out of scope --> destructed })); } -ListIterator RemoteFDB::list(const FDBToolRequest& request, const int depth) { - return forwardApiCall(ListHelper(depth), request); +ListIterator RemoteFDB::list(const FDBToolRequest& request, const std::string& tracingID, const int depth) { + return forwardApiCall(ListHelper(depth), request, tracingID); } -AxesIterator RemoteFDB::axesIterator(const FDBToolRequest& request, const int depth) { - return forwardApiCall(AxesHelper(depth), request); +AxesIterator RemoteFDB::axesIterator(const FDBToolRequest& request, const std::string& tracingID, const int depth) { + return forwardApiCall(AxesHelper(depth), request, tracingID); } -ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) { - return forwardApiCall(InspectHelper(), request); +ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) { + return forwardApiCall(InspectHelper(), request, tracingID); } -StatsIterator RemoteFDB::stats(const FDBToolRequest& request) { - return forwardApiCall(StatsHelper(), request); +StatsIterator RemoteFDB::stats(const FDBToolRequest& request, const std::string& tracingID) { + return forwardApiCall(StatsHelper(), request, tracingID); } -WipeStateIterator RemoteFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) { - return forwardApiCall(WipeHelper(doit, porcelain, unsafeWipeAll), request); +WipeStateIterator RemoteFDB::wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit, + bool porcelain, bool unsafeWipeAll) { + return forwardApiCall(WipeHelper(doit, porcelain, unsafeWipeAll), request, tracingID); } void RemoteFDB::print(std::ostream& s) const { @@ -314,68 +315,73 @@ void RemoteFDB::print(std::ostream& s) const { } // Client -const eckit::Configuration& RemoteFDB::clientConfig() const { +const Configuration& RemoteFDB::clientConfig() const { return config(); } -bool RemoteFDB::handle(remote::Message message, uint32_t requestID) { +bool RemoteFDB::handle(Message message, uint32_t requestID) { switch (message) { - case fdb5::remote::Message::Complete: { - + case Message::Complete: { + std::lock_guard lock(messageMutex_); auto it = messageQueues_.find(requestID); if (it == messageQueues_.end()) { return false; } it->second->close(); - // Remove entry (shared_ptr --> message queue will be destroyed when it - // goes out of scope in the worker thread). + // Remove entry (shared_ptr --> message queue will be destroyed when it goes out of scope in the worker + // thread). messageQueues_.erase(it); return true; } - case fdb5::remote::Message::Error: { - - std::ostringstream ss; - ss << "RemoteFDB - client id: " << clientId() - << " - received an error without error description for requestID " << requestID << std::endl; - throw RemoteFDBException(ss.str(), controlEndpoint()); - + case Message::Error: { + std::lock_guard lock(messageMutex_); + // Received Error message without error description. Remove the corresponding entry from the message queue + // and let the caller know & complain + auto it = messageQueues_.find(requestID); + if (it != messageQueues_.end()) { + it->second->interrupt( + std::make_exception_ptr(RemoteFDBException("no error description provided", controlEndpoint()))); + // Remove entry (shared_ptr --> message queue will be destroyed when it goes out of scope in the worker + // thread). + messageQueues_.erase(it); + } return false; } default: + Log::warning() << *this << " - Received unexpected [message=" << message << ",requestID=" << requestID + << "]" << std::endl; return false; } } -bool RemoteFDB::handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) { +bool RemoteFDB::handle(Message message, uint32_t requestID, Buffer&& payload) { switch (message) { - case fdb5::remote::Message::Blob: { + case Message::Blob: { + std::lock_guard lock(messageMutex_); auto it = messageQueues_.find(requestID); if (it == messageQueues_.end()) { return false; } - it->second->emplace(std::move(payload)); return true; } - - case fdb5::remote::Message::Error: { - + case Message::Error: { + std::lock_guard lock(messageMutex_); auto it = messageQueues_.find(requestID); - if (it == messageQueues_.end()) { - return false; + if (it != messageQueues_.end()) { + std::string errmsg{static_cast(payload.data()), payload.size()}; + it->second->interrupt(std::make_exception_ptr(RemoteFDBException(errmsg, controlEndpoint()))); + // Remove entry (shared_ptr --> message queue will be destroyed when it goes out of scope in the worker + // thread). + messageQueues_.erase(it); } - std::string msg; - msg.resize(payload.size(), ' '); - payload.copy(&msg[0], payload.size()); - it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, controlEndpoint()))); - // Remove entry (shared_ptr --> message queue will be destroyed when it - // goes out of scope in the worker thread). - messageQueues_.erase(it); - return true; + return false; } default: + Log::warning() << *this << " - Received unexpected [message=" << message << ",requestID=" << requestID + << ",payloadSize=" << payload.size() << "]" << std::endl; return false; } } diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h index e479e5827..13f859b3a 100644 --- a/src/fdb5/api/RemoteFDB.h +++ b/src/fdb5/api/RemoteFDB.h @@ -28,10 +28,12 @@ namespace fdb5 { +using namespace remote; + //---------------------------------------------------------------------------------------------------------------------- class Archiver; -class RemoteFDB : public LocalFDB, public remote::Client { +class RemoteFDB : public LocalFDB, public Client { public: // types @@ -40,30 +42,36 @@ class RemoteFDB : public LocalFDB, public remote::Client { public: // method RemoteFDB(const eckit::Configuration& config, const std::string& name); - ~RemoteFDB() override {} + ~RemoteFDB() override; - ListIterator inspect(const metkit::mars::MarsRequest& request) override; + ListIterator inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) override; - ListIterator list(const FDBToolRequest& request, int depth) override; + ListIterator list(const FDBToolRequest& request, const std::string& tracingID, int depth) override; - AxesIterator axesIterator(const FDBToolRequest& request, int depth = 3) override; + AxesIterator axesIterator(const FDBToolRequest& request, const std::string& tracingID, int depth = 3) override; - DumpIterator dump(const FDBToolRequest& request, bool simple) override { NOTIMP; } + DumpIterator dump(const FDBToolRequest& request, const std::string& tracingID, bool simple) override { NOTIMP; } - StatusIterator status(const FDBToolRequest& request) override { NOTIMP; } + StatusIterator status(const FDBToolRequest& request, const std::string& tracingID) override { NOTIMP; } - WipeStateIterator wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) override; + WipeStateIterator wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit, bool porcelain, + bool unsafeWipeAll) override; - PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) override { NOTIMP; } + PurgeIterator purge(const FDBToolRequest& request, const std::string& tracingID, bool doit, + bool porcelain) override { + NOTIMP; + } - StatsIterator stats(const FDBToolRequest& request) override; + StatsIterator stats(const FDBToolRequest& request, const std::string& tracingID) override; - ControlIterator control(const FDBToolRequest& request, ControlAction action, + ControlIterator control(const FDBToolRequest& request, const std::string& tracingID, ControlAction action, ControlIdentifiers identifiers) override { NOTIMP; } - MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; } + MoveIterator move(const FDBToolRequest& request, const std::string& tracingID, const eckit::URI& dest) override { + NOTIMP; + } const eckit::net::Endpoint& storeEndpoint() const; const eckit::net::Endpoint& storeEndpoint(const eckit::net::Endpoint& fieldLocationEndpoint) const; @@ -71,15 +79,15 @@ class RemoteFDB : public LocalFDB, public remote::Client { private: // methods template - auto forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) + auto forwardApiCall(const HelperClass& helper, const FDBToolRequest& request, const std::string& tracingID) -> APIIterator; void print(std::ostream& s) const override; // Client const eckit::Configuration& clientConfig() const override; - bool handle(remote::Message message, uint32_t requestID) override; - bool handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) override; + bool handle(Message message, uint32_t requestID) override; + bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override; private: // members @@ -93,6 +101,7 @@ class RemoteFDB : public LocalFDB, public remote::Client { // The shared_ptr allows this removal to be asynchronous with the actual task // cleaning up and returning to the client. std::unordered_map> messageQueues_; + std::mutex messageMutex_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/api/SelectFDB.cc b/src/fdb5/api/SelectFDB.cc index 12d2463c9..a62d2ba67 100644 --- a/src/fdb5/api/SelectFDB.cc +++ b/src/fdb5/api/SelectFDB.cc @@ -52,9 +52,9 @@ FDBBase& SelectFDB::FDBLane::get() { return *fdb_; } -void SelectFDB::FDBLane::flush() { +void SelectFDB::FDBLane::flush(const std::string& tracingID) { if (fdb_) { - fdb_->flush(); + fdb_->flush(tracingID); } } @@ -83,27 +83,27 @@ SelectFDB::SelectFDB(const Config& config, const std::string& name) : FDBBase(co SelectFDB::~SelectFDB() {} -void SelectFDB::archive(const Key& key, const void* data, size_t length) { +void SelectFDB::archive(const Key& key, const void* data, size_t length, const std::string& tracingID) { for (auto& lane : subFdbs_) { if (lane.matches(key, Matcher::DontMatchOnMissing)) { - lane.get().archive(key, data, length); + lane.get().archive(key, data, length, tracingID); return; } } std::ostringstream ss; - ss << "No matching fdb for key: " << key; + ss << "tracingID: " << tracingID << " - No matching fdb for key: " << key; throw eckit::UserError(ss.str(), Here()); } -ListIterator SelectFDB::inspect(const MarsRequest& request) { +ListIterator SelectFDB::inspect(const MarsRequest& request, const std::string& tracingID) { std::queue> lists; for (auto& lane : subFdbs_) { if (lane.matches(request, Matcher::DontMatchOnMissing)) { - lists.push(lane.get().inspect(request)); + lists.push(lane.get().inspect(request, tracingID)); } } @@ -128,32 +128,36 @@ auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) return QueryIterator(new APIAggregateIterator(std::move(iterQueue))); } -ListIterator SelectFDB::list(const FDBToolRequest& request, const int level) { - LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::list() >> " << request << std::endl; - return queryInternal(request, - [level](FDBBase& fdb, const FDBToolRequest& request) { return fdb.list(request, level); }); +ListIterator SelectFDB::list(const FDBToolRequest& request, const std::string& tracingID, const int level) { + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - SelectFDB::list() >> " << request << std::endl; + return queryInternal(request, [tracingID, level](FDBBase& fdb, const FDBToolRequest& request) { + return fdb.list(request, tracingID, level); + }); } -DumpIterator SelectFDB::dump(const FDBToolRequest& request, bool simple) { - LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::dump() >> " << request << std::endl; - return queryInternal(request, - [simple](FDBBase& fdb, const FDBToolRequest& request) { return fdb.dump(request, simple); }); +DumpIterator SelectFDB::dump(const FDBToolRequest& request, const std::string& tracingID, bool simple) { + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - SelectFDB::dump() >> " << request << std::endl; + return queryInternal(request, [tracingID, simple](FDBBase& fdb, const FDBToolRequest& request) { + return fdb.dump(request, tracingID, simple); + }); } -StatusIterator SelectFDB::status(const FDBToolRequest& request) { - LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::status() >> " << request << std::endl; - return queryInternal(request, [](FDBBase& fdb, const FDBToolRequest& request) { return fdb.status(request); }); +StatusIterator SelectFDB::status(const FDBToolRequest& request, const std::string& tracingID) { + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - SelectFDB::status() >> " << request << std::endl; + return queryInternal( + request, [tracingID](FDBBase& fdb, const FDBToolRequest& request) { return fdb.status(request, tracingID); }); } -WipeStateIterator SelectFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) { - LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::wipe() >> " << request << std::endl; +WipeStateIterator SelectFDB::wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit, + bool porcelain, bool unsafeWipeAll) { + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - SelectFDB::wipe() >> " << request << std::endl; FDBLane* matchingLane = nullptr; for (auto& lane : subFdbs_) { if (lane.matches(request.request(), Matcher::MatchOnMissing)) { if (matchingLane != nullptr) { std::stringstream ss; - ss << "Multiple matching lanes for request " << request.request(); + ss << "tracingID: " << tracingID << " - Multiple matching lanes for request " << request.request(); ss << " - wipe request must not match multiple SelectFDB lanes."; throw eckit::UserError(ss.str(), Here()); } @@ -164,42 +168,44 @@ WipeStateIterator SelectFDB::wipe(const FDBToolRequest& request, bool doit, bool if (matchingLane == nullptr) { std::stringstream ss; - ss << "No matching lane for request " << request.request(); + ss << "tracingID: " << tracingID << " - No matching lane for request " << request.request(); throw eckit::UserError(ss.str(), Here()); } - return matchingLane->get().wipe(request, doit, porcelain, unsafeWipeAll); + return matchingLane->get().wipe(request, tracingID, doit, porcelain, unsafeWipeAll); } -PurgeIterator SelectFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) { - LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::purge() >> " << request << std::endl; - return queryInternal(request, [doit, porcelain](FDBBase& fdb, const FDBToolRequest& request) { - return fdb.purge(request, doit, porcelain); +PurgeIterator SelectFDB::purge(const FDBToolRequest& request, const std::string& tracingID, bool doit, bool porcelain) { + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - SelectFDB::purge() >> " << request << std::endl; + return queryInternal(request, [tracingID, doit, porcelain](FDBBase& fdb, const FDBToolRequest& request) { + return fdb.purge(request, tracingID, doit, porcelain); }); } -StatsIterator SelectFDB::stats(const FDBToolRequest& request) { - LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::stats() >> " << request << std::endl; - return queryInternal(request, [](FDBBase& fdb, const FDBToolRequest& request) { return fdb.stats(request); }); +StatsIterator SelectFDB::stats(const FDBToolRequest& request, const std::string& tracingID) { + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - SelectFDB::stats() >> " << request << std::endl; + return queryInternal( + request, [tracingID](FDBBase& fdb, const FDBToolRequest& request) { return fdb.stats(request, tracingID); }); } -ControlIterator SelectFDB::control(const FDBToolRequest& request, ControlAction action, +ControlIterator SelectFDB::control(const FDBToolRequest& request, const std::string& tracingID, ControlAction action, ControlIdentifiers identifiers) { - LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::control >> " << request << std::endl; - return queryInternal(request, [action, identifiers](FDBBase& fdb, const FDBToolRequest& request) { - return fdb.control(request, action, identifiers); + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - SelectFDB::control >> " << request << std::endl; + return queryInternal(request, [tracingID, action, identifiers](FDBBase& fdb, const FDBToolRequest& request) { + return fdb.control(request, tracingID, action, identifiers); }); } -AxesIterator SelectFDB::axesIterator(const FDBToolRequest& request, int level) { - LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::axesIterator() >> " << request << std::endl; - return queryInternal( - request, [level](FDBBase& fdb, const FDBToolRequest& request) { return fdb.axesIterator(request, level); }); +AxesIterator SelectFDB::axesIterator(const FDBToolRequest& request, const std::string& tracingID, int level) { + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - SelectFDB::axesIterator() >> " << request << std::endl; + return queryInternal(request, [tracingID, level](FDBBase& fdb, const FDBToolRequest& request) { + return fdb.axesIterator(request, tracingID, level); + }); } -void SelectFDB::flush() { +void SelectFDB::flush(const std::string& tracingID) { for (auto& lane : subFdbs_) { - lane.flush(); + lane.flush(tracingID); } } diff --git a/src/fdb5/api/SelectFDB.h b/src/fdb5/api/SelectFDB.h index 9e01170e5..341410fa7 100644 --- a/src/fdb5/api/SelectFDB.h +++ b/src/fdb5/api/SelectFDB.h @@ -47,7 +47,7 @@ class SelectFDB : public FDBBase { FDBBase& get(); - void flush(); + void flush(const std::string& tracingID); template // T is either a mars request or a Key bool matches(const T& vals, metkit::mars::Matcher::MatchMissingPolicy matchOnMissing) const; @@ -60,30 +60,34 @@ class SelectFDB : public FDBBase { ~SelectFDB() override; - void archive(const Key& key, const void* data, size_t length) override; + void archive(const Key& key, const void* data, size_t length, const std::string& tracingID) override; - ListIterator inspect(const metkit::mars::MarsRequest& request) override; + ListIterator inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) override; - ListIterator list(const FDBToolRequest& request, int level) override; + ListIterator list(const FDBToolRequest& request, const std::string& tracingID, int level) override; - DumpIterator dump(const FDBToolRequest& request, bool simple) override; + DumpIterator dump(const FDBToolRequest& request, const std::string& tracingID, bool simple) override; - StatusIterator status(const FDBToolRequest& request) override; + StatusIterator status(const FDBToolRequest& request, const std::string& tracingID) override; - WipeStateIterator wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) override; + WipeStateIterator wipe(const FDBToolRequest& request, const std::string& tracingID, bool doit, bool porcelain, + bool unsafeWipeAll) override; - PurgeIterator purge(const FDBToolRequest& request, bool doit, bool porcelain) override; + PurgeIterator purge(const FDBToolRequest& request, const std::string& tracingID, bool doit, + bool porcelain) override; - StatsIterator stats(const FDBToolRequest& request) override; + StatsIterator stats(const FDBToolRequest& request, const std::string& tracingID) override; - ControlIterator control(const FDBToolRequest& request, ControlAction action, + ControlIterator control(const FDBToolRequest& request, const std::string& tracingID, ControlAction action, ControlIdentifiers identifiers) override; - MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; } + MoveIterator move(const FDBToolRequest& request, const std::string& tracingID, const eckit::URI& dest) override { + NOTIMP; + } - AxesIterator axesIterator(const FDBToolRequest& request, int level) override; + AxesIterator axesIterator(const FDBToolRequest& request, const std::string& tracingID, int level) override; - void flush() override; + void flush(const std::string& tracingID) override; private: // methods diff --git a/src/fdb5/api/local/AxesVisitor.cc b/src/fdb5/api/local/AxesVisitor.cc index e3279caea..595bd08a1 100644 --- a/src/fdb5/api/local/AxesVisitor.cc +++ b/src/fdb5/api/local/AxesVisitor.cc @@ -21,8 +21,9 @@ namespace fdb5::api::local { //---------------------------------------------------------------------------------------------------------------------- -AxesVisitor::AxesVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, int level) : - QueryVisitor(queue, request), level_(level) {} +AxesVisitor::AxesVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, int level) : + QueryVisitor(queue, request, tracingID), level_(level) {} bool AxesVisitor::preVisitDatabase(const eckit::URI& uri, const Schema& schema) { // If level == 1, avoid constructing the Catalogue/Store objects, so just interrogate the URIs diff --git a/src/fdb5/api/local/AxesVisitor.h b/src/fdb5/api/local/AxesVisitor.h index c41d7d495..08ddf7b3a 100644 --- a/src/fdb5/api/local/AxesVisitor.h +++ b/src/fdb5/api/local/AxesVisitor.h @@ -37,7 +37,8 @@ namespace api::local { class AxesVisitor : public QueryVisitor { public: - AxesVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, int level); + AxesVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, int level); bool visitIndexes() override { return true; } diff --git a/src/fdb5/api/local/ControlVisitor.cc b/src/fdb5/api/local/ControlVisitor.cc index ee817a07b..524ab9feb 100644 --- a/src/fdb5/api/local/ControlVisitor.cc +++ b/src/fdb5/api/local/ControlVisitor.cc @@ -19,8 +19,8 @@ namespace local { //---------------------------------------------------------------------------------------------------------------------- ControlVisitor::ControlVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, - ControlAction action, ControlIdentifiers identifiers) : - QueryVisitor(queue, request), action_(action), identifiers_(identifiers) {} + const std::string& tracingID, ControlAction action, ControlIdentifiers identifiers) : + QueryVisitor(queue, request, tracingID), action_(action), identifiers_(identifiers) {} bool ControlVisitor::visitDatabase(const Catalogue& catalogue) { diff --git a/src/fdb5/api/local/ControlVisitor.h b/src/fdb5/api/local/ControlVisitor.h index 3945ba86a..2b5ea524f 100644 --- a/src/fdb5/api/local/ControlVisitor.h +++ b/src/fdb5/api/local/ControlVisitor.h @@ -29,8 +29,8 @@ namespace local { class ControlVisitor : public QueryVisitor { public: - ControlVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, ControlAction action, - ControlIdentifiers identifiers); + ControlVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, ControlAction action, ControlIdentifiers identifiers); bool visitIndexes() override { return false; } bool visitEntries() override { return false; } diff --git a/src/fdb5/api/local/DumpVisitor.h b/src/fdb5/api/local/DumpVisitor.h index 2a45212e4..220c74503 100644 --- a/src/fdb5/api/local/DumpVisitor.h +++ b/src/fdb5/api/local/DumpVisitor.h @@ -36,8 +36,9 @@ class DumpVisitor : public QueryVisitor { public: - DumpVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool simple) : - QueryVisitor(queue, request), out_(new QueueStringLogTarget(queue)), simple_(simple) {} + DumpVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, bool simple) : + QueryVisitor(queue, request, tracingID), out_(new QueueStringLogTarget(queue)), simple_(simple) {} bool visitIndexes() override { return false; } bool visitEntries() override { return false; } diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index 7ad949ec6..821386c0d 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -48,8 +48,9 @@ struct ListVisitor : public QueryVisitor { public: - ListVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, int level) : - QueryVisitor(queue, request), level_(level) {} + ListVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, int level) : + QueryVisitor(queue, request, tracingID), level_(level) {} /// @todo remove this with better logic bool preVisitDatabase(const eckit::URI& uri, const Schema& schema) override { diff --git a/src/fdb5/api/local/MoveVisitor.cc b/src/fdb5/api/local/MoveVisitor.cc index 12f1d42e3..e835e2f73 100644 --- a/src/fdb5/api/local/MoveVisitor.cc +++ b/src/fdb5/api/local/MoveVisitor.cc @@ -36,8 +36,8 @@ namespace local { //---------------------------------------------------------------------------------------------------------------------- MoveVisitor::MoveVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, - const eckit::URI& dest) : - QueryVisitor(queue, request), dest_(dest) {} + const std::string& tracingID, const eckit::URI& dest) : + QueryVisitor(queue, request, tracingID), dest_(dest) {} bool MoveVisitor::visitDatabase(const Catalogue& catalogue) { if (catalogue.key().match(request_)) { diff --git a/src/fdb5/api/local/MoveVisitor.h b/src/fdb5/api/local/MoveVisitor.h index aefdcd7e4..3df17f6f9 100644 --- a/src/fdb5/api/local/MoveVisitor.h +++ b/src/fdb5/api/local/MoveVisitor.h @@ -35,7 +35,8 @@ class MoveVisitor : public QueryVisitor { public: // methods - MoveVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, const eckit::URI& dest); + MoveVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, const eckit::URI& dest); bool visitIndexes() override { return false; } bool visitEntries() override { return false; } diff --git a/src/fdb5/api/local/PurgeVisitor.cc b/src/fdb5/api/local/PurgeVisitor.cc index 789386c3b..8208d0f1d 100644 --- a/src/fdb5/api/local/PurgeVisitor.cc +++ b/src/fdb5/api/local/PurgeVisitor.cc @@ -30,9 +30,9 @@ namespace local { //---------------------------------------------------------------------------------------------------------------------- -PurgeVisitor::PurgeVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool doit, - bool porcelain) : - QueryVisitor(queue, request), +PurgeVisitor::PurgeVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, bool doit, bool porcelain) : + QueryVisitor(queue, request, tracingID), out_(new QueueStringLogTarget(queue)), doit_(doit), porcelain_(porcelain) {} diff --git a/src/fdb5/api/local/PurgeVisitor.h b/src/fdb5/api/local/PurgeVisitor.h index ab59af5e7..d9599e47e 100644 --- a/src/fdb5/api/local/PurgeVisitor.h +++ b/src/fdb5/api/local/PurgeVisitor.h @@ -38,8 +38,8 @@ namespace local { class PurgeVisitor : public QueryVisitor { public: - PurgeVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool doit, - bool porcelain); + PurgeVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, bool doit, bool porcelain); bool visitDatabase(const Catalogue& catalogue) override; bool visitIndex(const Index& index) override; diff --git a/src/fdb5/api/local/QueryVisitor.h b/src/fdb5/api/local/QueryVisitor.h index c2a03fa04..bcffe577e 100644 --- a/src/fdb5/api/local/QueryVisitor.h +++ b/src/fdb5/api/local/QueryVisitor.h @@ -42,8 +42,9 @@ class QueryVisitor : public EntryVisitor { using ValueType = T; - QueryVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request) : - queue_(queue), request_(request) {} + QueryVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID) : + queue_(queue), request_(request), tracingID_(tracingID) {} protected: // methods @@ -62,6 +63,7 @@ class QueryVisitor : public EntryVisitor { eckit::Queue& queue_; metkit::mars::MarsRequest request_; + std::string tracingID_; private: // members diff --git a/src/fdb5/api/local/WipeVisitor.cc b/src/fdb5/api/local/WipeVisitor.cc index db7a7a364..86f140c58 100644 --- a/src/fdb5/api/local/WipeVisitor.cc +++ b/src/fdb5/api/local/WipeVisitor.cc @@ -33,8 +33,9 @@ namespace fdb5::api::local { //---------------------------------------------------------------------------------------------------------------------- WipeCatalogueVisitor::WipeCatalogueVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, bool doit) : - QueryVisitor(queue, request), doit_(doit) {} + const metkit::mars::MarsRequest& request, const std::string& tracingID, + bool doit) : + QueryVisitor(queue, request, tracingID), doit_(doit) {} bool WipeCatalogueVisitor::visitDatabase(const Catalogue& catalogue) { diff --git a/src/fdb5/api/local/WipeVisitor.h b/src/fdb5/api/local/WipeVisitor.h index df3a4f305..170639487 100644 --- a/src/fdb5/api/local/WipeVisitor.h +++ b/src/fdb5/api/local/WipeVisitor.h @@ -29,7 +29,8 @@ class WipeCatalogueVisitor : public QueryVisitor { public: // methods - WipeCatalogueVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool doit); + WipeCatalogueVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const std::string& tracingID, bool doit); bool visitEntries() override { return false; } bool visitDatabase(const Catalogue& catalogue) override; diff --git a/src/fdb5/daos/DaosCatalogueReader.cc b/src/fdb5/daos/DaosCatalogueReader.cc index e6114819b..02ba19825 100644 --- a/src/fdb5/daos/DaosCatalogueReader.cc +++ b/src/fdb5/daos/DaosCatalogueReader.cc @@ -121,10 +121,10 @@ std::optional DaosCatalogueReader::computeAxis(const std::string& keyword) return std::nullopt; } -bool DaosCatalogueReader::retrieve(const Key& key, Field& field) const { +bool DaosCatalogueReader::retrieve(const Key& key, Field& field, const std::string& tracingID) const { - LOG_DEBUG_LIB(LibFdb5) << "Trying to retrieve key " << key << std::endl; - LOG_DEBUG_LIB(LibFdb5) << "Scanning index " << current_.location() << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - Trying to retrieve key " << key << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - Scanning index " << current_.location() << std::endl; if (!current_.mayContain(key)) { return false; diff --git a/src/fdb5/daos/DaosCatalogueReader.h b/src/fdb5/daos/DaosCatalogueReader.h index b5cd7fc8e..d827e8faa 100644 --- a/src/fdb5/daos/DaosCatalogueReader.h +++ b/src/fdb5/daos/DaosCatalogueReader.h @@ -34,11 +34,11 @@ class DaosCatalogueReader : public DaosCatalogue, public CatalogueReader { void deselectIndex() override; bool open() override; - void flush(size_t archivedFields) override {} + void flush(size_t archivedFields, const std::string& tracingID) override {} void clean() override {} void close() override {} - bool retrieve(const Key& key, Field& field) const override; + bool retrieve(const Key& key, Field& field, const std::string& tracingID) const override; void print(std::ostream& out) const override { NOTIMP; } diff --git a/src/fdb5/daos/DaosCatalogueWriter.cc b/src/fdb5/daos/DaosCatalogueWriter.cc index 2b961db26..d762ad87a 100644 --- a/src/fdb5/daos/DaosCatalogueWriter.cc +++ b/src/fdb5/daos/DaosCatalogueWriter.cc @@ -188,7 +188,7 @@ void DaosCatalogueWriter::deselectIndex() { void DaosCatalogueWriter::clean() { - flush(0); + flush(0, ""); deselectIndex(); } @@ -211,7 +211,7 @@ const Index& DaosCatalogueWriter::currentIndex() { /// @todo: other writers may be simultaneously updating the axes KeyValues in DAOS. Should these /// new updates be retrieved and put into in-memory axes from time to time, e.g. every /// time a value is put in an axis KeyValue? -void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, +void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, const std::string& tracingID, std::shared_ptr fieldLocation) { if (current_.null()) { @@ -319,7 +319,7 @@ void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, } } -void DaosCatalogueWriter::flush(size_t archivedFields) { +void DaosCatalogueWriter::flush(size_t archivedFields, const std::string& tracingID) { if (!current_.null()) { current_ = Index(); diff --git a/src/fdb5/daos/DaosCatalogueWriter.h b/src/fdb5/daos/DaosCatalogueWriter.h index ab9a6a167..e2b590cae 100644 --- a/src/fdb5/daos/DaosCatalogueWriter.h +++ b/src/fdb5/daos/DaosCatalogueWriter.h @@ -55,11 +55,12 @@ class DaosCatalogueWriter : public DaosCatalogue, public CatalogueWriter { void deselectIndex() override; bool open() override { NOTIMP; } - void flush(size_t archivedFields) override; + void flush(size_t archivedFields, const std::string& tracingID) override; void clean() override; void close() override; - void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override; + void archive(const Key& idxKey, const Key& datumKey, const std::string& tracingID, + std::shared_ptr fieldLocation) override; void print(std::ostream& out) const override { NOTIMP; } diff --git a/src/fdb5/daos/DaosFieldLocation.cc b/src/fdb5/daos/DaosFieldLocation.cc index 118e7c63e..cc113cfd4 100644 --- a/src/fdb5/daos/DaosFieldLocation.cc +++ b/src/fdb5/daos/DaosFieldLocation.cc @@ -40,7 +40,7 @@ std::shared_ptr DaosFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); } -eckit::DataHandle* DaosFieldLocation::dataHandle() const { +eckit::DataHandle* DaosFieldLocation::dataHandle(const std::string& tracingID) const { return fdb5::DaosArrayName(uri_).dataHandle(offset(), length()); } diff --git a/src/fdb5/daos/DaosFieldLocation.h b/src/fdb5/daos/DaosFieldLocation.h index f0be9378d..497e98eeb 100644 --- a/src/fdb5/daos/DaosFieldLocation.h +++ b/src/fdb5/daos/DaosFieldLocation.h @@ -32,7 +32,7 @@ class DaosFieldLocation : public FieldLocation { DaosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); DaosFieldLocation(eckit::Stream&); - eckit::DataHandle* dataHandle() const override; + eckit::DataHandle* dataHandle(const std::string& tracingID) const override; std::shared_ptr make_shared() const override; diff --git a/src/fdb5/daos/DaosLazyFieldLocation.cc b/src/fdb5/daos/DaosLazyFieldLocation.cc index 4d5bb6e71..3ccef9451 100644 --- a/src/fdb5/daos/DaosLazyFieldLocation.cc +++ b/src/fdb5/daos/DaosLazyFieldLocation.cc @@ -28,9 +28,9 @@ std::shared_ptr DaosLazyFieldLocation::make_shared() const return std::make_shared(std::move(*this)); } -eckit::DataHandle* DaosLazyFieldLocation::dataHandle() const { +eckit::DataHandle* DaosLazyFieldLocation::dataHandle(const std::string& tracingID) const { - return realise()->dataHandle(); + return realise()->dataHandle(tracingID); } void DaosLazyFieldLocation::print(std::ostream& out) const { diff --git a/src/fdb5/daos/DaosLazyFieldLocation.h b/src/fdb5/daos/DaosLazyFieldLocation.h index 3a79315bb..1230f47ef 100644 --- a/src/fdb5/daos/DaosLazyFieldLocation.h +++ b/src/fdb5/daos/DaosLazyFieldLocation.h @@ -34,7 +34,7 @@ class DaosLazyFieldLocation : public FieldLocation { DaosLazyFieldLocation(const fdb5::DaosLazyFieldLocation& rhs); DaosLazyFieldLocation(const fdb5::DaosKeyValueName& index, const std::string& key); - eckit::DataHandle* dataHandle() const override; + eckit::DataHandle* dataHandle(const std::string& tracingID) const override; std::shared_ptr make_shared() const override; diff --git a/src/fdb5/daos/DaosStore.cc b/src/fdb5/daos/DaosStore.cc index 5412f227d..e593e55b0 100644 --- a/src/fdb5/daos/DaosStore.cc +++ b/src/fdb5/daos/DaosStore.cc @@ -9,6 +9,7 @@ */ #include "eckit/config/Resource.h" +#include "eckit/log/Bytes.h" #include "fdb5/database/WipeState.h" @@ -112,12 +113,13 @@ bool DaosStore::exists() const { } /// @todo: never used in actual fdb-read? -eckit::DataHandle* DaosStore::retrieve(Field& field) const { +eckit::DataHandle* DaosStore::retrieve(Field& field, const std::string& tracingID) const { - return field.dataHandle(); + return field.dataHandle(tracingID); } -std::unique_ptr DaosStore::archive(const Key&, const void* data, eckit::Length length) { +std::unique_ptr DaosStore::archive(const Key&, const std::string& tracingID, const void* data, + eckit::Length length) { /// @note: performed RPCs: /// - open pool if not cached (daos_pool_connect) -- always skipped as it is cached after selectDatabase. @@ -131,6 +133,10 @@ std::unique_ptr DaosStore::archive(const Key&, const void* std::unique_ptr h(n.dataHandle()); + if (!tracingID.empty()) { + LOG_DEBUG_LIB(LibFdb5) << tracingID << " - archiving " << eckit::Bytes(length) << "to " << *h << std::endl; + } + /// @note: performed RPCs: /// - open pool if not cached (daos_pool_connect) -- always skipped, opened above /// - check if container exists if not cached/open (daos_cont_open) -- always skipped, cached/open above @@ -154,7 +160,15 @@ std::unique_ptr DaosStore::archive(const Key&, const void* /// - close (daos_array_close here) -- always performed } -size_t DaosStore::flush() { +size_t DaosStore::flush(const std::string& tracingID) { + if (archivedFields_ == 0) { + return 0; + } + + if (!tracingID.empty()) { + LOG_DEBUG_LIB(LibFdb5) << tracingID << " - flushing " << archivedFields_ << " fields" << std::endl; + } + size_t archived = archivedFields_; archivedFields_ = 0; return archived; diff --git a/src/fdb5/daos/DaosStore.h b/src/fdb5/daos/DaosStore.h index 228f26e7e..44b8636db 100644 --- a/src/fdb5/daos/DaosStore.h +++ b/src/fdb5/daos/DaosStore.h @@ -39,7 +39,7 @@ class DaosStore : public Store, public DaosCommon { std::set asCollocatedDataURIs(const std::set&) const override; bool open() override { return true; } - size_t flush() override; + size_t flush(const std::string& tracingID) override; void close() override {}; void checkUID() const override { /* nothing to do */ } @@ -61,8 +61,9 @@ class DaosStore : public Store, public DaosCommon { bool exists() const override; - eckit::DataHandle* retrieve(Field& field) const override; - std::unique_ptr archive(const Key& key, const void* data, eckit::Length length) override; + eckit::DataHandle* retrieve(Field& field, const std::string& tracingID) const override; + std::unique_ptr archive(const Key& key, const std::string& tracingID, const void* data, + eckit::Length length) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc index 191890df9..21ee8435c 100644 --- a/src/fdb5/database/ArchiveVisitor.cc +++ b/src/fdb5/database/ArchiveVisitor.cc @@ -16,21 +16,23 @@ namespace fdb5 { -ArchiveVisitor::ArchiveVisitor(Archiver& owner, const Key& initialFieldKey, const void* data, size_t size, - const ArchiveCallback& callback) : - BaseArchiveVisitor(owner, initialFieldKey), data_(data), size_(size), callback_(callback) {} +ArchiveVisitor::ArchiveVisitor(Archiver& owner, const Key& initialFieldKey, const std::string& tracingID, + const void* data, size_t size, const ArchiveCallback& callback) : + BaseArchiveVisitor(owner, initialFieldKey, tracingID), data_(data), size_(size), callback_(callback) {} -std::shared_ptr ArchiveVisitor::create(Archiver& owner, const Key& dataKey, const void* data, - size_t size, const ArchiveCallback& callback) { - return std::shared_ptr(new ArchiveVisitor(owner, dataKey, data, size, callback)); +std::shared_ptr ArchiveVisitor::create(Archiver& owner, const Key& dataKey, + const std::string& tracingID, const void* data, size_t size, + const ArchiveCallback& callback) { + return std::shared_ptr(new ArchiveVisitor(owner, dataKey, tracingID, data, size, callback)); } void ArchiveVisitor::callbacks(std::shared_ptr catalogue, const Key& idxKey, const Key& datumKey, + const std::string& tracingID, std::shared_ptr>> p, std::shared_ptr fieldLocation) { p->set_value(fieldLocation); ASSERT(catalogue); - catalogue->archive(idxKey, datumKey, std::move(fieldLocation)); + catalogue->archive(idxKey, datumKey, tracingID, std::move(fieldLocation)); } bool ArchiveVisitor::selectDatum(const Key& datumKey, const Key& fullKey) { @@ -43,9 +45,9 @@ bool ArchiveVisitor::selectDatum(const Key& datumKey, const Key& fullKey) { std::shared_ptr self = shared_from_this(); auto writer = catalogue(); - store()->archiveCb(idxKey, data_, size_, + store()->archiveCb(idxKey, tracingID_, data_, size_, [self, idxKey, datumKey, p, writer](std::unique_ptr loc) mutable { - self->callbacks(writer, idxKey, datumKey, p, std::move(loc)); + self->callbacks(writer, idxKey, datumKey, self->tracingID_, p, std::move(loc)); }); callback_(initialFieldKey(), data_, size_, p->get_future()); diff --git a/src/fdb5/database/ArchiveVisitor.h b/src/fdb5/database/ArchiveVisitor.h index 6bc238a94..2188d9d3c 100644 --- a/src/fdb5/database/ArchiveVisitor.h +++ b/src/fdb5/database/ArchiveVisitor.h @@ -33,12 +33,14 @@ class ArchiveVisitor : public BaseArchiveVisitor, public std::enable_shared_from public: // methods - static std::shared_ptr create(Archiver& owner, const Key& dataKey, const void* data, size_t size, + static std::shared_ptr create(Archiver& owner, const Key& dataKey, const std::string& tracingID, + const void* data, size_t size, const ArchiveCallback& callback = CALLBACK_ARCHIVE_NOOP); protected: // methods - ArchiveVisitor(Archiver& owner, const Key& dataKey, const void* data, size_t size, const ArchiveCallback& callback); + ArchiveVisitor(Archiver& owner, const Key& dataKey, const std::string& tracingID, const void* data, size_t size, + const ArchiveCallback& callback); bool selectDatum(const Key& datumKey, const Key& fullKey) override; @@ -47,7 +49,7 @@ class ArchiveVisitor : public BaseArchiveVisitor, public std::enable_shared_from private: // methods void callbacks(std::shared_ptr catalogue, const Key& idxKey, const Key& datumKey, - std::shared_ptr>> p, + const std::string& tracingID, std::shared_ptr>> p, std::shared_ptr fieldLocation); private: // members diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc index e60f870d7..9f9cb4d11 100644 --- a/src/fdb5/database/Archiver.cc +++ b/src/fdb5/database/Archiver.cc @@ -28,21 +28,23 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Archiver::Archiver(const Config& dbConfig, const ArchiveCallback& callback) : - dbConfig_(dbConfig), db_(nullptr), callback_(callback) {} +Archiver::Archiver(const std::string& tracingID, const Config& dbConfig, const ArchiveCallback& callback) : + tracingID_(tracingID), dbConfig_(dbConfig), db_(nullptr), callback_(callback) {} Archiver::~Archiver() { - flush(); // certify that all sessions are flushed before closing them + flush(tracingID_); // certify that all sessions are flushed before closing them } -void Archiver::archive(const Key& key, const void* data, size_t len) { - auto visitor = ArchiveVisitor::create(*this, key, data, len, callback_); - archive(key, *visitor); +void Archiver::archive(const Key& key, const void* data, size_t len, const std::string& tracingID) { + tracingID_ = tracingID; + auto visitor = ArchiveVisitor::create(*this, key, tracingID, data, len, callback_); + archive(key, *visitor, tracingID); } -void Archiver::archive(const Key& key, BaseArchiveVisitor& visitor) { +void Archiver::archive(const Key& key, BaseArchiveVisitor& visitor, const std::string& tracingID) { std::lock_guard lock(flushMutex_); + tracingID_ = tracingID; visitor.rule(nullptr); dbConfig_.schema().expand(key, visitor); @@ -57,15 +59,15 @@ void Archiver::archive(const Key& key, BaseArchiveVisitor& visitor) { rule->check(key); } -void Archiver::flushDatabase(Database& db) { +void Archiver::flushDatabase(Database& db, const std::string& tracingID) { // flush the store, pass the number of flushed fields to the catalogue - db.catalogue_->flush(db.store_->flush()); + db.catalogue_->flush(db.store_->flush(tracingID), tracingID); } -void Archiver::flush() { +void Archiver::flush(const std::string& tracingID) { std::lock_guard lock(flushMutex_); for (auto i = databases_.begin(); i != databases_.end(); ++i) { - flushDatabase(i->second); + flushDatabase(i->second, tracingID); } } @@ -98,7 +100,7 @@ void Archiver::selectDatabase(const Key& dbKey) { // flushing before evicting from cache std::lock_guard lock(flushMutex_); - flushDatabase(databases_[oldK]); + flushDatabase(databases_[oldK], tracingID_); eckit::Log::info() << "Closing database " << *databases_[oldK].catalogue_ << std::endl; databases_.erase(oldK); diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h index 6d970d9fe..affc7ff10 100644 --- a/src/fdb5/database/Archiver.h +++ b/src/fdb5/database/Archiver.h @@ -51,7 +51,8 @@ class Archiver { public: // methods - Archiver(const Config& dbConfig = Config().expandConfig(), const ArchiveCallback& callback = CALLBACK_ARCHIVE_NOOP); + Archiver(const std::string& tracingID, const Config& dbConfig = Config().expandConfig(), + const ArchiveCallback& callback = CALLBACK_ARCHIVE_NOOP); Archiver(const Archiver&) = delete; Archiver& operator=(const Archiver&) = delete; @@ -60,12 +61,12 @@ class Archiver { virtual ~Archiver(); - void archive(const Key& key, BaseArchiveVisitor& visitor); - void archive(const Key& key, const void* data, size_t len); + void archive(const Key& key, BaseArchiveVisitor& visitor, const std::string& tracingID); + void archive(const Key& key, const void* data, size_t len, const std::string& tracingID); /// Flushes all buffers and closes all data handles into a consistent DB state /// @note always safe to call - void flush(); + void flush(const std::string& tracingID); friend std::ostream& operator<<(std::ostream& s, const Archiver& x) { x.print(s); @@ -74,7 +75,7 @@ class Archiver { protected: // methods - virtual void flushDatabase(Database& db); + virtual void flushDatabase(Database& db, const std::string& tracingID); private: // methods @@ -85,6 +86,7 @@ class Archiver { protected: // members std::map databases_; + std::string tracingID_; private: // members diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc index 9d4530477..b288e406c 100644 --- a/src/fdb5/database/BaseArchiveVisitor.cc +++ b/src/fdb5/database/BaseArchiveVisitor.cc @@ -18,8 +18,8 @@ namespace fdb5 { -BaseArchiveVisitor::BaseArchiveVisitor(Archiver& owner, const Key& initialFieldKey) : - WriteVisitor(owner.prev_), owner_(owner), initialFieldKey_(initialFieldKey) { +BaseArchiveVisitor::BaseArchiveVisitor(Archiver& owner, const Key& initialFieldKey, const std::string& tracingID) : + WriteVisitor(owner.prev_), tracingID_(tracingID), owner_(owner), initialFieldKey_(initialFieldKey) { checkMissingKeysOnWrite_ = eckit::Resource("checkMissingKeysOnWrite", true); } diff --git a/src/fdb5/database/BaseArchiveVisitor.h b/src/fdb5/database/BaseArchiveVisitor.h index 87b153ca3..749f51e41 100644 --- a/src/fdb5/database/BaseArchiveVisitor.h +++ b/src/fdb5/database/BaseArchiveVisitor.h @@ -34,7 +34,7 @@ class BaseArchiveVisitor : public WriteVisitor { public: // methods - BaseArchiveVisitor(Archiver& owner, const Key& initialFieldKey); + BaseArchiveVisitor(Archiver& owner, const Key& initialFieldKey, const std::string& tracingID); protected: // methods @@ -52,6 +52,10 @@ class BaseArchiveVisitor : public WriteVisitor { const Key& initialFieldKey() const { return initialFieldKey_; } +protected: // members + + std::string tracingID_; + private: // members Archiver& owner_; diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h index 608e016ae..d4f2e1511 100644 --- a/src/fdb5/database/Catalogue.h +++ b/src/fdb5/database/Catalogue.h @@ -86,7 +86,7 @@ class Catalogue { virtual std::string type() const = 0; virtual bool open() = 0; - virtual void flush(size_t archivedFields) = 0; + virtual void flush(size_t archivedFields, const std::string& tracingID) = 0; virtual void clean() = 0; virtual void close() = 0; @@ -179,7 +179,7 @@ class CatalogueReader : virtual public Catalogue { virtual DbStats stats() const = 0; std::optional> axis(const std::string& keyword) const; - virtual bool retrieve(const Key& key, Field& field) const = 0; + virtual bool retrieve(const Key& key, Field& field, const std::string& tracingID) const = 0; protected: // methods @@ -204,7 +204,7 @@ class CatalogueWriter : virtual public Catalogue { virtual bool createIndex(const Key& idxKey, size_t datumKeySize) = 0; virtual const Index& currentIndex() = 0; virtual const Key currentIndexKey(); - virtual void archive(const Key& idxKey, const Key& datumKey, + virtual void archive(const Key& idxKey, const Key& datumKey, const std::string& tracingID, std::shared_ptr fieldLocation) = 0; virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0; @@ -364,7 +364,7 @@ class NullCatalogue : public Catalogue { std::string type() const override { NOTIMP; } bool open() override { NOTIMP; } - void flush(size_t archivedFields) override { NOTIMP; } + void flush(size_t archivedFields, const std::string& tracingID) override { NOTIMP; } void clean() override { NOTIMP; } void close() override { NOTIMP; } diff --git a/src/fdb5/database/Field.h b/src/fdb5/database/Field.h index bad13c62d..e2cf5bc78 100644 --- a/src/fdb5/database/Field.h +++ b/src/fdb5/database/Field.h @@ -46,7 +46,7 @@ class Field { const FieldDetails& details = FieldDetails()); Field(const FieldLocation&& location, time_t timestamp, const FieldDetails& details = FieldDetails()); - eckit::DataHandle* dataHandle() const { return location_->dataHandle(); } + eckit::DataHandle* dataHandle(const std::string& tracingID) const { return location_->dataHandle(tracingID); } const FieldLocation& location() const { return *location_; } diff --git a/src/fdb5/database/FieldLocation.h b/src/fdb5/database/FieldLocation.h index b54133e27..828c5bd3e 100644 --- a/src/fdb5/database/FieldLocation.h +++ b/src/fdb5/database/FieldLocation.h @@ -13,13 +13,17 @@ /// @author Simon Smart /// @date Nov 2016 -#ifndef fdb5_FieldLocation_H -#define fdb5_FieldLocation_H +#pragma once -#include +#include +#include #include +#include +#include +#include #include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/URI.h" #include "eckit/io/Length.h" #include "eckit/memory/Owned.h" #include "eckit/serialisation/Streamable.h" @@ -54,7 +58,7 @@ class FieldLocation : public eckit::OwnedLock, public eckit::Streamable { virtual eckit::Length length() const { return length_; } const Key& remapKey() const { return remapKey_; } - virtual eckit::DataHandle* dataHandle() const = 0; + virtual eckit::DataHandle* dataHandle(const std::string& tracingID = "") const = 0; /// Create a (shared) copy of the current object, for storage in a general container. virtual std::shared_ptr make_shared() const = 0; @@ -179,5 +183,3 @@ class FieldLocationPrinter : public FieldLocationVisitor { //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 - -#endif diff --git a/src/fdb5/database/Inspector.cc b/src/fdb5/database/Inspector.cc index 2f4916109..02e4d8fc0 100644 --- a/src/fdb5/database/Inspector.cc +++ b/src/fdb5/database/Inspector.cc @@ -56,10 +56,10 @@ static void purgeCatalogue(Key& key, CatalogueReader*& db) { Inspector::Inspector(const Config& dbConfig) : databases_(Resource("fdbMaxOpenDatabases", 16), &purgeCatalogue), dbConfig_(dbConfig) {} -ListIterator Inspector::inspect(const metkit::mars::MarsRequest& request) const { +ListIterator Inspector::inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) const { auto iterator = std::make_unique(); - MultiRetrieveVisitor visitor(*iterator, databases_, dbConfig_); + MultiRetrieveVisitor visitor(*iterator, databases_, dbConfig_, tracingID); const auto& schema = dbConfig_.schema(); LOG_DEBUG_LIB(LibFdb5) << "Using schema: " << schema << std::endl; diff --git a/src/fdb5/database/Inspector.h b/src/fdb5/database/Inspector.h index 407b0581f..1fd1d6af2 100644 --- a/src/fdb5/database/Inspector.h +++ b/src/fdb5/database/Inspector.h @@ -76,7 +76,7 @@ class Inspector { /// Retrieves the data selected by the MarsRequest to the provided DataHandle /// @returns data handle to read from - ListIterator inspect(const metkit::mars::MarsRequest& request) const; + ListIterator inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) const; /// Give read access to a range of entries according to a request diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc index cd8961afb..1936720e2 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.cc +++ b/src/fdb5/database/MultiRetrieveVisitor.cc @@ -27,8 +27,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- MultiRetrieveVisitor::MultiRetrieveVisitor(InspectIterator& iterator, eckit::CacheLRU& databases, - const Config& config) : - databases_(databases), iterator_(iterator), config_(config) {} + const Config& config, const std::string& tracingID) : + ReadVisitor(tracingID), databases_(databases), iterator_(iterator), config_(config) {} MultiRetrieveVisitor::~MultiRetrieveVisitor() {} @@ -42,7 +42,7 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& dbKey, const Key& /* fullKe if (catalogue_) { if (dbKey == catalogue_->key()) { - eckit::Log::info() << "This is the current db" << std::endl; + eckit::Log::info() << "tracingID: " << tracingID_ << " - This is the current db" << std::endl; return true; } } @@ -50,7 +50,7 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& dbKey, const Key& /* fullKe /* is the DB already open ? */ if (databases_.exists(dbKey)) { - LOG_DEBUG_LIB(LibFdb5) << "FDB5 Reusing database " << dbKey << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID_ << " - FDB5 Reusing database " << dbKey << std::endl; catalogue_ = databases_.access(dbKey); return true; } @@ -62,16 +62,16 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& dbKey, const Key& /* fullKe // If this database is locked for retrieval then it "does not exist" if (!newCatalogue->enabled(ControlIdentifier::Retrieve)) { std::ostringstream ss; - ss << "Catalogue " << *newCatalogue << " is LOCKED for retrieval"; + ss << "tracingID: " << tracingID_ << " - Catalogue " << *newCatalogue << " is LOCKED for retrieval"; eckit::Log::warning() << ss.str() << std::endl; return false; } - LOG_DEBUG_LIB(LibFdb5) << "MultiRetrieveVisitor::selectDatabase opening database " << dbKey - << " (type=" << newCatalogue->type() << ")" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID_ << " - MultiRetrieveVisitor::selectDatabase opening database " + << dbKey << " (type=" << newCatalogue->type() << ")" << std::endl; if (!newCatalogue->open()) { - LOG_DEBUG_LIB(LibFdb5) << "Database does not exist " << dbKey << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID_ << " - Database does not exist " << dbKey << std::endl; return false; } else { @@ -83,16 +83,17 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& dbKey, const Key& /* fullKe bool MultiRetrieveVisitor::selectIndex(const Key& idxKey) { ASSERT(catalogue_); - LOG_DEBUG_LIB(LibFdb5) << "selectIndex " << idxKey << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID_ << " - selectIndex " << idxKey << std::endl; return catalogue_->selectIndex(idxKey); } bool MultiRetrieveVisitor::selectDatum(const Key& datumKey, const Key& fullKey) { ASSERT(catalogue_); - LOG_DEBUG_LIB(LibFdb5) << "selectDatum " << datumKey << ", " << fullKey << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID_ << " - selectDatum " << datumKey << ", " << fullKey + << std::endl; Field field; - if (catalogue_->retrieve(datumKey, field)) { + if (catalogue_->retrieve(datumKey, field, tracingID_)) { Key simplifiedKey; for (const auto& [keyword, value] : datumKey) { diff --git a/src/fdb5/database/MultiRetrieveVisitor.h b/src/fdb5/database/MultiRetrieveVisitor.h index 9137c6729..362bb0a4d 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.h +++ b/src/fdb5/database/MultiRetrieveVisitor.h @@ -37,7 +37,7 @@ class MultiRetrieveVisitor : public ReadVisitor { public: // methods MultiRetrieveVisitor(InspectIterator& queue, eckit::CacheLRU& databases, - const Config& config); + const Config& config, const std::string& tracingID); ~MultiRetrieveVisitor(); diff --git a/src/fdb5/database/ReadVisitor.h b/src/fdb5/database/ReadVisitor.h index 355a0385f..af1daa357 100644 --- a/src/fdb5/database/ReadVisitor.h +++ b/src/fdb5/database/ReadVisitor.h @@ -38,7 +38,7 @@ class ReadVisitor { public: // methods - ReadVisitor() {} + ReadVisitor(const std::string& tracingID) : tracingID_(tracingID) {} ReadVisitor(const ReadVisitor&) = delete; ReadVisitor& operator=(const ReadVisitor&) = delete; @@ -67,6 +67,8 @@ class ReadVisitor { CatalogueReader* catalogue_{}; + std::string tracingID_; + friend std::ostream& operator<<(std::ostream& s, const ReadVisitor& x) { x.print(s); return s; diff --git a/src/fdb5/database/ReindexVisitor.cc b/src/fdb5/database/ReindexVisitor.cc index 505b39e94..0552e4653 100644 --- a/src/fdb5/database/ReindexVisitor.cc +++ b/src/fdb5/database/ReindexVisitor.cc @@ -11,13 +11,14 @@ namespace fdb5 { -ReindexVisitor::ReindexVisitor(Reindexer& owner, const Key& initialFieldKey, const FieldLocation& fieldLocation) : - BaseArchiveVisitor(owner, initialFieldKey), fieldLocation_(std::move(fieldLocation)) {} +ReindexVisitor::ReindexVisitor(Reindexer& owner, const Key& initialFieldKey, const std::string& tracingID, + const FieldLocation& fieldLocation) : + BaseArchiveVisitor(owner, initialFieldKey, tracingID), fieldLocation_(std::move(fieldLocation)) {} bool ReindexVisitor::selectDatum(const Key& datumKey, const Key& fullKey) { checkMissingKeys(fullKey); const Key idxKey = catalogue()->currentIndexKey(); - catalogue()->archive(idxKey, datumKey, fieldLocation_.make_shared()); + catalogue()->archive(idxKey, datumKey, tracingID_, fieldLocation_.make_shared()); return true; } diff --git a/src/fdb5/database/ReindexVisitor.h b/src/fdb5/database/ReindexVisitor.h index 51185752b..82db91ebb 100644 --- a/src/fdb5/database/ReindexVisitor.h +++ b/src/fdb5/database/ReindexVisitor.h @@ -27,7 +27,8 @@ class ReindexVisitor : public BaseArchiveVisitor { public: // methods - ReindexVisitor(Reindexer& owner, const Key& initialFieldKey, const FieldLocation& fieldLocation); + ReindexVisitor(Reindexer& owner, const Key& initialFieldKey, const std::string& tracingID, + const FieldLocation& fieldLocation); protected: // methods diff --git a/src/fdb5/database/Reindexer.cc b/src/fdb5/database/Reindexer.cc index db749addd..083cbf0b3 100644 --- a/src/fdb5/database/Reindexer.cc +++ b/src/fdb5/database/Reindexer.cc @@ -15,19 +15,21 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Reindexer::Reindexer(const Config& dbConfig) : Archiver(dbConfig) {} +Reindexer::Reindexer(const std::string& tracingID, const Config& dbConfig) : Archiver(tracingID, dbConfig) {} Reindexer::~Reindexer() { - flush(); + flush(tracingID_); } -void Reindexer::reindex(const Key& key, const FieldLocation& fieldLocation) { - ReindexVisitor visitor{*this, key, fieldLocation}; - archive(key, visitor); +void Reindexer::reindex(const Key& key, const FieldLocation& fieldLocation, const std::string& tracingID) { + ReindexVisitor visitor{*this, key, tracingID, fieldLocation}; + tracingID_ = tracingID; + archive(key, visitor, tracingID); } -void Reindexer::flushDatabase(Database& db) { - db.catalogue_->flush(db.catalogue_->archivedLocations()); +void Reindexer::flushDatabase(Database& db, const std::string& tracingID) { + tracingID_ = tracingID; + db.catalogue_->flush(db.catalogue_->archivedLocations(), tracingID); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Reindexer.h b/src/fdb5/database/Reindexer.h index 1942d2794..00ca1795a 100644 --- a/src/fdb5/database/Reindexer.h +++ b/src/fdb5/database/Reindexer.h @@ -24,16 +24,16 @@ class Reindexer : public Archiver { public: // methods - Reindexer(const Config& dbConfig = Config().expandConfig()); + Reindexer(const std::string& tracingID, const Config& dbConfig = Config().expandConfig()); virtual ~Reindexer(); // Write a field location to the catalogue without touching the store - void reindex(const Key& key, const FieldLocation& fieldLocation); + void reindex(const Key& key, const FieldLocation& fieldLocation, const std::string& tracingID); protected: // methods - void flushDatabase(Database& db) override; + void flushDatabase(Database& db, const std::string& tracingID) override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/RetrieveVisitor.cc b/src/fdb5/database/RetrieveVisitor.cc index aef4e2f04..ceb67555c 100644 --- a/src/fdb5/database/RetrieveVisitor.cc +++ b/src/fdb5/database/RetrieveVisitor.cc @@ -21,7 +21,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RetrieveVisitor::RetrieveVisitor(HandleGatherer& gatherer) : gatherer_(gatherer) {} +RetrieveVisitor::RetrieveVisitor(HandleGatherer& gatherer, const std::string& tracingID) : + ReadVisitor(tracingID), gatherer_(gatherer) {} // From Visitor @@ -34,7 +35,7 @@ bool RetrieveVisitor::selectDatabase(const Key& dbKey, const Key& /*fullKey*/) { } LOG_DEBUG_LIB(LibFdb5) << "RetrieveVisitor::selectDatabase " << dbKey << std::endl; - catalogue_ = CatalogueReaderFactory::instance().build(dbKey, fdb5::Config()).get(); + catalogue_ = CatalogueReaderFactory::instance().build(dbKey, fdb5::Config()).release(); // If this database is locked for retrieval then it "does not exist" if (!catalogue_->enabled(ControlIdentifier::Retrieve)) { @@ -63,8 +64,8 @@ bool RetrieveVisitor::selectDatum(const Key& datumKey, const Key& /*fullKey*/) { Field field; eckit::DataHandle* dh = nullptr; - if (catalogue_->retrieve(datumKey, field)) { - dh = store().retrieve(field); + if (catalogue_->retrieve(datumKey, field, tracingID_)) { + dh = store().retrieve(field, tracingID_); } if (dh) { diff --git a/src/fdb5/database/RetrieveVisitor.h b/src/fdb5/database/RetrieveVisitor.h index 361e15c4c..45b8d39cf 100644 --- a/src/fdb5/database/RetrieveVisitor.h +++ b/src/fdb5/database/RetrieveVisitor.h @@ -30,7 +30,7 @@ class RetrieveVisitor : public ReadVisitor { public: // methods - explicit RetrieveVisitor(HandleGatherer& gatherer); + explicit RetrieveVisitor(HandleGatherer& gatherer, const std::string& tracingID); protected: // methods diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc index 1be5d166d..0ecbaa64c 100644 --- a/src/fdb5/database/Store.cc +++ b/src/fdb5/database/Store.cc @@ -29,13 +29,13 @@ std::ostream& operator<<(std::ostream& s, const Store& x) { return s; } -void Store::archiveCb(const Key& key, const void* data, eckit::Length length, +void Store::archiveCb(const Key& idxKey, const std::string& tracingID, const void* data, eckit::Length length, std::function fieldLocation)> catalogue_archive) { - catalogue_archive(archive(key, data, length)); + catalogue_archive(archive(idxKey, tracingID, data, length)); } -std::unique_ptr Store::archive(const Key& /*key*/, const void* /*data*/, - eckit::Length /*length*/) { +std::unique_ptr Store::archive(const Key& /*idxKey*/, const std::string& /*tracingID*/, + const void* /*data*/, eckit::Length /*length*/) { NOTIMP; } diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h index e8a5dd7a8..3fb05747c 100644 --- a/src/fdb5/database/Store.h +++ b/src/fdb5/database/Store.h @@ -15,6 +15,10 @@ #pragma once #include +#include +#include +#include +#include #include "eckit/distributed/Transport.h" #include "eckit/filesystem/URI.h" @@ -38,11 +42,12 @@ class Store { virtual ~Store() = default; - virtual eckit::DataHandle* retrieve(Field& field) const = 0; + virtual eckit::DataHandle* retrieve(Field& field, const std::string& tracingID) const = 0; virtual void archiveCb( - const Key& idxKey, const void* data, eckit::Length length, + const Key& idxKey, const std::string& tracingID, const void* data, eckit::Length length, std::function fieldLocation)> catalogue_archive); - virtual std::unique_ptr archive(const Key& idxKey, const void* data, eckit::Length length); + virtual std::unique_ptr archive(const Key& idxKey, const std::string& tracingID, + const void* data, eckit::Length length); virtual void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit = true) const = 0; @@ -52,7 +57,7 @@ class Store { virtual std::string type() const = 0; virtual bool open() = 0; - virtual size_t flush() = 0; + virtual size_t flush(const std::string& tracingID) = 0; virtual void close() = 0; // virtual std::string owner() const = 0; diff --git a/src/fdb5/database/WipeState.cc b/src/fdb5/database/WipeState.cc index e656da2fb..b2572ebac 100644 --- a/src/fdb5/database/WipeState.cc +++ b/src/fdb5/database/WipeState.cc @@ -11,8 +11,6 @@ namespace fdb5 { // ----------------------------------------------------------------------------------------------- -WipeState::WipeState() {} - WipeState::WipeState(eckit::Stream& s) { // deleteURIs_ @@ -222,7 +220,8 @@ void StoreWipeState::failIfSigned() const { } } -StoreWipeState::StoreWipeState(eckit::URI uri) : storeURI_(std::move(uri)) {} +StoreWipeState::StoreWipeState(eckit::URI uri /*, const std::string& tracingID*/) : + /*WipeState(tracingID),*/ storeURI_(std::move(uri)) {} StoreWipeState::StoreWipeState(eckit::Stream& s) : WipeState(s) { diff --git a/src/fdb5/database/WipeState.h b/src/fdb5/database/WipeState.h index 8858aad17..4fe4a50c9 100644 --- a/src/fdb5/database/WipeState.h +++ b/src/fdb5/database/WipeState.h @@ -41,7 +41,7 @@ using URIMap = std::map>; class WipeState { public: - WipeState(); + WipeState() = default; WipeState(std::set safeURIs, URIMap deleteURIs) : deleteURIs_(std::move(deleteURIs)), safeURIs_(std::move(safeURIs)) {} @@ -121,7 +121,8 @@ class StoreWipeState : public WipeState { public: StoreWipeState() = default; - StoreWipeState(eckit::URI uri); + // StoreWipeState(const std::string& tracingID) : WipeState(tracingID) {} + StoreWipeState(eckit::URI uri /*, const std::string& tracingID*/); StoreWipeState(eckit::Stream& s); // Non-copyable @@ -212,13 +213,14 @@ class CatalogueWipeState : public WipeState { /// @todo: Can we remove some of these constructors? - CatalogueWipeState() : WipeState() {} + CatalogueWipeState() = default; + // CatalogueWipeState(const std::string& tracingID) : WipeState(tracingID) {} - CatalogueWipeState(const Key& dbKey) : WipeState(), dbKey_(dbKey) {} + CatalogueWipeState(const Key& dbKey /*, const std::string& tracingID*/) : /*WipeState(tracingID),*/ dbKey_(dbKey) {} CatalogueWipeState(const Key& dbKey, std::set safeURIs, URIMap deleteURIs, - std::set indexesToMask) : - WipeState(std::move(safeURIs), std::move(deleteURIs)), + std::set indexesToMask /*, const std::string& tracingID*/) : + WipeState(std::move(safeURIs), std::move(deleteURIs) /*, tracingID*/), dbKey_(dbKey), indexesToMask_(std::move(indexesToMask)) {} diff --git a/src/fdb5/io/FieldHandle.cc b/src/fdb5/io/FieldHandle.cc index d4a3d233c..5c62312e5 100644 --- a/src/fdb5/io/FieldHandle.cc +++ b/src/fdb5/io/FieldHandle.cc @@ -26,7 +26,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -FieldHandle::FieldHandle(ListIterator& it) : +FieldHandle::FieldHandle(ListIterator& it, const std::string& tracingID) : datahandles_({}), totalSize_(0), currentIdx_(0), @@ -70,7 +70,7 @@ FieldHandle::FieldHandle(ListIterator& it) : ListElement element; if (cube.find(i, element)) { eckit::Length len = element.location().length(); - eckit::DataHandle* dh = element.location().dataHandle(); + eckit::DataHandle* dh = element.location().dataHandle(tracingID); datahandles_.push_back(std::make_pair(len, dh)); totalSize_ += len; @@ -86,7 +86,7 @@ FieldHandle::FieldHandle(ListIterator& it) : else { while (it.next(el)) { eckit::Length len = el.location().length(); - eckit::DataHandle* dh = el.location().dataHandle(); + eckit::DataHandle* dh = el.location().dataHandle(tracingID); datahandles_.push_back(std::make_pair(len, dh)); totalSize_ += len; diff --git a/src/fdb5/io/FieldHandle.h b/src/fdb5/io/FieldHandle.h index 4d2750f31..7ab22bb32 100644 --- a/src/fdb5/io/FieldHandle.h +++ b/src/fdb5/io/FieldHandle.h @@ -27,7 +27,7 @@ class FieldHandle : public eckit::DataHandle { // -- Contructors - FieldHandle(ListIterator& it); + FieldHandle(ListIterator& it, const std::string& tracingID); // -- Destructor diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index 0d847973e..4b99302d4 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -26,10 +26,13 @@ ::eckit::Reanimator RadosFieldLocation::reanimator_; RadosFieldLocation::RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length) : FieldLocation(eckit::URI("rados", path), offset, length) {} -RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri) : FieldLocation(uri) {} +RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, + std::optional> tracingID) : + FieldLocation(uri, tracingID) {} -RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length) : - FieldLocation(uri, offset, length) {} +RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, + std::optional> tracingID) : + FieldLocation(uri, offset, length, Key(), tracingID) {} RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : FieldLocation(rhs.uri_) {} diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index d54725544..0ef22f16a 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -47,7 +47,7 @@ bool RadosStore::exists() const { return true; } -eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const { +eckit::DataHandle* RadosStore::retrieve(Field& field, const std::string& tracingID, Key& remapKey) const { return remapKey.empty() ? field.dataHandle() : field.dataHandle(remapKey); } diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 300475a03..bd4ad6e01 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -48,7 +48,7 @@ class RadosStore : public Store { std::string type() const override { return "rados"; } bool exists() const override; - eckit::DataHandle* retrieve(Field& field, Key& remapKey) const override; + eckit::DataHandle* retrieve(Field& field, const std::string& tracingID, Key& remapKey) const override; std::unique_ptr archive(const uint32_t, const Key& key, const void* data, eckit::Length length) override; diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc index 4d7b2d846..cd1a8891e 100644 --- a/src/fdb5/remote/Connection.cc +++ b/src/fdb5/remote/Connection.cc @@ -15,7 +15,7 @@ namespace fdb5::remote { Connection::Connection() : single_(false) {} void Connection::teardown() { - closingSocket_ = true; + closingSocket_.store(true); if (!valid()) { return; @@ -91,11 +91,29 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) const { && readUnsafe(socket, &tail, sizeof(tail))) { ASSERT(tail == MessageHeader::EndMarker); + + if (hdr.message == Message::Exit) { + closingSocket_.store(true); + } + if (hdr.message == Message::Error) { + eckit::net::Endpoint remoteEndpoint{socket.remoteHost(), socket.remotePort()}; + std::ostringstream ss; + if (payload.size() == 0) { + ss << "Received an error without error description for clientID " << hdr.clientID() << " requestID " + << hdr.requestID << std::endl; + throw RemoteFDBException(ss.str(), remoteEndpoint); + } + std::string errmsg{static_cast(payload.data()), payload.size()}; + ss << "Received error message: \"" << errmsg << "\" from " << remoteEndpoint << " for clientID " + << hdr.clientID() << " requestID " << hdr.requestID << std::endl; + eckit::Log::warning() << ss.str(); + } return payload; } } hdr.message = Message::Exit; + closingSocket_.store(true); return eckit::Buffer{0}; } @@ -130,7 +148,7 @@ void Connection::write(const Message msg, const bool control, const uint32_t cli void Connection::error(std::string_view msg, uint32_t clientID, uint32_t requestID) const { eckit::Log::error() << "[clientID=" << clientID << ",requestID=" << requestID << "] " << msg << std::endl; - write(Message::Error, false, clientID, requestID, msg.data(), msg.length()); + write(Message::Error, true, clientID, requestID, msg.data(), msg.length()); } eckit::Buffer Connection::readControl(MessageHeader& hdr) const { diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h index f851b8eaf..595fe8066 100644 --- a/src/fdb5/remote/Connection.h +++ b/src/fdb5/remote/Connection.h @@ -19,6 +19,7 @@ #include #include "eckit/exception/Exceptions.h" +#include "eckit/net/Endpoint.h" #include "eckit/net/TCPSocket.h" #include "eckit/os/BackTrace.h" #include "eckit/serialisation/MemoryStream.h" @@ -50,6 +51,15 @@ class TCPException : public eckit::Exception { //---------------------------------------------------------------------------------------------------------------------- +class RemoteFDBException : public eckit::RemoteException { +public: + + RemoteFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint) : + eckit::RemoteException(msg, endpoint) {} +}; + +//---------------------------------------------------------------------------------------------------------------------- + class Connection { public: // types @@ -103,10 +113,11 @@ class Connection { protected: // members bool single_; + bool tracingEnabled_{true}; private: // members - bool closingSocket_ = false; + mutable std::atomic closingSocket_{false}; mutable std::mutex controlMutex_; mutable std::mutex dataMutex_; diff --git a/src/fdb5/remote/RemoteConfiguration.cc b/src/fdb5/remote/RemoteConfiguration.cc index 438b68b1f..49f000204 100644 --- a/src/fdb5/remote/RemoteConfiguration.cc +++ b/src/fdb5/remote/RemoteConfiguration.cc @@ -48,6 +48,8 @@ RemoteConfiguration::RemoteConfiguration(const eckit::Configuration& config) { else { preferSingleConnection_ = std::nullopt; } + + tracingEnabled_ = config.has("tracing") ? config.getBool("tracing") : true; } RemoteConfiguration::RemoteConfiguration(eckit::Stream& s) { @@ -91,10 +93,12 @@ RemoteConfiguration::RemoteConfiguration(eckit::Stream& s) { else { preferSingleConnection_ = std::nullopt; } -} -bool RemoteConfiguration::singleConnection() const { - return singleConnection_; + if (v.contains("TracingEnabled")) { + eckit::Value te = v["TracingEnabled"]; + ASSERT(te.isBool()); + tracingEnabled_ = te; + } } eckit::Stream& operator<<(eckit::Stream& s, const RemoteConfiguration& r) { @@ -104,6 +108,9 @@ eckit::Stream& operator<<(eckit::Stream& s, const RemoteConfiguration& r) { if (r.preferSingleConnection_) { val["PreferSingleConnection"] = eckit::toValue(r.preferSingleConnection_.value()); } + if (r.tracingEnabled_) { + val["TracingEnabled"] = eckit::toValue(r.tracingEnabled_.value()); + } s << val; return s; } @@ -163,6 +170,17 @@ RemoteConfiguration RemoteConfiguration::common(RemoteConfiguration& clientConf, agreedConf.numberOfConnections_ = {ncSelected}; agreedConf.singleConnection_ = (ncSelected == 1); + if (clientConf.tracingEnabled_ && serverConf.tracingEnabled_) { + agreedConf.tracingEnabled_ = clientConf.tracingEnabled() && serverConf.tracingEnabled(); + } + LOG_DEBUG_LIB(LibFdb5) + << "Protocol negotiation - TracingEnabled: client " + << (clientConf.tracingEnabled_ ? std::to_string(clientConf.tracingEnabled_.value()) : "undefined") + << ", server " + << (serverConf.tracingEnabled_ ? std::to_string(serverConf.tracingEnabled_.value()) : "undefined") + << ", agreed " + << (agreedConf.tracingEnabled_ ? std::to_string(agreedConf.tracingEnabled_.value()) : "undefined") << std::endl; + return agreedConf; } diff --git a/src/fdb5/remote/RemoteConfiguration.h b/src/fdb5/remote/RemoteConfiguration.h index add9cc6f0..58009395b 100644 --- a/src/fdb5/remote/RemoteConfiguration.h +++ b/src/fdb5/remote/RemoteConfiguration.h @@ -55,7 +55,8 @@ class RemoteConfiguration { static RemoteConfiguration common(RemoteConfiguration& clientConf, RemoteConfiguration& serverConf); - bool singleConnection() const; + bool singleConnection() const { return singleConnection_; } + bool tracingEnabled() const { return tracingEnabled_ ? tracingEnabled_.value() : false; } friend eckit::Stream& operator<<(eckit::Stream& s, const RemoteConfiguration& r); @@ -65,6 +66,7 @@ class RemoteConfiguration { std::vector numberOfConnections_; std::optional preferSingleConnection_; + std::optional tracingEnabled_; bool singleConnection_{false}; }; diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc index b6c9d8e6e..b2a11a0ed 100644 --- a/src/fdb5/remote/RemoteFieldLocation.cc +++ b/src/fdb5/remote/RemoteFieldLocation.cc @@ -54,7 +54,6 @@ RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, remoteLocation.offset(), remoteLocation.length(), remoteLocation.remapKey()) {} RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri) : FieldLocation(uri) { - ASSERT(uri.scheme() == RemoteFieldLocation::typeName()); } @@ -65,7 +64,9 @@ RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri, const eckit::Off ASSERT(uri.scheme() == RemoteFieldLocation::typeName()); } -RemoteFieldLocation::RemoteFieldLocation(eckit::Stream& s) : FieldLocation(s) {} +RemoteFieldLocation::RemoteFieldLocation(eckit::Stream& s) : FieldLocation(s) { + ASSERT(uri().scheme() == RemoteFieldLocation::typeName()); +} RemoteFieldLocation::RemoteFieldLocation(const RemoteFieldLocation& rhs) : FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} @@ -96,7 +97,7 @@ eckit::URI RemoteFieldLocation::internalURI(const eckit::URI& uri) { return remote; } -eckit::DataHandle* RemoteFieldLocation::dataHandle() const { +eckit::DataHandle* RemoteFieldLocation::dataHandle(const std::string& tracingID) const { if (fdb5::LibFdb5::instance().debug()) { eckit::Log::debug() << "RemoteFieldLocation::dataHandle for location: "; @@ -110,7 +111,7 @@ eckit::DataHandle* RemoteFieldLocation::dataHandle() const { std::unique_ptr loc( FieldLocationFactory::instance().build(remote.scheme(), remote, offset_, length_, remapKey_)); - return store.dataHandle(*loc); + return store.dataHandle(*loc, tracingID); } void RemoteFieldLocation::visit(FieldLocationVisitor& visitor) const { diff --git a/src/fdb5/remote/RemoteFieldLocation.h b/src/fdb5/remote/RemoteFieldLocation.h index ecbb5a8c5..23d60655b 100644 --- a/src/fdb5/remote/RemoteFieldLocation.h +++ b/src/fdb5/remote/RemoteFieldLocation.h @@ -40,7 +40,7 @@ class RemoteFieldLocation : public FieldLocation { static const char* typeName() { return "fdb"; } - eckit::DataHandle* dataHandle() const override; + eckit::DataHandle* dataHandle(const std::string& tracingID) const override; std::shared_ptr make_shared() const override; void visit(FieldLocationVisitor& visitor) const override; @@ -61,8 +61,6 @@ class RemoteFieldLocation : public FieldLocation { private: // methods void print(std::ostream& out) const override; - -private: // members }; diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc index e3e51e784..3454e2bd7 100644 --- a/src/fdb5/remote/client/Client.cc +++ b/src/fdb5/remote/client/Client.cc @@ -59,7 +59,8 @@ Client::Client(const eckit::Configuration& config, } void Client::refreshConnection() { - if (connection_->valid()) { + if (connection_->valid()) { // NO ==> this way we refresh the connection only for one of the clients sharing the + // connection. return; } eckit::Log::warning() << "Connection to " << connection_->controlEndpoint() @@ -69,8 +70,14 @@ void Client::refreshConnection() { connection_->add(*this); } +void Client::deregister() { + if (!deregistered_.exchange(true)) { + connection_->remove(id_); + } +} + Client::~Client() { - connection_->remove(id_); + deregister(); } void Client::controlWriteCheckResponse(const Message msg, const uint32_t requestID, const bool dataListener, @@ -86,8 +93,16 @@ void Client::controlWriteCheckResponse(const Message msg, const uint32_t request } auto f = connection_->controlWrite(*this, msg, requestID, dataListener, payloads); - f.wait(); - ASSERT(f.get().size() == 0); + try { + f.wait(); + ASSERT(f.get().size() == 0); + } + catch (const std::exception& e) { + std::ostringstream ss; + ss << "Error while waiting for response to control message " << msg << " with requestID " << requestID << ": " + << e.what(); + throw RemoteFDBException(ss.str(), connection_->controlEndpoint()); + } } eckit::Buffer Client::controlWriteReadResponse(const Message msg, const uint32_t requestID, const void* const payload, @@ -103,8 +118,16 @@ eckit::Buffer Client::controlWriteReadResponse(const Message msg, const uint32_t } auto f = connection_->controlWrite(*this, msg, requestID, false, payloads); - f.wait(); - return eckit::Buffer{f.get()}; + try { + f.wait(); + return eckit::Buffer{f.get()}; + } + catch (const std::exception& e) { + std::ostringstream ss; + ss << "Error while waiting for response to control message " << msg << " with requestID " << requestID << ": " + << e.what(); + throw RemoteFDBException(ss.str(), connection_->controlEndpoint()); + } } void Client::dataWrite(Message msg, uint32_t requestID, PayloadList payloads) { diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h index 320a6a62c..b402312b5 100644 --- a/src/fdb5/remote/client/Client.h +++ b/src/fdb5/remote/client/Client.h @@ -17,6 +17,7 @@ #include "fdb5/remote/Messages.h" #include "fdb5/remote/client/ClientConnection.h" +#include #include #include // std::pair #include @@ -27,15 +28,6 @@ namespace fdb5::remote { //---------------------------------------------------------------------------------------------------------------------- -class RemoteFDBException : public eckit::RemoteException { -public: - - RemoteFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint) : - eckit::RemoteException(msg, endpoint) {} -}; - -//---------------------------------------------------------------------------------------------------------------------- - class Client { public: // types @@ -86,11 +78,18 @@ class Client { virtual bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) = 0; virtual void closeConnection() {} + bool tracingEnabled() const { return connection_->tracingEnabled(); } + // Create a new connection if the current one is invalid void refreshConnection(); protected: + /// Deregister this client from its connection. Idempotent. + /// Derived classes with state accessed by handle() should call this + /// in their destructor, before that state is destroyed. + void deregister(); + std::shared_ptr connection_; private: @@ -100,6 +99,7 @@ class Client { private: uint32_t id_; + std::atomic deregistered_{false}; mutable std::mutex blockingRequestMutex_; }; diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc index bbd8eb669..75a804ee7 100644 --- a/src/fdb5/remote/client/ClientConnection.cc +++ b/src/fdb5/remote/client/ClientConnection.cc @@ -34,6 +34,7 @@ #include "fdb5/remote/client/Client.h" #include "fdb5/remote/client/ClientConnectionRouter.h" +using namespace eckit; using namespace eckit::literals; namespace fdb5::remote { @@ -44,19 +45,19 @@ class DataWriteRequest { public: - DataWriteRequest() : client_(nullptr), msg_(Message::None), id_(0), data_(eckit::Buffer(0)) {} + DataWriteRequest() : client_(nullptr), msg_(Message::None), id_(0), data_(Buffer(0)) {} - DataWriteRequest(Client* client, Message msg, uint32_t id, eckit::Buffer&& data) : + DataWriteRequest(Client* client, Message msg, uint32_t id, Buffer&& data) : client_(client), msg_(msg), id_(id), data_(std::move(data)) {} Client* client_; Message msg_; uint32_t id_; - eckit::Buffer data_; + Buffer data_; }; -ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint) : +ClientConnection::ClientConnection(const net::Endpoint& controlEndpoint, const std::string& defaultEndpoint) : controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), @@ -110,15 +111,15 @@ uint32_t ClientConnection::generateRequestID() { return ++id_; } -bool ClientConnection::connect(const eckit::Configuration& config, bool singleAttempt) { +bool ClientConnection::connect(const Configuration& config, bool singleAttempt) { if (connected_) { - eckit::Log::warning() << "ClientConnection::connect() called when already connected" << std::endl; + Log::warning() << "ClientConnection::connect() called when already connected" << std::endl; return connected_; } - int fdbMaxConnectRetries = (singleAttempt ? 1 : eckit::Resource("fdbMaxConnectRetries", 3)); - int fdbConnectTimeout = eckit::Resource("fdbConnectTimeout", (singleAttempt ? 2 : 5)); // 0 = No timeout + int fdbMaxConnectRetries = (singleAttempt ? 1 : Resource("fdbMaxConnectRetries", 3)); + int fdbConnectTimeout = Resource("fdbConnectTimeout", (singleAttempt ? 2 : 5)); // 0 = No timeout try { // Connect to server, and check that the server is happy on the response @@ -126,7 +127,7 @@ bool ClientConnection::connect(const eckit::Configuration& config, bool singleAt controlClient_.connect(controlEndpoint_, fdbMaxConnectRetries, fdbConnectTimeout); writeControlStartupMessage(config); - eckit::SessionID serverSession = verifyServerStartupResponse(); + SessionID serverSession = verifyServerStartupResponse(); if (!single_) { // Connect to the specified data port @@ -141,7 +142,7 @@ bool ClientConnection::connect(const eckit::Configuration& config, bool singleAt connected_ = true; } - catch (eckit::TooManyRetries& e) { + catch (TooManyRetries& e) { if (controlClient_.isConnected()) { controlClient_.close(); } @@ -174,23 +175,22 @@ void ClientConnection::disconnect() { } } -const eckit::net::Endpoint& ClientConnection::controlEndpoint() const { +const net::Endpoint& ClientConnection::controlEndpoint() const { return controlEndpoint_; } -RemoteConfiguration ClientConnection::availableFunctionality(const eckit::Configuration& config) const { +RemoteConfiguration ClientConnection::availableFunctionality(const Configuration& config) const { return RemoteConfiguration{config}; } //---------------------------------------------------------------------------------------------------------------------- -std::future ClientConnection::controlWrite(const Client& client, const Message msg, - const uint32_t requestID, const bool /*dataListener*/, - const PayloadList payloads) const { - std::future f; +std::future ClientConnection::controlWrite(const Client& client, const Message msg, const uint32_t requestID, + const bool /*dataListener*/, const PayloadList payloads) const { + std::future f; { std::lock_guard lock(promisesMutex_); - auto pp = promises_.emplace(requestID, std::promise{}).first; + auto pp = promises_.emplace(requestID, std::promise{}).first; f = pp->second.get_future(); } Connection::write(msg, true, client.clientId(), requestID, payloads); @@ -205,7 +205,7 @@ void ClientConnection::dataWrite(DataWriteRequest& request) const { void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, PayloadList payloads) { - static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 320); + static size_t maxQueueLength = Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 320); { // retrieve or add client to the list @@ -219,7 +219,7 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r // Reset the queue after previous done/errors ASSERT(!dataWriteQueue_); - dataWriteQueue_ = std::make_unique>(maxQueueLength); + dataWriteQueue_ = std::make_unique>(maxQueueLength); dataWriteThread_ = std::thread([this] { dataWriteThreadLoop(); }); } } @@ -230,7 +230,7 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r payloadLength += payload.length; } - eckit::Buffer buffer{payloadLength}; + Buffer buffer{payloadLength}; uint32_t offset = 0; for (const auto& payload : payloads) { buffer.copy(payload.data, payload.length, offset); @@ -242,7 +242,7 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r void ClientConnection::dataWriteThreadLoop() { - eckit::Timer timer; + Timer timer; DataWriteRequest element; try { @@ -264,12 +264,12 @@ void ClientConnection::dataWriteThreadLoop() { // They will be released when flush() is called. } -void ClientConnection::writeControlStartupMessage(const eckit::Configuration& config) { +void ClientConnection::writeControlStartupMessage(const Configuration& config) { - eckit::Buffer payload(4096); - eckit::MemoryStream s(payload); + Buffer payload(4096); + MemoryStream s(payload); s << sessionID_; - s << eckit::net::Endpoint(controlEndpoint_.hostname(), controlEndpoint_.port()); + s << net::Endpoint(controlEndpoint_.hostname(), controlEndpoint_.port()); s << LibFdb5::instance().remoteProtocolVersion().used(); s << availableFunctionality(config); @@ -278,10 +278,10 @@ void ClientConnection::writeControlStartupMessage(const eckit::Configuration& co Connection::write(Message::Startup, true, 0, 0, payload, s.position()); } -void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSession) { +void ClientConnection::writeDataStartupMessage(const SessionID& serverSession) { - eckit::Buffer payload(1_KiB); - eckit::MemoryStream s(payload); + Buffer payload(1_KiB); + MemoryStream s(payload); s << sessionID_; s << serverSession; @@ -291,45 +291,53 @@ void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSes Connection::write(Message::Startup, false, 0, 0, payload, s.position()); } -eckit::SessionID ClientConnection::verifyServerStartupResponse() { +SessionID ClientConnection::verifyServerStartupResponse() { MessageHeader hdr; - eckit::Buffer payload = Connection::readControl(hdr); + Buffer payload = Connection::readControl(hdr); ASSERT(hdr.requestID == 0); - eckit::MemoryStream s(payload); - eckit::SessionID clientSession(s); - eckit::SessionID serverSession(s); - eckit::net::Endpoint dataEndpoint(s); - eckit::LocalConfiguration serverFunctionality(s); + MemoryStream s(payload); + SessionID clientSession(s); + SessionID serverSession(s); + net::Endpoint dataEndpoint(s); + LocalConfiguration serverFunctionality(s); dataEndpoint_ = dataEndpoint; LOG_DEBUG_LIB(LibFdb5) << "verifyServerStartupResponse - Received from server " << clientSession << " " << serverSession << " " << dataEndpoint << std::endl; if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) { - eckit::Log::warning() << "Data and control interface hostnames do not match. " << dataEndpoint_.hostname() - << " /= " << controlEndpoint_.hostname() << std::endl; + Log::warning() << "Data and control interface hostnames do not match. " << dataEndpoint_.hostname() + << " /= " << controlEndpoint_.hostname() << std::endl; } if (clientSession != sessionID_) { std::ostringstream ss; ss << "Session ID does not match session received from server: " << sessionID_ << " != " << clientSession; - throw eckit::BadValue(ss.str(), Here()); + throw BadValue(ss.str(), Here()); } if (serverFunctionality.has("NumberOfConnections") && serverFunctionality.getInt("NumberOfConnections") == 1) { single_ = true; } + tracingEnabled_ = serverFunctionality.has("TracingEnabled"); if (single_ && !(dataEndpoint_ == controlEndpoint_)) { - eckit::Log::warning() << "Returned control interface does not match. " << dataEndpoint_ - << " /= " << controlEndpoint_ << std::endl; + Log::warning() << "Returned control interface does not match. " << dataEndpoint_ << " /= " << controlEndpoint_ + << std::endl; } return serverSession; } +std::string msgHeader(MessageHeader& hdr, net::Endpoint& endpoint) { + std::ostringstream ss; + ss << (hdr.control() ? "CONTROL" : "DATA") << " connection=" << endpoint << " [message=" << hdr.message + << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]"; + return ss.str(); +} + void ClientConnection::listeningControlThreadLoop() { try { @@ -338,91 +346,104 @@ void ClientConnection::listeningControlThreadLoop() { while (true) { - eckit::Buffer payload = Connection::readControl(hdr); - - LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message - << ",clientID=" << hdr.clientID() << ",control=" << hdr.control() - << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" - << std::endl; + Buffer payload = Connection::readControl(hdr); + LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop - " + << msgHeader(hdr, controlEndpoint_) << std::endl; if (hdr.message == Message::Exit) { - LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop() -- Control thread stopping" + LOG_DEBUG_LIB(LibFdb5) << "CONTROL connection=" << controlEndpoint_ << " - thread stopping" << std::endl; return; } - else { - if (hdr.clientID()) { - bool handled = false; - ASSERT(hdr.control() || single_); + if (hdr.clientID()) { + ASSERT(hdr.control() || single_); + bool found = false; + bool handled = false; + { + // is the message a response to a blocking request? + // acquire the mutex and look for the request ID in the promises map + // only hold the mutex for the promise lookup/fulfillment, then release before calling handle(). std::lock_guard lock(promisesMutex_); - auto pp = promises_.find(hdr.requestID); if (pp != promises_.end()) { - if (hdr.payloadSize == 0) { - ASSERT(hdr.message == Message::Received); - pp->second.set_value(eckit::Buffer(0)); + found = true; + if (hdr.message == Message::Error) { // this is an error response to a blocking request, + // set the exception on the promise + std::string errmsg = + (hdr.payloadSize == 0) + ? "remote error - no error message provided" + : std::string{static_cast(payload.data()), payload.size()}; + try { + pp->second.set_exception( + std::make_exception_ptr(RemoteFDBException(errmsg, controlEndpoint()))); + } + catch (const std::exception& e) { + Log::error() + << "ERROR: " << msgHeader(hdr, controlEndpoint_) << " - received error \"" << errmsg + << "\" for blocking request - unable to set the exception on the promise: " + << e.what() << std::endl; + } } else { - pp->second.set_value(std::move(payload)); + if (hdr.payloadSize == 0) { + ASSERT(hdr.message == Message::Received); + pp->second.set_value(Buffer(0)); + } + else { + pp->second.set_value(std::move(payload)); + } + handled = true; } promises_.erase(pp); - handled = true; } - else { - Client* client = nullptr; - { - std::lock_guard lock(clientsMutex_); - - auto it = clients_.find(hdr.clientID()); - if (it == clients_.end()) { - std::ostringstream ss; - ss << "ERROR: CONTROL connection=" << controlEndpoint_ - << " received [clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID - << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl; - ss << "ClientID (" << hdr.clientID() << ") not found. ABORTING"; - eckit::Log::status() << ss.str() << std::endl; - eckit::Log::error() << "Retrieving... " << ss.str() << std::endl; - throw eckit::SeriousBug(ss.str(), Here()); - } - client = it->second; - } + } + if (!found) { + // if not a response to a blocking request, then it must be a message for a client, look up the + // client and call handle() + std::lock_guard lock(clientsMutex_); + auto it = clients_.find(hdr.clientID()); + if (it == clients_.end()) { + std::ostringstream ss; + ss << "ERROR: " << msgHeader(hdr, controlEndpoint_) << " - ClientID not found. ABORTING"; + Log::status() << ss.str() << std::endl; + Log::error() << ss.str() << std::endl; + throw SeriousBug(ss.str(), Here()); + } - if (hdr.payloadSize == 0) { - handled = client->handle(hdr.message, hdr.requestID); - } - else { - handled = client->handle(hdr.message, hdr.requestID, std::move(payload)); - } + auto* client = it->second; + if (hdr.payloadSize == 0) { + handled = client->handle(hdr.message, hdr.requestID); + } + else { + handled = client->handle(hdr.message, hdr.requestID, std::move(payload)); } + } - if (!handled) { - std::ostringstream ss; - if (hdr.message == Message::Error) { - ss << "RemoteFDB received an unhandled error on CONTROL connection. [clientID=" - << hdr.clientID() << ",requestID=" << hdr.requestID << "]"; - if (hdr.payloadSize) { - std::string msg; - msg.resize(payload.size(), ' '); - payload.copy(msg.data(), payload.size()); - ss << ": " << msg; - } - throw RemoteFDBException(ss.str(), controlEndpoint_); - } - else { - ss << "ERROR: CONTROL connection=" << controlEndpoint_ - << "Unexpected message recieved [message=" << hdr.message - << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << "]. ABORTING"; - eckit::Log::status() << ss.str() << std::endl; - eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl; - throw eckit::SeriousBug(ss.str(), Here()); + if (!handled) { + std::ostringstream ss; + ss << "ERROR: " << msgHeader(hdr, controlEndpoint_); + + if (hdr.message == Message::Error) { + ss << " - received an unhandled error"; + if (hdr.payloadSize) { + std::string errmsg{static_cast(payload.data()), payload.size()}; + ss << ": \"" << errmsg << "\""; } + Log::status() << ss.str() << std::endl; + Log::error() << ss.str() << std::endl; + throw RemoteFDBException(ss.str(), controlEndpoint_); + } + else { + ss << " - received unexpected message. ABORTING"; + Log::status() << ss.str() << std::endl; + Log::error() << ss.str() << std::endl; + throw SeriousBug(ss.str(), Here()); } } } } - // We don't want to let exceptions escape inside a worker thread. } catch (const std::exception& e) { @@ -433,14 +454,6 @@ void ClientConnection::listeningControlThreadLoop() { } } -void ClientConnection::closeConnection() { - LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::closeConnection() -- Data thread stopping" << std::endl; - std::lock_guard lock(clientsMutex_); - for (auto& [id, client] : clients_) { - client->closeConnection(); - } -} - void ClientConnection::listeningDataThreadLoop() { try { @@ -451,66 +464,64 @@ void ClientConnection::listeningDataThreadLoop() { while (true) { - eckit::Buffer payload = Connection::readData(hdr); - - LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningDataThreadLoop - got [message=" << hdr.message - << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" + Buffer payload = Connection::readData(hdr); + LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningDataThreadLoop - " << msgHeader(hdr, dataEndpoint_) << std::endl; if (hdr.message == Message::Exit) { - closeConnection(); + LOG_DEBUG_LIB(LibFdb5) << "DATA connection=" << dataEndpoint_ << " - thread stopping" << std::endl; + std::lock_guard lock(clientsMutex_); + for (auto& [id, client] : clients_) { + client->closeConnection(); + } return; } - else { - if (hdr.clientID()) { - bool handled = false; - Client* client = nullptr; - { - std::lock_guard lock(clientsMutex_); - - auto it = clients_.find(hdr.clientID()); - if (it == clients_.end()) { - std::ostringstream ss; - ss << "ERROR: DATA connection=" << dataEndpoint_ << " received [clientID=" << hdr.clientID() - << ",requestID=" << hdr.requestID << ",message=" << hdr.message - << ",payload=" << hdr.payloadSize << "]" << std::endl; - ss << "ClientID (" << hdr.clientID() << ") not found. ABORTING"; - eckit::Log::status() << ss.str() << std::endl; - eckit::Log::error() << "Retrieving... " << ss.str() << std::endl; - throw eckit::SeriousBug(ss.str(), Here()); - } - client = it->second; - } - ASSERT(client); - ASSERT(!hdr.control()); - if (hdr.payloadSize == 0) { - handled = client->handle(hdr.message, hdr.requestID); - } - else { - handled = client->handle(hdr.message, hdr.requestID, std::move(payload)); - } + if (hdr.clientID()) { + ASSERT(!hdr.control()); - if (!handled) { - std::ostringstream ss; - if (hdr.message == Message::Error) { - ss << "RemoteFDB received an unhandled error on DATA connection. [clientID=" - << hdr.clientID() << ",requestID=" << hdr.requestID << "]"; - if (hdr.payloadSize) { - std::string msg; - msg.resize(payload.size(), ' '); - payload.copy(msg.data(), payload.size()); - ss << ": " << msg; - } - throw RemoteFDBException(ss.str(), dataEndpoint_); - } - else { - ss << "ERROR: DATA connection=" << dataEndpoint_ << " Unexpected message recieved (" - << hdr.message << "). ABORTING"; - eckit::Log::status() << ss.str() << std::endl; - eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl; - throw eckit::SeriousBug(ss.str(), Here()); + bool handled = false; + + // Hold clientsMutex_ across handle() to prevent the Client + // from being destroyed (via remove()) while handle() is in flight. + std::lock_guard lock(clientsMutex_); + + auto it = clients_.find(hdr.clientID()); + if (it == clients_.end()) { + std::ostringstream ss; + ss << "ERROR: " << msgHeader(hdr, dataEndpoint_) << " - ClientID not found. ABORTING"; + Log::status() << ss.str() << std::endl; + Log::error() << ss.str() << std::endl; + throw SeriousBug(ss.str(), Here()); + } + + auto* client = it->second; + if (hdr.payloadSize == 0) { + handled = client->handle(hdr.message, hdr.requestID); + } + else { + handled = client->handle(hdr.message, hdr.requestID, std::move(payload)); + } + + if (!handled) { + std::ostringstream ss; + ss << "ERROR: " << msgHeader(hdr, dataEndpoint_); + + if (hdr.message == Message::Error) { + ss << " - received an unhandled error"; + if (hdr.payloadSize) { + std::string errmsg{static_cast(payload.data()), payload.size()}; + ss << ": \"" << errmsg << "\""; } + Log::status() << ss.str() << std::endl; + Log::error() << ss.str() << std::endl; + throw RemoteFDBException(ss.str(), dataEndpoint_); + } + else { + ss << " - received unexpected message. ABORTING"; + Log::status() << ss.str() << std::endl; + Log::error() << ss.str() << std::endl; + throw SeriousBug(ss.str(), Here()); } } } diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h index d0ffb102b..7f15168aa 100644 --- a/src/fdb5/remote/client/ClientConnection.h +++ b/src/fdb5/remote/client/ClientConnection.h @@ -60,6 +60,8 @@ class ClientConnection : protected Connection { const eckit::net::Endpoint& controlEndpoint() const; const std::string& defaultEndpoint() const { return defaultEndpoint_; } + bool tracingEnabled() const { return tracingEnabled_; } + using Connection::valid; private: // methods @@ -81,7 +83,6 @@ class ClientConnection : protected Connection { void listeningControlThreadLoop(); void listeningDataThreadLoop(); void dataWriteThreadLoop(); - void closeConnection(); const eckit::net::TCPSocket& controlSocket() const override { return controlClient_; } @@ -105,8 +106,6 @@ class ClientConnection : protected Connection { std::thread listeningControlThread_; std::thread listeningDataThread_; - std::mutex requestMutex_; - // requestID std::mutex idMutex_; uint32_t id_; diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc index 01e861c43..d34dd6951 100644 --- a/src/fdb5/remote/client/ClientConnectionRouter.cc +++ b/src/fdb5/remote/client/ClientConnectionRouter.cc @@ -1,7 +1,8 @@ #include "fdb5/remote/client/ClientConnectionRouter.h" -namespace { +#include +namespace { class ConnectionError : public eckit::Exception { public: @@ -24,7 +25,11 @@ ConnectionError::ConnectionError(const eckit::net::Endpoint& endpoint) { reason(s.str()); eckit::Log::status() << what() << std::endl; } + +std::mutex initMutex; +std::unique_ptr instance_{nullptr}; } // namespace + namespace fdb5::remote { //---------------------------------------------------------------------------------------------------------------------- @@ -86,7 +91,7 @@ std::shared_ptr ClientConnectionRouter::connection( std::shared_ptr ClientConnectionRouter::refresh(const eckit::Configuration& config, const std::shared_ptr& connection) { - std::lock_guard lock(connectionMutex_); + std::lock_guard lock(connectionMutex_); const auto iter = connections_.find(connection->controlEndpoint()); if (iter == connections_.end() || !iter->second->valid()) { auto newConnection = @@ -107,11 +112,20 @@ void ClientConnectionRouter::deregister(ClientConnection& connection) { if (it != connections_.end() && &connection == it->second.get()) { connections_.erase(it); } + if (connections_.empty()) { + std::lock_guard lock(initMutex); + instance_.reset(); + } } ClientConnectionRouter& ClientConnectionRouter::instance() { static ClientConnectionRouter router; return router; + // std::lock_guard lock(initMutex); + // if (!instance_) { + // instance_.reset(new ClientConnectionRouter()); + // } + // return *instance_; } void ClientConnectionRouter::teardown(std::exception_ptr e) { diff --git a/src/fdb5/remote/client/ReadLimiter.cc b/src/fdb5/remote/client/ReadLimiter.cc index 51e34eae3..b139bd8dc 100644 --- a/src/fdb5/remote/client/ReadLimiter.cc +++ b/src/fdb5/remote/client/ReadLimiter.cc @@ -9,39 +9,62 @@ */ #include "fdb5/remote/client/ReadLimiter.h" + +#include +#include #include + #include "eckit/config/Resource.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Log.h" + +#include "fdb5/LibFdb5.h" #include "fdb5/remote/client/RemoteStore.h" namespace fdb5::remote { //---------------------------------------------------------------------------------------------------------------------- namespace { -ReadLimiter* instance_ = nullptr; +std::mutex instanceMutex_; +std::unique_ptr instance_{nullptr}; } // namespace -bool ReadLimiter::isInitialised() { - return instance_ != nullptr; +void ReadLimiter::init(size_t memoryLimit) { + std::lock_guard lock(instanceMutex_); + if (instance_ == nullptr) { + instance_.reset(new ReadLimiter(memoryLimit)); + } } ReadLimiter& ReadLimiter::instance() { - ASSERT(instance_); + std::lock_guard lock(instanceMutex_); + if (instance_ == nullptr) { + instance_.reset(new ReadLimiter(defaultReadLimit())); + } return *instance_; } -void ReadLimiter::init(size_t memoryLimit) { - if (!instance_) { - instance_ = new ReadLimiter(memoryLimit); - } +size_t ReadLimiter::defaultReadLimit() { + static size_t limit = eckit::Resource("$FDB_READ_LIMIT;fdbReadLimit", size_t{1_GiB}); // 1 GiB default + return limit; } -ReadLimiter::ReadLimiter(size_t memoryLimit) : memoryUsed_{0}, memoryLimit_{memoryLimit} {} +ReadLimiter::ReadLimiter(size_t memoryLimit) : memoryUsed_{0}, memoryLimit_{memoryLimit} { + LOG_DEBUG_LIB(LibFdb5) << "ReadLimiter initialized with memory limit: " << eckit::Bytes(memoryLimit_) << std::endl; +} -void ReadLimiter::add(RemoteStore* client, uint32_t id, const FieldLocation& fieldLocation, const Key& remapKey) { +void ReadLimiter::add(RemoteStore* client, uint32_t id, + std::optional> tracingID, + const FieldLocation& fieldLocation, const Key& remapKey) { eckit::Buffer requestBuffer(4096); eckit::MemoryStream s(requestBuffer); + /// add tracing ID to the request buffer (if supported) s << fieldLocation; s << remapKey; + if (tracingID.has_value()) { + s << tracingID.value().get(); + } + size_t requestSize = s.position(); size_t resultSize = fieldLocation.length(); @@ -53,7 +76,7 @@ void ReadLimiter::add(RemoteStore* client, uint32_t id, const FieldLocation& fie } { - std::lock_guard lock(mutex_); + std::lock_guard lock(mutex_); requests_.emplace_back(RequestInfo{client, id, std::move(requestBuffer), requestSize, resultSize}); } @@ -61,7 +84,7 @@ void ReadLimiter::add(RemoteStore* client, uint32_t id, const FieldLocation& fie } bool ReadLimiter::tryNextRequest() { - std::lock_guard lock(mutex_); + std::lock_guard lock(mutex_); if (requests_.empty()) { return false; } @@ -84,7 +107,7 @@ bool ReadLimiter::tryNextRequest() { void ReadLimiter::finishRequest(uint32_t clientID, uint32_t requestID) { { - std::lock_guard lock(mutex_); + std::lock_guard lock(mutex_); auto it = activeRequests_.find(clientID); if (it == activeRequests_.end()) { @@ -104,37 +127,38 @@ void ReadLimiter::finishRequest(uint32_t clientID, uint32_t requestID) { /// @note: Only called when a RemoteStore is destroyed, which is currently on exit. void ReadLimiter::evictClient(size_t clientID) { - { - std::lock_guard lock(mutex_); + std::lock_guard lock(instanceMutex_); + if (instance_ != nullptr) { + auto& instance = *instance_; + std::lock_guard lock(instance.mutex_); // Remove the client's active requests - auto it = activeRequests_.find(clientID); + auto it = instance.activeRequests_.find(clientID); - if (it != activeRequests_.end()) { + if (it != instance.activeRequests_.end()) { for (auto requestID : it->second) { - memoryUsed_ -= resultSizes_[{clientID, requestID}]; - resultSizes_.erase({clientID, requestID}); + instance.memoryUsed_ -= instance.resultSizes_[{clientID, requestID}]; + instance.resultSizes_.erase({clientID, requestID}); } - activeRequests_.erase(it); + instance.activeRequests_.erase(it); } // Clean up any pending requests attributed to this client ///@note O(n), room for optimisation. - auto it2 = requests_.begin(); - while (it2 != requests_.end()) { + auto it2 = instance.requests_.begin(); + while (it2 != instance.requests_.end()) { if (it2->client->id() == clientID) { - it2 = requests_.erase(it2); + it2 = instance.requests_.erase(it2); } else { ++it2; // Only increment if we didn't erase } } + instance.tryNextRequest(); } - - tryNextRequest(); } void ReadLimiter::print(std::ostream& out) const { - std::lock_guard lock(mutex_); + std::lock_guard lock(mutex_); out << "ReadLimiter(memoryUsed=" << memoryUsed_ << ", memoryLimit=" << memoryLimit_ << ") {" << std::endl; diff --git a/src/fdb5/remote/client/ReadLimiter.h b/src/fdb5/remote/client/ReadLimiter.h index bcacc403a..797a1d587 100644 --- a/src/fdb5/remote/client/ReadLimiter.h +++ b/src/fdb5/remote/client/ReadLimiter.h @@ -13,8 +13,9 @@ #include "eckit/io/Buffer.h" #include "eckit/serialisation/MemoryStream.h" +#include "fdb5/database/FieldLocation.h" #include "fdb5/database/Key.h" -#include "fdb5/remote/RemoteFieldLocation.h" +#include "fdb5/remote/client/RemoteStore.h" #include #include @@ -40,8 +41,7 @@ struct RequestInfo { class ReadLimiter { public: - static bool isInitialised(); - + static void init(size_t memoryLimit); static ReadLimiter& instance(); ReadLimiter(const ReadLimiter&) = delete; @@ -49,10 +49,9 @@ class ReadLimiter { ReadLimiter(ReadLimiter&&) = delete; ReadLimiter& operator=(ReadLimiter&&) = delete; - static void init(size_t memoryLimit); - // Add a new request to the queue of requests to be sent. Will not be sent until we know we have buffer space. - void add(RemoteStore* client, uint32_t id, const FieldLocation& fieldLocation, + void add(RemoteStore* client, uint32_t id, std::optional> tracingID, + const FieldLocation& fieldLocation, const Key& remapKey); // use const *? // Attempt to send the next request in the queue. Returns true if a request was sent. @@ -66,7 +65,7 @@ class ReadLimiter { // request). /// @todo: This is somewhat pointless right now because the RemoteStores appear to be infinitely long lived... /// Revisit if this changes. - void evictClient(size_t clientID); + static void evictClient(size_t clientID); // Debugging void print(std::ostream& out) const; @@ -75,12 +74,14 @@ class ReadLimiter { ReadLimiter(size_t memoryLimit); + static size_t defaultReadLimit(); + // Send the request to the server void sendRequest(const RequestInfo& request) const; private: - mutable std::mutex mutex_; + mutable std::recursive_mutex mutex_; size_t memoryUsed_; const size_t memoryLimit_; diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc index a6b5c3713..f3ea53805 100644 --- a/src/fdb5/remote/client/RemoteCatalogue.cc +++ b/src/fdb5/remote/client/RemoteCatalogue.cc @@ -66,7 +66,7 @@ RemoteCatalogue::RemoteCatalogue(const eckit::URI& /*uri*/, const Config& config RemoteCatalogue::~RemoteCatalogue() = default; -void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, +void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, const std::string& tracingID, std::shared_ptr fieldLocation) { ASSERT(!datumKey.empty()); @@ -83,6 +83,9 @@ void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, Buffer buffer(defaultBufferSizeArchive); MemoryStream stream(buffer); + if (tracingEnabled()) { + stream << tracingID; + } stream << idxKey; stream << datumKey; stream << *fieldLocation; @@ -132,7 +135,7 @@ const Rule& RemoteCatalogue::rule() const { return rule_.value().get(); } -void RemoteCatalogue::flush(size_t archivedFields) { +void RemoteCatalogue::flush(size_t archivedFields, const std::string& tracingID) { std::lock_guard lock(archiveMutex_); ASSERT(archivedFields == numLocations_); @@ -142,9 +145,13 @@ void RemoteCatalogue::flush(size_t archivedFields) { eckit::Buffer sendBuf(defaultBufferSizeFlush); eckit::MemoryStream s(sendBuf); + if (tracingEnabled()) { + s << tracingID; + } s << numLocations_; - LOG_DEBUG_LIB(LibFdb5) << " RemoteCatalogue::flush - flushing " << numLocations_ << " fields" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - RemoteCatalogue::flush - flushing " << numLocations_ + << " fields" << std::endl; // The flush call is blocking controlWriteCheckResponse(Message::Flush, generateRequestID(), false, sendBuf, s.position()); @@ -196,14 +203,14 @@ const eckit::Configuration& RemoteCatalogue::clientConfig() const { } bool RemoteCatalogue::handle(Message message, uint32_t requestID) { - Log::warning() << *this << " - Received [message=" << ((uint)message) << ",requestID=" << requestID << "]" - << std::endl; + Log::warning() << *this << " - Received unexpected [message=" << ((uint)message) << ",requestID=" << requestID + << "]" << std::endl; return false; } bool RemoteCatalogue::handle(Message message, uint32_t requestID, eckit::Buffer&& payload) { - LOG_DEBUG_LIB(LibFdb5) << *this << " - Received [message=" << ((uint)message) << ",requestID=" << requestID - << ",payloadSize=" << payload.size() << "]" << std::endl; + Log::warning() << *this << " - Received unexpected [message=" << message << ",requestID=" << requestID + << ",payloadSize=" << payload.size() << "]" << std::endl; return false; } diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h index b71a23bae..e79e3862a 100644 --- a/src/fdb5/remote/client/RemoteCatalogue.h +++ b/src/fdb5/remote/client/RemoteCatalogue.h @@ -40,14 +40,17 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C // From CatalogueWriter const Index& currentIndex() override; bool createIndex(const Key& idxKey, size_t datumKeySize) override; - void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override; + void archive(const Key& idxKey, const Key& datumKey, const std::string& tracingID, + std::shared_ptr fieldLocation) override; void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override; void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override; void reconsolidate() override; // From CatalogueReader DbStats stats() const override { return {}; } - bool retrieve(const Key& /*key*/, Field& /*field*/) const override { return false; } + bool retrieve(const Key& /*key*/, Field& /*field*/, const std::string& /*tracingID*/) const override { + return false; + } // From Catalogue bool selectIndex(const Key& idxKey) override; @@ -74,7 +77,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C std::string type() const override { return typeName(); } bool open() override; - void flush(size_t archivedFields) override; + void flush(size_t archivedFields, const std::string& tracingID) override; void clean() override; void close() override; bool exists() const override; diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc index d4836d26e..8d75f2264 100644 --- a/src/fdb5/remote/client/RemoteStore.cc +++ b/src/fdb5/remote/client/RemoteStore.cc @@ -24,8 +24,10 @@ #include "fdb5/remote/client/ReadLimiter.h" #include "fdb5/rules/Rule.h" +#include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" +#include "eckit/io/Buffer.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "eckit/log/Log.h" @@ -33,6 +35,7 @@ #include "eckit/runtime/Main.h" #include "eckit/serialisation/MemoryStream.h" #include "eckit/serialisation/Reanimator.h" +#include "eckit/utils/Literals.h" #include #include @@ -127,9 +130,7 @@ class FDBRemoteDataHandle : public DataHandle { // If we are in the DataHandle, then there MUST be data to read RemoteStore::StoredMessage msg = std::make_pair(remote::Message{}, eckit::Buffer{0}); - // eckit::Log::info() << "RemoteDataHandle::read() -- popping next" << std::endl; ASSERT(queue_->pop(msg) != -1); - // eckit::Log::info() << "RemoteDataHandle::read() -- popped next" << std::endl; // Handle any remote errors communicated from the server if (msg.first == Message::Error) { @@ -205,9 +206,13 @@ class FDBRemoteDataHandle : public DataHandle { Client::EndpointList storeEndpoints(const Config& config) { ASSERT(config.has("stores")); - ASSERT(config.has("fieldLocationEndpoints")); const auto stores = config.getStringVector("stores"); - const auto fieldLocationEndpoints = config.getStringVector("fieldLocationEndpoints"); + + // endpoints used in RemoteFieldLocations can differ from the store endpoints (e.g. different networs; caninical + // store names) if so, the canonical names must be provided in the fieldLocationEndpoints config, otherwhise the + // store endpoints are used. + const auto fieldLocationEndpoints = + config.has("fieldLocationEndpoints") ? config.getStringVector("fieldLocationEndpoints") : stores; ASSERT(stores.size() == fieldLocationEndpoints.size()); @@ -218,7 +223,6 @@ Client::EndpointList storeEndpoints(const Config& config) { } return out; } - } // namespace //---------------------------------------------------------------------------------------------------------------------- @@ -234,6 +238,7 @@ RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) : } RemoteStore::~RemoteStore() { + // If we have launched a thread with an async and we manage to get here, this is // an error. n.b. if we don't do something, we will block in the destructor // of std::future. @@ -242,9 +247,9 @@ RemoteStore::~RemoteStore() { eckit::Main::instance().terminate(); } - if (ReadLimiter::isInitialised()) { - ReadLimiter::instance().evictClient(id()); - } + ReadLimiter::evictClient(id()); + + deregister(); } eckit::URI RemoteStore::uri() const { @@ -273,15 +278,15 @@ bool RemoteStore::exists() const { return result; } -eckit::DataHandle* RemoteStore::retrieve(Field& field) const { - return field.dataHandle(); +eckit::DataHandle* RemoteStore::retrieve(Field& field, const std::string& tracingID) const { + return field.dataHandle(tracingID); } void RemoteStore::archiveCb( - const Key& key, const void* data, eckit::Length length, + const Key& idxKey, const std::string& tracingID, const void* data, eckit::Length length, std::function fieldLocation)> catalogue_archive) { - ASSERT(!key.empty()); + ASSERT(!idxKey.empty()); ASSERT(data); ASSERT(static_cast(length) != 0ll); @@ -297,8 +302,11 @@ void RemoteStore::archiveCb( eckit::Buffer keyBuffer(defaultBufferSizeKey); eckit::MemoryStream keyStream(keyBuffer); + if (!tracingID.empty()) { + keyStream << tracingID; + } keyStream << dbKey_; - keyStream << key; + keyStream << idxKey; PayloadList payloads; payloads.emplace_back(keyStream.position(), keyBuffer.data()); @@ -313,7 +321,7 @@ bool RemoteStore::open() { return true; } -size_t RemoteStore::flush() { +size_t RemoteStore::flush(const std::string& tracingID) { // Flush only does anything if there is an ongoing archive(); if (locations_.archived() == 0) { @@ -328,9 +336,14 @@ size_t RemoteStore::flush() { Buffer sendBuf(defaultBufferSizeFlush); MemoryStream s(sendBuf); + // check if the server supports the tracingID, if so include it in the payload + if (tracingEnabled()) { + s << tracingID; + } s << locations; - LOG_DEBUG_LIB(LibFdb5) << " RemoteStore::flush - flushing " << locations << " fields" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - RemoteStore::flush - flushing " << locations + << " fields" << std::endl; // The flush call is blocking controlWriteCheckResponse(Message::Flush, generateRequestID(), false, sendBuf, s.position()); @@ -351,16 +364,12 @@ void RemoteStore::print(std::ostream& out) const { } void RemoteStore::closeConnection() { + std::lock_guard lock(messageMutex_); for (auto& kv : messageQueues_) { if (!kv.second->closed()) { kv.second->interrupt(std::make_exception_ptr(eckit::Exception("Unexpected closure of store", Here()))); } } - for (auto& kv : retrieveMessageQueues_) { - if (!kv.second->closed()) { - kv.second->interrupt(std::make_exception_ptr(eckit::Exception("Unexpected closure of store", Here()))); - } - } } const eckit::Configuration& RemoteStore::clientConfig() const { @@ -371,38 +380,31 @@ bool RemoteStore::handle(Message message, uint32_t requestID) { switch (message) { case Message::Complete: { - // eckit::Log::info() << "RemoteStore::handle COMPLETE" << std::endl; - auto it = messageQueues_.find(requestID); - if (it != messageQueues_.end()) { - // eckit::Log::info() << "RemoteStore::handle COMPLETE close and erase queue" << std::endl; - it->second->close(); - - // Remove entry (shared_ptr --> message queue will be destroyed when it - // goes out of scope in the worker thread). - messageQueues_.erase(it); - // eckit::Log::info() << "RemoteStore::handle COMPLETE closed and erased queue" << std::endl; + std::lock_guard lock(messageMutex_); + auto id = messageQueues_.find(requestID); + if (id == messageQueues_.end()) { + return false; } - else { - std::lock_guard lock(retrieveMessageMutex_); - auto id = retrieveMessageQueues_.find(requestID); - ASSERT(id != retrieveMessageQueues_.end()); - id->second->emplace(std::make_pair(message, Buffer(0))); - - retrieveMessageQueues_.erase(id); - } + id->second->emplace(std::make_pair(message, Buffer(0))); + messageQueues_.erase(id); return true; } case Message::Error: { - - std::ostringstream ss; - ss << "RemoteStore client id: " << id() << " - received an error without error description for requestID " - << requestID << std::endl; - throw RemoteFDBException(ss.str(), controlEndpoint()); - + // Received Error message without error description. Remove the corresponding entry from the message queue + // and let the caller know & complain + std::lock_guard lock(messageMutex_); + auto it = messageQueues_.find(requestID); + if (it != messageQueues_.end()) { + it->second->interrupt( + std::make_exception_ptr(RemoteFDBException("no error description provided", controlEndpoint()))); + messageQueues_.erase(it); + } return false; } default: + Log::warning() << *this << " - Received unexpected [message=" << ((uint)message) + << ",requestID=" << requestID << "]" << std::endl; return false; } } @@ -410,9 +412,13 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa switch (message) { - case Message::Store: { // received a Field location from the remote store, can forward to the archiver for the - // indexing + case Message::Store: { + // received a FieldLocation from the remote store, can forward to the archiver for the indexing MemoryStream s(payload); + std::string tracingID{}; + if (tracingEnabled()) { + s >> tracingID; + } std::unique_ptr location(eckit::Reanimator::reanimate(s)); if (defaultEndpoint().empty()) { return locations_.location(requestID, std::move(location)); @@ -424,58 +430,53 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa } } case Message::Blob: { - auto it = messageQueues_.find(requestID); - if (it != messageQueues_.end()) { - it->second->emplace(message, std::move(payload)); - } - else { - std::lock_guard lock(retrieveMessageMutex_); - auto id = retrieveMessageQueues_.find(requestID); - ASSERT(id != retrieveMessageQueues_.end()); - id->second->emplace(std::make_pair(message, std::move(payload))); - } + std::lock_guard lock(messageMutex_); + auto id = messageQueues_.find(requestID); + ASSERT(id != messageQueues_.end()); + id->second->emplace(std::make_pair(message, std::move(payload))); return true; } case Message::Error: { - + std::lock_guard lock(messageMutex_); auto it = messageQueues_.find(requestID); if (it != messageQueues_.end()) { - std::string msg; - msg.resize(payload.size(), ' '); - payload.copy(&msg[0], payload.size()); - it->second->interrupt(std::make_exception_ptr(RemoteFDBException(msg, controlEndpoint()))); - - // Remove entry (shared_ptr --> message queue will be destroyed when it - // goes out of scope in the worker thread). + std::string errmsg{static_cast(payload.data()), payload.size()}; + it->second->interrupt(std::make_exception_ptr(RemoteFDBException(errmsg, controlEndpoint()))); messageQueues_.erase(it); } - return true; + return false; } default: + Log::warning() << *this << " - Received unexpected [message=" << message << ",requestID=" << requestID + << ",payloadSize=" << payload.size() << "]" << std::endl; return false; } } -eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation) { - return dataHandle(fieldLocation, Key()); +eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const std::string& tracingID) { + return dataHandle(fieldLocation, tracingID, Key()); } -eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) { +eckit::DataHandle* RemoteStore::dataHandle(const FieldLocation& fieldLocation, const std::string& tracingID, + const Key& remapKey) { + if (tracingID.empty()) { + throw RemoteFDBException("No tracing ID provided for data handle request for fieldLocation: ", + controlEndpoint()); + } uint32_t id = generateRequestID(); static size_t queueSize = 320; std::shared_ptr queue = nullptr; { - std::lock_guard lock(retrieveMessageMutex_); - - auto entry = retrieveMessageQueues_.emplace(id, std::make_shared(queueSize)); + std::lock_guard lock(messageMutex_); + auto entry = messageQueues_.emplace(id, std::make_shared(queueSize)); ASSERT(entry.second); queue = entry.first->second; } - ReadLimiter::instance().add(this, id, fieldLocation, remapKey); + ReadLimiter::instance().add(this, id, tracingID, fieldLocation, remapKey); return new FDBRemoteDataHandle(id, this->id(), fieldLocation.length(), queue, controlEndpoint()); } diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h index 9b6eb27e2..c6872e623 100644 --- a/src/fdb5/remote/client/RemoteStore.h +++ b/src/fdb5/remote/client/RemoteStore.h @@ -125,13 +125,14 @@ class RemoteStore : public Store, public Client { static eckit::URI uri(const eckit::URI& dataURI); bool open() override; - size_t flush() override; + size_t flush(const std::string& tracingID) override; void close() override; void checkUID() const override {} - eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation); - eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation, const Key& remapKey); + eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation, const std::string& tracingID); + eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation, const std::string& tracingID, + const Key& remapKey); bool canMoveTo(const Key& key, const Config& config, const eckit::URI& dest) const override { return false; } void moveTo(const Key& key, const Config& config, const eckit::URI& dest, @@ -159,9 +160,9 @@ class RemoteStore : public Store, public Client { bool exists() const override; - eckit::DataHandle* retrieve(Field& field) const override; + eckit::DataHandle* retrieve(Field& field, const std::string& tracingID) const override; void archiveCb( - const Key& key, const void* data, eckit::Length length, + const Key& idxKey, const std::string& tracingID, const void* data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; @@ -187,9 +188,8 @@ class RemoteStore : public Store, public Client { // The shared_ptr allows this removal to be asynchronous with the actual task // cleaning up and returning to the client. std::map> messageQueues_; - std::map> retrieveMessageQueues_; - std::mutex retrieveMessageMutex_; + std::mutex messageMutex_; Locations locations_; }; diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc index dc4e416fe..6217f22db 100644 --- a/src/fdb5/remote/server/CatalogueHandler.cc +++ b/src/fdb5/remote/server/CatalogueHandler.cc @@ -189,7 +189,7 @@ template struct BaseHelper { virtual size_t encodeBufferSize(const ValueType&) const { return 4096; } void extraDecode(eckit::Stream&) {} - ValueType apiCall(FDB&, const FDBToolRequest&) const { NOTIMP; } + ValueType apiCall(FDB&, const FDBToolRequest&, const std::string&) const { NOTIMP; } struct Encoded { size_t position; @@ -205,7 +205,9 @@ struct BaseHelper { }; struct ListHelper : public BaseHelper { - ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.list(request, false, depth_); } + ListIterator apiCall(FDB& fdb, const FDBToolRequest& request, const std::string& tracingID) const { + return fdb.list(request, ListMode::Full, tracingID, depth_); + } void extraDecode(eckit::Stream& s) { s >> depth_; } @@ -218,7 +220,9 @@ struct AxesHelper : public BaseHelper { virtual size_t encodeBufferSize(const AxesElement& el) const { return el.encodeSize(); } void extraDecode(eckit::Stream& s) { s >> level_; } - AxesIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.axesIterator(request, level_); } + AxesIterator apiCall(FDB& fdb, const FDBToolRequest& request, const std::string& tracingID) const { + return fdb.axesIterator(request, tracingID, level_); + } private: @@ -264,10 +268,8 @@ struct WipeHelper : public BaseHelper { s >> unsafeWipeAll_; } - WipeStateIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { - // XXX: I'm inclined to say that in a multi-server scenario, unsafe wipe all is a bad idea. - ASSERT(!unsafeWipeAll_); - return fdb.internal_->wipe(request, doit_, false, unsafeWipeAll_); + WipeStateIterator apiCall(FDB& fdb, const FDBToolRequest& request, const std::string& tracingID) const { + return fdb.internal_->wipe(request, tracingID, doit_, false, unsafeWipeAll_); } private: @@ -277,11 +279,15 @@ struct WipeHelper : public BaseHelper { }; struct InspectHelper : public BaseHelper { - ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.inspect(request.request()); } + ListIterator apiCall(FDB& fdb, const FDBToolRequest& request, const std::string& tracingID) const { + return fdb.inspect(request.request(), tracingID); + } }; struct StatsHelper : public BaseHelper { - StatsIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.stats(request); } + StatsIterator apiCall(FDB& fdb, const FDBToolRequest& request, const std::string& tracingID) const { + return fdb.stats(request, tracingID); + } }; template @@ -290,7 +296,15 @@ void CatalogueHandler::handleApiCall(uint32_t clientID, uint32_t requestID, ecki MemoryStream s(payload); + std::string tracingID; FDBToolRequest request(s); + if (tracingEnabled_) { + s >> tracingID; + } + if (tracingEnabled_ && !tracingID.empty()) { + Log::info() << "Received: " << tracingID << " - " << request.request() << std::endl; + } + helper.extraDecode(s); // Construct worker thread to feed responses back to client @@ -304,7 +318,8 @@ void CatalogueHandler::handleApiCall(uint32_t clientID, uint32_t requestID, ecki } } - workerThreads_.emplace(requestID, std::async(std::launch::async, [request, clientID, requestID, helper, this]() { + workerThreads_.emplace(requestID, + std::async(std::launch::async, [request, clientID, requestID, helper, tracingID, this]() { try { FDB* fdb = nullptr; { @@ -314,7 +329,7 @@ void CatalogueHandler::handleApiCall(uint32_t clientID, uint32_t requestID, ecki fdb = &it->second; ASSERT(fdb); } - auto iterator = helper.apiCall(*fdb, request); + auto iterator = helper.apiCall(*fdb, request, tracingID); typename decltype(iterator)::value_type elem; while (iterator.next(elem)) { @@ -464,14 +479,21 @@ void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffe ASSERT(payload.size() > 0); + std::string tracingID; size_t numArchived = 0; + MemoryStream s(payload); + if (agreedConf().tracingEnabled()) { + s >> tracingID; + } s >> numArchived; if (numArchived == 0) { return; } + Log::info() << (tracingID.empty() ? "" : tracingID + " - ") << "flushing " << numArchived << " fields" << std::endl; + auto it = catalogues_.find(clientID); ASSERT(it != catalogues_.end()); @@ -493,17 +515,21 @@ void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffe it->second.locationsArchived = 0; } - it->second.catalogue->flush(numArchived); + it->second.catalogue->flush(numArchived, tracingID); - Log::info() << "Flush complete" << std::endl; - Log::status() << "Flush complete" << std::endl; + Log::info() << (tracingID.empty() ? "" : tracingID + " - ") << "flush complete" << std::endl; + Log::status() << (tracingID.empty() ? "" : tracingID + " - ") << "flush complete" << std::endl; } // A helper function to make archiveThreadLoop a bit cleaner void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) { - MemoryStream s(data, length); + std::string tracingID; + MemoryStream s(data, length); + if (agreedConf().tracingEnabled()) { + s >> tracingID; + } fdb5::Key idxKey(s); fdb5::Key datumKey(s); @@ -512,6 +538,7 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque LOG_DEBUG_LIB(LibFdb5) << "CatalogueHandler::archiveBlob key: " << idxKey << datumKey << " location: " << location->uri() << std::endl; + std::map::iterator it; { std::lock_guard lock(handlerMutex_); @@ -523,10 +550,12 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque } } + eckit::Log::info() << (tracingID.empty() ? "" : tracingID + " - ") << "archiving " << *location << " to " + << it->second.catalogue->key() << idxKey << datumKey << std::endl; if (!it->second.catalogue->selectIndex(idxKey)) { it->second.catalogue->createIndex(idxKey, datumKey.keys().size()); } - it->second.catalogue->archive(idxKey, datumKey, std::move(location)); + it->second.catalogue->archive(idxKey, datumKey, tracingID, std::move(location)); { std::lock_guard lock(fieldLocationsMutex_); it->second.locationsArchived++; diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc index 6655c2f2d..b62786456 100644 --- a/src/fdb5/remote/server/ServerConnection.cc +++ b/src/fdb5/remote/server/ServerConnection.cc @@ -181,6 +181,7 @@ void ServerConnection::initialiseConnections() { try { agreedConf_ = RemoteConfiguration::common(clientConf, serverConf); single_ = agreedConf_.singleConnection(); + tracingEnabled_ = agreedConf_.tracingEnabled(); } catch (const eckit::Exception& e) { error(e.what(), hdr.clientID(), hdr.requestID); diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h index 903d3fac4..28db640fa 100644 --- a/src/fdb5/remote/server/ServerConnection.h +++ b/src/fdb5/remote/server/ServerConnection.h @@ -65,11 +65,13 @@ struct readLocationElem { uint32_t clientID; uint32_t requestID; std::unique_ptr readLocation; + std::optional tracingID; - readLocationElem() : clientID(0), requestID(0), readLocation(nullptr) {} + readLocationElem() : clientID(0), requestID(0), readLocation(nullptr), tracingID(std::nullopt) {} - readLocationElem(uint32_t clientID, uint32_t requestID, std::unique_ptr readLocation) : - clientID(clientID), requestID(requestID), readLocation(std::move(readLocation)) {} + readLocationElem(uint32_t clientID, uint32_t requestID, std::unique_ptr readLocation, + std::optional tracingID) : + clientID(clientID), requestID(requestID), readLocation(std::move(readLocation)), tracingID(tracingID) {} }; struct ArchiveElem { diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc index 54efa7c2d..49fd18891 100644 --- a/src/fdb5/remote/server/StoreHandler.cc +++ b/src/fdb5/remote/server/StoreHandler.cc @@ -145,13 +145,18 @@ void StoreHandler::read(uint32_t clientID, uint32_t requestID, const eckit::Buff MemoryStream s(payload); std::unique_ptr location(eckit::Reanimator::reanimate(s)); - - LOG_DEBUG_LIB(LibFdb5) << "Queuing for read: " << requestID << " " << *location << std::endl; + Key remapKey(s); + std::optional tracingID; + if (true /* tracingEnabled_ */) { + std::string tracingIDstr; + s >> tracingIDstr; + tracingID = tracingIDstr; + } std::unique_ptr dh; - dh.reset(location->dataHandle()); + dh.reset(location->dataHandle(tracingID.has_value() ? tracingID.value() : "")); - readLocationQueue_.emplace(readLocationElem(clientID, requestID, std::move(dh))); + readLocationQueue_.emplace(readLocationElem(clientID, requestID, std::move(dh), tracingID)); } void StoreHandler::readLocationThreadLoop() { @@ -162,12 +167,12 @@ void StoreHandler::readLocationThreadLoop() { // send the data back to the client. // Forward the API call - writeToParent(elem.clientID, elem.requestID, std::move(elem.readLocation)); + writeToParent(elem.clientID, elem.requestID, std::move(elem.readLocation), elem.tracingID); } } void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t requestID, - std::unique_ptr dh) { + std::unique_ptr dh, std::optional& tracingID) { try { Log::status() << "Reading: " << requestID << std::endl; // Write the data to the parent, in chunks if necessary. @@ -179,6 +184,9 @@ void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t request dh->openForRead(); LOG_DEBUG_LIB(LibFdb5) << "Reading: " << requestID << " dh size: " << dh->size() << std::endl; + Log::info() << LogFormatSetter{Log::normalFormat} << (tracingID.has_value() ? (tracingID.value() + " - ") : "") + << "reading: " << *dh << std::endl; + while ((dataRead = dh->read(writeBuffer, writeBuffer.size())) != 0) { write(Message::Blob, false, clientID, requestID, writeBuffer, dataRead); } @@ -205,8 +213,12 @@ void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t request void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) { - MemoryStream s(data, length); + std::string tracingID; + MemoryStream s(data, length); + if (true /* tracingEnabled_ */) { + s >> tracingID; + } fdb5::Key dbKey(s); fdb5::Key idxKey(s); @@ -218,7 +230,11 @@ void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID Store& ss = store(clientID, dbKey); - std::shared_ptr location = ss.archive(idxKey, charData + s.position(), length - s.position()); + std::shared_ptr location = + ss.archive(idxKey, tracingID, charData + s.position(), length - s.position()); + if (!tracingID.empty()) { + Log::info() << tracingID << " - archiving " << *location << std::endl; + } std::promise> promise; promise.set_value(location); @@ -235,6 +251,9 @@ void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID eckit::Buffer buffer(16_KiB); MemoryStream stream(buffer); + if (true /* tracingEnabled_ */) { + stream << tracingID; + } stream << (*location); Connection::write(Message::Store, true, clientID, requestID, buffer, stream.position()); } @@ -242,13 +261,20 @@ void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buffer& payload) { size_t numArchived = 0; - + std::string tracingID; if (requestID != 0) { ASSERT(payload.size() > 0); MemoryStream stream(payload); + + if (true /* agreedConf().tracingEnabled()*/) { + stream >> tracingID; + } stream >> numArchived; } + if (!tracingID.empty()) { + Log::info() << tracingID << " - flushing " << numArchived << " fields" << std::endl; + } if (numArchived > 0) { ASSERT(archiveFuture_.valid()); @@ -256,13 +282,13 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf std::lock_guard lock(handlerMutex_); auto it = stores_.find(clientID); ASSERT(it != stores_.end()); - it->second.store->flush(); + it->second.store->flush(tracingID); flushCallback_(); } - Log::info() << "Flush complete" << std::endl; - Log::status() << "Flush complete" << std::endl; + Log::info() << (tracingID.empty() ? "" : (tracingID + " - ")) << "flush complete" << std::endl; + Log::status() << (tracingID.empty() ? "" : (tracingID + " - ")) << "flush complete" << std::endl; } bool StoreHandler::remove(bool control, uint32_t clientID) { diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h index 8b4f10a58..ccb0e9e55 100644 --- a/src/fdb5/remote/server/StoreHandler.h +++ b/src/fdb5/remote/server/StoreHandler.h @@ -46,7 +46,8 @@ class StoreHandler : public ServerConnection, public CallbackRegistry { void readLocationThreadLoop(); - void writeToParent(uint32_t clientID, uint32_t requestID, std::unique_ptr dh); + void writeToParent(uint32_t clientID, uint32_t requestID, std::unique_ptr dh, + std::optional& tracingID); bool remove(bool control, uint32_t clientID) override; diff --git a/src/fdb5/toc/AdoptVisitor.cc b/src/fdb5/toc/AdoptVisitor.cc index e3d1323c6..574f25aa0 100644 --- a/src/fdb5/toc/AdoptVisitor.cc +++ b/src/fdb5/toc/AdoptVisitor.cc @@ -21,9 +21,9 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -AdoptVisitor::AdoptVisitor(Archiver& owner, const Key& initialFieldKey, const PathName& path, Offset offset, - Length length) : - BaseArchiveVisitor(owner, initialFieldKey), path_(path), offset_(offset), length_(length) { +AdoptVisitor::AdoptVisitor(Archiver& owner, const Key& initialFieldKey, const std::string& tracingID, + const PathName& path, Offset offset, Length length) : + BaseArchiveVisitor(owner, initialFieldKey, tracingID), path_(path), offset_(offset), length_(length) { ASSERT(offset_ >= Offset(0)); ASSERT(length_ > Length(0)); } diff --git a/src/fdb5/toc/AdoptVisitor.h b/src/fdb5/toc/AdoptVisitor.h index e8b8ebc50..c8801491f 100644 --- a/src/fdb5/toc/AdoptVisitor.h +++ b/src/fdb5/toc/AdoptVisitor.h @@ -34,8 +34,8 @@ class AdoptVisitor : public BaseArchiveVisitor { public: // methods - AdoptVisitor(Archiver& owner, const Key& initialFieldKey, const eckit::PathName& path, eckit::Offset offset, - eckit::Length length); + AdoptVisitor(Archiver& owner, const Key& initialFieldKey, const std::string& tracingID, const eckit::PathName& path, + eckit::Offset offset, eckit::Length length); protected: // methods diff --git a/src/fdb5/toc/TocCatalogueReader.cc b/src/fdb5/toc/TocCatalogueReader.cc index c1369bd36..343d66a43 100644 --- a/src/fdb5/toc/TocCatalogueReader.cc +++ b/src/fdb5/toc/TocCatalogueReader.cc @@ -110,9 +110,10 @@ void TocCatalogueReader::close() { } } -bool TocCatalogueReader::retrieve(const Key& key, Field& field) const { - LOG_DEBUG_LIB(LibFdb5) << "Trying to retrieve key " << key << " " << key.names() << std::endl; - LOG_DEBUG_LIB(LibFdb5) << "Scanning indexes " << matching_.size() << std::endl; +bool TocCatalogueReader::retrieve(const Key& key, Field& field, const std::string& tracingID) const { + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - Trying to retrieve key " << key << " " << key.names() + << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tracingID: " << tracingID << " - Scanning indexes " << matching_.size() << std::endl; for (const auto& m : matching_) { const Index& idx(m->first); diff --git a/src/fdb5/toc/TocCatalogueReader.h b/src/fdb5/toc/TocCatalogueReader.h index 4862e86c4..beea08d79 100644 --- a/src/fdb5/toc/TocCatalogueReader.h +++ b/src/fdb5/toc/TocCatalogueReader.h @@ -56,11 +56,11 @@ class TocCatalogueReader : public TocCatalogue, public CatalogueReader { void deselectIndex() override; bool open() override; - void flush(size_t archivedFields) override {} + void flush(size_t archivedFields, const std::string& tracingID) override {} void clean() override {} void close() override; - bool retrieve(const Key& key, Field& field) const override; + bool retrieve(const Key& key, Field& field, const std::string& tracingID) const override; void print(std::ostream& out) const override; diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc index 7a42d488f..160a741cb 100644 --- a/src/fdb5/toc/TocCatalogueWriter.cc +++ b/src/fdb5/toc/TocCatalogueWriter.cc @@ -52,6 +52,7 @@ TocCatalogueWriter::~TocCatalogueWriter() { // selectIndex is called during schema traversal and in case of out-of-order fieldLocation archival bool TocCatalogueWriter::selectIndex(const Key& idxKey) { + std::lock_guard lock(archiveMutex_); currentIndexKey_ = idxKey; @@ -126,7 +127,7 @@ void TocCatalogueWriter::clean() { LOG_DEBUG_LIB(LibFdb5) << "Closing path " << directory_ << std::endl; - flush(archivedLocations_); // closes the TOC entries & indexes but not data files + flush(archivedLocations_, ""); // closes the TOC entries & indexes but not data files compactSubTocIndexes(); @@ -139,6 +140,8 @@ void TocCatalogueWriter::close() { } void TocCatalogueWriter::index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) { + std::lock_guard lock(archiveMutex_); + archivedLocations_++; if (current_.null()) { @@ -321,8 +324,9 @@ bool TocCatalogueWriter::enabled(const ControlIdentifier& controlIdentifier) con return TocCatalogue::enabled(controlIdentifier); } -void TocCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, +void TocCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, const std::string& tracingID, std::shared_ptr fieldLocation) { + std::lock_guard lock(archiveMutex_); archivedLocations_++; @@ -342,6 +346,9 @@ void TocCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, } } + LOG_DEBUG_LIB(LibFdb5) << (tracingID.empty() ? "" : tracingID + " - ") << "archiving " << *fieldLocation + << std::endl; + Field field(std::move(fieldLocation), currentIndex().timestamp()); current_.put(datumKey, field); @@ -351,7 +358,7 @@ void TocCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, } } -void TocCatalogueWriter::flush(size_t archivedFields) { +void TocCatalogueWriter::flush(size_t archivedFields, const std::string& tracingID) { ASSERT(archivedFields == archivedLocations_); if (archivedLocations_ == 0) { diff --git a/src/fdb5/toc/TocCatalogueWriter.h b/src/fdb5/toc/TocCatalogueWriter.h index b1a4e68e7..b402f745b 100644 --- a/src/fdb5/toc/TocCatalogueWriter.h +++ b/src/fdb5/toc/TocCatalogueWriter.h @@ -16,12 +16,13 @@ #ifndef fdb5_TocCatalogueWriter_H #define fdb5_TocCatalogueWriter_H +#include + #include "eckit/os/AutoUmask.h" #include "fdb5/database/Index.h" -#include "fdb5/toc/TocRecord.h" - #include "fdb5/toc/TocCatalogue.h" +#include "fdb5/toc/TocRecord.h" #include "fdb5/toc/TocSerialisationVersion.h" namespace fdb5 { @@ -70,11 +71,12 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter { void deselectIndex() override; bool open() override; - void flush(size_t archivedFields) override; + void flush(size_t archivedFields, const std::string& tracingID) override; void clean() override; void close() override; - void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override; + void archive(const Key& idxKey, const Key& datumKey, const std::string& tracingID, + std::shared_ptr fieldLocation) override; void reconsolidateIndexesAndTocs(); void print(std::ostream& out) const override; @@ -110,6 +112,8 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter { eckit::AutoUmask umask_; size_t archivedLocations_; + + std::recursive_mutex archiveMutex_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/toc/TocFieldLocation.cc b/src/fdb5/toc/TocFieldLocation.cc index 8ac190377..12a3b7ca9 100644 --- a/src/fdb5/toc/TocFieldLocation.cc +++ b/src/fdb5/toc/TocFieldLocation.cc @@ -50,7 +50,7 @@ std::shared_ptr TocFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); } -eckit::DataHandle* TocFieldLocation::dataHandle() const { +eckit::DataHandle* TocFieldLocation::dataHandle(const std::string& tracingID) const { if (remapKey_.empty()) { return uri_.path().partHandle(offset(), length()); } diff --git a/src/fdb5/toc/TocFieldLocation.h b/src/fdb5/toc/TocFieldLocation.h index c816cd0b7..57a4fe08a 100644 --- a/src/fdb5/toc/TocFieldLocation.h +++ b/src/fdb5/toc/TocFieldLocation.h @@ -37,7 +37,7 @@ class TocFieldLocation : public FieldLocation { TocFieldLocation(const UriStore& store, const FieldRef& ref); TocFieldLocation(eckit::Stream&); - eckit::DataHandle* dataHandle() const override; + eckit::DataHandle* dataHandle(const std::string& tracingID) const override; std::shared_ptr make_shared() const override; diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc index 52d72af39..3a57e7513 100644 --- a/src/fdb5/toc/TocStore.cc +++ b/src/fdb5/toc/TocStore.cc @@ -135,11 +135,12 @@ bool TocStore::exists() const { return directory_.exists(); } -eckit::DataHandle* TocStore::retrieve(Field& field) const { - return field.dataHandle(); +eckit::DataHandle* TocStore::retrieve(Field& field, const std::string& tracingID) const { + return field.dataHandle(tracingID); } -std::unique_ptr TocStore::archive(const Key& idxKey, const void* data, eckit::Length length) { +std::unique_ptr TocStore::archive(const Key& idxKey, const std::string& tracingID, + const void* data, eckit::Length length) { archivedFields_++; eckit::PathName dataPath = getDataPath(idxKey); @@ -148,6 +149,11 @@ std::unique_ptr TocStore::archive(const Key& idxKey, const eckit::Offset position = dh.position(); + if (!tracingID.empty()) { + LOG_DEBUG_LIB(LibFdb5) << tracingID << " - archiving " << eckit::Bytes(length) << "to " << dataPath + << std::endl; + } + long len = dh.write(data, length); ASSERT(len == static_cast(length)); @@ -155,11 +161,15 @@ std::unique_ptr TocStore::archive(const Key& idxKey, const return std::make_unique(dataPath, position, length, Key()); } -size_t TocStore::flush() { +size_t TocStore::flush(const std::string& tracingID) { if (archivedFields_ == 0) { return 0; } + if (!tracingID.empty()) { + LOG_DEBUG_LIB(LibFdb5) << tracingID << " - flushing " << archivedFields_ << " fields" << std::endl; + } + // ensure consistent state before writing Toc entry flushDataHandles(); diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h index 904dd77e5..8a17e6b91 100644 --- a/src/fdb5/toc/TocStore.h +++ b/src/fdb5/toc/TocStore.h @@ -51,7 +51,7 @@ class TocStore : public Store, public TocCommon { std::set asCollocatedDataURIs(const std::set&) const override; bool open() override { return true; } - size_t flush() override; + size_t flush(const std::string& tracingID) override; void close() override; void checkUID() const override { TocCommon::checkUID(); } @@ -74,8 +74,9 @@ class TocStore : public Store, public TocCommon { std::string type() const override { return "file"; } bool exists() const override; - eckit::DataHandle* retrieve(Field& field) const override; - std::unique_ptr archive(const Key& idxKey, const void* data, eckit::Length length) override; + eckit::DataHandle* retrieve(Field& field, const std::string& tracingID) const override; + std::unique_ptr archive(const Key& idxKey, const std::string& tracingID, const void* data, + eckit::Length length) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; diff --git a/src/fdb5/tools/compare/grib/Utils.cc b/src/fdb5/tools/compare/grib/Utils.cc index 2f8b5548b..7f931da3c 100644 --- a/src/fdb5/tools/compare/grib/Utils.cc +++ b/src/fdb5/tools/compare/grib/Utils.cc @@ -70,7 +70,7 @@ std::unique_ptr extractGribMessage(const fdb5::ListElement& gribLoc) const auto length = gribLoc.length(); std::unique_ptr buffer = std::make_unique(length); - auto dh = std::unique_ptr(gribLoc.location().dataHandle()); + auto dh = std::unique_ptr(gribLoc.location().dataHandle("")); dh->openForRead(); eckit::AutoClose closer(*dh); diff --git a/tests/fdb/api/ApiSpy.h b/tests/fdb/api/ApiSpy.h index 10a10d0a4..2732d2bd6 100644 --- a/tests/fdb/api/ApiSpy.h +++ b/tests/fdb/api/ApiSpy.h @@ -86,66 +86,70 @@ class ApiSpy : public fdb5::FDBBase { } ~ApiSpy() override { knownSpies().erase(std::find(knownSpies().begin(), knownSpies().end(), this)); } - void archive(const fdb5::Key& key, const void* data, size_t length) override { + void archive(const fdb5::Key& key, const void* data, size_t length, const std::string& tracingID) override { counts_.archive += 1; archives_.push_back(std::make_tuple(key, data, length)); } - fdb5::ListIterator inspect(const metkit::mars::MarsRequest& request) override { + fdb5::ListIterator inspect(const metkit::mars::MarsRequest& request, const std::string& tracingID) override { counts_.inspect += 1; retrieves_.push_back(request); return fdb5::ListIterator(0); } - fdb5::ListIterator list(const fdb5::FDBToolRequest& /* request */, const int level) override { + fdb5::ListIterator list(const fdb5::FDBToolRequest& /* request */, const std::string& tracingID, + const int level) override { counts_.list += 1; ASSERT(level == 3); return fdb5::ListIterator(0); } - fdb5::AxesIterator axesIterator(const fdb5::FDBToolRequest& request, int level = 3) override { + fdb5::AxesIterator axesIterator(const fdb5::FDBToolRequest& request, const std::string& tracingID, + int level = 3) override { counts_.axes += 1; return fdb5::AxesIterator(0); } - fdb5::DumpIterator dump(const fdb5::FDBToolRequest& request, bool simple) override { + fdb5::DumpIterator dump(const fdb5::FDBToolRequest& request, const std::string& tracingID, bool simple) override { counts_.dump += 1; return fdb5::DumpIterator(0); } - fdb5::StatusIterator status(const fdb5::FDBToolRequest& request) override { + fdb5::StatusIterator status(const fdb5::FDBToolRequest& request, const std::string& tracingID) override { counts_.status += 1; return fdb5::StatusIterator(0); } - fdb5::WipeStateIterator wipe(const fdb5::FDBToolRequest& request, bool doit, bool verbose, - bool unsafeWipeAll) override { + fdb5::WipeStateIterator wipe(const fdb5::FDBToolRequest& request, const std::string& tracingID, bool doit, + bool verbose, bool unsafeWipeAll) override { counts_.wipe += 1; return fdb5::WipeStateIterator(0); } - fdb5::PurgeIterator purge(const fdb5::FDBToolRequest& request, bool doit, bool verbose) override { + fdb5::PurgeIterator purge(const fdb5::FDBToolRequest& request, const std::string& tracingID, bool doit, + bool verbose) override { counts_.purge += 1; return fdb5::PurgeIterator(0); } - fdb5::StatsIterator stats(const fdb5::FDBToolRequest& request) override { + fdb5::StatsIterator stats(const fdb5::FDBToolRequest& request, const std::string& tracingID) override { counts_.stats += 1; return fdb5::StatsIterator(0); } - fdb5::MoveIterator move(const fdb5::FDBToolRequest& request, const eckit::URI& dest) override { + fdb5::MoveIterator move(const fdb5::FDBToolRequest& request, const std::string& tracingID, + const eckit::URI& dest) override { counts_.move += 1; return fdb5::MoveIterator(0); } - fdb5::StatusIterator control(const fdb5::FDBToolRequest& request, fdb5::ControlAction action, - fdb5::ControlIdentifiers identifiers) override { + fdb5::StatusIterator control(const fdb5::FDBToolRequest& request, const std::string& tracingID, + fdb5::ControlAction action, fdb5::ControlIdentifiers identifiers) override { counts_.control += 1; return fdb5::StatusIterator(0); } - void flush() override { counts_.flush += 1; } + void flush(const std::string& tracingID) override { counts_.flush += 1; } // For diagnostics diff --git a/tests/fdb/daos/test_daos_catalogue.cc b/tests/fdb/daos/test_daos_catalogue.cc index 7dfb00acc..a07b63ee3 100644 --- a/tests/fdb/daos/test_daos_catalogue.cc +++ b/tests/fdb/daos/test_daos_catalogue.cc @@ -226,7 +226,7 @@ CASE("DaosCatalogue tests") { EXPECT(cat_kv.has(index_key.valuesToString())); fdb5::CatalogueWriter& catw = dcatw; - catw.archive(index_key, field_key, std::move(loc)); + catw.archive(index_key, field_key, "archive_test_daos_catalogue_tracingID", std::move(loc)); EXPECT(index_kv.has(field_key.valuesToString())); fdb5::DaosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; fdb5::DaosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; @@ -248,7 +248,7 @@ CASE("DaosCatalogue tests") { fdb5::Field f; fdb5::CatalogueReader& catr = dcatr; - catr.retrieve(field_key, f); + catr.retrieve(field_key, f, "test_daos_catalogue_tracingID"); EXPECT_EQUAL(f.location().uri().name(), eckit::URI("daos", "test_uri").name()); EXPECT_EQUAL(f.location().offset(), eckit::Offset(0)); EXPECT_EQUAL(f.location().length(), eckit::Length(1)); @@ -318,7 +318,8 @@ CASE("DaosCatalogue tests") { fdb5::DaosStore dstore{db_key, config}; fdb5::Store& store = static_cast(dstore); - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + std::string tracingID = "archive_test_daos_store_tracingID"; + std::unique_ptr loc(store.archive(index_key, tracingID, data, sizeof(data))); /// @todo: there are two cont create with label here /// @todo: again, daos_fini happening before cont and pool close @@ -330,27 +331,28 @@ CASE("DaosCatalogue tests") { cat.deselectIndex(); cat.selectIndex(index_key); fdb5::CatalogueWriter& catw = dcatw; - catw.archive(index_key, field_key, std::move(loc)); + catw.archive(index_key, field_key, tracingID, std::move(loc)); /// flush store before flushing catalogue - dstore.flush(); // not necessary if using a DAOS store + dstore.flush("flush_test_daos_catalogue_tracingID"); // not necessary if using a DAOS store } // find data + std::string retrieve_tracingID = "retrieve_test_daos_catalogue_tracingID"; fdb5::Field field; { fdb5::DaosCatalogueReader dcatr{db_key, config}; fdb5::Catalogue& cat = dcatr; cat.selectIndex(index_key); fdb5::CatalogueReader& catr = dcatr; - catr.retrieve(field_key, field); + catr.retrieve(field_key, field, retrieve_tracingID); } std::cout << "Read location: " << field.location() << std::endl; // retrieve data - std::unique_ptr dh(store.retrieve(field)); + std::unique_ptr dh(store.retrieve(field, retrieve_tracingID)); EXPECT(dynamic_cast(dh.get())); eckit::MemoryHandle mh; @@ -365,7 +367,7 @@ CASE("DaosCatalogue tests") { fdb5::Catalogue& cat = static_cast(dcat); eckit::Queue queue(100); metkit::mars::MarsRequest r = db_key.request("retrieve"); - fdb5::api::local::WipeCatalogueVisitor wv{queue, r, true}; + fdb5::api::local::WipeCatalogueVisitor wv{queue, r, "wipe_test_daos_catalogue_tracingID", true}; cat.visitEntries(wv, false); } @@ -416,7 +418,8 @@ CASE("DaosCatalogue tests") { fdb5::TocStore tstore{db_key, config}; fdb5::Store& store = static_cast(tstore); - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + std::string tracingID = "archive_test_daos_store_tracingID"; + std::unique_ptr loc(store.archive(index_key, tracingID, data, sizeof(data))); /// @todo: there are two cont create with label here /// @todo: again, daos_fini happening before cont and pool close @@ -428,27 +431,28 @@ CASE("DaosCatalogue tests") { cat.deselectIndex(); cat.selectIndex(index_key); fdb5::CatalogueWriter& catw = dcatw; - catw.archive(index_key, field_key, std::move(loc)); + catw.archive(index_key, field_key, tracingID, std::move(loc)); /// flush store before flushing catalogue - tstore.flush(); + tstore.flush("flush_test_daos_catalogue_tracingID"); } // find data + std::string retrieve_tracingID = "retrieve_test_daos_catalogue_tracingID"; fdb5::Field field; { fdb5::DaosCatalogueReader dcatr{db_key, config}; fdb5::Catalogue& cat = dcatr; cat.selectIndex(index_key); fdb5::CatalogueReader& catr = dcatr; - catr.retrieve(field_key, field); + catr.retrieve(field_key, field, retrieve_tracingID); } std::cout << "Read location: " << field.location() << std::endl; // retrieve data - std::unique_ptr dh(store.retrieve(field)); + std::unique_ptr dh(store.retrieve(field, retrieve_tracingID)); std::vector test(dh->size()); dh->openForRead(); @@ -478,7 +482,7 @@ CASE("DaosCatalogue tests") { fdb5::Catalogue& cat = static_cast(dcat); eckit::Queue queue(100); metkit::mars::MarsRequest r = db_key.request("retrieve"); - fdb5::api::local::WipeCatalogueVisitor wv{queue, r, true}; + fdb5::api::local::WipeCatalogueVisitor wv{queue, r, "test_daos_catalogue_tracingID", true}; cat.visitEntries(wv, false); } diff --git a/tests/fdb/daos/test_daos_store.cc b/tests/fdb/daos/test_daos_store.cc index 4638e9668..9508631cd 100644 --- a/tests/fdb/daos/test_daos_store.cc +++ b/tests/fdb/daos/test_daos_store.cc @@ -182,16 +182,18 @@ CASE("DaosStore tests") { /// DaosManager is configured with client config from the file fdb5::DaosStore dstore{db_key, config}; fdb5::Store& store = dstore; - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + std::string tracingID = "archive_test_daos_store_tracingID"; + std::unique_ptr loc(store.archive(index_key, tracingID, data, sizeof(data))); /// @todo: two cont create with label happen here /// @todo: again, daos_fini happening before cont and pool close - dstore.flush(); // not necessary for DAOS store + dstore.flush("flush_test_daos_store_tracingID"); // not necessary for DAOS store // retrieve fdb5::Field field(std::move(loc), std::time(nullptr)); std::cout << "Read location: " << field.location() << std::endl; - std::unique_ptr dh(store.retrieve(field)); + std::string retrieve_tracingID = "retrieve_test_daos_store_tracingID"; + std::unique_ptr dh(store.retrieve(field, retrieve_tracingID)); EXPECT(dynamic_cast(dh.get())); eckit::MemoryHandle mh; @@ -259,7 +261,8 @@ CASE("DaosStore tests") { fdb5::DaosStore dstore{db_key, config}; fdb5::Store& store = static_cast(dstore); - std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + std::string tracingID = "archive_test_daos_store_tracingID"; + std::unique_ptr loc(store.archive(index_key, tracingID, data, sizeof(data))); /// @todo: there are two cont create with label here /// @todo: again, daos_fini happening before cont and pool close @@ -272,26 +275,27 @@ CASE("DaosStore tests") { cat.deselectIndex(); cat.selectIndex(index_key); // const fdb5::Index& idx = tcat.currentIndex(); - static_cast(tcat).archive(index_key, field_key, std::move(loc)); + static_cast(tcat).archive(index_key, field_key, tracingID, std::move(loc)); /// flush store before flushing catalogue - dstore.flush(); // not necessary if using a DAOS store + dstore.flush("flush_test_daos_store_tracingID"); // not necessary if using a DAOS store } // find data + std::string retrieve_tracingID = "retrieve_test_daos_catalogue_tracingID"; fdb5::Field field; { fdb5::TocCatalogueReader tcat{db_key, config}; fdb5::Catalogue& cat = static_cast(tcat); cat.selectIndex(index_key); - static_cast(tcat).retrieve(field_key, field); + static_cast(tcat).retrieve(field_key, field, retrieve_tracingID); } std::cout << "Read location: " << field.location() << std::endl; // retrieve data - std::unique_ptr dh(store.retrieve(field)); + std::unique_ptr dh(store.retrieve(field, retrieve_tracingID)); EXPECT(dynamic_cast(dh.get())); eckit::MemoryHandle mh; @@ -318,7 +322,7 @@ CASE("DaosStore tests") { fdb5::Catalogue& cat = static_cast(tcat); metkit::mars::MarsRequest r = db_key.request("retrieve"); eckit::Queue queue(100); - fdb5::api::local::WipeCatalogueVisitor wv{queue, r, true}; + fdb5::api::local::WipeCatalogueVisitor wv{queue, r, "test_daos_store_tracingID", true}; cat.visitEntries(wv, false); } diff --git a/tests/fdb/test_service.cc b/tests/fdb/test_service.cc index 313d8437a..67f6de8af 100644 --- a/tests/fdb/test_service.cc +++ b/tests/fdb/test_service.cc @@ -90,9 +90,10 @@ struct FixtureService { std::string data_str = data.str(); fdb5::Key k{p}; - auto visitor = - ArchiveVisitor::create(fdb, k, static_cast(data_str.c_str()), data_str.size()); - fdb.archive(k, *visitor); + std::string tracingID = "archive_param_" + param + "_step_" + str(step) + "_level_" + str(level); + auto visitor = ArchiveVisitor::create(fdb, k, tracingID, static_cast(data_str.c_str()), + data_str.size()); + fdb.archive(k, *visitor, tracingID); } } } @@ -210,7 +211,7 @@ CASE("test_fdb_service") { FixtureService f; SECTION("test_fdb_service_write") { - fdb5::Archiver fdb; + fdb5::Archiver fdb("test_fdb_service_write_tracingID"); f.p["class"] = "rd"; f.p["stream"] = "oper"; @@ -387,7 +388,7 @@ CASE("test_fdb_service_subtoc") { fdb5::Config config(expanded, userConf); SECTION("test_fdb_service_subtoc_write") { - fdb5::Archiver fdb(config); + fdb5::Archiver fdb("test_fdb_service_subtoc_write_tracingID", config); f.p["class"] = "rd"; f.p["stream"] = "oper"; diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc index 9b539009d..52aa22dd5 100644 --- a/tests/fdb/type/test_toKey.cc +++ b/tests/fdb/type/test_toKey.cc @@ -129,8 +129,9 @@ CASE("Step & ClimateDaily - expansion") { } fdb5::Config conf = config.expandConfig(); - fdb5::Archiver archiver(conf); - auto visitor = fdb5::ArchiveVisitor::create(archiver, key, data, 4); + std::string tracingID = "test_toKey_tracingID"; + fdb5::Archiver archiver(tracingID, conf); + auto visitor = fdb5::ArchiveVisitor::create(archiver, key, tracingID, data, 4); config.schema().expand(key, *visitor); fdb5::TypedKey tKey(visitor->rule()->registry()); @@ -264,8 +265,9 @@ CASE("Expver, Time & ClimateDaily - string ctor - expansion") { EXPECT_EQUAL(key.valuesToString(), "ei:0001:dacl:g:pb:pl:20210427:0600:0:99:100:50:129.128"); { - fdb5::Archiver archiver; - auto visitor = fdb5::ArchiveVisitor::create(archiver, key, data, 4); + std::string tracingID = "test_toKey_tracingID"; + fdb5::Archiver archiver(tracingID); + auto visitor = fdb5::ArchiveVisitor::create(archiver, key, tracingID, data, 4); config.schema().expand(key, *visitor); fdb5::TypedKey tKey(visitor->rule()->registry()); tKey.pushFrom(key); @@ -293,8 +295,9 @@ CASE("ClimateMonthly - string ctor - expansion") { EXPECT_EQUAL(key.valuesToString(), "op:0001:mnth:g:cl:pl:20210427:0000:50:129.128"); { - fdb5::Archiver archiver; - auto visitor = fdb5::ArchiveVisitor::create(archiver, key, data, 4); + std::string tracingID = "test_toKey_tracingID"; + fdb5::Archiver archiver(tracingID); + auto visitor = fdb5::ArchiveVisitor::create(archiver, key, tracingID, data, 4); config.schema().expand(key, *visitor); fdb5::TypedKey tKey(visitor->rule()->registry()); tKey.pushFrom(key); @@ -326,8 +329,9 @@ CASE("Date - string ctor - expansion") { EXPECT_EQUAL(key.valuesToString(), "od:0001:oper:ofb:" + t(now.yyyymmdd()) + ":0000:mhs:3001"); { - fdb5::Archiver archiver; - auto visitor = fdb5::ArchiveVisitor::create(archiver, key, data, 4); + std::string tracingID = "test_toKey_tracingID"; + fdb5::Archiver archiver(tracingID); + auto visitor = fdb5::ArchiveVisitor::create(archiver, key, tracingID, data, 4); config.schema().expand(key, *visitor); fdb5::TypedKey tKey(visitor->rule()->registry()); tKey.pushFrom(key);