diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 109b7f1c99..5738382229 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -2313,6 +2313,20 @@ func readPromPPFeatures(logger log.Logger) { case "shrink_shard_copier": pp_storage.ShrinkShardCopier = true _ = level.Info(logger).Log("msg", "[FEATURE] Shrink shard copier is enabled.") + + case "select_func_optimization": + if err := querier.SetSelectFuncOptimize(strings.TrimSpace(fvalue)); err != nil { + level.Error(logger).Log( + "msg", "[FEATURE] Error parsing select_func_optimization value", + "err", err, + ) + continue + } + + level.Info(logger).Log( + "msg", "[FEATURE] Select function optimization is set.", + "optimization", fvalue, + ) } } } diff --git a/pp-pkg/storage/adapter.go b/pp-pkg/storage/adapter.go index 4ae0973ad9..81d7041458 100644 --- a/pp-pkg/storage/adapter.go +++ b/pp-pkg/storage/adapter.go @@ -311,7 +311,14 @@ func (ar *Adapter) Querier(mint, maxt int64) (storage.Querier, error) { queriers = append( queriers, - querier.NewQuerier(head, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, ar.storageQuerierMetrics), + querier.NewQuerierWithOutSelectFuncOptimize( + head, + querier.NewNoOpShardedDeduplicator, + mint, + maxt, + nil, + ar.storageQuerierMetrics, + ), ) } diff --git a/pp-pkg/storage/batch_storage_test.go b/pp-pkg/storage/batch_storage_test.go index 7574c892bf..efad08e61b 100644 --- a/pp-pkg/storage/batch_storage_test.go +++ b/pp-pkg/storage/batch_storage_test.go @@ -10,6 +10,7 @@ import ( "github.com/jonboulle/clockwork" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pp/go/storage/storagetest" + "github.com/prometheus/prometheus/storage" "github.com/stretchr/testify/suite" "github.com/prometheus/prometheus/model/labels" @@ -17,7 +18,6 @@ import ( pp_model "github.com/prometheus/prometheus/pp/go/model" pp_storage "github.com/prometheus/prometheus/pp/go/storage" "github.com/prometheus/prometheus/pp/go/storage/catalog" - "github.com/prometheus/prometheus/storage" ) const ( @@ -202,7 +202,7 @@ func (s *BatchStorageSuite) TestCommit_WithSamplesAdded() { StartTimestampMs: 0, EndTimestampMs: 5000, LabelSetIDs: []uint32{0}, - }) + }, cppbridge.NoDownsampling, &storage.SelectHints{}) s.Require().Equal(cppbridge.DataStorageQueryStatusSuccess, queryResult.Status) s.Equal(storagetest.SamplesMap{ 0: []cppbridge.Sample{ diff --git a/pp/bare_bones/algorithm.h b/pp/bare_bones/algorithm.h index fc650bbfaf..846d126927 100644 --- a/pp/bare_bones/algorithm.h +++ b/pp/bare_bones/algorithm.h @@ -1,5 +1,6 @@ #pragma once +#include #include namespace BareBones { @@ -20,6 +21,12 @@ constexpr bool is_in(const Value& value, Args&&... args) { return (... || (value == std::forward(args))); } +template +constexpr ResultType build_bitmask() { + static_assert((... && std::has_single_bit(static_cast(Values))), "each bitmask value must be a power of two"); + return (... | static_cast(Values)); +} + template auto lexicographical_compare_three_way(const Range1& range1, const Range2& range2, Comparator&& comparator) { using result_type = decltype(comparator(*range1.begin(), *range2.begin())); diff --git a/pp/entrypoint/aggregation_iterator.cpp b/pp/entrypoint/aggregation_iterator.cpp new file mode 100644 index 0000000000..9b3bddbecf --- /dev/null +++ b/pp/entrypoint/aggregation_iterator.cpp @@ -0,0 +1,31 @@ +#include "aggregation_iterator.h" + +#include "series_data/serialization.h" + +extern "C" void prompp_series_data_serialization_serialized_data_aggregation_iterator_ctor(void* args) { + struct Arguments { + entrypoint::series_data::AggregationIterator* iterator; + entrypoint::series_data::SerializedDataPtr serialized_data; + uint32_t chunk_ref; + }; + + const auto in = static_cast(args); + std::construct_at(in->iterator, in->serialized_data->aggregation_iterator(in->chunk_ref)); +} + +extern "C" void prompp_series_data_serialization_serialized_data_aggregation_iterator_next(void* iterator) { + using series_data::decoder::DecodeIteratorSentinel; + + ++(*static_cast(iterator)); +} + +extern "C" void prompp_series_data_serialization_serialized_data_aggregation_iterator_reset(void* args) { + struct Arguments { + entrypoint::series_data::AggregationIterator* iterator; + entrypoint::series_data::SerializedDataPtr serialized_data; + uint32_t chunk_ref; + }; + + const Arguments* in = static_cast(args); + *in->iterator = in->serialized_data->aggregation_iterator(in->chunk_ref); +} diff --git a/pp/entrypoint/aggregation_iterator.h b/pp/entrypoint/aggregation_iterator.h new file mode 100644 index 0000000000..5032ff53f9 --- /dev/null +++ b/pp/entrypoint/aggregation_iterator.h @@ -0,0 +1,38 @@ +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Create a aggregation iterator for corresponding chunk_ref. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * chunk_ref uint32 // inner chunk id. + * } + * + */ +void prompp_series_data_serialization_serialized_data_aggregation_iterator_ctor(void* args); + +/** + * @brief Advance aggregation iterator. + * + * @param iterator uintptr // pointer to aggregation iterator + * + */ +void prompp_series_data_serialization_serialized_data_aggregation_iterator_next(void* iterator); + +/** + * @brief Reset a aggregation iterator for corresponding chunk_ref. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * iterator uintptr // pointer to aggregation iterator + * chunkRef uint32 // inner chunk id. + * } + * + */ +void prompp_series_data_serialization_serialized_data_aggregation_iterator_reset(void* args); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/pp/entrypoint/go_constants.cpp b/pp/entrypoint/go_constants.cpp index 9b6164e8b3..b435d7fd78 100644 --- a/pp/entrypoint/go_constants.cpp +++ b/pp/entrypoint/go_constants.cpp @@ -1,8 +1,8 @@ #include "go_constants.h" -#include "head/serialization.h" #include "metrics/storage.h" #include "prometheus/relabeler.h" +#include "series_data/serialization.h" #include "wal/output_decoder.h" #include "wal/segment_samples_storage.h" @@ -14,7 +14,8 @@ static_assert(sizeof(roaring::Roaring) == Sizeof_RoaringBitset); static_assert(sizeof(PromPP::Prometheus::Relabel::InnerSeries) == Sizeof_InnerSeries); -static_assert(sizeof(entrypoint::head::SerializedDataIterator) == Sizeof_SerializedDataIterator); +static_assert(sizeof(entrypoint::series_data::SamplesIterator) == Sizeof_SerializedDataSamplesIterator); +static_assert(sizeof(entrypoint::series_data::AggregationIterator) == Sizeof_SerializedDataAggregationIterator); static_assert(sizeof(metrics::Storage::Iterator) == Sizeof_MetricsIterator); diff --git a/pp/entrypoint/go_constants.h b/pp/entrypoint/go_constants.h index 4a85d6ea79..0b248ef774 100644 --- a/pp/entrypoint/go_constants.h +++ b/pp/entrypoint/go_constants.h @@ -5,7 +5,8 @@ #define Sizeof_InnerSeries (Sizeof_SizeT + Sizeof_BareBonesVector + Sizeof_RoaringBitset) #define Sizeof_GoLabels 16 -#define Sizeof_SerializedDataIterator 152 +#define Sizeof_SerializedDataSamplesIterator 152 +#define Sizeof_SerializedDataAggregationIterator 208 #define Sizeof_MetricsIterator 24 diff --git a/pp/entrypoint/head/serialization.h b/pp/entrypoint/head/serialization.h deleted file mode 100644 index 3168a8ff03..0000000000 --- a/pp/entrypoint/head/serialization.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include "series_data/serialization/serialized_data.h" - -namespace entrypoint::head { - -class SerializedDataGo { - public: - explicit SerializedDataGo(const series_data::DataStorage& storage, const series_data::querier::QueriedChunkList& queried_chunks) - : data_{series_data::serialization::DataSerializer{storage}.serialize(queried_chunks)} {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE auto get_buffer_view() const noexcept { return data_view_.get_buffer_view(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE auto get_chunks_view() const noexcept { return data_view_.get_chunks_view(); } - - [[nodiscard]] PROMPP_ALWAYS_INLINE auto next() noexcept { return data_view_.next_series(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE auto iterator(uint32_t chunk_id) const noexcept { return data_view_.create_series_iterator(chunk_id); } - - private: - series_data::serialization::SerializedData data_; - series_data::serialization::SerializedDataView data_view_{data_}; -}; - -using SerializedDataPtr = std::unique_ptr; -using SerializedDataIterator = series_data::serialization::SerializedDataView::SeriesIterator; - -static_assert(sizeof(SerializedDataPtr) == sizeof(void*)); - -} // namespace entrypoint::head \ No newline at end of file diff --git a/pp/entrypoint/head_wal.cpp b/pp/entrypoint/head_wal.cpp index ed15fb41ae..bfa3ee44d1 100644 --- a/pp/entrypoint/head_wal.cpp +++ b/pp/entrypoint/head_wal.cpp @@ -5,8 +5,8 @@ #include "exception.hpp" #include "hashdex.hpp" #include "head/lss.h" -#include "head/series_data.h" #include "primitives/go_slice.h" +#include "series_data/series_data.h" #include "wal/decoder.h" #include "wal/encoder.h" #include "wal/wal.h" diff --git a/pp/entrypoint/samples_iterator.cpp b/pp/entrypoint/samples_iterator.cpp new file mode 100644 index 0000000000..217c2ae915 --- /dev/null +++ b/pp/entrypoint/samples_iterator.cpp @@ -0,0 +1,43 @@ +#include "samples_iterator.h" + +#include "series_data/serialization.h" + +extern "C" void prompp_series_data_serialization_serialized_data_samples_iterator_ctor(void* args) { + struct Arguments { + entrypoint::series_data::SamplesIterator* iterator; + entrypoint::series_data::SerializedDataPtr serialized_data; + uint32_t chunk_ref; + }; + + const auto in = static_cast(args); + std::construct_at(in->iterator, in->serialized_data->samples_iterator(in->chunk_ref)); +} + +extern "C" void prompp_series_data_serialization_serialized_data_samples_iterator_next(void* iterator) { + using series_data::decoder::DecodeIteratorSentinel; + + ++(*static_cast(iterator)); +} + +extern "C" void prompp_series_data_serialization_serialized_data_samples_iterator_seek(void* args) { + using series_data::decoder::DecodeIteratorSentinel; + + struct Arguments { + entrypoint::series_data::SamplesIterator* iterator; + int64_t target_timestamp; + }; + + const Arguments* in = static_cast(args); + in->iterator->seek_to(in->target_timestamp); +} + +extern "C" void prompp_series_data_serialization_serialized_data_samples_iterator_reset(void* args) { + struct Arguments { + entrypoint::series_data::SamplesIterator* iterator; + entrypoint::series_data::SerializedDataPtr serialized_data; + uint32_t chunk_ref; + }; + + const Arguments* in = static_cast(args); + *in->iterator = in->serialized_data->samples_iterator(in->chunk_ref); +} diff --git a/pp/entrypoint/samples_iterator.h b/pp/entrypoint/samples_iterator.h new file mode 100644 index 0000000000..c3c53b10a8 --- /dev/null +++ b/pp/entrypoint/samples_iterator.h @@ -0,0 +1,49 @@ +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Create a samples iterator for corresponding chunk_ref. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * chunk_ref uint32 // inner chunk id. + * } + * + */ +void prompp_series_data_serialization_serialized_data_samples_iterator_ctor(void* args); + +/** + * @brief Advance samples iterator. + * + * @param iterator uintptr // pointer to samples iterator + * + */ +void prompp_series_data_serialization_serialized_data_samples_iterator_next(void* iterator); + +/** + * @brief Advance samples iterator until referenced sample is gte targetTimestamp. + * + * @param args { + * iterator uintptr // pointer to samples iterator + * targetTimestamp int64 // target timestamp + * } + * + */ +void prompp_series_data_serialization_serialized_data_samples_iterator_seek(void* args); + +/** + * @brief Reset a samples iterator for corresponding chunk_ref. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * iterator uintptr // pointer to samples iterator + * chunkRef uint32 // inner chunk id. + * } + * + */ +void prompp_series_data_serialization_serialized_data_samples_iterator_reset(void* args); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/pp/entrypoint/series_data/aggregation_iterator.h b/pp/entrypoint/series_data/aggregation_iterator.h new file mode 100644 index 0000000000..76a6b377f6 --- /dev/null +++ b/pp/entrypoint/series_data/aggregation_iterator.h @@ -0,0 +1,248 @@ +#pragma once + +#include "prometheus/promql/window_function.h" +#include "series_data/decoder/decorator/changes_iterator.h" +#include "series_data/decoder/decorator/delta_iterator.h" +#include "series_data/decoder/decorator/downsampling_decode_iterator.h" +#include "series_data/decoder/decorator/irate_iterator.h" +#include "series_data/decoder/decorator/last_over_step.h" +#include "series_data/decoder/decorator/last_over_time.h" +#include "series_data/decoder/decorator/max_over_time.h" +#include "series_data/decoder/decorator/min_over_time.h" +#include "series_data/decoder/decorator/rate_iterator.h" +#include "series_data/decoder/decorator/resets_iterator.h" +#include "series_data/decoder/decorator/sum_over_time.h" +#include "series_data/decoder/decorator/window_function_iterator.h" +#include "series_data/decoder/universal_decode_iterator.h" +#include "series_data/serialization/serialized_data.h" + +namespace entrypoint::series_data { + +template +concept invalidatable = requires(Iterator iterator) { + { iterator.invalidate_sample() }; +}; + +class AggregationIterator { + public: + using DecodeIteratorSentinel = ::series_data::decoder::DecodeIteratorSentinel; + using UniversalDecodeIterator = ::series_data::decoder::UniversalDecodeIterator; + using SeriesIterator = ::series_data::serialization::SerializedDataView::SeriesIterator; + using DownsamplingIterator = ::series_data::decoder::decorator::DownsamplingDecodeIterator; + + template + using WindowFunctionIterator = ::series_data::decoder::decorator::WindowFunctionIterator; + using MinOverTimeIterator = WindowFunctionIterator<::series_data::decoder::decorator::MinOverTimeIterator>; + using MaxOverTimeIterator = WindowFunctionIterator<::series_data::decoder::decorator::MaxOverTimeIterator>; + using LastOverTimeIterator = WindowFunctionIterator<::series_data::decoder::decorator::LastOverTimeIterator>; + using LastOverStepIterator = WindowFunctionIterator<::series_data::decoder::decorator::LastOverStepIterator, + ::series_data::decoder::decorator::StepLookbackDeltaWindowCalculator>; + using SumOverTimeIterator = WindowFunctionIterator<::series_data::decoder::decorator::SumOverTimeIterator>; + using RateIterator = WindowFunctionIterator<::series_data::decoder::decorator::RateIterator>; + using IRateIterator = WindowFunctionIterator<::series_data::decoder::decorator::IRateIterator>; + using ChangesIterator = WindowFunctionIterator<::series_data::decoder::decorator::ChangesIterator>; + using DeltaIterator = WindowFunctionIterator<::series_data::decoder::decorator::DeltaIterator>; + using ResetsIterator = WindowFunctionIterator<::series_data::decoder::decorator::ResetsIterator>; + + enum class Type : uint8_t { + kSeries = 0, + kDownsampling, + kMinOverTime, + kMaxOverTime, + kLastOverTime, + kLastOverStep, + kSumOverTime, + kRate, + kIRate, + kChanges, + kDelta, + kResets, + }; + + DECODE_ITERATOR_TYPE_TRAITS(); + +#define DEFINE_CONSTRUCTOR(AggregationIteratorType, field, type) \ + template \ + explicit AggregationIterator(std::in_place_type_t, Args&&... args) \ + : iterator_{.field{std::forward(args)...}}, type_{Type::type} {} + + DEFINE_CONSTRUCTOR(SeriesIterator, series, kSeries) + DEFINE_CONSTRUCTOR(DownsamplingIterator, downsampling, kDownsampling) + DEFINE_CONSTRUCTOR(MinOverTimeIterator, min_over_time, kMinOverTime) + DEFINE_CONSTRUCTOR(MaxOverTimeIterator, max_over_time, kMaxOverTime) + DEFINE_CONSTRUCTOR(LastOverTimeIterator, last_over_time, kLastOverTime) + DEFINE_CONSTRUCTOR(LastOverStepIterator, last_over_step, kLastOverStep) + DEFINE_CONSTRUCTOR(SumOverTimeIterator, sum_over_time, kSumOverTime) + DEFINE_CONSTRUCTOR(RateIterator, rate, kRate) + DEFINE_CONSTRUCTOR(IRateIterator, irate, kIRate) + DEFINE_CONSTRUCTOR(ChangesIterator, changes, kChanges) + DEFINE_CONSTRUCTOR(DeltaIterator, delta, kDelta) + DEFINE_CONSTRUCTOR(ResetsIterator, resets, kResets) + +#undef DEFINE_CONSTRUCTOR + + template + PROMPP_ALWAYS_INLINE decltype(auto) visit(Visitor&& visitor) const { + switch (type_) { + case Type::kSeries: { + return std::forward(visitor)(iterator_.series); + } + + case Type::kDownsampling: { + return std::forward(visitor)(iterator_.downsampling); + } + + case Type::kMinOverTime: { + return std::forward(visitor)(iterator_.min_over_time); + } + + case Type::kMaxOverTime: { + return std::forward(visitor)(iterator_.max_over_time); + } + + case Type::kLastOverTime: { + return std::forward(visitor)(iterator_.last_over_time); + } + + case Type::kLastOverStep: { + return std::forward(visitor)(iterator_.last_over_step); + } + + case Type::kSumOverTime: { + return std::forward(visitor)(iterator_.sum_over_time); + } + + case Type::kRate: { + return std::forward(visitor)(iterator_.rate); + } + + case Type::kIRate: { + return std::forward(visitor)(iterator_.irate); + } + + case Type::kChanges: { + return std::forward(visitor)(iterator_.changes); + } + + case Type::kDelta: { + return std::forward(visitor)(iterator_.delta); + } + + default: { + return std::forward(visitor)(iterator_.resets); + } + } + } + + template + PROMPP_ALWAYS_INLINE decltype(auto) visit(Visitor&& visitor) { + return const_cast(this)->visit( + [&](const Iterator& iterator) PROMPP_LAMBDA_INLINE { return std::forward(visitor)(const_cast(iterator)); }); + } + + PROMPP_ALWAYS_INLINE const ::series_data::encoder::Sample& operator*() const { + return visit([](const auto& iterator) PROMPP_LAMBDA_INLINE -> const auto& { return *iterator; }); + } + PROMPP_ALWAYS_INLINE const ::series_data::encoder::Sample* operator->() const { + return visit([](const auto& iterator) PROMPP_LAMBDA_INLINE -> const auto* { return iterator.operator->(); }); + } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel& sentinel) const { + return visit([&sentinel](const auto& iterator) PROMPP_LAMBDA_INLINE { return iterator == sentinel; }); + } + + PROMPP_ALWAYS_INLINE AggregationIterator& operator++() { + visit([](Iterator& iterator) PROMPP_LAMBDA_INLINE { + ++iterator; + + if constexpr (invalidatable) { + if (iterator == DecodeIteratorSentinel{}) [[unlikely]] { + iterator.invalidate_sample(); + } + } + }); + return *this; + } + + PROMPP_ALWAYS_INLINE AggregationIterator operator++(int) { + const auto result = *this; + ++*this; + return result; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE Type type() const noexcept { return type_; } + + private: + union { + SeriesIterator series; + DownsamplingIterator downsampling; + MinOverTimeIterator min_over_time; + MaxOverTimeIterator max_over_time; + LastOverTimeIterator last_over_time; + LastOverStepIterator last_over_step; + SumOverTimeIterator sum_over_time; + RateIterator rate; + IRateIterator irate; + ChangesIterator changes; + DeltaIterator delta; + ResetsIterator resets; + } iterator_; + + Type type_; +}; + +struct SelectHints { + ::series_data::decoder::decorator::WindowFunctionParameters function_parameters; + PromPP::Prometheus::promql::WindowFunction window_function{PromPP::Prometheus::promql::WindowFunction::kNone}; +}; + +PROMPP_ALWAYS_INLINE AggregationIterator create_aggregation_iterator(::series_data::serialization::SerializedDataView::SeriesIterator&& iterator, + const SelectHints& select_hints, + PromPP::Primitives::Timestamp downsampling_ms) { + if (downsampling_ms != ::series_data::decoder::decorator::kNoDownsampling) [[unlikely]] { + return AggregationIterator(std::in_place_type, std::move(iterator), downsampling_ms); + } + + using enum PromPP::Prometheus::promql::WindowFunction; + + switch (select_hints.window_function) { + case kRate: + case kIncrease: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kIrate: + case kIdelta: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kMinOverTime: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kMaxOverTime: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kLastOverTime: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kLastOverStep: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kSumOverTime: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kDelta: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kResets: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + case kChanges: + return AggregationIterator(std::in_place_type, std::move(iterator), select_hints.function_parameters); + + default: + return AggregationIterator(std::in_place_type, std::move(iterator)); + } +} + +} // namespace entrypoint::series_data diff --git a/pp/entrypoint/series_data/querier.h b/pp/entrypoint/series_data/querier.h index 58d9c6060c..bfe9a36d20 100644 --- a/pp/entrypoint/series_data/querier.h +++ b/pp/entrypoint/series_data/querier.h @@ -2,11 +2,13 @@ #include "bare_bones/bitset.h" #include "entrypoint/go_constants.h" +#include "entrypoint/series_data/aggregation_iterator.h" #include "primitives/go_slice.h" #include "primitives/primitives.h" +#include "prometheus/query.h" +#include "serialization.h" #include "series_data/querier/instant_querier.h" #include "series_data/querier/querier.h" -#include "series_data/serialization/serialized_data.h" namespace entrypoint::series_data { @@ -54,6 +56,7 @@ struct SampleWithGoLabels : public ::series_data::encoder::Sample { using InstantQuerierWithArgumentsWrapperEntrypoint = InstantQuerierWithArgumentsWrapper, std::span>; +using GoSelectHints = PromPP::Prometheus::GenericSelectHints; class RangeQuerierWithArgumentsWrapperV2 { using DataStorage = ::series_data::DataStorage; @@ -64,8 +67,22 @@ class RangeQuerierWithArgumentsWrapperV2 { using BytesStream = PromPP::Primitives::Go::BytesStream; public: - RangeQuerierWithArgumentsWrapperV2(DataStorage& storage, const Query& query, head::SerializedDataPtr* serialized_data) - : querier_(storage), query_(&query), serialized_data_(serialized_data) {} + RangeQuerierWithArgumentsWrapperV2(DataStorage& storage, + const Query& query, + const GoSelectHints& hints, + SerializedDataPtr* serialized_data, + PromPP::Primitives::Timestamp downsampling_ms) + : select_hints_{ + .function_parameters = {.interval = {.min = hints.interval.min - 1, .max = hints.interval.max}, + .step = hints.step_ms, + .range = hints.range_ms, + .lookback_delta = hints.lookback_delta}, + .window_function = PromPP::Prometheus::promql::window_function_from_string(static_cast(hints.func)), + }, + querier_(storage), + query_(&query), + serialized_data_(serialized_data), + downsampling_ms_(downsampling_ms) {} void query() noexcept { querier_.query(*query_); @@ -74,23 +91,30 @@ class RangeQuerierWithArgumentsWrapperV2 { } } - PROMPP_ALWAYS_INLINE void query_finalize() const noexcept { serialize_chunks(); } + PROMPP_ALWAYS_INLINE void query_finalize() noexcept { serialize_chunks(); } [[nodiscard]] const BareBones::Bitset& series_to_load() const noexcept { return querier_.get_series_to_load(); } [[nodiscard]] bool need_loading() const noexcept { return querier_.need_loading(); } [[nodiscard]] DataStorage& storage() noexcept { return querier_.get_storage(); } private: + SelectHints select_hints_; ::series_data::querier::Querier querier_; const Query* query_; - head::SerializedDataPtr* serialized_data_; + SerializedDataPtr* serialized_data_; + PromPP::Primitives::Timestamp downsampling_ms_; - PROMPP_ALWAYS_INLINE void serialize_chunks() const noexcept { - std::construct_at(serialized_data_, std::make_unique(querier_.get_storage(), querier_.chunks())); + PROMPP_ALWAYS_INLINE void serialize_chunks() noexcept { + std::construct_at(serialized_data_, + std::make_unique(querier_.get_storage(), querier_.chunks(), std::move(select_hints_), downsampling_ms_)); } }; -enum class QuerierType : uint8_t { kInstantQuerier = 0, kRangeQuerier, kRangeQuerierV2 }; +enum class QuerierType : uint8_t { + kInstantQuerier = 0, + kRangeQuerier, + kRangeQuerierV2, +}; using QuerierVariant = std::variant; using QuerierVariantPtr = std::unique_ptr; diff --git a/pp/entrypoint/series_data/serialization.h b/pp/entrypoint/series_data/serialization.h new file mode 100644 index 0000000000..d12c45c13e --- /dev/null +++ b/pp/entrypoint/series_data/serialization.h @@ -0,0 +1,43 @@ +#pragma once + +#include "aggregation_iterator.h" +#include "primitives/go_slice.h" +#include "primitives/primitives.h" +#include "prometheus/query.h" +#include "series_data/serialization/serialized_data.h" + +namespace entrypoint::series_data { + +using SamplesIterator = ::series_data::serialization::SerializedDataView::SeriesIterator; + +class SerializedDataGo { + public: + explicit SerializedDataGo(const ::series_data::DataStorage& storage, + const ::series_data::querier::QueriedChunkList& queried_chunks, + SelectHints&& select_hints, + PromPP::Primitives::Timestamp downsampling_ms) + : data_{::series_data::serialization::DataSerializer{storage}.serialize(queried_chunks)}, + select_hints_(std::move(select_hints)), + downsampling_ms_(downsampling_ms) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE auto get_buffer_view() const noexcept { return data_view_.get_buffer_view(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE auto get_chunks_view() const noexcept { return data_view_.get_chunks_view(); } + + [[nodiscard]] PROMPP_ALWAYS_INLINE auto next() noexcept { return data_view_.next_series(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE SamplesIterator samples_iterator(uint32_t chunk_id) const noexcept { return data_view_.create_series_iterator(chunk_id); } + [[nodiscard]] PROMPP_ALWAYS_INLINE AggregationIterator aggregation_iterator(uint32_t chunk_id) const noexcept { + return create_aggregation_iterator(data_view_.create_series_iterator(chunk_id), select_hints_, downsampling_ms_); + } + + private: + ::series_data::serialization::SerializedData data_; + ::series_data::serialization::SerializedDataView data_view_{data_}; + const SelectHints select_hints_; + PromPP::Primitives::Timestamp downsampling_ms_{}; +}; + +using SerializedDataPtr = std::unique_ptr; + +static_assert(sizeof(SerializedDataPtr) == sizeof(void*)); + +} // namespace entrypoint::series_data \ No newline at end of file diff --git a/pp/entrypoint/head/series_data.h b/pp/entrypoint/series_data/series_data.h similarity index 95% rename from pp/entrypoint/head/series_data.h rename to pp/entrypoint/series_data/series_data.h index 0b76fc2939..b5b934fd5b 100644 --- a/pp/entrypoint/head/series_data.h +++ b/pp/entrypoint/series_data/series_data.h @@ -1,6 +1,5 @@ #pragma once -#include "data_storage.h" #include "series_data/encoder.h" #include "series_data/outdated_chunk_merger.h" diff --git a/pp/entrypoint/series_data_data_storage.cpp b/pp/entrypoint/series_data_data_storage.cpp index e46ba77273..9192cc55fa 100644 --- a/pp/entrypoint/series_data_data_storage.cpp +++ b/pp/entrypoint/series_data_data_storage.cpp @@ -6,7 +6,6 @@ #include "head/chunk_recoder.h" #include "head/data_storage.h" #include "head/lss.h" -#include "head/serialization.h" #include "primitives/go_slice.h" #include "series_data/data_storage.h" #include "series_data/decoder.h" @@ -14,6 +13,7 @@ #include "series_data/querier.h" #include "series_data/querier/instant_querier.h" #include "series_data/querier/querier.h" +#include "series_data/serialization.h" #include "series_data/unloading/loader.h" #include "series_data/unloading/unloader.h" #include "series_index/querier/selector_querier.h" @@ -115,6 +115,30 @@ extern "C" void prompp_series_data_data_storage_queried_series_set_bitset(void* new (res) Result{.result = result}; } +extern "C" void prompp_get_promql_optimized_functions(void* res) { + using PromPP::Prometheus::promql::FunctionType; + using PromPP::Prometheus::promql::kFunctions; + + struct GoFunction { + PromPP::Primitives::Go::String name; + FunctionType type; + }; + + static constexpr auto kGoFunctions = [] { + std::array functions; + for (size_t i = 0; i < functions.size(); ++i) { + functions[i] = {.name = PromPP::Primitives::Go::String(kFunctions[i].name), .type = kFunctions[i].type}; + } + return functions; + }(); + + struct Result { + SliceView functions; + }; + + new (res) Result{.functions = SliceView{kGoFunctions.data(), kGoFunctions.size(), kGoFunctions.size()}}; +} + extern "C" void prompp_series_data_data_storage_query_v2(void* args, void* res) { using Query = series_data::querier::Query>; using entrypoint::series_data::RangeQuerierWithArgumentsWrapperV2; @@ -123,18 +147,21 @@ extern "C" void prompp_series_data_data_storage_query_v2(void* args, void* res) struct Arguments { DataStoragePtr data_storage; Query query; + PromPP::Primitives::Timestamp downsampling_ms; + entrypoint::series_data::GoSelectHints* hints; }; struct Result { QuerierVariantPtr querier{}; QueryStatus status{}; - entrypoint::head::SerializedDataPtr* serialized_data{}; + entrypoint::series_data::SerializedDataPtr* serialized_data{}; }; const auto in = static_cast(args); const auto out = static_cast(res); - RangeQuerierWithArgumentsWrapperV2 querier(*in->data_storage, in->query, out->serialized_data); + RangeQuerierWithArgumentsWrapperV2 querier(*in->data_storage, in->query, in->hints ? *in->hints : entrypoint::series_data::GoSelectHints{}, + out->serialized_data, in->downsampling_ms); querier.query(); if (querier.need_loading()) { @@ -145,6 +172,27 @@ extern "C" void prompp_series_data_data_storage_query_v2(void* args, void* res) } } +extern "C" void prompp_series_data_serialized_data_next(void* args, void* res) { + struct Arguments { + entrypoint::series_data::SerializedDataPtr serialized_data; + }; + + using Result = struct { + uint32_t series_id; + uint32_t chunk_ref; + }; + const auto out = new (res) Result{}; + std::tie(out->series_id, out->chunk_ref) = static_cast(args)->serialized_data->next(); +} + +extern "C" void prompp_series_data_serialized_data_dtor(void* args) { + struct Arguments { + entrypoint::series_data::SerializedDataPtr serialized_data; + }; + + static_cast(args)->~Arguments(); +} + extern "C" void prompp_series_data_data_storage_instant_query(void* args, void* res) { using entrypoint::series_data::InstantQuerierWithArgumentsWrapperEntrypoint; using PromPP::Primitives::Timestamp; @@ -244,6 +292,7 @@ extern "C" void prompp_series_data_chunk_recoder_ctor(void* args, void* res) { uint32_t ls_id_batch_size; DataStoragePtr data_storage; PromPP::Primitives::TimeInterval time_interval; + PromPP::Primitives::Timestamp downsampling_ms; }; struct Result { ChunkRecoderVariantPtr chunk_recoder; @@ -255,13 +304,14 @@ extern "C" void prompp_series_data_chunk_recoder_ctor(void* args, void* res) { new (res) Result{ .chunk_recoder = std::make_unique( std::in_place_type, - ChunkRecoderIterator{ls_id_set.begin(), ls_id_set.end(), in->ls_id_batch_size, in->data_storage.get(), in->time_interval}, in->time_interval), + ChunkRecoderIterator{ls_id_set.begin(), ls_id_set.end(), in->ls_id_batch_size, in->data_storage.get(), in->time_interval}, in->time_interval, + in->downsampling_ms), }; } extern "C" void prompp_series_data_serialized_chunk_recoder_ctor(void* args, void* res) { struct Arguments { - entrypoint::head::SerializedDataPtr* serialized_data; + entrypoint::series_data::SerializedDataPtr* serialized_data; PromPP::Primitives::TimeInterval time_interval; }; struct Result { @@ -273,7 +323,7 @@ extern "C" void prompp_series_data_serialized_chunk_recoder_ctor(void* args, voi .chunk_recoder = std::make_unique( std::in_place_type, series_data::chunk::SerializedChunkIterator{in->serialized_data->get()->get_buffer_view(), in->serialized_data->get()->get_chunks_view()}, - in->time_interval), + in->time_interval, series_data::decoder::decorator::kNoDownsampling), }; } diff --git a/pp/entrypoint/series_data_data_storage.h b/pp/entrypoint/series_data_data_storage.h index 142ce80c5f..719f94007d 100644 --- a/pp/entrypoint/series_data_data_storage.h +++ b/pp/entrypoint/series_data_data_storage.h @@ -92,23 +92,62 @@ void prompp_series_data_data_storage_queried_series_set_bitset(void* args, void* */ void prompp_series_data_data_storage_allocated_memory(void* args, void* res); +/** + * @brief Get optimized promql functions list + * + * @param res { + * functions []struct { + * name string + * type uint8 + * } // serialized data + * } + */ +void prompp_get_promql_optimized_functions(void* res); + /** * @brief Queries data storage and serializes result (new serialization model). + * If args.downsamplingMs != 0 than DownsamplingIterator will be created regardless of the args.hints * * @param args { - * dataStorage uintptr // pointer to constructed data storage - * query DataStorageQuery // query + * dataStorage uintptr // pointer to constructed data storage + * query DataStorageQuery // query + * downsamplingMs int64 // downsampling interval in milliseconds (0 - downsampling is disabled) + * hints *storage.SelectHints // select hints * } * * @param res { - * Querier uintptr // pointer to constructed Querier if data loading is needed. + * querier uintptr // pointer to constructed Querier if data loading is needed. * // If constructed (!= 0) it must be destroyed by calling prompp_series_data_data_storage_query_final. - * Status uint8 // status of a query (0 - Success, 1 - Data loading is needed) + * status uint8 // status of a query (0 - Success, 1 - Data loading is needed) * serializedData uintptr // pointer to serialized data * } */ void prompp_series_data_data_storage_query_v2(void* args, void* res); +/** + * @brief Get next series_id in serialized data. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * } + * + * @param res { + * series_id uint32 // series id (UINT32_MAX if no more series). + * chunk_ref uint32 // inner chunk id. + * } + */ +void prompp_series_data_serialized_data_next(void* args, void* res); + +/** + * @brief Destroy serialized data object. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * } + * + */ +void prompp_series_data_serialized_data_dtor(void* args); + /** * @brief return instant series at given timestamp for label sets. * @@ -163,10 +202,11 @@ void prompp_series_data_data_storage_dtor(void* args); * lss uintptr // pointer to constructed label sets * lsIdBatchSize uint32 // size of ls batch for recoding * dataStorage uintptr // pointer to constructed data storage - * time_interval struct { closed interval [min, max] + * timeInterval struct { // closed interval [min, max] * min int64 * max int64 * } + * downsamplingMs int64 // downsampling interval in milliseconds (0 - downsampling is disabled) * } * @param res { * chunk_recoder uintptr // pointer to chunk recoder diff --git a/pp/entrypoint/series_data_encoder.cpp b/pp/entrypoint/series_data_encoder.cpp index 6b369ee04a..196b272122 100644 --- a/pp/entrypoint/series_data_encoder.cpp +++ b/pp/entrypoint/series_data_encoder.cpp @@ -1,9 +1,9 @@ #include "series_data_encoder.h" -#include "head/series_data.h" #include "primitives/primitives.h" #include "prometheus/relabeler.h" #include "series_data/data_storage.h" +#include "series_data/series_data.h" extern "C" void prompp_series_data_encoder_ctor(void* args, void* res) { struct Arguments { diff --git a/pp/entrypoint/series_data_serialization_serialized_data.cpp b/pp/entrypoint/series_data_serialization_serialized_data.cpp deleted file mode 100644 index cf42d872ec..0000000000 --- a/pp/entrypoint/series_data_serialization_serialized_data.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "series_data_serialization_serialized_data.h" - -#include "head/serialization.h" - -extern "C" void prompp_series_data_serialization_serialized_data_next(void* args, void* res) { - struct Arguments { - entrypoint::head::SerializedDataPtr serialized_data; - }; - - using Result = struct { - uint32_t series_id; - uint32_t chunk_ref; - }; - const auto out = new (res) Result{}; - std::tie(out->series_id, out->chunk_ref) = static_cast(args)->serialized_data->next(); -} - -extern "C" void prompp_series_data_serialization_serialized_data_iterator_ctor(void* args) { - struct Arguments { - entrypoint::head::SerializedDataIterator* iterator; - entrypoint::head::SerializedDataPtr serialized_data; - uint32_t chunk_ref; - }; - - const auto in = static_cast(args); - new (in->iterator) entrypoint::head::SerializedDataIterator(in->serialized_data->iterator(in->chunk_ref)); -} - -extern "C" void prompp_series_data_serialization_serialized_data_iterator_next(void* iterator) { - using series_data::decoder::DecodeIteratorSentinel; - - ++(*static_cast(iterator)); -} - -extern "C" void prompp_series_data_serialization_serialized_data_iterator_seek(void* args) { - using series_data::decoder::DecodeIteratorSentinel; - - struct Arguments { - entrypoint::head::SerializedDataIterator* iterator; - int64_t target_timestamp; - }; - - const Arguments* in = static_cast(args); - in->iterator->seek_to(in->target_timestamp); -} - -extern "C" void prompp_series_data_serialization_serialized_data_iterator_reset(void* args) { - struct Arguments { - entrypoint::head::SerializedDataIterator* iterator; - entrypoint::head::SerializedDataPtr serialized_data; - uint32_t chunk_ref; - }; - - const Arguments* in = static_cast(args); - in->iterator->reset(in->serialized_data->get_buffer_view(), in->serialized_data->get_chunks_view(), in->chunk_ref); -} - -extern "C" void prompp_series_data_serialization_serialized_data_dtor(void* args) { - struct Arguments { - entrypoint::head::SerializedDataPtr serialized_data; - }; - - static_cast(args)->~Arguments(); -} \ No newline at end of file diff --git a/pp/entrypoint/series_data_serialization_serialized_data.h b/pp/entrypoint/series_data_serialization_serialized_data.h deleted file mode 100644 index e13230d1d7..0000000000 --- a/pp/entrypoint/series_data_serialization_serialized_data.h +++ /dev/null @@ -1,73 +0,0 @@ -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @brief Get next series_id in serialized data. - * - * @param args { - * serializedData uintptr // pointer to serialized data. - * } - * - * @param res { - * series_id uint32 // series id (UINT32_MAX if no more series). - * chunk_ref uint32 // inner chunk id. - * } - */ -void prompp_series_data_serialization_serialized_data_next(void* args, void* res); - -/** - * @brief Create a decode iterator for corresponding chunk_ref. - * - * @param args { - * serializedData uintptr // pointer to serialized data. - * chunk_ref uint32 // inner chunk id. - * } - * - */ -void prompp_series_data_serialization_serialized_data_iterator_ctor(void* args); - -/** - * @brief Advance decode iterator. - * - * @param iterator uintptr // pointer to decode iterator - * - */ -void prompp_series_data_serialization_serialized_data_iterator_next(void* iterator); - -/** - * @brief Advance decode iterator until referenced sample is gte targetTimestamp. - * - * @param args { - * iterator uintptr // pointer to decode iterator - * targetTimestamp int64 // target timestamp - * } - * - */ -void prompp_series_data_serialization_serialized_data_iterator_seek(void* args); - -/** - * @brief Reset a decode iterator for corresponding chunk_ref. - * - * @param args { - * serializedData uintptr // pointer to serialized data. - * iterator uintptr // pointer to decode iterator - * chunkRef uint32 // inner chunk id. - * } - * - */ -void prompp_series_data_serialization_serialized_data_iterator_reset(void* args); - -/** - * @brief Destroy serialized data object. - * - * @param args { - * serializedData uintptr // pointer to serialized data. - * } - * - */ -void prompp_series_data_serialization_serialized_data_dtor(void* args); - -#ifdef __cplusplus -} // extern "C" -#endif diff --git a/pp/go/cppbridge/common.go b/pp/go/cppbridge/common.go index fe2a761ea4..675482386a 100644 --- a/pp/go/cppbridge/common.go +++ b/pp/go/cppbridge/common.go @@ -1,5 +1,12 @@ package cppbridge +const ( + PromqlCppThinningFunction = iota + 1 + PromqlCppSynthesizingFunction + PromqlCppCrossSeriesSynthesizingFunction + PromqlCppExtrapolatedRateFunction +) + // GetFlavor returns recognized architecture flavor // //revive:disable:confusing-naming // wrapper @@ -23,3 +30,7 @@ func GetMemInfo() MemInfo { func DumpMemoryProfile(filename string) bool { return dumpMemoryProfile(filename) == 0 } + +func GetPromqlCppFunctions() []PromqlCppFunction { + return getPromqlCppFunctions() +} diff --git a/pp/go/cppbridge/data_storage.go b/pp/go/cppbridge/data_storage.go index 01d073c9f8..159e901f66 100644 --- a/pp/go/cppbridge/data_storage.go +++ b/pp/go/cppbridge/data_storage.go @@ -3,6 +3,7 @@ package cppbridge import ( "runtime" "sync/atomic" + "unsafe" ) // DataStorage is Go wrapper around series_data::Data_storage. @@ -102,9 +103,10 @@ type DataStorageQuery struct { LabelSetIDs []uint32 } -func (ds *DataStorage) Query(query DataStorageQuery) DataStorageQueryResult { +func (ds *DataStorage) Query(query DataStorageQuery, downsamplingMs int64, selectHints unsafe.Pointer) DataStorageQueryResult { sd := NewDataStorageSerializedData(ds) - querier, status := seriesDataDataStorageQueryV2(ds.dataStorage, query, sd) + querier, status := seriesDataDataStorageQueryV2(ds.dataStorage, query, sd, downsamplingMs, selectHints) + runtime.KeepAlive(selectHints) return DataStorageQueryResult{ Querier: querier, Status: status, diff --git a/pp/go/cppbridge/entrypoint.go b/pp/go/cppbridge/entrypoint.go index d09411b1ea..ad3a0417f8 100644 --- a/pp/go/cppbridge/entrypoint.go +++ b/pp/go/cppbridge/entrypoint.go @@ -31,7 +31,8 @@ type ( CppStdVector = [C.Sizeof_StdVector]byte CppBareBonesVector = [C.Sizeof_BareBonesVector]byte CppRoaringBitset = [C.Sizeof_RoaringBitset]byte - CppSerializedDataIterator = [C.Sizeof_SerializedDataIterator]byte + CppSerializedDataSamplesIterator = [C.Sizeof_SerializedDataSamplesIterator]byte + CppSerializedDataAggregationIterator = [C.Sizeof_SerializedDataAggregationIterator]byte CppMetricsIterator = [C.Sizeof_MetricsIterator]byte CppSegmentSamplesStorage = [C.Sizeof_SegmentSamplesStorage]byte CppRemoteWriteMessageEncoder = [C.Sizeof_RemoteWriteMessageEncoder]byte @@ -2105,11 +2106,40 @@ type DataStorageQueryResult struct { SerializedData *DataStorageSerializedData } -func seriesDataDataStorageQueryV2(dataStorage uintptr, query DataStorageQuery, serializedData *DataStorageSerializedData) (querier uintptr, status uint8) { +type PromqlCppFunctionType uint8 + +type PromqlCppFunction struct { + Name string + Type PromqlCppFunctionType +} + +func getPromqlCppFunctions() []PromqlCppFunction { + res := struct { + functions []PromqlCppFunction + }{nil} + + testGC() + fastcgo.UnsafeCall1( + C.prompp_get_promql_optimized_functions, + uintptr(unsafe.Pointer(&res)), + ) + + return res.functions +} + +func seriesDataDataStorageQueryV2( + dataStorage uintptr, + query DataStorageQuery, + serializedData *DataStorageSerializedData, + downsamplingMs int64, + selectHints unsafe.Pointer, +) (querier uintptr, status uint8) { args := struct { - dataStorage uintptr - query DataStorageQuery - }{dataStorage, query} + dataStorage uintptr + query DataStorageQuery + downsamplingMs int64 + selectHints uintptr + }{dataStorage, query, downsamplingMs, uintptr(selectHints)} res := struct { Querier uintptr @@ -2190,7 +2220,7 @@ func seriesDataSerializedDataDtor(serializedData uintptr) { testGC() fastcgo.UnsafeCall1( - C.prompp_series_data_serialization_serialized_data_dtor, + C.prompp_series_data_serialized_data_dtor, uintptr(unsafe.Pointer(&args)), ) } @@ -2206,7 +2236,7 @@ func seriesDataSerializedDataNext(serializedData uintptr) (uint32, uint32) { testGC() fastcgo.UnsafeCall2( - C.prompp_series_data_serialization_serialized_data_next, + C.prompp_series_data_serialized_data_next, uintptr(unsafe.Pointer(&args)), uintptr(unsafe.Pointer(&res)), ) @@ -2214,7 +2244,11 @@ func seriesDataSerializedDataNext(serializedData uintptr) (uint32, uint32) { return res.seriesID, res.chunkRef } -func seriesDataSerializedDataIteratorCtor(iterator *DataStorageSerializedDataIterator, serializedData uintptr, chunkRef uint32) { +func seriesDataSerializedDataSamplesIteratorCtor( + iterator *DataStorageSerializedDataSamplesIterator, + serializedData uintptr, + chunkRef uint32, +) { args := struct { iterator uintptr serializedData uintptr @@ -2223,20 +2257,23 @@ func seriesDataSerializedDataIteratorCtor(iterator *DataStorageSerializedDataIte testGC() fastcgo.UnsafeCall1( - C.prompp_series_data_serialization_serialized_data_iterator_ctor, + C.prompp_series_data_serialization_serialized_data_samples_iterator_ctor, uintptr(unsafe.Pointer(&args)), ) } -func seriesDataSerializedDataIteratorNext(iterator *DataStorageSerializedDataIterator) { +func seriesDataSerializedDataSamplesIteratorNext(iterator *DataStorageSerializedDataSamplesIterator) { testGC() fastcgo.UnsafeCall1( - C.prompp_series_data_serialization_serialized_data_iterator_next, + C.prompp_series_data_serialization_serialized_data_samples_iterator_next, uintptr(unsafe.Pointer(iterator)), ) } -func seriesDataSerializedDataIteratorSeek(iterator *DataStorageSerializedDataIterator, targetTimestamp int64) { +func seriesDataSerializedDataSamplesIteratorSeek( + iterator *DataStorageSerializedDataSamplesIterator, + targetTimestamp int64, +) { args := struct { iterator uintptr targetTimestamp int64 @@ -2244,12 +2281,60 @@ func seriesDataSerializedDataIteratorSeek(iterator *DataStorageSerializedDataIte testGC() fastcgo.UnsafeCall1( - C.prompp_series_data_serialization_serialized_data_iterator_seek, + C.prompp_series_data_serialization_serialized_data_samples_iterator_seek, + uintptr(unsafe.Pointer(&args)), + ) +} + +func seriesDataSerializedDataSamplesIteratorReset( + iterator *DataStorageSerializedDataSamplesIterator, + serializedData uintptr, + chunkRef uint32, +) { + args := struct { + iterator uintptr + serializedData uintptr + chunkRef uint32 + }{uintptr(unsafe.Pointer(iterator)), serializedData, chunkRef} + + testGC() + fastcgo.UnsafeCall1( + C.prompp_series_data_serialization_serialized_data_samples_iterator_reset, + uintptr(unsafe.Pointer(&args)), + ) +} + +func seriesDataSerializedDataAggregationIteratorCtor( + iterator *DataStorageSerializedDataAggregationIterator, + serializedData uintptr, + chunkRef uint32, +) { + args := struct { + iterator uintptr + serializedData uintptr + chunkRef uint32 + }{uintptr(unsafe.Pointer(iterator)), serializedData, chunkRef} + + testGC() + fastcgo.UnsafeCall1( + C.prompp_series_data_serialization_serialized_data_aggregation_iterator_ctor, uintptr(unsafe.Pointer(&args)), ) } -func seriesDataSerializedDataIteratorReset(iterator *DataStorageSerializedDataIterator, serializedData uintptr, chunkRef uint32) { +func seriesDataSerializedDataAggregationIteratorNext(iterator *DataStorageSerializedDataAggregationIterator) { + testGC() + fastcgo.UnsafeCall1( + C.prompp_series_data_serialization_serialized_data_aggregation_iterator_next, + uintptr(unsafe.Pointer(iterator)), + ) +} + +func seriesDataSerializedDataAggregationIteratorReset( + iterator *DataStorageSerializedDataAggregationIterator, + serializedData uintptr, + chunkRef uint32, +) { args := struct { iterator uintptr serializedData uintptr @@ -2258,7 +2343,7 @@ func seriesDataSerializedDataIteratorReset(iterator *DataStorageSerializedDataIt testGC() fastcgo.UnsafeCall1( - C.prompp_series_data_serialization_serialized_data_iterator_reset, + C.prompp_series_data_serialization_serialized_data_aggregation_iterator_reset, uintptr(unsafe.Pointer(&args)), ) } @@ -2422,13 +2507,14 @@ func seriesDataEncoderDtor(encoder uintptr) { ) } -func seriesDataChunkRecoderCtor(lss uintptr, lsIdBatchSize uint32, dataStorage uintptr, timeInterval TimeInterval) uintptr { +func seriesDataChunkRecoderCtor(lss uintptr, lsIdBatchSize uint32, dataStorage uintptr, timeInterval TimeInterval, downsamplingMs int64) uintptr { args := struct { lss uintptr lsIdBatchSize uint32 dataStorage uintptr TimeInterval - }{lss, lsIdBatchSize, dataStorage, timeInterval} + downsamplingMs int64 + }{lss, lsIdBatchSize, dataStorage, timeInterval, downsamplingMs} var res struct { chunkRecoder uintptr } diff --git a/pp/go/cppbridge/entrypoint.h b/pp/go/cppbridge/entrypoint.h index 67d9cf0cd5..f9e5517183 100755 --- a/pp/go/cppbridge/entrypoint.h +++ b/pp/go/cppbridge/entrypoint.h @@ -2,6 +2,44 @@ extern "C" { #endif +/** + * @brief Create a aggregation iterator for corresponding chunk_ref. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * chunk_ref uint32 // inner chunk id. + * } + * + */ +void prompp_series_data_serialization_serialized_data_aggregation_iterator_ctor(void* args); + +/** + * @brief Advance aggregation iterator. + * + * @param iterator uintptr // pointer to aggregation iterator + * + */ +void prompp_series_data_serialization_serialized_data_aggregation_iterator_next(void* iterator); + +/** + * @brief Reset a aggregation iterator for corresponding chunk_ref. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * iterator uintptr // pointer to aggregation iterator + * chunkRef uint32 // inner chunk id. + * } + * + */ +void prompp_series_data_serialization_serialized_data_aggregation_iterator_reset(void* args); + +#ifdef __cplusplus +} // extern "C" +#endif +#ifdef __cplusplus +extern "C" { +#endif + /** * @brief Free memory allocated for response as []byte * @@ -40,7 +78,8 @@ void prompp_dump_memory_profile(void* args, void* res); #define Sizeof_InnerSeries (Sizeof_SizeT + Sizeof_BareBonesVector + Sizeof_RoaringBitset) #define Sizeof_GoLabels 16 -#define Sizeof_SerializedDataIterator 152 +#define Sizeof_SerializedDataSamplesIterator 152 +#define Sizeof_SerializedDataAggregationIterator 208 #define Sizeof_MetricsIterator 24 @@ -1409,6 +1448,55 @@ void prompp_remote_write_encode_message(void* args); extern "C" { #endif +/** + * @brief Create a samples iterator for corresponding chunk_ref. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * chunk_ref uint32 // inner chunk id. + * } + * + */ +void prompp_series_data_serialization_serialized_data_samples_iterator_ctor(void* args); + +/** + * @brief Advance samples iterator. + * + * @param iterator uintptr // pointer to samples iterator + * + */ +void prompp_series_data_serialization_serialized_data_samples_iterator_next(void* iterator); + +/** + * @brief Advance samples iterator until referenced sample is gte targetTimestamp. + * + * @param args { + * iterator uintptr // pointer to samples iterator + * targetTimestamp int64 // target timestamp + * } + * + */ +void prompp_series_data_serialization_serialized_data_samples_iterator_seek(void* args); + +/** + * @brief Reset a samples iterator for corresponding chunk_ref. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * iterator uintptr // pointer to samples iterator + * chunkRef uint32 // inner chunk id. + * } + * + */ +void prompp_series_data_serialization_serialized_data_samples_iterator_reset(void* args); + +#ifdef __cplusplus +} // extern "C" +#endif +#ifdef __cplusplus +extern "C" { +#endif + /** * @brief Construct a new series data DataStorage * @@ -1499,23 +1587,62 @@ void prompp_series_data_data_storage_queried_series_set_bitset(void* args, void* */ void prompp_series_data_data_storage_allocated_memory(void* args, void* res); +/** + * @brief Get optimized promql functions list + * + * @param res { + * functions []struct { + * name string + * type uint8 + * } // serialized data + * } + */ +void prompp_get_promql_optimized_functions(void* res); + /** * @brief Queries data storage and serializes result (new serialization model). + * If args.downsamplingMs != 0 than DownsamplingIterator will be created regardless of the args.hints * * @param args { - * dataStorage uintptr // pointer to constructed data storage - * query DataStorageQuery // query + * dataStorage uintptr // pointer to constructed data storage + * query DataStorageQuery // query + * downsamplingMs int64 // downsampling interval in milliseconds (0 - downsampling is disabled) + * hints *storage.SelectHints // select hints * } * * @param res { - * Querier uintptr // pointer to constructed Querier if data loading is needed. + * querier uintptr // pointer to constructed Querier if data loading is needed. * // If constructed (!= 0) it must be destroyed by calling prompp_series_data_data_storage_query_final. - * Status uint8 // status of a query (0 - Success, 1 - Data loading is needed) + * status uint8 // status of a query (0 - Success, 1 - Data loading is needed) * serializedData uintptr // pointer to serialized data * } */ void prompp_series_data_data_storage_query_v2(void* args, void* res); +/** + * @brief Get next series_id in serialized data. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * } + * + * @param res { + * series_id uint32 // series id (UINT32_MAX if no more series). + * chunk_ref uint32 // inner chunk id. + * } + */ +void prompp_series_data_serialized_data_next(void* args, void* res); + +/** + * @brief Destroy serialized data object. + * + * @param args { + * serializedData uintptr // pointer to serialized data. + * } + * + */ +void prompp_series_data_serialized_data_dtor(void* args); + /** * @brief return instant series at given timestamp for label sets. * @@ -1570,10 +1697,11 @@ void prompp_series_data_data_storage_dtor(void* args); * lss uintptr // pointer to constructed label sets * lsIdBatchSize uint32 // size of ls batch for recoding * dataStorage uintptr // pointer to constructed data storage - * time_interval struct { closed interval [min, max] + * timeInterval struct { // closed interval [min, max] * min int64 * max int64 * } + * downsamplingMs int64 // downsampling interval in milliseconds (0 - downsampling is disabled) * } * @param res { * chunk_recoder uintptr // pointer to chunk recoder @@ -1813,79 +1941,6 @@ void prompp_series_data_encoder_dtor(void* args); extern "C" { #endif -/** - * @brief Get next series_id in serialized data. - * - * @param args { - * serializedData uintptr // pointer to serialized data. - * } - * - * @param res { - * series_id uint32 // series id (UINT32_MAX if no more series). - * chunk_ref uint32 // inner chunk id. - * } - */ -void prompp_series_data_serialization_serialized_data_next(void* args, void* res); - -/** - * @brief Create a decode iterator for corresponding chunk_ref. - * - * @param args { - * serializedData uintptr // pointer to serialized data. - * chunk_ref uint32 // inner chunk id. - * } - * - */ -void prompp_series_data_serialization_serialized_data_iterator_ctor(void* args); - -/** - * @brief Advance decode iterator. - * - * @param iterator uintptr // pointer to decode iterator - * - */ -void prompp_series_data_serialization_serialized_data_iterator_next(void* iterator); - -/** - * @brief Advance decode iterator until referenced sample is gte targetTimestamp. - * - * @param args { - * iterator uintptr // pointer to decode iterator - * targetTimestamp int64 // target timestamp - * } - * - */ -void prompp_series_data_serialization_serialized_data_iterator_seek(void* args); - -/** - * @brief Reset a decode iterator for corresponding chunk_ref. - * - * @param args { - * serializedData uintptr // pointer to serialized data. - * iterator uintptr // pointer to decode iterator - * chunkRef uint32 // inner chunk id. - * } - * - */ -void prompp_series_data_serialization_serialized_data_iterator_reset(void* args); - -/** - * @brief Destroy serialized data object. - * - * @param args { - * serializedData uintptr // pointer to serialized data. - * } - * - */ -void prompp_series_data_serialization_serialized_data_dtor(void* args); - -#ifdef __cplusplus -} // extern "C" -#endif -#ifdef __cplusplus -extern "C" { -#endif - /** * @brief Construct a new WAL Decoder * diff --git a/pp/go/cppbridge/head.go b/pp/go/cppbridge/head.go index 2edf09d561..ab9e35aa6e 100644 --- a/pp/go/cppbridge/head.go +++ b/pp/go/cppbridge/head.go @@ -10,6 +10,8 @@ const ( NormalNaN uint64 = 0x7ff8000000000001 StaleNaN uint64 = 0x7ff0000000000002 + + NoDownsampling = 0 ) func IsStaleNaN(v float64) bool { @@ -117,8 +119,19 @@ type ChunkRecoder struct { serializedData *DataStorageSerializedData } -func NewChunkRecoder(lss *LabelSetStorage, lsIdBatchSize uint32, dataStorage *DataStorage, timeInterval TimeInterval) *ChunkRecoder { - return initializeChunkRecoder(lss, dataStorage, nil, seriesDataChunkRecoderCtor(lss.Pointer(), lsIdBatchSize, dataStorage.dataStorage, timeInterval)) +func NewChunkRecoder( + lss *LabelSetStorage, + lsIdBatchSize uint32, + dataStorage *DataStorage, + timeInterval TimeInterval, + downsamplingMs int64, +) *ChunkRecoder { + return initializeChunkRecoder( + lss, + dataStorage, + nil, + seriesDataChunkRecoderCtor(lss.Pointer(), lsIdBatchSize, dataStorage.dataStorage, timeInterval, downsamplingMs), + ) } func NewSerializedChunkRecoder(serializedData *DataStorageSerializedData, timeInterval TimeInterval) *ChunkRecoder { @@ -246,39 +259,83 @@ func (sd *DataStorageSerializedData) Next() (uint32, uint32) { return seriesDataSerializedDataNext(sd.serializedData) } -type DataStorageSerializedDataIteratorControlBlock struct { - Timestamp int64 - Value float64 +type DataStorageSerializedDataSamplesIteratorControlBlock struct { + timestamp int64 + value float64 remainingSamples uint8 } -type DataStorageSerializedDataIterator struct { - DataStorageSerializedDataIteratorControlBlock - cppInternalData [unsafe.Sizeof(CppSerializedDataIterator{}) - unsafe.Sizeof(DataStorageSerializedDataIteratorControlBlock{})]byte +type DataStorageSerializedDataSamplesIterator struct { + DataStorageSerializedDataSamplesIteratorControlBlock + cppInternalData [unsafe.Sizeof(CppSerializedDataSamplesIterator{}) - unsafe.Sizeof(DataStorageSerializedDataSamplesIteratorControlBlock{})]byte } -func NewDataStorageSerializedDataIterator(serializedData *DataStorageSerializedData, chunkRef uint32) DataStorageSerializedDataIterator { - it := DataStorageSerializedDataIterator{} - seriesDataSerializedDataIteratorCtor(&it, serializedData.serializedData, chunkRef) +func NewDataStorageSerializedDataSamplesIterator(serializedData *DataStorageSerializedData, chunkRef uint32) DataStorageSerializedDataSamplesIterator { + it := DataStorageSerializedDataSamplesIterator{} + seriesDataSerializedDataSamplesIteratorCtor(&it, serializedData.serializedData, chunkRef) return it } -func (it *DataStorageSerializedDataIterator) Next() { - seriesDataSerializedDataIteratorNext(it) +func (it *DataStorageSerializedDataSamplesIterator) Next() { + seriesDataSerializedDataSamplesIteratorNext(it) } -func (it *DataStorageSerializedDataIterator) Seek(timestamp int64) { - seriesDataSerializedDataIteratorSeek(it, timestamp) +func (it *DataStorageSerializedDataSamplesIterator) Seek(timestamp int64) { + seriesDataSerializedDataSamplesIteratorSeek(it, timestamp) } -func (it *DataStorageSerializedDataIterator) Reset(serializedData *DataStorageSerializedData, chunkRef uint32) { - seriesDataSerializedDataIteratorReset(it, serializedData.serializedData, chunkRef) +func (it *DataStorageSerializedDataSamplesIterator) Reset(serializedData *DataStorageSerializedData, chunkRef uint32) { + seriesDataSerializedDataSamplesIteratorReset(it, serializedData.serializedData, chunkRef) } -func (it *DataStorageSerializedDataIterator) HasData() bool { +func (it *DataStorageSerializedDataSamplesIterator) HasData() bool { return it.remainingSamples != 0 } +func (it *DataStorageSerializedDataSamplesIterator) Timestamp() int64 { + return it.timestamp +} + +func (it *DataStorageSerializedDataSamplesIterator) Value() float64 { + return it.value +} + +type DataStorageSerializedDataAggregationIteratorControlBlock struct { + timestamp int64 + value float64 +} + +type DataStorageSerializedDataAggregationIterator struct { + DataStorageSerializedDataAggregationIteratorControlBlock + cppInternalData [unsafe.Sizeof(CppSerializedDataAggregationIterator{}) - unsafe.Sizeof(DataStorageSerializedDataAggregationIteratorControlBlock{})]byte +} + +func NewDataStorageSerializedDataAggregationIterator(serializedData *DataStorageSerializedData, chunkRef uint32) DataStorageSerializedDataAggregationIterator { + it := DataStorageSerializedDataAggregationIterator{} + seriesDataSerializedDataAggregationIteratorCtor(&it, serializedData.serializedData, chunkRef) + return it +} + +func (it *DataStorageSerializedDataAggregationIterator) Next() { + seriesDataSerializedDataAggregationIteratorNext(it) +} + +func (it *DataStorageSerializedDataAggregationIterator) Reset(serializedData *DataStorageSerializedData, chunkRef uint32) { + seriesDataSerializedDataAggregationIteratorReset(it, serializedData.serializedData, chunkRef) +} + +func (it *DataStorageSerializedDataAggregationIterator) HasData() bool { + return it.timestamp != math.MinInt64 +} + +func (it *DataStorageSerializedDataAggregationIterator) Timestamp() int64 { + return it.timestamp +} + +func (it *DataStorageSerializedDataAggregationIterator) Value() float64 { + return it.value +} + // UnloadedDataLoader is Go wrapper around series_data::Loader. type UnloadedDataLoader struct { loader uintptr diff --git a/pp/go/cppbridge/head_test.go b/pp/go/cppbridge/head_test.go index cf1b201884..f3470745c0 100644 --- a/pp/go/cppbridge/head_test.go +++ b/pp/go/cppbridge/head_test.go @@ -5,6 +5,7 @@ import ( "unsafe" "github.com/prometheus/prometheus/pp/go/storage/querier" + "github.com/prometheus/prometheus/storage" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/pp/go/cppbridge" @@ -38,7 +39,7 @@ func (s *HeadSuite) TestChunkRecoder() { s.encoder.Encode(0, 2, 1.0) s.encoder.Encode(1, 3, 2.0) s.encoder.Encode(1, 4, 2.0) - recoder := cppbridge.NewChunkRecoder(s.lss, 2, s.dataStorage, cppbridge.TimeInterval{MinT: 0, MaxT: 4}) + recoder := cppbridge.NewChunkRecoder(s.lss, 2, s.dataStorage, cppbridge.TimeInterval{MinT: 0, MaxT: 4}, cppbridge.NoDownsampling) // Act chunk2 := recoder.RecodeNextChunk() @@ -78,7 +79,7 @@ func (s *HeadSuite) TestChunkRecoderWithBatchIterator() { s.encoder.Encode(1, 3, 2.0) s.encoder.Encode(1, 4, 2.0) - recoder := cppbridge.NewChunkRecoder(s.lss, 1, s.dataStorage, cppbridge.TimeInterval{MinT: 0, MaxT: 4}) + recoder := cppbridge.NewChunkRecoder(s.lss, 1, s.dataStorage, cppbridge.TimeInterval{MinT: 0, MaxT: 4}, cppbridge.NoDownsampling) // Act chunk2 := recoder.RecodeNextChunk() @@ -123,8 +124,8 @@ func (s *HeadSuite) TestSerializedChunkRecoder() { result := s.dataStorage.Query(cppbridge.DataStorageQuery{ StartTimestampMs: timeInterval.MinT, EndTimestampMs: timeInterval.MaxT, - LabelSetIDs: []uint32{0, 1}}, - ) + LabelSetIDs: []uint32{0, 1}, + }, cppbridge.NoDownsampling, unsafe.Pointer(&storage.SelectHints{})) recoder := cppbridge.NewSerializedChunkRecoder(result.SerializedData, timeInterval) // Act diff --git a/pp/go/storage/appender/appender_test.go b/pp/go/storage/appender/appender_test.go index fe1ace831f..5e7bfbade0 100644 --- a/pp/go/storage/appender/appender_test.go +++ b/pp/go/storage/appender/appender_test.go @@ -19,6 +19,7 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/head/shard" "github.com/prometheus/prometheus/pp/go/storage/head/task" "github.com/prometheus/prometheus/pp/go/storage/storagetest" + prom_storage "github.com/prometheus/prometheus/storage" "github.com/stretchr/testify/suite" ) @@ -123,7 +124,7 @@ func (s *AppenderSuite) getHeadData(labelSetIDs []uint32) headStorageData { StartTimestampMs: 0, EndTimestampMs: math.MaxInt64, LabelSetIDs: labelSetIDs, - }) + }, cppbridge.NoDownsampling, &prom_storage.SelectHints{}) data.dsResult = append(data.dsResult, dsResult) data.shards = append(data.shards, storageData{ diff --git a/pp/go/storage/block/block.go b/pp/go/storage/block/block.go index ab89a8437a..a48955bfb9 100644 --- a/pp/go/storage/block/block.go +++ b/pp/go/storage/block/block.go @@ -62,10 +62,16 @@ func NewChunkIterator( lss *cppbridge.LabelSetStorage, lsIdBatchSize uint32, ds *cppbridge.DataStorage, - minT, maxT int64, + minT, maxT, downsamplingMs int64, ) ChunkIterator { return ChunkIterator{ - r: cppbridge.NewChunkRecoder(lss, lsIdBatchSize, ds, cppbridge.TimeInterval{MinT: minT, MaxT: maxT}), + r: cppbridge.NewChunkRecoder( + lss, + lsIdBatchSize, + ds, + cppbridge.TimeInterval{MinT: minT, MaxT: maxT}, + downsamplingMs, + ), hasMoreData: true, } } diff --git a/pp/go/storage/block/writer.go b/pp/go/storage/block/writer.go index b0b08b5d7b..f2cae41989 100644 --- a/pp/go/storage/block/writer.go +++ b/pp/go/storage/block/writer.go @@ -99,18 +99,7 @@ func (w *Writer[TShard]) createWriters(sd TShard) (blockWriters, error) { maxT = timeInterval.MaxT } - var chunkIterator ChunkIterator - _ = sd.DataStorage().WithRLock(func(*cppbridge.DataStorage) error { - chunkIterator = NewChunkIterator(sd.LSS().Target(), LsIdBatchSize, sd.DataStorage().Raw(), minT, maxT) - return nil - }) - - writer, err := newBlockWriter( - w.dataDir, - w.maxBlockChunkSegmentSize, - NewIndexWriter(sd.LSS().Target()), - chunkIterator, - ) + writer, err := w.createWriter(w.dataDir, sd, sd.LSS().Target(), minT, maxT, cppbridge.NoDownsampling) if err != nil { return blockWriters{}, errors.Join(err, writers.Close()) } @@ -121,6 +110,26 @@ func (w *Writer[TShard]) createWriters(sd TShard) (blockWriters, error) { return writers, nil } +func (w *Writer[TShard]) createWriter( + dataDir string, + sd TShard, + lss *cppbridge.LabelSetStorage, + minT, maxT, downsamplingMs int64, +) (blockWriter, error) { + var chunkIterator ChunkIterator + _ = sd.DataStorage().WithRLock(func(ds *cppbridge.DataStorage) error { + chunkIterator = NewChunkIterator(lss, LsIdBatchSize, ds, minT, maxT, downsamplingMs) + return nil + }) + + return newBlockWriter( + dataDir, + w.maxBlockChunkSegmentSize, + NewIndexWriter(lss), + chunkIterator, + ) +} + // recodeAndWriteChunks recodes and writes chunks for the shard. func (*Writer[TShard]) recodeAndWriteChunks(sd TShard, writers blockWriters) error { var loader *cppbridge.UnloadedDataRevertableLoader diff --git a/pp/go/storage/head/shard/data_storage.go b/pp/go/storage/head/shard/data_storage.go index 8941bf5ef1..ddc6818449 100644 --- a/pp/go/storage/head/shard/data_storage.go +++ b/pp/go/storage/head/shard/data_storage.go @@ -2,8 +2,10 @@ package shard import ( "sync" + "unsafe" "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/storage" ) // DataStorage samles storage with labels IDs. @@ -73,9 +75,9 @@ func (ds *DataStorage) MergeOutOfOrderChunks() { ds.locker.Unlock() } -func (ds *DataStorage) Query(query cppbridge.DataStorageQuery) cppbridge.DataStorageQueryResult { +func (ds *DataStorage) Query(query cppbridge.DataStorageQuery, downsamplingMs int64, hints *storage.SelectHints) cppbridge.DataStorageQueryResult { ds.locker.RLock() - result := ds.dataStorage.Query(query) + result := ds.dataStorage.Query(query, downsamplingMs, unsafe.Pointer(hints)) ds.locker.RUnlock() return result } diff --git a/pp/go/storage/loader_test.go b/pp/go/storage/loader_test.go index 713a47474b..a69ea9cc5f 100644 --- a/pp/go/storage/loader_test.go +++ b/pp/go/storage/loader_test.go @@ -23,6 +23,8 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/storagetest" "github.com/prometheus/prometheus/pp/go/util" "github.com/stretchr/testify/suite" + + prom_storage "github.com/prometheus/prometheus/storage" ) type idGeneratorStub struct { @@ -211,7 +213,7 @@ func (s *HeadLoadSuite) TestLoadWithDisabledDataUnloading() { StartTimestampMs: 0, EndTimestampMs: 2, LabelSetIDs: []uint32{0}, - }) + }, cppbridge.NoDownsampling, &prom_storage.SelectHints{}) err := loadedHead.Close() // Assert @@ -263,7 +265,7 @@ func (s *HeadLoadSuite) TestAppendAfterLoad() { StartTimestampMs: 0, EndTimestampMs: 4, LabelSetIDs: []uint32{0}, - }) + }, cppbridge.NoDownsampling, &prom_storage.SelectHints{}) err := loadedHead.Close() diff --git a/pp/go/storage/querier/aggr_series_set.go b/pp/go/storage/querier/aggr_series_set.go new file mode 100644 index 0000000000..0884d8c260 --- /dev/null +++ b/pp/go/storage/querier/aggr_series_set.go @@ -0,0 +1,262 @@ +package querier + +import ( + "math" + + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/util/annotations" +) + +// +// AggrSeriesSet +// + +// AggrSeriesSet contains a set of aggregated series. +// [storage.SeriesSet] interface implementation. +type AggrSeriesSet struct { + labelSetSnapshot *cppbridge.LabelSetSnapshot + serializedData *cppbridge.DataStorageSerializedData + mint int64 + maxt int64 + + lastIndexFromLSSQueryResult int + series []AggrSeries +} + +// NewAggrSeriesSet init new [AggrSeriesSet]. +func NewAggrSeriesSet( + labelSetSnapshot *cppbridge.LabelSetSnapshot, + serializedData *cppbridge.DataStorageSerializedData, + lssQueryResult *cppbridge.LSSQueryResult, + mint, maxt int64, +) *AggrSeriesSet { + return &AggrSeriesSet{ + labelSetSnapshot: labelSetSnapshot, + serializedData: serializedData, + mint: mint, + maxt: maxt, + series: make([]AggrSeries, 0, lssQueryResult.Len()), + lastIndexFromLSSQueryResult: 0, + } +} + +// At returns the current aggregated series. +// [storage.SeriesSet] interface implementation. +func (ss *AggrSeriesSet) At() storage.Series { + return &ss.series[len(ss.series)-1] +} + +// Err returns the error of the [AggrSeriesSet] - always nil. +// [storage.SeriesSet] interface implementation. +func (*AggrSeriesSet) Err() error { + return nil +} + +// Next advances the iterator by one and returns false if there are no more values. +// [storage.SeriesSet] interface implementation. +func (ss *AggrSeriesSet) Next() bool { + if ss.serializedData == nil { + return false + } + + seriesID, chunkRef := ss.serializedData.Next() + if seriesID == math.MaxUint32 { + return false + } + + builder := builderPool.Get().(*labels.ScratchBuilder) + builder.Reset() + ss.series = append(ss.series, NewAggrSeries( + labels.NewLabelsWithLSS(ss.labelSetSnapshot, seriesID, builder), + ss.serializedData, + ss.mint, + ss.maxt, + chunkRef, + )) + builderPool.Put(builder) + + return true +} + +// Warnings returns the warnings of the [AggrSeriesSet] - always nil. +// [storage.SeriesSet] interface implementation. +func (*AggrSeriesSet) Warnings() annotations.Annotations { + return nil +} + +// +// AggrSeries +// + +// AggrSeries represents a time series with aggregated samples. +// [storage.Series] interface implementation. +type AggrSeries struct { + labelSet labels.Labels + serializedData *cppbridge.DataStorageSerializedData + mint int64 + maxt int64 + chunkRef uint32 +} + +// NewAggrSeries init new [AggrSeries]. +func NewAggrSeries( + labelSet labels.Labels, + serializedData *cppbridge.DataStorageSerializedData, + mint, maxt int64, + chunkRef uint32, +) AggrSeries { + return AggrSeries{ + mint: mint, + maxt: maxt, + labelSet: labelSet, + serializedData: serializedData, + chunkRef: chunkRef, + } +} + +// Iterator returns an iterator that iterates over the aggregated of the samples of [AggrSeries]. +// [storage.Series] interface implementation. +func (s *AggrSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { + aggrChunkIterator, ok := it.(*AggrChunkIterator) + if !ok { + return NewAggrChunkIterator( + s.serializedData, + s.mint, + s.maxt, + s.chunkRef, + ) + } + + aggrChunkIterator.reset(s.serializedData, s.mint, s.maxt, s.chunkRef) + return aggrChunkIterator +} + +// Labels returns the labels of the [AggrSeries]. +// [storage.Series] interface implementation. +func (s *AggrSeries) Labels() labels.Labels { + return s.labelSet +} + +// +// AggrChunkIterator +// + +// AggrChunkIterator iterates over the aggregations of a time series, that can only get the next value. +type AggrChunkIterator struct { + serializedData *cppbridge.DataStorageSerializedData + chunkIterator cppbridge.DataStorageSerializedDataAggregationIterator + mint int64 + maxt int64 + isInitialized bool +} + +// NewAggrChunkIterator init new [AggrChunkIterator]. +func NewAggrChunkIterator( + serializedData *cppbridge.DataStorageSerializedData, + mint, maxt int64, + chunkRef uint32, +) *AggrChunkIterator { + return &AggrChunkIterator{ + serializedData: serializedData, + chunkIterator: cppbridge.NewDataStorageSerializedDataAggregationIterator(serializedData, chunkRef), + mint: mint, + maxt: maxt, + } +} + +// At returns the current timestamp/value pair if the value is a float. +// [chunkenc.Iterator] interface implementation. +// +//nolint:gocritic // unnamedResult not need +func (it *AggrChunkIterator) At() (int64, float64) { + return it.chunkIterator.Timestamp(), it.chunkIterator.Value() +} + +// AtFloatHistogram returns the current timestamp/value pair if the value is a histogram with floating-point counts. +// [chunkenc.Iterator] interface implementation. +func (*AggrChunkIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { + return 0, nil +} + +// AtHistogram returns the current timestamp/value pair if the value is a histogram with integer counts. +// [chunkenc.Iterator] interface implementation. +func (*AggrChunkIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) { + return 0, nil +} + +// AtT returns the current timestamp. +// [chunkenc.Iterator] interface implementation. +func (it *AggrChunkIterator) AtT() int64 { + return it.chunkIterator.Timestamp() +} + +// Err returns the current error - always nil. +// [chunkenc.Iterator] interface implementation. +func (*AggrChunkIterator) Err() error { + return nil +} + +// Next advances the iterator by one and returns the type of the value. +// [chunkenc.Iterator] interface implementation. +func (it *AggrChunkIterator) Next() chunkenc.ValueType { + if it.nextValue() == chunkenc.ValNone { + return chunkenc.ValNone + } + + if it.AtT() > it.maxt { + return chunkenc.ValNone + } + + return chunkenc.ValFloat +} + +// Seek advances the iterator forward to the first sample with a timestamp equal or greater than t. +// [chunkenc.Iterator] interface implementation. +func (it *AggrChunkIterator) Seek(t int64) chunkenc.ValueType { + it.isInitialized = true + if t > it.AtT() { + return it.Next() + } + + if it.AtT() > it.maxt { + return chunkenc.ValNone + } + + return chunkenc.ValFloat +} + +// nextValue advances the iterator by one and returns the type of the value. +func (it *AggrChunkIterator) nextValue() chunkenc.ValueType { + if !it.isInitialized { + if !it.chunkIterator.HasData() { + return chunkenc.ValNone + } + + it.isInitialized = true + return chunkenc.ValFloat + } + + it.chunkIterator.Next() + if !it.chunkIterator.HasData() { + return chunkenc.ValNone + } + + return chunkenc.ValFloat +} + +// reset resets the iterator to the beginning of the serialized data. +func (it *AggrChunkIterator) reset( + serializedData *cppbridge.DataStorageSerializedData, + mint, maxt int64, + chunkRef uint32, +) { + it.serializedData = serializedData + it.mint = mint + it.maxt = maxt + it.isInitialized = false + it.chunkIterator.Reset(serializedData, chunkRef) +} diff --git a/pp/go/storage/querier/chunk_querier.go b/pp/go/storage/querier/chunk_querier.go index 84ddff2289..e943779be8 100644 --- a/pp/go/storage/querier/chunk_querier.go +++ b/pp/go/storage/querier/chunk_querier.go @@ -123,7 +123,7 @@ func (q *ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead]) LabelValues( func (q *ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead]) Select( ctx context.Context, _ bool, - _ *storage.SelectHints, + hints *storage.SelectHints, matchers ...*labels.Matcher, ) storage.ChunkSeriesSet { release, err := q.head.AcquireQuery(ctx) @@ -151,7 +151,7 @@ func (q *ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead]) Select( shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) - queryDataStorage(dsQueryChunkQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt) + queryDataStorage(dsQueryChunkQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) chunkSeriesSets := poolProvider.GetChunkSeriesSet() defer poolProvider.PutChunkSeriesSet(chunkSeriesSets) diff --git a/pp/go/storage/querier/interface.go b/pp/go/storage/querier/interface.go index c001d5adfa..c822592ced 100644 --- a/pp/go/storage/querier/interface.go +++ b/pp/go/storage/querier/interface.go @@ -7,6 +7,7 @@ import ( "github.com/prometheus/prometheus/pp/go/model" "github.com/prometheus/prometheus/pp/go/storage/head/poolprovider" "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/storage" ) // @@ -51,6 +52,8 @@ type DataStorage interface { // Query returns serialized chunks from data storage. Query( query cppbridge.DataStorageQuery, + downsamplingMs int64, + hints *storage.SelectHints, ) cppbridge.DataStorageQueryResult // WithRLock calls fn on raw [cppbridge.DataStorage] with read lock. diff --git a/pp/go/storage/querier/merge_series_set_test.go b/pp/go/storage/querier/merge_series_set_test.go index 447de5e9b4..a384025163 100644 --- a/pp/go/storage/querier/merge_series_set_test.go +++ b/pp/go/storage/querier/merge_series_set_test.go @@ -76,8 +76,8 @@ func (s *MergeShardSeriesSetSuite) TestMergeShardSeriesSetScenarios() { esets := make([]storage.SeriesSet, 0, bm.numShards) asets := make([]storage.SeriesSet, 0, bm.numShards) for i := 0; i < bm.numShards; i++ { - esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) - asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, cppbridge.NoDownsampling, matcher)) + asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, cppbridge.NoDownsampling, matcher)) } assertMergeShardSeriesSetsEqual(s, esets, asets) }) @@ -90,8 +90,8 @@ func (s *MergeShardSeriesSetSuite) TestMergeShardSeriesSetScenarios() { esets = append(esets, &querier.SeriesSet{}) asets = append(asets, &querier.SeriesSet{}) } - esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) - asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, cppbridge.NoDownsampling, matcher)) + asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, cppbridge.NoDownsampling, matcher)) } assertMergeShardSeriesSetsEqual(s, esets, asets) }) @@ -100,10 +100,10 @@ func (s *MergeShardSeriesSetSuite) TestMergeShardSeriesSetScenarios() { esets := make([]storage.SeriesSet, 0, bm.numShards) asets := make([]storage.SeriesSet, 0, bm.numShards) for i := 0; i < bm.numShards; i++ { - esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, cppbridge.NoDownsampling, matcher)) } for i := bm.numShards - 1; i >= 0; i-- { - asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, cppbridge.NoDownsampling, matcher)) } assertMergeShardSeriesSetsEqual(s, esets, asets) }) @@ -141,7 +141,7 @@ func BenchmarkMergeSeriesSet(b *testing.B) { b.StopTimer() seriesSets = seriesSets[:0] for i := 0; i < bm.numShards; i++ { - seriesSets = append(seriesSets, queryOpt(b, head.lsses[i], head.dss[i], start, end, matcher)) + seriesSets = append(seriesSets, queryOpt(b, head.lsses[i], head.dss[i], start, end, cppbridge.NoDownsampling, matcher)) } b.StartTimer() diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index aa8979d2eb..b9cc1e998e 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -40,6 +40,74 @@ const ( DefaultInstantQueryValueNotFoundTimestampValue int64 = 0 ) +// +// queryOptimizeType +// + +// queryOptimizeType is the type for query optimization. +type queryOptimizeType uint8 + +const ( + // dropPointOptimizeType is the option for drop point functions optimization. + dropPointOptimizeType queryOptimizeType = 1 << iota + + // newPointOptimizeType is the option for new point functions optimization. + // Optimization creates a new point at the end of the window or step. + newPointOptimizeType + + // crossSeriesOptimizeType is the option for cross-series functions optimization. + // A new series is created. + crossSeriesOptimizeType +) + +const ( + // noneOptimizeType is the option without any optimization. + noneOptimizeType queryOptimizeType = 0 + + // allOptimizeType is the option for all functions optimization. + allOptimizeType queryOptimizeType = dropPointOptimizeType | newPointOptimizeType | crossSeriesOptimizeType +) + +// SetSelectFuncOptimize sets the select func optimization option by name. +func SetSelectFuncOptimize(opt string) error { + switch opt { + case "none": + selectFuncOptimize = noneOptimizeType + return nil + + case "drop_point": + selectFuncOptimize = dropPointOptimizeType + return nil + + case "new_point": + selectFuncOptimize = newPointOptimizeType + return nil + + case "cross": + selectFuncOptimize = crossSeriesOptimizeType + return nil + + case "all": + selectFuncOptimize = allOptimizeType + return nil + + default: + return fmt.Errorf( + "invalid select func optimization option: '%s', valid options are: "+ + "'none', 'drop_point', 'new_point', 'cross', 'all'", opt, + ) + } +} + +// selectFuncOptimize is the option for selecting functions optimization. +var selectFuncOptimize = noneOptimizeType + +// emptySelectHints is an empty select hints, it's used when no optimization is needed. +var emptySelectHints = &storage.SelectHints{} + +// emptySeriesSet is an empty series set. +var emptySeriesSet = &SeriesSet{} + // // Querier // @@ -58,6 +126,7 @@ type Querier[ deduplicatorCtor deduplicatorCtor closer func() error metrics *Metrics + queryOptimize queryOptimizeType } // NewQuerier init new [Querier]. @@ -73,6 +142,49 @@ func NewQuerier[ mint, maxt int64, closer func() error, metrics *Metrics, +) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { + return newQuerierWithSelectFuncOptimize(head, deduplicatorCtor, mint, maxt, closer, metrics, selectFuncOptimize) +} + +// NewQuerierWithOutSelectFuncOptimize init new [Querier] without select func optimization. +func NewQuerierWithOutSelectFuncOptimize[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + head THead, + deduplicatorCtor deduplicatorCtor, + mint, maxt int64, + closer func() error, + metrics *Metrics, +) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { + return newQuerierWithSelectFuncOptimize( + head, + deduplicatorCtor, + mint, + maxt, + closer, + metrics, + selectFuncOptimize&dropPointOptimizeType, + ) +} + +// newQuerierWithSelectFuncOptimize init new [Querier] with select func optimization. +func newQuerierWithSelectFuncOptimize[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + head THead, + deduplicatorCtor deduplicatorCtor, + mint, maxt int64, + closer func() error, + metrics *Metrics, + queryOptimize queryOptimizeType, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { return &Querier[TTask, TDataStorage, TLSS, TShard, THead]{ mint: mint, @@ -81,6 +193,7 @@ func NewQuerier[ deduplicatorCtor: deduplicatorCtor, closer: closer, metrics: metrics, + queryOptimize: queryOptimize, } } @@ -253,7 +366,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectInstant( func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( ctx context.Context, _ bool, - _ *storage.SelectHints, + hints *storage.SelectHints, matchers ...*labels.Matcher, ) storage.SeriesSet { start := time.Now() @@ -288,9 +401,53 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } + hints = SwitchFuncOptimize(hints, q.queryOptimize) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) - queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt) + queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) + + if isAggregationSeriesFunc(hints) { + return q.makeAggrSeriesSet(lssQueryResults, snapshots, shardedSerializedData) + } + + return q.makeSeriesSet(lssQueryResults, snapshots, shardedSerializedData) +} + +// makeAggrSeriesSet makes the aggregated series set. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggrSeriesSet( + lssQueryResults []*cppbridge.LSSQueryResult, + snapshots []*cppbridge.LabelSetSnapshot, + shardedSerializedData []*cppbridge.DataStorageSerializedData, +) storage.SeriesSet { + poolProvider := q.head.PoolProvider() + + seriesSets := poolProvider.GetSeriesSet() + defer poolProvider.PutSeriesSet(seriesSets) + for shardID, serializedData := range shardedSerializedData { + if serializedData != nil { + seriesSets[shardID] = NewAggrSeriesSet( + snapshots[shardID], + serializedData, + lssQueryResults[shardID], + q.mint, + q.maxt, + ) + continue + } + + seriesSets[shardID] = emptySeriesSet + } + + return NewMergeShardSeriesSet(seriesSets) +} + +// makeSeriesSet makes the series set. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeSeriesSet( + lssQueryResults []*cppbridge.LSSQueryResult, + snapshots []*cppbridge.LabelSetSnapshot, + shardedSerializedData []*cppbridge.DataStorageSerializedData, +) storage.SeriesSet { + poolProvider := q.head.PoolProvider() seriesSets := poolProvider.GetSeriesSet() defer poolProvider.PutSeriesSet(seriesSets) @@ -305,12 +462,67 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( ) continue } - seriesSets[shardID] = &SeriesSet{} + + seriesSets[shardID] = emptySeriesSet } return NewMergeShardSeriesSet(seriesSets) } +// SwitchFuncOptimize switch the function optimization hints. +func SwitchFuncOptimize(hints *storage.SelectHints, queryOptimize queryOptimizeType) *storage.SelectHints { + if hints == nil { + return emptySelectHints + } + + if hints.IsSubquery { + return emptySelectHints + } + + if funcOptimizeMap[hints.Func]&queryOptimize != 0 && isNotWithpout(hints) { + return hints + } + + return emptySelectHints +} + +// isNotWithpout checks if the hints is not without by. +func isNotWithpout(hints *storage.SelectHints) bool { + return hints.By || len(hints.Grouping) == 0 +} + +// funcOptimizeMap is the map of the function to the query optimization type. +var funcOptimizeMap = func() map[string]queryOptimizeType { + optimizeType := func(Type cppbridge.PromqlCppFunctionType) queryOptimizeType { + switch Type { + case cppbridge.PromqlCppThinningFunction: + return dropPointOptimizeType + case cppbridge.PromqlCppSynthesizingFunction: + return newPointOptimizeType + case cppbridge.PromqlCppCrossSeriesSynthesizingFunction: + return crossSeriesOptimizeType + + default: + return noneOptimizeType + } + } + + cppFunctions := cppbridge.GetPromqlCppFunctions() + functions := make(map[string]queryOptimizeType, len(cppFunctions)) + for _, function := range cppFunctions { + if oType := optimizeType(function.Type); oType != noneOptimizeType { + functions[function.Name] = oType + } + } + + return functions +}() + +// isAggregationSeriesFunc checks if the function is an aggregation series function. +func isAggregationSeriesFunc(hints *storage.SelectHints) bool { + return funcOptimizeMap[hints.Func]&dropPointOptimizeType == dropPointOptimizeType +} + // convertPrometheusMatchersToPPMatchers converts prometheus matchers to pp matchers. func convertPrometheusMatchersToPPMatchers(matchers ...*labels.Matcher) []model.LabelMatcher { promppMatchers := make([]model.LabelMatcher, len(matchers)) @@ -336,6 +548,7 @@ func queryDataStorage[ lssQueryResults []*cppbridge.LSSQueryResult, shardedSerializedData []*cppbridge.DataStorageSerializedData, mint, maxt int64, + hints *storage.SelectHints, ) { loadAndQueryWaiter := NewLoadAndQueryWaiter[TTask, TDataStorage, TLSS, TShard, THead](head) tDataStorageQuery := head.CreateTask( @@ -347,11 +560,15 @@ func queryDataStorage[ return nil } - result := s.DataStorage().Query(cppbridge.DataStorageQuery{ - StartTimestampMs: mint, - EndTimestampMs: maxt, - LabelSetIDs: lssQueryResult.IDs(), - }) + result := s.DataStorage().Query( + cppbridge.DataStorageQuery{ + StartTimestampMs: mint, + EndTimestampMs: maxt, + LabelSetIDs: lssQueryResult.IDs(), + }, + cppbridge.NoDownsampling, + hints, + ) if result.Status == cppbridge.DataStorageQueryStatusNeedDataLoad { loadAndQueryWaiter.Add(s, result.Querier) } diff --git a/pp/go/storage/querier/querier_optimize_quick_test.go b/pp/go/storage/querier/querier_optimize_quick_test.go new file mode 100644 index 0000000000..194cb7ec76 --- /dev/null +++ b/pp/go/storage/querier/querier_optimize_quick_test.go @@ -0,0 +1,434 @@ +//go:build !asan + +package querier_test + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + "testing/quick" + "time" + + "github.com/go-kit/log" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/promql" + "github.com/stretchr/testify/suite" +) + +// +// constants +// + +// defaultJitterMs is the default jitter. +const defaultJitterMs = 600000 + +// +// QueryParam +// + +// QueryParam is the struct for query parameter. +type QueryParam struct { + Start time.Time + End time.Time + Step time.Duration +} + +// Generate generates a random query parameter. +func (qp QueryParam) Generate(rd *rand.Rand, _ int) reflect.Value { + qp.gen(rd) + + return reflect.ValueOf(qp) +} + +// gen generates a random query parameter. +func (qp *QueryParam) gen(rd *rand.Rand) { + qp.Start = time.UnixMilli(defaultStartMs - defaultJitterMs + rd.Int63n(2*defaultJitterMs)) + qp.End = qp.Start.Add( + time.Duration(rd.Int63n((defaultStep*defaultCountOfSteps).Milliseconds()+defaultJitterMs) * 1e6), // ms to ns + ) + + qp.Step = 100 * time.Millisecond + + diff := qp.End.Sub(qp.Start) + if diff <= qp.Step { + return + } + + rndStep := time.Duration(rd.Int63n(diff.Milliseconds()) * 1e6) // ms to ns + if rndStep <= qp.Step { + return + } + + qp.Step = rndStep +} + +// +// SubQueryParams +// + +type SubQueryParams struct { + QueryParam + SubQueryStep time.Duration + SubQueryRange time.Duration +} + +// Generate generates a random query parameter. +func (sqp SubQueryParams) Generate(rd *rand.Rand, _ int) reflect.Value { + sqp.subGen(rd) + + return reflect.ValueOf(sqp) +} + +// subGen generates a random subquery parameter. +func (sqp *SubQueryParams) subGen(rd *rand.Rand) { + sqp.gen(rd) + + sqp.SubQueryStep = 100 * time.Millisecond + sqp.SubQueryRange = 100 * time.Millisecond + diff := sqp.End.Sub(sqp.Start) + + if diff > sqp.SubQueryStep { + rndStep := time.Duration(rd.Int63n(diff.Milliseconds()) * 1e6) // ms to ns + if rndStep > sqp.SubQueryStep { + sqp.SubQueryStep = rndStep + } + } + + if diff > sqp.SubQueryRange { + rndStep := time.Duration(rd.Int63n(diff.Milliseconds()) * 1e6) // ms to ns + if rndStep > sqp.SubQueryRange { + sqp.SubQueryRange = rndStep + } + } +} + +// +// ModifierQueryParams +// + +type ModifierQueryParams struct { + SubQueryParams + ModifierAt time.Time +} + +// Generate generates a random query parameter. +// +//nolint:gocritic // hugeParam // this is a test function +func (mqp ModifierQueryParams) Generate(rd *rand.Rand, _ int) reflect.Value { + mqp.modGen(rd) + + return reflect.ValueOf(mqp) +} + +// modGen generates a random modifier parameter. +func (mqp *ModifierQueryParams) modGen(rd *rand.Rand) { + mqp.subGen(rd) + + shiftMs := 100 * time.Millisecond + diff := mqp.End.Sub(mqp.Start) + + mqp.ModifierAt = mqp.Start.Add(shiftMs) + if diff <= shiftMs { + return + } + + rndShiftMs := time.Duration(rd.Int63n(diff.Milliseconds()) * 1e6) // ms to ns + if rndShiftMs <= shiftMs { + return + } + + mqp.ModifierAt = mqp.Start.Add(rndShiftMs) +} + +// +// OffsetQueryParams +// + +type OffsetQueryParams struct { + ModifierQueryParams + Offset time.Duration +} + +// Generate generates a random query parameter. +// +//nolint:gocritic // hugeParam // this is a test function +func (oqp OffsetQueryParams) Generate(rd *rand.Rand, _ int) reflect.Value { + oqp.offsetGen(rd) + + return reflect.ValueOf(oqp) +} + +// offsetGen generates a random offset parameter. +func (oqp *OffsetQueryParams) offsetGen(rd *rand.Rand) { + oqp.modGen(rd) + + oqp.Offset = time.Duration(0) + diff := oqp.End.Sub(oqp.Start) + + if diff == oqp.Offset { + return + } + + oqp.Offset = time.Duration(rd.Int63n(diff.Milliseconds()) * 1e6) // ms to ns +} + +// +// QuickQuerierOptimizeSuite +// + +type QuickQuerierOptimizeSuite struct { + QuerierOptimizeSuite + + quickQE *promql.Engine +} + +func TestQuickQuerierOptimizeSuite(t *testing.T) { + suite.Run(t, new(QuickQuerierOptimizeSuite)) +} + +func (s *QuickQuerierOptimizeSuite) SetupSuite() { + s.QuerierOptimizeSuite.SetupSuite() + + s.quickQE = promql.NewEngine(promql.EngineOpts{ + Logger: log.NewNopLogger(), + MaxSamples: 500000, + Timeout: 10 * time.Second, + LookbackDelta: s.lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { return s.lookbackDelta.Milliseconds() }, + EnableAtModifier: true, + EnableNegativeOffset: true, + }) +} + +func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickQueryParam() { + ctx := s.T().Context() + + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + f := func(qp QueryParam) bool { + query := qFunc.toQueryString( + metricName, + subQuery{window: model.Duration(s.step * 4), step: 0}, + modifierNone, + newOffset(0), + ) + + res, err := queryRange(ctx, "none", s.quickQE, s, s.queryOpts, query, qp.Start, qp.End, qp.Step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.quickQE, s, s.queryOpts, query, qp.Start, qp.End, qp.Step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + return s.Equal(res.res, resOpt.res) + } + + s.Require().NoError(quick.Check(f, nil)) + } + } +} + +func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickSubQueryParams() { + ctx := s.T().Context() + + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + f := func(sqp SubQueryParams) bool { + query := qFunc.toQueryString( + metricName, + subQuery{window: model.Duration(sqp.SubQueryRange), step: 0}, + modifierNone, + newOffset(0), + ) + + res, err := queryRange(ctx, "none", s.quickQE, s, s.queryOpts, query, sqp.Start, sqp.End, sqp.Step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.quickQE, s, s.queryOpts, query, sqp.Start, sqp.End, sqp.Step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + return s.Equal(res.res, resOpt.res) + } + + s.Require().NoError(quick.Check(f, nil)) + } + } +} + +func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickModifierQueryParams() { + ctx := s.T().Context() + + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + f := func(mqp ModifierQueryParams) bool { + query := qFunc.toQueryString( + metricName, + subQuery{window: model.Duration(mqp.SubQueryRange), step: 0}, + modifier(fmt.Sprintf(modifierAt, mqp.ModifierAt.UnixMilli())), + newOffset(0), + ) + + res, err := queryRange(ctx, "none", s.quickQE, s, s.queryOpts, query, mqp.Start, mqp.End, mqp.Step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.quickQE, s, s.queryOpts, query, mqp.Start, mqp.End, mqp.Step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + return s.Equal(res.res, resOpt.res) + } + + s.Require().NoError(quick.Check(f, nil)) + } + } +} + +func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickOffsetQueryParams() { + ctx := s.T().Context() + + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + f := func(oqp OffsetQueryParams) bool { + query := qFunc.toQueryString( + metricName, + subQuery{window: model.Duration(oqp.SubQueryRange), step: 0}, + modifier(fmt.Sprintf(modifierAt, oqp.ModifierAt.UnixMilli())), + newOffset(oqp.Offset), + ) + + res, err := queryRange(ctx, "none", s.quickQE, s, s.queryOpts, query, oqp.Start, oqp.End, oqp.Step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.quickQE, s, s.queryOpts, query, oqp.Start, oqp.End, oqp.Step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + return s.Equal(res.res, resOpt.res) + } + + s.Require().NoError(quick.Check(f, nil)) + } + } +} + +func (s *QuickQuerierOptimizeSuite) TestQueryInstantQuickQueryParam() { + ctx := s.T().Context() + + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + f := func(qp QueryParam) bool { + query := qFunc.toQueryString( + metricName, + subQuery{window: model.Duration(s.step * 4), step: 0}, + modifierNone, + newOffset(0), + ) + + res, err := queryInstant(ctx, "none", s.quickQE, s, s.queryOpts, query, qp.Start) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryInstant(ctx, "all", s.quickQE, s, s.queryOpts, query, qp.Start) + s.Require().NoError(err) + defer resOpt.qry.Close() + + return s.Equal(res.res, resOpt.res) + } + + s.Require().NoError(quick.Check(f, nil)) + } + } +} + +func (s *QuickQuerierOptimizeSuite) TestQueryInstantQuickSubQueryParams() { + ctx := s.T().Context() + + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + f := func(sqp SubQueryParams) bool { + query := qFunc.toQueryString( + metricName, + subQuery{window: model.Duration(sqp.SubQueryRange), step: 0}, + modifierNone, + newOffset(0), + ) + + res, err := queryInstant(ctx, "none", s.quickQE, s, s.queryOpts, query, sqp.Start) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryInstant(ctx, "all", s.quickQE, s, s.queryOpts, query, sqp.Start) + s.Require().NoError(err) + defer resOpt.qry.Close() + + return s.Equal(res.res, resOpt.res) + } + + s.Require().NoError(quick.Check(f, nil)) + } + } +} + +func (s *QuickQuerierOptimizeSuite) TestQueryInstantQuickModifierQueryParams() { + ctx := s.T().Context() + + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + f := func(mqp ModifierQueryParams) bool { + query := qFunc.toQueryString( + metricName, + subQuery{window: model.Duration(mqp.SubQueryRange), step: 0}, + modifier(fmt.Sprintf(modifierAt, mqp.ModifierAt.UnixMilli())), + newOffset(0), + ) + + res, err := queryInstant(ctx, "none", s.quickQE, s, s.queryOpts, query, mqp.Start) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryInstant(ctx, "all", s.quickQE, s, s.queryOpts, query, mqp.Start) + s.Require().NoError(err) + defer resOpt.qry.Close() + + return s.Equal(res.res, resOpt.res) + } + + s.Require().NoError(quick.Check(f, nil)) + } + } +} + +func (s *QuickQuerierOptimizeSuite) TestQueryInstantQuickOffsetQueryParams() { + ctx := s.T().Context() + + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + f := func(oqp OffsetQueryParams) bool { + query := qFunc.toQueryString( + metricName, + subQuery{window: model.Duration(oqp.SubQueryRange), step: 0}, + modifier(fmt.Sprintf(modifierAt, oqp.ModifierAt.UnixMilli())), + newOffset(oqp.Offset), + ) + + res, err := queryInstant(ctx, "none", s.quickQE, s, s.queryOpts, query, oqp.Start) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryInstant(ctx, "all", s.quickQE, s, s.queryOpts, query, oqp.Start) + s.Require().NoError(err) + defer resOpt.qry.Close() + + return s.Equal(res.res, resOpt.res) + } + + s.Require().NoError(quick.Check(f, nil)) + } + } +} diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go new file mode 100644 index 0000000000..19d9e46520 --- /dev/null +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -0,0 +1,827 @@ +package querier_test + +import ( + "context" + "fmt" + "math" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/value" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/querier" + "github.com/prometheus/prometheus/pp/go/storage/storagetest" + "github.com/prometheus/prometheus/promql" + prom_storage "github.com/prometheus/prometheus/storage" +) + +// +// SwitchFuncOptimizeSuite +// + +type SwitchFuncOptimizeSuite struct { + suite.Suite +} + +func TestSwitchFuncOptimizeSuite(t *testing.T) { + suite.Run(t, new(SwitchFuncOptimizeSuite)) +} + +func (s *SwitchFuncOptimizeSuite) TestNone() { + tests := []struct { + hints *prom_storage.SelectHints + expected *prom_storage.SelectHints + }{ + { + hints: &prom_storage.SelectHints{}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: false, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum_over_time"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time", IsSubquery: true}, + expected: &prom_storage.SelectHints{}, + }, + } + + for _, test := range tests { + result := querier.SwitchFuncOptimize(test.hints, 0) + s.Require().Equal(test.expected, result) + } +} + +func (s *SwitchFuncOptimizeSuite) TestDropPoint() { + tests := []struct { + hints *prom_storage.SelectHints + expected *prom_storage.SelectHints + }{ + { + hints: &prom_storage.SelectHints{}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: false, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum_over_time"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time"}, + expected: &prom_storage.SelectHints{Func: "max_over_time"}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time", IsSubquery: true}, + expected: &prom_storage.SelectHints{}, + }, + } + + for _, test := range tests { + result := querier.SwitchFuncOptimize(test.hints, 1) + s.Require().Equal(test.expected, result) + } +} + +func (s *SwitchFuncOptimizeSuite) TestNewPoint() { + tests := []struct { + hints *prom_storage.SelectHints + expected *prom_storage.SelectHints + }{ + { + hints: &prom_storage.SelectHints{}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: false, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum_over_time"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time", IsSubquery: true}, + expected: &prom_storage.SelectHints{}, + }, + } + + for _, test := range tests { + result := querier.SwitchFuncOptimize(test.hints, 2) + s.Require().Equal(test.expected, result) + } +} + +func (s *SwitchFuncOptimizeSuite) TestCrossSeries() { + tests := []struct { + hints *prom_storage.SelectHints + expected *prom_storage.SelectHints + }{ + { + hints: &prom_storage.SelectHints{}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: false, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum_over_time"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time", IsSubquery: true}, + expected: &prom_storage.SelectHints{}, + }, + } + + for _, test := range tests { + result := querier.SwitchFuncOptimize(test.hints, 4) + s.Require().Equal(test.expected, result) + } +} + +func (s *SwitchFuncOptimizeSuite) TestAll() { + tests := []struct { + hints *prom_storage.SelectHints + expected *prom_storage.SelectHints + }{ + { + hints: &prom_storage.SelectHints{}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: true, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum", By: false, Grouping: []string{"label"}}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "sum_over_time"}, + expected: &prom_storage.SelectHints{}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time"}, + expected: &prom_storage.SelectHints{Func: "max_over_time"}, + }, + { + hints: &prom_storage.SelectHints{Func: "max_over_time", IsSubquery: true}, + expected: &prom_storage.SelectHints{}, + }, + } + + for _, test := range tests { + result := querier.SwitchFuncOptimize(test.hints, 7) + s.Require().Equal(test.expected, result) + } +} + +// +// Constants +// + +const ( + // defaultStartMs is the default start time. + defaultStartMs = 1779290789000 + + // defaultStep is the default step. + defaultStep = 15 * time.Second +) + +// +// modifier +// + +// modifier is the modifier for query string. +type modifier string + +const ( + // modifierNone is the modifier for query string empty. + modifierNone modifier = "" + + // modifierAt is the modifier for query string At. + modifierAt string = " @ %d" + + // modifierEnd is the modifier for query string End. + modifierEnd modifier = " @ end()" + + // modifierStart is the modifier for query string Start. + modifierStart modifier = " @ start()" +) + +// +// offset +// + +// offset is the offset for query string. +type offset struct { + duration model.Duration + negative bool +} + +// newOffset creates a new [offset]. +func newOffset(duration time.Duration) offset { + if duration < 0 { + return offset{duration: model.Duration(-duration), negative: true} + } + + return offset{duration: model.Duration(duration)} +} + +// String converts the offset to query string. +func (o offset) String() string { + if o.duration == 0 { + return "" + } + + if o.negative { + return fmt.Sprintf(" offset -%s", o.duration) + } + + return fmt.Sprintf(" offset %s", o.duration) +} + +// +// queryFunc +// + +// queryFunc is the struct for query function. +type queryFunc struct { + name string + needRange bool +} + +// toQueryString converts the query function to query string. +func (q *queryFunc) toQueryString(metricName string, sq subQuery, mod modifier, o offset) string { + if q.needRange { + return fmt.Sprintf("%s(%s%s%s%s)", q.name, metricName, sq.toQueryString(), mod, o) + } + + return fmt.Sprintf("%s(%s%s%s)", q.name, metricName, mod, o) +} + +// +// subQuery +// + +// subQuery is the struct for subquery. +type subQuery struct { + window model.Duration + step model.Duration + defaultStep bool +} + +// toQueryString converts the subquery to query string. +func (s *subQuery) toQueryString() string { + if s.step == 0 { + if s.defaultStep { + return fmt.Sprintf("[%s:]", s.window) + } + + return fmt.Sprintf("[%s]", s.window) + } + + return fmt.Sprintf("[%s:%s]", s.window, s.step) +} + +// +// query +// + +// queryResult is a result of a query. +type queryResult struct { + qry promql.Query + res *promql.Result +} + +// queryRange executes a range query and returns the result. +func queryRange( + ctx context.Context, + optimization string, + queryEngine *promql.Engine, + q prom_storage.Queryable, + opts promql.QueryOpts, + query string, + start, end time.Time, + step time.Duration, +) (*queryResult, error) { + if err := querier.SetSelectFuncOptimize(optimization); err != nil { + return nil, err + } + + qry, err := queryEngine.NewRangeQuery(ctx, q, opts, query, start, end, step) + if err != nil { + return nil, err + } + + res := qry.Exec(ctx) + if res.Err != nil { + qry.Close() + return nil, res.Err + } + + return &queryResult{qry: qry, res: res}, nil +} + +// queryInstant executes an instant query and returns the result. +func queryInstant( + ctx context.Context, + optimization string, + queryEngine *promql.Engine, + q prom_storage.Queryable, + opts promql.QueryOpts, + query string, + ts time.Time, +) (*queryResult, error) { + if err := querier.SetSelectFuncOptimize(optimization); err != nil { + return nil, err + } + qry, err := queryEngine.NewInstantQuery(ctx, q, opts, query, ts) + if err != nil { + return nil, err + } + res := qry.Exec(ctx) + if res.Err != nil { + qry.Close() + return nil, res.Err + } + + return &queryResult{qry: qry, res: res}, nil +} + +// +// QuerierOptimizeSuite +// + +type QuerierOptimizeSuite struct { + suite.Suite + + dataDir string + head *storage.Head + start time.Time + end time.Time + step time.Duration + + lookbackDelta time.Duration + queryOpts promql.QueryOpts + metricNames []string + queryFuncs []queryFunc +} + +func (s *QuerierOptimizeSuite) SetupSuite() { + s.start = time.UnixMilli(defaultStartMs) + s.step = defaultStep + s.end = s.start.Add(s.step * defaultCountOfSteps) // 480 steps + + s.dataDir = s.createDataDirectory() + s.head = s.mustCreateHead(0) + s.fillHead() + + s.lookbackDelta = 5 * time.Minute + s.queryOpts = promql.NewPrometheusQueryOpts(false, s.lookbackDelta) + + s.queryFuncs = []queryFunc{ + // {name: "rate", needRange: true}, // - + // {name: "irate", needRange: true}, // - + // {name: "delta", needRange: true}, // - + // {name: "idelta", needRange: true}, // - + // {name: "increase", needRange: true}, // - + {name: "min_over_time", needRange: true}, // + + {name: "max_over_time", needRange: true}, // + + {name: "last_over_time", needRange: true}, // + + // {name: "sum_over_time", needRange: true}, // - + // {name: "resets", needRange: true}, // - + {name: "changes", needRange: true}, // + + } + + q, err := s.Querier(s.start.UnixMilli(), s.end.UnixMilli()) + s.Require().NoError(err) + + names, _, err := q.LabelValues(s.T().Context(), "__name__", &prom_storage.LabelHints{}) + s.Require().NoError(err) + + s.metricNames = querier.DeduplicateAndSortStringSlices(names) + s.Require().NoError(q.Close()) +} + +func (s *QuerierOptimizeSuite) TearDownSuite() { + s.Require().NoError(s.head.Close()) +} + +func (s *QuerierOptimizeSuite) createDataDirectory() string { + dataDir := filepath.Join(s.T().TempDir(), "data") + s.Require().NoError(os.MkdirAll(dataDir, os.ModeDir)) + return dataDir +} + +func (s *QuerierOptimizeSuite) mustCreateCatalog() *catalog.Catalog { + l, err := catalog.NewFileLogV2(filepath.Join(s.dataDir, "catalog.log")) + s.Require().NoError(err) + + c, err := catalog.New( + clockwork.NewFakeClock(), + l, + &catalog.DefaultIDGenerator{}, + catalog.DefaultMaxLogFileSize, + nil, + ) + s.Require().NoError(err) + + return c +} + +func (s *QuerierOptimizeSuite) mustCreateHead(unloadDataStorageInterval time.Duration) *storage.Head { + h, err := storage.NewBuilder( + s.mustCreateCatalog(), + s.dataDir, + maxSegmentSize, + prometheus.DefaultRegisterer, + unloadDataStorageInterval, + ).Build(0, numberOfShards) + s.Require().NoError(err) + return h +} + +func (s *QuerierOptimizeSuite) appendTimeSeries(timeSeries []storagetest.TimeSeries) { + storagetest.MustAppendTimeSeries(&s.Suite, s.head, timeSeries) +} + +func (s *QuerierOptimizeSuite) fillHead() { + countOfSamples := (s.end.UnixMilli()-s.start.UnixMilli())/s.step.Milliseconds() + 1 + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "sin_cos_metric", "value", "sin", "inc", "tick_usecond"), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }, + { + Labels: labels.FromStrings("__name__", "sin_cos_metric", "value", "sin", "inc", "tick_second"), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }, + { + Labels: labels.FromStrings("__name__", "sin_cos_metric", "value", "sin_stalenan", "inc", "tick_second"), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }, + { + Labels: labels.FromStrings("__name__", "sin_cos_metric", "value", "cos", "inc", "tick_usecond"), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }, + { + Labels: labels.FromStrings("__name__", "sin_cos_metric", "value", "cos", "inc", "tick_second"), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }, + { + Labels: labels.FromStrings("__name__", "counter_metric", "value", "inc"), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }, + { + Labels: labels.FromStrings("__name__", "counter_metric", "value", "with_stalenan"), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }, + { + Labels: labels.FromStrings("__name__", "counter_metric", "value", "with_resets"), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }, + } + + floatStaleNaN := math.Float64frombits(value.StaleNaN) + resetsCounter := 1 + valueCounter := 1 + for ts := s.start; !ts.After(s.end); ts = ts.Add(s.step) { + tsMilli := ts.UnixMilli() + timeSeries[0].Samples = append(timeSeries[0].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: math.Sin(float64(ts.UnixMilli())) * 10}, + ) + timeSeries[1].Samples = append(timeSeries[1].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: math.Sin(float64(ts.Second())) * 10}, + ) + + if valueCounter%5 == 0 { + timeSeries[2].Samples = append(timeSeries[2].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: floatStaleNaN}, + ) + } else { + timeSeries[2].Samples = append(timeSeries[2].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: math.Sin(float64(ts.Second())) * 10}, + ) + } + + timeSeries[3].Samples = append(timeSeries[3].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: math.Cos(float64(ts.UnixMilli())) * 10}, + ) + timeSeries[4].Samples = append(timeSeries[4].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: math.Cos(float64(ts.Second())) * 10}, + ) + timeSeries[5].Samples = append(timeSeries[5].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: float64(valueCounter)}, + ) + + if valueCounter%5 == 0 { + timeSeries[6].Samples = append(timeSeries[6].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: floatStaleNaN}, + ) + } else { + timeSeries[6].Samples = append(timeSeries[6].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: float64(valueCounter + 1)}, + ) + } + + if resetsCounter%10 == 0 { + resetsCounter = 1 + } + timeSeries[7].Samples = append(timeSeries[7].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: float64(resetsCounter)}, + ) + + resetsCounter++ + valueCounter++ + } + + s.appendTimeSeries(timeSeries) +} + +// Querier implements the [prom_storage.Queryable] interface. +func (s *QuerierOptimizeSuite) Querier(mint, maxt int64) (prom_storage.Querier, error) { + return querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, nil), nil +} + +// +// MatrixQuerierOptimizeSuiteSuite +// + +type MatrixQuerierOptimizeSuiteSuite struct { + QuerierOptimizeSuite + + queryEngine *promql.Engine + steps []time.Duration + subQueries []subQuery + modifiers []modifier + offsets []offset +} + +func TestMatrixQuerierOptimizeSuiteSuite(t *testing.T) { + suite.Run(t, new(MatrixQuerierOptimizeSuiteSuite)) +} + +func (s *MatrixQuerierOptimizeSuiteSuite) SetupSuite() { + s.QuerierOptimizeSuite.SetupSuite() + + s.queryEngine = promql.NewEngine(promql.EngineOpts{ + Logger: log.NewNopLogger(), + MaxSamples: 10000, + Timeout: 10 * time.Second, + LookbackDelta: s.lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { return s.lookbackDelta.Milliseconds() }, + EnableAtModifier: true, + EnableNegativeOffset: true, + }) + + s.steps = defaultSteps + s.subQueries = defaultSubQueries + s.modifiers = defaultModifiers + s.offsets = defaultOffsets + + q, err := s.Querier(s.start.UnixMilli(), s.end.UnixMilli()) + s.Require().NoError(err) + + names, _, err := q.LabelValues(s.T().Context(), "__name__", &prom_storage.LabelHints{}) + s.Require().NoError(err) + + s.metricNames = querier.DeduplicateAndSortStringSlices(names) + s.Require().NoError(q.Close()) +} + +// rangeArgs runs the given function for all combinations of +// query functions, metric names, steps, subqueries, modifiers and offsets. +// +//revive:disable-next-line:cognitive-complexity // matrix test +func (s *MatrixQuerierOptimizeSuiteSuite) rangeArgs(fn func( + qFunc queryFunc, + metricName string, + step time.Duration, + subq subQuery, + mod modifier, + o offset, +), +) { + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + for _, step := range s.steps { + for _, subq := range s.subQueries { + for _, mod := range s.modifiers { + for _, o := range s.offsets { + fn(qFunc, metricName, step, subq, mod, o) + } + } + } + } + } + } +} + +// rangeArgsWithStep runs the given function for all combinations of +// query functions, metric names, subqueries, modifiers and offsets. +// +//revive:disable-next-line:cognitive-complexity // matrix test +func (s *MatrixQuerierOptimizeSuiteSuite) rangeArgsWithStep(fn func( + qFunc queryFunc, + metricName string, + subq subQuery, + mod modifier, + o offset, +), +) { + for _, qFunc := range s.queryFuncs { + for _, metricName := range s.metricNames { + for _, subq := range s.subQueries { + for _, mod := range s.modifiers { + for _, o := range s.offsets { + fn(qFunc, metricName, subq, mod, o) + } + } + } + } + } +} + +func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRange() { + ctx := s.T().Context() + + s.rangeArgs(func(qFunc queryFunc, metricName string, step time.Duration, subq subQuery, mod modifier, o offset) { + query := qFunc.toQueryString(metricName, subq, mod, o) + s.Run(fmt.Sprintf("%s_step_%s", query, step), func() { + res, err := queryRange(ctx, "none", s.queryEngine, s, s.queryOpts, query, s.start, s.end, step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.queryEngine, s, s.queryOpts, query, s.start, s.end, step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + s.Require().Equal(res.res, resOpt.res) + }) + }) +} + +func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantStart() { + ctx := s.T().Context() + + s.rangeArgsWithStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { + query := qFunc.toQueryString(metricName, subq, mod, o) + s.Run(query, func() { + res, err := queryInstant(ctx, "none", s.queryEngine, s, s.queryOpts, query, s.start) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryInstant(ctx, "all", s.queryEngine, s, s.queryOpts, query, s.start) + s.Require().NoError(err) + defer resOpt.qry.Close() + + s.Require().Equal(res.res, resOpt.res) + }) + }) +} + +func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantMiddle() { + ctx := s.T().Context() + + s.rangeArgsWithStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { + query := qFunc.toQueryString(metricName, subq, mod, o) + s.Run(query, func() { + res, err := queryInstant(ctx, "none", s.queryEngine, s, s.queryOpts, query, s.start.Add(s.step*90)) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryInstant(ctx, "all", s.queryEngine, s, s.queryOpts, query, s.start.Add(s.step*90)) + s.Require().NoError(err) + defer resOpt.qry.Close() + + s.Require().Equal(res.res, resOpt.res) + }) + }) +} + +func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantEnd() { + ctx := s.T().Context() + + s.rangeArgsWithStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { + query := qFunc.toQueryString(metricName, subq, mod, o) + s.Run(query, func() { + res, err := queryInstant(ctx, "none", s.queryEngine, s, s.queryOpts, query, s.end) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryInstant(ctx, "all", s.queryEngine, s, s.queryOpts, query, s.end) + s.Require().NoError(err) + defer resOpt.qry.Close() + + s.Require().Equal(res.res, resOpt.res) + }) + }) +} + +func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle() { + ctx := s.T().Context() + queryF := "max_over_time(((min_over_time(%s[60s])))[60s:])" + start := s.start.Add(12 * time.Second) + for _, metricName := range s.metricNames { + query := fmt.Sprintf(queryF, metricName) + s.Run(query, func() { + res, err := queryRange(ctx, "none", s.queryEngine, s, s.queryOpts, query, start, s.end, s.step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.queryEngine, s, s.queryOpts, query, start, s.end, s.step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + s.Require().Equal(res.res, resOpt.res) + }) + } +} diff --git a/pp/go/storage/querier/querier_optimize_variables_asan_test.go b/pp/go/storage/querier/querier_optimize_variables_asan_test.go new file mode 100644 index 0000000000..0e30c607dd --- /dev/null +++ b/pp/go/storage/querier/querier_optimize_variables_asan_test.go @@ -0,0 +1,38 @@ +//go:build asan + +package querier_test + +import ( + "time" + + "github.com/prometheus/common/model" +) + +const ( + // defaultCountOfSteps is the default count of steps. + defaultCountOfSteps = 64 +) + +// defaultSteps is the default steps. +var defaultSteps = []time.Duration{ + defaultStep - time.Second, // [14s] + defaultStep * 2, // [30s] +} + +// defaultSubQueries is the default subqueries. +var defaultSubQueries = []subQuery{ + {window: model.Duration(defaultStep), step: 0}, // [15s] + {window: model.Duration(defaultStep*4 + time.Second), step: 0}, // [61s] +} + +// defaultModifiers is the default modifiers. +var defaultModifiers = []modifier{ + modifierNone, + modifierEnd, +} + +// defaultOffsets is the default offsets. +var defaultOffsets = []offset{ + newOffset(0), + newOffset(defaultStep * defaultCountOfSteps / 2), +} diff --git a/pp/go/storage/querier/querier_optimize_variables_test.go b/pp/go/storage/querier/querier_optimize_variables_test.go new file mode 100644 index 0000000000..c4aa406414 --- /dev/null +++ b/pp/go/storage/querier/querier_optimize_variables_test.go @@ -0,0 +1,80 @@ +//go:build !asan + +package querier_test + +import ( + "fmt" + "time" + + "github.com/prometheus/common/model" +) + +const ( + // defaultCountOfSteps is the default count of steps. + defaultCountOfSteps = 480 +) + +// defaultSteps is the default steps. +var defaultSteps = []time.Duration{ + defaultStep - time.Second, // [14s] + defaultStep, // [15s] + (defaultStep - time.Second) * 2, // [29s] + defaultStep * 2, // [30s] + (defaultStep - time.Second) * 4, // [59s] + defaultStep * 4, // [60s] + (defaultStep - time.Second) * 5, // [70s] +} + +// defaultSubQueries is the default subqueries. +var defaultSubQueries = []subQuery{ + {window: model.Duration(defaultStep), step: 0}, // [15s] + {window: model.Duration(defaultStep * 4), step: 0}, // [60s] + {window: model.Duration(defaultStep*4 - time.Second), step: 0}, // [59s] + {window: model.Duration(defaultStep*4 + time.Second), step: 0}, // [61s] + {window: model.Duration(defaultStep * 4), step: 0, defaultStep: true}, // [60s:] + {window: model.Duration(defaultStep * 16), step: model.Duration(defaultStep * 4)}, // [240s:60s] + {window: model.Duration(defaultStep*16 - time.Second), step: 0}, // [239s] + {window: model.Duration(defaultStep * 16), step: 0}, // [240s] + {window: model.Duration(defaultStep*16 + time.Second), step: 0}, // [241s] +} + +// defaultModifiers is the default modifiers. +var defaultModifiers = []modifier{ + modifierNone, + modifier(fmt.Sprintf(modifierAt, defaultStartMs/1e3+defaultStep*defaultCountOfSteps/2e9)), // middle of the range + modifierEnd, + modifierStart, +} + +// defaultOffsets is the default offsets. +var defaultOffsets = []offset{ + newOffset(0), + newOffset(defaultStep * defaultCountOfSteps / 2), + newOffset(-defaultStep * defaultCountOfSteps / 2), + newOffset(defaultStep * defaultCountOfSteps), + newOffset(-defaultStep * defaultCountOfSteps), +} + +// TestQueryRangeWithStep long test the query range with step. +func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeWithStep() { + ctx := s.T().Context() + w := s.end.Sub(s.start) / 3 + stepWindow := w * 2 / 3 + + for st, end := s.start, s.start.Add(w); !end.After(s.end); st, end = st.Add(stepWindow), end.Add(stepWindow) { + s.rangeArgs(func(qFunc queryFunc, mName string, step time.Duration, subq subQuery, mod modifier, o offset) { + query := qFunc.toQueryString(mName, subq, mod, o) + s.Run(fmt.Sprintf("%s_step_%s", query, step), func() { + res, err := queryRange(ctx, "none", s.queryEngine, s, s.queryOpts, query, st, end, step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.queryEngine, s, s.queryOpts, query, st, end, step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + s.Require().Equal(res.res, resOpt.res) + }) + }) + } +} diff --git a/pp/go/storage/querier/querier_test.go b/pp/go/storage/querier/querier_test.go index b45da5872e..977868af5a 100644 --- a/pp/go/storage/querier/querier_test.go +++ b/pp/go/storage/querier/querier_test.go @@ -9,6 +9,8 @@ import ( "github.com/jonboulle/clockwork" "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/suite" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/prometheus/prometheus/pp/go/storage" @@ -19,7 +21,6 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/pp/go/storage/storagetest" prom_storage "github.com/prometheus/prometheus/storage" - "github.com/stretchr/testify/suite" ) const ( @@ -119,7 +120,7 @@ func (s *QuerierSuite) TestRangeQuery() { matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") // Act - seriesSet := q.Select(s.context, false, nil, matcher) + seriesSet := q.Select(s.context, false, &prom_storage.SelectHints{}, matcher) // Assert s.Equal(timeSeries, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) @@ -147,7 +148,7 @@ func (s *QuerierSuite) TestRangeQueryWithoutMatching() { matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "unknown_metric") // Act - seriesSet := q.Select(s.context, false, nil, matcher) + seriesSet := q.Select(s.context, false, &prom_storage.SelectHints{}, matcher) // Assert s.Equal([]storagetest.TimeSeries{}, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) @@ -202,7 +203,7 @@ func (s *QuerierSuite) TestRangeQueryWithDataStorageLoading() { // Act s.Require().NoError(services.UnloadUnusedSeriesDataWithHead(s.head)) s.appendTimeSeries(timeSeriesAfterUnload) - seriesSet := q.Select(s.context, false, nil, matcher) + seriesSet := q.Select(s.context, false, &prom_storage.SelectHints{}, matcher) // Assert timeSeries[0].AppendSamples(timeSeriesAfterUnload[0].Samples...) @@ -238,7 +239,7 @@ func (s *QuerierSuite) TestInstantQuery() { matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") // Act - seriesSet := q.Select(s.context, false, nil, matcher) + seriesSet := q.Select(s.context, false, &prom_storage.SelectHints{}, matcher) // Assert s.Equal(timeSeries, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) @@ -293,7 +294,7 @@ func (s *QuerierSuite) TestInstantQueryWithDataStorageLoading() { // Act s.Require().NoError(services.UnloadUnusedSeriesDataWithHead(s.head)) s.appendTimeSeries(timeSeriesAfterUnload) - seriesSet := q.Select(s.context, false, nil, matcher) + seriesSet := q.Select(s.context, false, &prom_storage.SelectHints{}, matcher) // Assert s.Equal([]storagetest.TimeSeries{ diff --git a/pp/go/storage/querier/series.go b/pp/go/storage/querier/series.go index 33e7efd390..8c90fe7989 100644 --- a/pp/go/storage/querier/series.go +++ b/pp/go/storage/querier/series.go @@ -23,7 +23,7 @@ var builderPool = sync.Pool{ // ChunkIterator iterates over the samples of a time series, that can only get the next value with limit. type ChunkIterator struct { serializedData *cppbridge.DataStorageSerializedData - chunkIterator cppbridge.DataStorageSerializedDataIterator + chunkIterator cppbridge.DataStorageSerializedDataSamplesIterator mint int64 maxt int64 isInitialized bool @@ -33,12 +33,12 @@ type ChunkIterator struct { func NewChunkIterator(serializedData *cppbridge.DataStorageSerializedData, chunkRef uint32, mint, maxt int64) *ChunkIterator { it := &ChunkIterator{ serializedData: serializedData, - chunkIterator: cppbridge.NewDataStorageSerializedDataIterator(serializedData, chunkRef), + chunkIterator: cppbridge.NewDataStorageSerializedDataSamplesIterator(serializedData, chunkRef), mint: mint, maxt: maxt, } - if it.chunkIterator.Timestamp < mint { + if it.chunkIterator.Timestamp() < mint { it.chunkIterator.Seek(mint) } @@ -52,7 +52,7 @@ func (it *ChunkIterator) Reset(serializedData *cppbridge.DataStorageSerializedDa it.isInitialized = false it.chunkIterator.Reset(serializedData, chunkRef) - if it.chunkIterator.Timestamp < mint { + if it.chunkIterator.Timestamp() < mint { it.chunkIterator.Seek(mint) } } @@ -61,7 +61,7 @@ func (it *ChunkIterator) Reset(serializedData *cppbridge.DataStorageSerializedDa // //nolint:gocritic // unnamedResult not need func (it *ChunkIterator) At() (int64, float64) { - return it.chunkIterator.Timestamp, it.chunkIterator.Value + return it.chunkIterator.Timestamp(), it.chunkIterator.Value() } // AtFloatHistogram returns the current timestamp/value pair if the value is a histogram with floating-point counts. @@ -76,7 +76,7 @@ func (it *ChunkIterator) AtHistogram(h *histogram.Histogram) (int64, *histogram. // AtT returns the current timestamp. func (it *ChunkIterator) AtT() int64 { - return it.chunkIterator.Timestamp + return it.chunkIterator.Timestamp() } // Err returns the current error. diff --git a/pp/go/storage/querier/series_bench_test.go b/pp/go/storage/querier/series_bench_test.go index c2ac78aabb..7ef27ee490 100644 --- a/pp/go/storage/querier/series_bench_test.go +++ b/pp/go/storage/querier/series_bench_test.go @@ -29,7 +29,7 @@ func iterateSeriesSet(seriesSet storage.SeriesSet) { } } -func queryOpt(t testing.TB, lss *shard.LSS, ds *shard.DataStorage, start, end int64, matchers ...model.LabelMatcher) *querier.SeriesSet { +func queryOpt(t testing.TB, lss *shard.LSS, ds *shard.DataStorage, start, end, downsamplingMs int64, matchers ...model.LabelMatcher) *querier.SeriesSet { selector, snapshot, err := lss.QuerySelector(0, matchers) require.NoError(t, err) if selector == 0 || snapshot == nil { @@ -45,7 +45,7 @@ func queryOpt(t testing.TB, lss *shard.LSS, ds *shard.DataStorage, start, end in StartTimestampMs: start, EndTimestampMs: end, LabelSetIDs: lssQueryResult.IDs(), - }) + }, downsamplingMs, &storage.SelectHints{}) require.Equal(t, cppbridge.DataStorageQueryStatusSuccess, dsQueryResult.Status) return querier.NewSeriesSet(start, end, lssQueryResult, snapshot, dsQueryResult.SerializedData) @@ -69,7 +69,7 @@ func BenchmarkSeriesSetOpt(b *testing.B) { b.StopTimer() runtime.GC() runtime.GC() - seriesSet := queryOpt(b, lss, ds, start, end, matcher) + seriesSet := queryOpt(b, lss, ds, start, end, cppbridge.NoDownsampling, matcher) b.StartTimer() iterateSeriesSet(seriesSet) diff --git a/pp/go/storage/querier/series_test.go b/pp/go/storage/querier/series_test.go index b04851802b..4d29a4fa17 100644 --- a/pp/go/storage/querier/series_test.go +++ b/pp/go/storage/querier/series_test.go @@ -1,9 +1,11 @@ package querier_test import ( + "math" "testing" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/prometheus/prometheus/pp/go/model" "github.com/prometheus/prometheus/pp/go/storage/head/shard" @@ -71,7 +73,13 @@ func (s *SeriesSetTestSuite) SetupTest() { } } -func (s *SeriesSetTestSuite) query(lss *shard.LSS, ds *shard.DataStorage, start, end int64, matchers ...model.LabelMatcher) *querier.SeriesSet { +func (s *SeriesSetTestSuite) query( + lss *shard.LSS, + ds *shard.DataStorage, + start, end, downsamplingMs int64, + hints *storage.SelectHints, + matchers ...model.LabelMatcher, +) *querier.SeriesSet { selector, snapshot, err := lss.QuerySelector(0, matchers) require.NoError(s.T(), err) if selector == 0 || snapshot == nil { @@ -87,12 +95,40 @@ func (s *SeriesSetTestSuite) query(lss *shard.LSS, ds *shard.DataStorage, start, StartTimestampMs: start, EndTimestampMs: end, LabelSetIDs: lssQueryResult.IDs(), - }) + }, downsamplingMs, hints) require.Equal(s.T(), cppbridge.DataStorageQueryStatusSuccess, dsQueryResult.Status) return querier.NewSeriesSet(start, end, lssQueryResult, snapshot, dsQueryResult.SerializedData) } +func (s *SeriesSetTestSuite) queryAggr( + lss *shard.LSS, + ds *shard.DataStorage, + start, end, downsamplingMs int64, + hints *storage.SelectHints, + matchers ...model.LabelMatcher, +) *querier.AggrSeriesSet { + selector, snapshot, err := lss.QuerySelector(0, matchers) + s.Require().NoError(err) + if selector == 0 || snapshot == nil { + return &querier.AggrSeriesSet{} + } + + lssQueryResult := snapshot.Query(selector) + if lssQueryResult.Status() == cppbridge.LSSQueryStatusNoMatch { + return &querier.AggrSeriesSet{} + } + + dsQueryResult := ds.Query(cppbridge.DataStorageQuery{ + StartTimestampMs: start, + EndTimestampMs: end, + LabelSetIDs: lssQueryResult.IDs(), + }, downsamplingMs, hints) + + s.Require().Equal(cppbridge.DataStorageQueryStatusSuccess, dsQueryResult.Status) + return querier.NewAggrSeriesSet(snapshot, dsQueryResult.SerializedData, lssQueryResult, start, end) +} + func (s *SeriesSetTestSuite) nextSample(iterator chunkenc.Iterator) cppbridge.Sample { s.Equal(chunkenc.ValFloat, iterator.Next()) ts, v := iterator.At() @@ -118,7 +154,7 @@ func (s *SeriesSetTestSuite) TestQueryAllValues() { storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) // Act - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal(s.T(), expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, false)) @@ -139,7 +175,7 @@ func (s *SeriesSetTestSuite) TestQueryNoValues() { storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) // Act - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal(s.T(), expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, false)) @@ -160,7 +196,7 @@ func (s *SeriesSetTestSuite) TestQuerySingleSeries() { storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) // Act - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal(s.T(), expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, false)) @@ -181,7 +217,7 @@ func (s *SeriesSetTestSuite) TestQuerySingleSample() { storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) // Act - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal(s.T(), expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, false)) @@ -202,7 +238,7 @@ func (s *SeriesSetTestSuite) TestQueryCutByUpperLimit() { storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) // Act - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal(s.T(), expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, false)) @@ -223,7 +259,7 @@ func (s *SeriesSetTestSuite) TestQueryCutByLowerLimit() { storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) // Act - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal(s.T(), expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, false)) @@ -253,7 +289,7 @@ func (s *SeriesSetTestSuite) TestQueryLargeChunks() { storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, timeSeries...) // Act - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal(s.T(), expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, false)) @@ -272,7 +308,7 @@ func (s *SeriesSetTestSuite) TestQueryEmptyStorage() { expected := []storagetest.TimeSeries{} // Act - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal(s.T(), expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, false)) @@ -327,8 +363,8 @@ func (s *SeriesSetTestSuite) TestQueryMergedSeriesSets() { storagetest.MustAppendTimeSeriesToLSSAndDataStorage(anotherLss, anotherDs, timeSeries2...) expected := append(timeSeries1, timeSeries2...) // Act - seriesSet1 := s.query(s.lss, s.ds, start, end, matcher) - seriesSet2 := s.query(anotherLss, anotherDs, start, end, matcher) + seriesSet1 := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) + seriesSet2 := s.query(anotherLss, anotherDs, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) // Assert require.Equal( @@ -352,7 +388,7 @@ func (s *SeriesSetTestSuite) TestSeriesSeek() { expected := s.timeSeries[:4] storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries[:4]...) - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) require.True(s.T(), seriesSet.Next()) series := seriesSet.At() require.Equal(s.T(), expected[0].Labels, series.Labels()) @@ -389,7 +425,7 @@ func (s *SeriesSetTestSuite) TestSeriesSeekOutOfRange() { var end int64 = 1000 storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries[:4]...) - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) require.True(s.T(), seriesSet.Next()) series := seriesSet.At() var iterator chunkenc.Iterator @@ -416,7 +452,7 @@ func (s *SeriesSetTestSuite) TestSeriesParallelRead() { expected := s.timeSeries storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) - seriesSet := s.query(s.lss, s.ds, start, end, matcher) + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, matcher) seriesSlice := make([]storage.Series, 0, 2) require.True(s.T(), seriesSet.Next()) seriesSlice = append(seriesSlice, seriesSet.At()) @@ -439,7 +475,7 @@ func (s *SeriesSetTestSuite) TestSeriesResetIterator() { var start int64 = 0 var end int64 = 50 - seriesSet := s.query(s.lss, s.ds, start, end, model.LabelMatcher{ + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, model.LabelMatcher{ Name: "__name__", Value: "metric", MatcherType: model.MatcherTypeExactMatch, @@ -466,7 +502,7 @@ func (s *SeriesSetTestSuite) TestSeriesResetIteratorWithMinTimestamp() { var start int64 = 12 var end int64 = 50 - seriesSet := s.query(s.lss, s.ds, start, end, model.LabelMatcher{ + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, &storage.SelectHints{}, model.LabelMatcher{ Name: "__name__", Value: "metric", MatcherType: model.MatcherTypeExactMatch, @@ -485,3 +521,227 @@ func (s *SeriesSetTestSuite) TestSeriesResetIteratorWithMinTimestamp() { s.Equal(cppbridge.Sample{Timestamp: 12, Value: 2}, s.nextSample(iterator)) s.Equal(chunkenc.ValNone, iterator.Next()) } + +func (s *SeriesSetTestSuite) TestDownsampling() { + // Arrange + const Downsampling = 100 + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 123, Value: 1.0}, + {Timestamp: 152, Value: 1.0}, + {Timestamp: 180, Value: 1.0}, + {Timestamp: 215, Value: 1.0}, + {Timestamp: 242, Value: 1.0}, + {Timestamp: 275, Value: 1.0}, + {Timestamp: 303, Value: 1.0}, + }, + }, + }...) + + // Act + seriesSet := s.queryAggr(s.lss, s.ds, 0, 400, Downsampling, &storage.SelectHints{}, model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }) + + // Assert + require.Equal(s.T(), []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 180, Value: 1.0}, + {Timestamp: 275, Value: 1.0}, + {Timestamp: 303, Value: 1.0}, + }, + }, + }, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) +} + +func (s *SeriesSetTestSuite) TestQueryWithoutHints() { + // Arrange + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 123, Value: 1.0}, + {Timestamp: 152, Value: 1.0}, + {Timestamp: 180, Value: 1.0}, + {Timestamp: 215, Value: 1.0}, + }, + }, + }...) + + // Act + seriesSet := s.query(s.lss, s.ds, 0, 400, cppbridge.NoDownsampling, nil, model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }) + + // Assert + require.Equal(s.T(), []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 123, Value: 1.0}, + {Timestamp: 152, Value: 1.0}, + {Timestamp: 180, Value: 1.0}, + {Timestamp: 215, Value: 1.0}, + }, + }, + }, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) +} + +func (s *SeriesSetTestSuite) TestMinOverTimeFunc() { + // Arrange + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 1.0}, + {Timestamp: 150, Value: 2.0}, + {Timestamp: 200, Value: 3.0}, + {Timestamp: 250, Value: 0.0}, + }, + }, + }...) + hints := &storage.SelectHints{ + Start: 101, + End: 200, + Func: "min_over_time", + } + + // Act + seriesSet := s.queryAggr(s.lss, s.ds, 0, 400, cppbridge.NoDownsampling, hints, model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }) + + // Assert + s.Require().Equal([]storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 150, Value: 2.0}, + }, + }, + }, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) +} + +func (s *SeriesSetTestSuite) TestMaxOverTimeFunc() { + // Arrange + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 1.0}, + {Timestamp: 150, Value: 2.0}, + {Timestamp: 200, Value: 3.0}, + {Timestamp: 250, Value: 0.0}, + }, + }, + }...) + hints := &storage.SelectHints{ + Start: 100, + End: 200, + Func: "max_over_time", + } + + // Act + seriesSet := s.queryAggr(s.lss, s.ds, 0, 400, cppbridge.NoDownsampling, hints, model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }) + + // Assert + require.Equal(s.T(), []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 200, Value: 3.0}, + }, + }, + }, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) +} + +func (s *SeriesSetTestSuite) TestLastOverTimeFunc() { + // Arrange + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 1.0}, + {Timestamp: 150, Value: 2.0}, + {Timestamp: 200, Value: 3.0}, + {Timestamp: 250, Value: math.Float64frombits(value.StaleNaN)}, + {Timestamp: 300, Value: 0.0}, + }, + }, + }...) + hints := &storage.SelectHints{ + Start: 100, + End: 250, + Func: "last_over_time", + } + + // Act + seriesSet := s.queryAggr(s.lss, s.ds, 0, 400, cppbridge.NoDownsampling, hints, model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }) + + // Assert + require.Equal(s.T(), []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 200, Value: 3.0}, + }, + }, + }, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) +} + +func (s *SeriesSetTestSuite) TestChangesFunc() { + // Arrange + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 1.0}, + {Timestamp: 150, Value: 2.0}, + {Timestamp: 200, Value: 3.0}, + {Timestamp: 250, Value: math.Float64frombits(value.StaleNaN)}, + {Timestamp: 300, Value: 0.0}, + }, + }, + }...) + hints := &storage.SelectHints{ + Start: 99, + End: 250, + Func: "changes", + } + + // Act + seriesSet := s.queryAggr(s.lss, s.ds, 0, 400, cppbridge.NoDownsampling, hints, model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }) + + // Assert + actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + s.Equal(labels.FromStrings("__name__", "metric", "job", "test"), actual[0].Labels) + s.Equal([]cppbridge.Sample{ + {Timestamp: 100, Value: 1.0}, + {Timestamp: 150, Value: 2.0}, + {Timestamp: 200, Value: 3.0}, + }, actual[0].Samples[:3]) + s.Equal(int64(250), actual[0].Samples[3].Timestamp) + s.Equal(value.StaleNaN, math.Float64bits(actual[0].Samples[3].Value)) +} diff --git a/pp/go/storage/storagetest/fixtures.go b/pp/go/storage/storagetest/fixtures.go index 831d377096..a0733fa316 100644 --- a/pp/go/storage/storagetest/fixtures.go +++ b/pp/go/storage/storagetest/fixtures.go @@ -170,13 +170,13 @@ func GetSamplesFromSerializedData(serializedData *cppbridge.DataStorageSerialize break } - iterator := cppbridge.NewDataStorageSerializedDataIterator(serializedData, chunkRef) + iterator := cppbridge.NewDataStorageSerializedDataSamplesIterator(serializedData, chunkRef) for { if !iterator.HasData() { break } - result[seriesID] = append(result[seriesID], cppbridge.Sample{Timestamp: iterator.Timestamp, Value: iterator.Value}) + result[seriesID] = append(result[seriesID], cppbridge.Sample{Timestamp: iterator.Timestamp(), Value: iterator.Value()}) iterator.Next() } } diff --git a/pp/head/benchmarks/chunk_recoder_benchmark.cpp b/pp/head/benchmarks/chunk_recoder_benchmark.cpp index 48c46e4565..2efb3129e2 100644 --- a/pp/head/benchmarks/chunk_recoder_benchmark.cpp +++ b/pp/head/benchmarks/chunk_recoder_benchmark.cpp @@ -25,14 +25,14 @@ const DataStorage& get_data_storage_for_benchmark() { return storage; } -void ChunkRecoder(benchmark::State& state) { +void ChunkRecoder(benchmark::State& state, PromPP::Primitives::Timestamp downsampling) { ZoneScoped; const auto& storage = get_data_storage_for_benchmark(); const auto time_interval = series_data::Decoder::get_time_interval(storage); const std::ranges::iota_view ls_id_set(0, storage.open_chunks.size()); - head::ChunkRecoder recoder(head::ChunkRecoderIterator{ls_id_set.begin(), ls_id_set.end(), storage.open_chunks.size(), &storage, time_interval}, - time_interval); + head::ChunkRecoder recoder(head::ChunkRecoderIterator{ls_id_set.begin(), ls_id_set.end(), storage.open_chunks.size(), &storage, time_interval}, time_interval, + downsampling); struct { TimeInterval interval; @@ -47,6 +47,17 @@ void ChunkRecoder(benchmark::State& state) { } } +void ChunkRecoder(benchmark::State& state) { + ChunkRecoder(state, series_data::decoder::decorator::kNoDownsampling); +} + +void ChunkRecoderWithDownsampling(benchmark::State& state) { + constexpr PromPP::Primitives::Timestamp kDownsampling5Minutes = std::chrono::duration_cast(std::chrono::minutes(5)).count(); + + ChunkRecoder(state, kDownsampling5Minutes); +} + BENCHMARK(ChunkRecoder)->ComputeStatistics("min", benchmark::min_time); +BENCHMARK(ChunkRecoderWithDownsampling)->ComputeStatistics("min", benchmark::min_time); } // namespace diff --git a/pp/head/chunk_recoder.h b/pp/head/chunk_recoder.h index 130d83705f..ddca7ac71f 100644 --- a/pp/head/chunk_recoder.h +++ b/pp/head/chunk_recoder.h @@ -7,6 +7,7 @@ #include "prometheus/tsdb/chunkenc/xor.h" #include "series_data/concepts.h" #include "series_data/decoder.h" +#include "series_data/decoder/decorator/downsampling_decode_iterator.h" #include "series_data/encoder/bit_sequence.h" namespace head { @@ -113,8 +114,8 @@ class ChunkRecoderIterator { template class ChunkRecoder { public: - explicit ChunkRecoder(ChunkIterator&& iterator, const PromPP::Primitives::TimeInterval& time_interval) - : iterator_(std::move(iterator)), time_interval_{time_interval} {} + explicit ChunkRecoder(ChunkIterator&& iterator, const PromPP::Primitives::TimeInterval& time_interval, PromPP::Primitives::Timestamp downsampling_ms) + : iterator_(std::move(iterator)), time_interval_{time_interval}, downsampling_ms_{downsampling_ms} {} PROMPP_ALWAYS_INLINE ChunkIterator& chunk_iterator() noexcept { return iterator_; } @@ -153,6 +154,7 @@ class ChunkRecoder { ChunkIterator iterator_; PromPP::Prometheus::tsdb::chunkenc::FixedSizeBStream stream_{kMaxStreamSize}; const PromPP::Primitives::TimeInterval time_interval_; + const PromPP::Primitives::Timestamp downsampling_ms_; PROMPP_ALWAYS_INLINE static void reset_info(ChunkInfoInterface auto& info) noexcept { info.interval.reset(0, 0); @@ -167,37 +169,48 @@ class ChunkRecoder { void recode_chunk(ChunkInfoInterface auto& info) { Encoder encoder; - series_data::Decoder::create_decode_iterator(*iterator_, [&](Iterator&& begin, auto&&) { - begin.template seek([&](int64_t timestamp, double value) { - if (timestamp > time_interval_.max) [[unlikely]] { - return series_data::decoder::SeekResult::kStop; - } - - if (timestamp < time_interval_.min) [[unlikely]] { - return series_data::decoder::SeekResult::kNext; - } - - if (encoder.state().state == BareBones::Encoding::Gorilla::GorillaState::kFirstPoint) [[unlikely]] { - info.interval.min = timestamp; - } - - if constexpr (std::is_same_v || - std::is_same_v) { - encoder.encode_constant_value(timestamp, value, stream_, stream_); - } else { - encoder.encode(timestamp, value, stream_, stream_); - } - - ++info.samples_count; - return series_data::decoder::SeekResult::kNext; + + if (downsampling_ms_ == series_data::decoder::decorator::kNoDownsampling) [[likely]] { + series_data::Decoder::create_decode_iterator( + *iterator_, [&](Iterator&& begin, auto&&) { recode_chunk(std::forward(begin), encoder, info); }); + } else { + series_data::Decoder::create_decode_iterator(*iterator_, [&](Iterator&& begin, auto&&) { + recode_chunk(series_data::decoder::decorator::DownsamplingDecodeIterator(std::forward(begin), downsampling_ms_), encoder, info); }); - }); + } if (info.samples_count > 0) [[likely]] { info.interval.max = encoder.last_timestamp(); info.series_id = iterator_->series_id(); } } + + template + void recode_chunk(Iterator&& it, Encoder& encoder, ChunkInfoInterface auto& info) { + it.template seek([&](int64_t timestamp, double value) { + if (timestamp > time_interval_.max) [[unlikely]] { + return series_data::decoder::SeekResult::kStop; + } + + if (timestamp < time_interval_.min) [[unlikely]] { + return series_data::decoder::SeekResult::kNext; + } + + if (encoder.state().state == BareBones::Encoding::Gorilla::GorillaState::kFirstPoint) [[unlikely]] { + info.interval.min = timestamp; + } + + if constexpr (std::is_same_v || + std::is_same_v) { + encoder.encode_constant_value(timestamp, value, stream_, stream_); + } else { + encoder.encode(timestamp, value, stream_, stream_); + } + + ++info.samples_count; + return series_data::decoder::SeekResult::kNext; + }); + } }; } // namespace head \ No newline at end of file diff --git a/pp/head/chunk_recoder_tests.cpp b/pp/head/chunk_recoder_tests.cpp index 0979047da7..bd2630f145 100644 --- a/pp/head/chunk_recoder_tests.cpp +++ b/pp/head/chunk_recoder_tests.cpp @@ -12,6 +12,7 @@ using head::kUnlimitedLsIdBatchSize; using PromPP::Primitives::kInvalidLabelSetID; using PromPP::Primitives::LabelSetID; using PromPP::Primitives::TimeInterval; +using PromPP::Primitives::Timestamp; using series_data::DataStorage; using series_data::Encoder; using series_data::serialization::DataSerializer; @@ -36,9 +37,9 @@ class ChunkRecoderFixture : public ::testing::Test { DataStorage storage_; LsIdSet ls_id_set_; - ChunkRecoder create_recoder(const LsIdSet& ls_id_set, uint32_t ls_id_batch_size, const TimeInterval& time_interval) { + ChunkRecoder create_recoder(const LsIdSet& ls_id_set, uint32_t ls_id_batch_size, const TimeInterval& time_interval, const Timestamp downsampling_ms) { ls_id_set_ = ls_id_set; - return ChunkRecoder{ChunkIterator{ls_id_set_.begin(), ls_id_set_.end(), ls_id_batch_size, &storage_, time_interval}, time_interval}; + return ChunkRecoder{ChunkIterator{ls_id_set_.begin(), ls_id_set_.end(), ls_id_batch_size, &storage_, time_interval}, time_interval, downsampling_ms}; } template @@ -53,7 +54,7 @@ class ChunkRecoderFixture : public ::testing::Test { TEST_F(ChunkRecoderFixture, EmptyStorage) { // Arrange - ChunkRecoder recoder({{}, {}, kUnlimitedLsIdBatchSize, &storage_, {}}, {}); + ChunkRecoder recoder({{}, {}, kUnlimitedLsIdBatchSize, &storage_, {}}, {}, 0); // Act const auto info1 = recode(recoder); @@ -70,7 +71,7 @@ TEST_F(ChunkRecoderFixture, StorageWithOneChunk) { encoder.encode(0, 1, 1.0); encoder.encode(0, 2, 1.0); - auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 2}); + auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 2}, 0); // Act const auto info1 = recode(recoder); @@ -96,7 +97,7 @@ TEST_F(ChunkRecoderFixture, StorageWithEmptyChunks) { encoder.encode(4, 3, 2.0); encoder.encode(4, 4, 2.0); - auto recoder = create_recoder({0, 1, 2, 3, 4}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 4}); + auto recoder = create_recoder({0, 1, 2, 3, 4}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 4}, 0); // Act const auto info1 = recode(recoder); @@ -129,7 +130,7 @@ TEST_F(ChunkRecoderFixture, ReverseOrderOfChunks) { encoder.encode(4, 3, 2.0); encoder.encode(4, 4, 2.0); - auto recoder = create_recoder({4, 3, 2, 1, 0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 4}); + auto recoder = create_recoder({4, 3, 2, 1, 0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 4}, 0); // Act const auto info1 = recode(recoder); @@ -163,7 +164,7 @@ TEST_F(ChunkRecoderFixture, ChunkWithFinalizedTimestampStream) { encoder.encode(1, 2, 1.0); encoder.encode(1, 3, 1.0); - auto recoder = create_recoder({0, 1}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 3}); + auto recoder = create_recoder({0, 1}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 3}, 0); // Act const auto info1 = recode(recoder); @@ -205,7 +206,7 @@ TEST_F(ChunkRecoderFixture, GorillaChunk) { encoder.encode(0, 3, 1.3); encoder.encode(0, 4, 1.4); - auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 4}); + auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 4}, 0); // Act const auto info = recode(recoder); @@ -228,7 +229,7 @@ TEST_F(ChunkRecoderFixture, NoChunksByTimeInterval) { encoder.encode(0, 1, 1.1); encoder.encode(0, 2, 1.2); - auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 0}); + auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 0}, 0); // Act const auto info = recode(recoder); @@ -247,7 +248,7 @@ TEST_F(ChunkRecoderFixture, PartialReencodingByTimeInterval) { encoder.encode(0, 4, 1.0); encoder.encode(1, 0, 1.0); - auto recoder = create_recoder({0, 1}, kUnlimitedLsIdBatchSize, {.min = 1, .max = 2}); + auto recoder = create_recoder({0, 1}, kUnlimitedLsIdBatchSize, {.min = 1, .max = 2}, 0); // Act const auto info = recode(recoder); @@ -270,7 +271,7 @@ TEST_F(ChunkRecoderFixture, EmptyFinalizedChunk) { encoder.encode(0, 2, 1.0); encoder.encode(0, 5, 1.0); - auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 3, .max = 4}); + auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 3, .max = 4}, 0); // Act const auto info = recode(recoder); @@ -286,7 +287,7 @@ TEST_F(ChunkRecoderFixture, EmptyFinalizedChunkNonEmptyOpenedChunk) { encoder.encode(0, 2, 1.0); encoder.encode(0, 5, 1.0); - auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 3, .max = 5}); + auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 3, .max = 5}, 0); // Act const auto info = recode(recoder); @@ -307,7 +308,7 @@ TEST_F(ChunkRecoderFixture, EmptyLssWithNonEmptyDataStorage) { Encoder encoder{storage_}; encoder.encode(0, 1, 1.0); - auto recoder = create_recoder({}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 1}); + auto recoder = create_recoder({}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 1}, 0); // Act const auto info1 = recode(recoder); @@ -320,7 +321,7 @@ TEST_F(ChunkRecoderFixture, EmptyLssWithNonEmptyDataStorage) { TEST_F(ChunkRecoderFixture, EmptyStorageWithNonEmptyLss) { // Arrange - const auto recoder = create_recoder({0, 1}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 1}); + const auto recoder = create_recoder({0, 1}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 1}, 0); // Act const bool has_more_data = recoder.has_more_data(); @@ -335,7 +336,7 @@ TEST_F(ChunkRecoderFixture, ConstantEncoderChunkWithStaleNanOnSecondPoint) { encoder.encode(0, 1, 0.0); encoder.encode(0, 2, STALE_NAN); - auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 2}); + auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 2}, 0); // Act const auto info = recode(recoder); @@ -358,7 +359,7 @@ TEST_F(ChunkRecoderFixture, ConstantEncoderChunkWithStaleNanOnThirdPoint) { encoder.encode(0, 2, 0.0); encoder.encode(0, 3, STALE_NAN); - auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 3}); + auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 3}, 0); // Act const auto info = recode(recoder); @@ -381,7 +382,7 @@ TEST_F(ChunkRecoderFixture, TwoDoubleConstantEncoderChunkWithStaleNan) { encoder.encode(0, 2, 1.0); encoder.encode(0, 3, STALE_NAN); - auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 3}); + auto recoder = create_recoder({0}, kUnlimitedLsIdBatchSize, {.min = 0, .max = 3}, 0); // Act const auto info = recode(recoder); @@ -409,7 +410,7 @@ TEST_F(ChunkRecoderFixture, RecodeSerializedChunks) { DataSerializer serializer{storage_}; const auto serialized_data = serializer.serialize(); - ChunkRecoder recoder(series_data::chunk::SerializedChunkIterator{serialized_data.bytes_buffer, serialized_data.chunks}, {.min = 0, .max = 4}); + ChunkRecoder recoder(series_data::chunk::SerializedChunkIterator{serialized_data.bytes_buffer, serialized_data.chunks}, {.min = 0, .max = 4}, 0); // Act const auto info1 = recode(recoder); @@ -444,7 +445,7 @@ TEST_F(ChunkRecoderFixture, RecodeWithLsIdBatchSize) { encoder.encode(3, 5, 3.0); encoder.encode(3, 6, 3.0); - auto recoder = create_recoder({0, 1, 2, 3}, 1, {.min = 0, .max = 4}); + auto recoder = create_recoder({0, 1, 2, 3}, 1, {.min = 0, .max = 4}, 0); // Act // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) @@ -476,4 +477,45 @@ TEST_F(ChunkRecoderFixture, RecodeWithLsIdBatchSize) { EXPECT_FALSE(next_batch_result3); } +TEST_F(ChunkRecoderFixture, RecodeWithDownsampling) { + // Arrange + Encoder encoder{storage_}; + encoder.encode(0, 100, 1.0); + encoder.encode(0, 150, 1.0); + encoder.encode(0, 200, 1.0); + encoder.encode(0, 250, 1.0); + encoder.encode(0, 300, 1.0); + + encoder.encode(1, 50, 1.0); + encoder.encode(1, 100, 1.0); + encoder.encode(1, 150, 1.0); + encoder.encode(1, 200, 1.0); + encoder.encode(1, 250, 1.0); + + auto recoder = create_recoder({0, 1}, 2, {.min = 0, .max = 300}, 100); + + // Act + // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) + const auto info1 = recode(recoder); + const auto info2 = recode(recoder); + + // Assert + EXPECT_EQ((RecodeInfo{ + .interval = {.min = 100, .max = 300}, + .series_id = 0, + .samples_count = 3, + .buffer = "\x00\x03\xC8\x01\x3F\xF0\x00\x00\x00\x00\x00\x00\x64\x00"s, + .has_more_data = true, + }), + info1); + EXPECT_EQ((RecodeInfo{ + .interval = {.min = 100, .max = 250}, + .series_id = 1, + .samples_count = 3, + .buffer = "\x00\x03\xC8\x01\x3F\xF0\x00\x00\x00\x00\x00\x00\x64\x5F\xE7\x00"s, + .has_more_data = false, + }), + info2); +} + } // namespace \ No newline at end of file diff --git a/pp/performance_tests/chunk_recoder_test.cpp b/pp/performance_tests/chunk_recoder_test.cpp deleted file mode 100644 index b70c11544b..0000000000 --- a/pp/performance_tests/chunk_recoder_test.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "chunk_recoder_test.h" - -#include -#include - -#include "bare_bones/gorilla.h" -#include "head/chunk_recoder.h" -#include "performance_tests/dummy_wal.h" -#include "primitives/snug_composites.h" -#include "series_data/encoder.h" - -namespace performance_tests { - -using PromPP::Primitives::TimeInterval; - -void ChunkRecoder::execute(const Config& config, [[maybe_unused]] Metrics& metrics) const { - DummyWal::Timeseries tmsr; - DummyWal dummy_wal(input_file_full_name(config)); - - PromPP::Primitives::SnugComposites::LabelSet::EncodingBimap label_set_bitmap; - series_data::DataStorage storage; - series_data::Encoder encoder{storage}; - TimeInterval time_interval; - - while (dummy_wal.read_next_segment()) { - while (dummy_wal.read_next(tmsr)) { - const auto ls_id = label_set_bitmap.find_or_emplace(tmsr.label_set()); - auto& sample = tmsr.samples()[0]; - encoder.encode(ls_id, sample.timestamp(), sample.value()); - - time_interval.min = std::min(time_interval.min, sample.timestamp()); - time_interval.max = std::max(time_interval.max, sample.timestamp()); - } - } - - const std::ranges::iota_view ls_id_set(0, label_set_bitmap.size()); - head::ChunkRecoder recoder(head::ChunkRecoderIterator{ls_id_set.begin(), ls_id_set.end(), label_set_bitmap.size(), &storage, time_interval}, time_interval); - - struct { - TimeInterval interval; - uint32_t series_id{}; - uint8_t samples_count{}; - } chunk_data; - - const auto start_tm = std::chrono::steady_clock::now(); - while (recoder.has_more_data()) { - recoder.recode_next_chunk(chunk_data); - } - - std::cout << "recode time: " << (std::chrono::steady_clock::now() - start_tm).count() << std::endl; -} - -} // namespace performance_tests \ No newline at end of file diff --git a/pp/performance_tests/chunk_recoder_test.h b/pp/performance_tests/chunk_recoder_test.h deleted file mode 100644 index 486a9c50fe..0000000000 --- a/pp/performance_tests/chunk_recoder_test.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "performance_tests/test_with_input_only.h" - -namespace performance_tests { - -struct ChunkRecoder : TestWithInputOnly { - std::string input_file_base_name() const final { return "dummy_wal"; } - bool has_output() const final { return false; } - std::string output_file_base_name() const final { return ""; } - void execute(const Config& config, Metrics& metrics) const final; -}; - -} // namespace performance_tests diff --git a/pp/performance_tests/performance_tests.cpp b/pp/performance_tests/performance_tests.cpp index 269126674e..7d700e5e9e 100644 --- a/pp/performance_tests/performance_tests.cpp +++ b/pp/performance_tests/performance_tests.cpp @@ -1,4 +1,3 @@ -#include "chunk_recoder_test.h" #include "full_load_lss_test.h" #include "full_save_lss_test.h" #include "load_gorilla_from_wal_and_calculate_hash_over_label_set_names_test.h" @@ -70,7 +69,6 @@ int main([[maybe_unused]] int argc, char* argv[]) { test_db.add(std::make_unique()); test_db.add(std::make_unique()); test_db.add(std::make_unique()); - test_db.add(std::make_unique()); test_db.add(std::make_unique()); test_db.add(std::make_unique()); test_db.add(std::make_unique()); diff --git a/pp/primitives/primitives.h b/pp/primitives/primitives.h index a2d79226b8..54d7e51800 100644 --- a/pp/primitives/primitives.h +++ b/pp/primitives/primitives.h @@ -2,6 +2,7 @@ #include #include +#include #include #include "bare_bones/preprocess.h" @@ -35,6 +36,8 @@ struct TimeInterval { return std::max(min, interval.min) <= std::min(max, interval.max); } + [[nodiscard]] PROMPP_ALWAYS_INLINE Timestamp difference() const noexcept { return max - min; } + bool operator==(const TimeInterval&) const noexcept = default; }; diff --git a/pp/prometheus/promql/function_names.gperf b/pp/prometheus/promql/function_names.gperf new file mode 100644 index 0000000000..70200431be --- /dev/null +++ b/pp/prometheus/promql/function_names.gperf @@ -0,0 +1,82 @@ +abs +absent +absent_over_time +acos +acosh +asin +asinh +atan +atanh +avg_over_time +ceil +changes +clamp +clamp_max +clamp_min +cos +cosh +count_over_time +day_of_month +day_of_week +day_of_year +days_in_month +deg +delta +deriv +exp +floor +histogram_avg +histogram_count +histogram_fraction +histogram_quantile +histogram_stddev +histogram_stdvar +histogram_sum +holt_winters +hour +idelta +increase +info +irate +label_join +label_replace +last_over_step +last_over_time +ln +log10 +log2 +mad_over_time +max_over_time +min_over_time +minute +month +op_defined +op_replace_nan +op_smoothie +op_zero_if_none +pi +predict_linear +present_over_time +quantile_over_time +rad +rate +resets +round +scalar +sgn +sin +sinh +sort +sort_by_label +sort_by_label_desc +sort_desc +sqrt +stddev_over_time +stdvar_over_time +sum_over_time +tan +tanh +time +timestamp +vector +year \ No newline at end of file diff --git a/pp/prometheus/promql/function_names_hash.h b/pp/prometheus/promql/function_names_hash.h new file mode 100644 index 0000000000..ffdfc04d7b --- /dev/null +++ b/pp/prometheus/promql/function_names_hash.h @@ -0,0 +1,74 @@ +/* C++ code produced by gperf version 3.2.1 */ +/* Command-line: gperf -L C++ function_names.gperf */ +/* Computed positions: -k'2-4,$' */ +#pragma once + +#include + +#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) && (')' == 41) && ('*' == 42) && \ + ('+' == 43) && (',' == 44) && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) && \ + ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) && ('=' == 61) && ('>' == 62) && \ + ('?' == 63) && ('A' == 65) && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) && \ + ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) && ('R' == 82) && ('S' == 83) && \ + ('T' == 84) && ('U' == 85) && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) && \ + ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) && ('g' == 103) && \ + ('h' == 104) && ('i' == 105) && ('j' == 106) && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) && ('o' == 111) && ('p' == 112) && \ + ('q' == 113) && ('r' == 114) && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) && ('w' == 119) && ('x' == 120) && ('y' == 121) && \ + ('z' == 122) && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126)) +/* The character set is not based on ISO-646. */ +#error "gperf generated tables don't work with this execution character set. Please report a bug to ." +#endif + +#define TOTAL_KEYWORDS 82 +#define MIN_WORD_LENGTH 2 +#define MAX_WORD_LENGTH 18 +#define MIN_HASH_VALUE 2 +#define MAX_HASH_VALUE 198 +/* maximum key range = 197, duplicates = 0 */ + +namespace PromPP::Prometheus::promql { +class FunctionNamesHash { + public: + static constexpr unsigned int hash(const char* str, size_t len); +}; + +constexpr unsigned int FunctionNamesHash::hash(const char* str, size_t len) { + constexpr static unsigned char asso_values[] = { + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 5, 5, 0, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 0, 199, 25, 45, 60, 20, 15, 75, 90, 5, 5, 199, 5, 45, 45, 0, 0, 60, 20, 0, 0, + 0, 30, 55, 199, 45, 10, 10, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, 199}; + unsigned int hval = len; + + switch (hval) { + default: + hval += asso_values[static_cast(str[3])]; +#if (defined __cplusplus && (__cplusplus >= 201703L || (__cplusplus >= 201103L && defined __clang__ && __clang_major__ + (__clang_minor__ >= 9) > 3))) || \ + (defined __STDC_VERSION__ && __STDC_VERSION__ >= 202000L && ((defined __GNUC__ && __GNUC__ >= 10) || (defined __clang__ && __clang_major__ >= 9))) + [[fallthrough]]; +#elif (defined __GNUC__ && __GNUC__ >= 7) || (defined __clang__ && __clang_major__ >= 10) + __attribute__((__fallthrough__)); +#endif + /*FALLTHROUGH*/ + case 3: + hval += asso_values[static_cast(str[2])]; +#if (defined __cplusplus && (__cplusplus >= 201703L || (__cplusplus >= 201103L && defined __clang__ && __clang_major__ + (__clang_minor__ >= 9) > 3))) || \ + (defined __STDC_VERSION__ && __STDC_VERSION__ >= 202000L && ((defined __GNUC__ && __GNUC__ >= 10) || (defined __clang__ && __clang_major__ >= 9))) + [[fallthrough]]; +#elif (defined __GNUC__ && __GNUC__ >= 7) || (defined __clang__ && __clang_major__ >= 10) + __attribute__((__fallthrough__)); +#endif + /*FALLTHROUGH*/ + case 2: + hval += asso_values[static_cast(str[1])]; + break; + } + return hval + asso_values[static_cast(str[len - 1])]; +} + +} // namespace PromPP::Prometheus::promql diff --git a/pp/prometheus/promql/window_function.h b/pp/prometheus/promql/window_function.h new file mode 100644 index 0000000000..c4bafa3427 --- /dev/null +++ b/pp/prometheus/promql/window_function.h @@ -0,0 +1,104 @@ +#pragma once + +#include +#include +#include + +#include "prometheus/promql/function_names_hash.h" + +namespace PromPP::Prometheus::promql { + +enum class WindowFunction : uint8_t { + kNone = 0, + kRate, + kIncrease, + kIrate, + kIdelta, + kMinOverTime, + kMaxOverTime, + kLastOverTime, + kLastOverStep, + kSumOverTime, + kDelta, + kResets, + kChanges, + kSum, + kMin, + kMax, +}; + +enum class FunctionType : uint8_t { + kNone = 0, + kThinning, + kSynthesizing, + kCrossSeriesSynthesizing, + kExtrapolatedRate, +}; + +struct Function { + std::string_view name; + FunctionType type; +}; + +// The order of the functions must match the order in the WindowFunction enum +constexpr std::array kFunctions = { + Function{.name = "", .type = FunctionType::kNone}, + Function{.name = "rate", .type = FunctionType::kExtrapolatedRate}, + Function{.name = "increase", .type = FunctionType::kExtrapolatedRate}, + Function{.name = "irate", .type = FunctionType::kNone}, + Function{.name = "idelta", .type = FunctionType::kNone}, + Function{.name = "min_over_time", .type = FunctionType::kThinning}, + Function{.name = "max_over_time", .type = FunctionType::kThinning}, + Function{.name = "last_over_time", .type = FunctionType::kThinning}, + Function{.name = "last_over_step", .type = FunctionType::kNone}, + Function{.name = "sum_over_time", .type = FunctionType::kNone}, + Function{.name = "delta", .type = FunctionType::kExtrapolatedRate}, + Function{.name = "resets", .type = FunctionType::kNone}, + Function{.name = "changes", .type = FunctionType::kThinning}, + Function{.name = "sum", .type = FunctionType::kNone}, + Function{.name = "min", .type = FunctionType::kNone}, + Function{.name = "max", .type = FunctionType::kNone}, +}; + +constexpr uint32_t function_name_hash(std::string_view str) { + return FunctionNamesHash::hash(str.data(), str.length()); +} + +[[nodiscard]] constexpr WindowFunction window_function_from_string(std::string_view name) noexcept { + if (name.empty()) { + return WindowFunction::kNone; + } + + const auto hash = function_name_hash(name); + +#define PROMQL_WINDOW_FUNC_CASE(kind) \ + case function_name_hash(kFunctions[static_cast(WindowFunction::kind)].name): { \ + if (name != kFunctions[static_cast(WindowFunction::kind)].name) [[unlikely]] { \ + break; \ + } \ + return WindowFunction::kind; \ + } + + switch (hash) { + PROMQL_WINDOW_FUNC_CASE(kRate) + PROMQL_WINDOW_FUNC_CASE(kIncrease) + PROMQL_WINDOW_FUNC_CASE(kIrate) + PROMQL_WINDOW_FUNC_CASE(kIdelta) + PROMQL_WINDOW_FUNC_CASE(kMinOverTime) + PROMQL_WINDOW_FUNC_CASE(kMaxOverTime) + PROMQL_WINDOW_FUNC_CASE(kLastOverTime) + PROMQL_WINDOW_FUNC_CASE(kLastOverStep) + PROMQL_WINDOW_FUNC_CASE(kSumOverTime) + PROMQL_WINDOW_FUNC_CASE(kDelta) + PROMQL_WINDOW_FUNC_CASE(kResets) + PROMQL_WINDOW_FUNC_CASE(kChanges) + default: + return WindowFunction::kNone; + } + +#undef PROMQL_WINDOW_FUNC_CASE + + return WindowFunction::kNone; +} + +} // namespace PromPP::Prometheus::promql \ No newline at end of file diff --git a/pp/prometheus/query.h b/pp/prometheus/query.h index 146805d557..0f257ed4ed 100644 --- a/pp/prometheus/query.h +++ b/pp/prometheus/query.h @@ -1,23 +1,39 @@ #pragma once +#include + #include "label_matcher.h" +#include "primitives/primitives.h" namespace PromPP::Prometheus { -struct ReadHints { - std::string func{}; - std::vector grouping; +template class Slice> +struct GenericSelectHints { + Primitives::TimeInterval interval{.min = 0, .max = 0}; + int64_t limit{}; + int64_t step_ms{}; - int64_t start_ms{}; - int64_t end_ms{}; + String func{}; + + Slice grouping; int64_t range_ms{}; + + uint64_t shard_count{}; + uint64_t shard_index{}; + + int64_t lookback_delta{}; + + bool disable_trimming{}; bool by{}; + bool is_subquery{}; - bool operator==(const ReadHints&) const noexcept = default; + bool operator==(const GenericSelectHints&) const noexcept = default; }; +using SelectHints = GenericSelectHints; + struct Query { - ReadHints hints{}; + SelectHints hints{}; LabelMatchers label_matchers; int64_t start_timestamp_ms{}; int64_t end_timestamp_ms{}; @@ -32,4 +48,4 @@ struct LabelValuesQuery { bool operator==(const LabelValuesQuery&) const noexcept = default; }; -} // namespace PromPP::Prometheus \ No newline at end of file +} // namespace PromPP::Prometheus diff --git a/pp/series_data/decoder/decorator/changes_iterator.h b/pp/series_data/decoder/decorator/changes_iterator.h new file mode 100644 index 0000000000..b54b3e5d5f --- /dev/null +++ b/pp/series_data/decoder/decorator/changes_iterator.h @@ -0,0 +1,90 @@ +#pragma once + +#include "primitives/primitives.h" +#include "series_data/decoder/universal_decode_iterator.h" + +namespace series_data::decoder::decorator { + +template +class ChangesIterator { + public: + DECODE_ITERATOR_TYPE_TRAITS(); + + explicit ChangesIterator(PromPP::Primitives::TimeInterval interval) : interval_(interval) {} + explicit ChangesIterator(Iterator&& iterator, PromPP::Primitives::TimeInterval interval) : iterator_(std::move(iterator)), interval_(interval) { + seek_to_first_sample(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE const PromPP::Primitives::TimeInterval& interval() const noexcept { return interval_; } + PROMPP_ALWAYS_INLINE void set_interval(const PromPP::Primitives::TimeInterval& interval) { + interval_ = interval; + seek_to_first_sample(); + } + + PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const { return iterator_.operator*(); } + PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const { return iterator_.operator->(); } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel&) const { return iterator_->timestamp == kInvalidTimestamp; } + + PROMPP_ALWAYS_INLINE ChangesIterator& operator++() { + seek_to_next_sample(); + return *this; + } + + PROMPP_ALWAYS_INLINE ChangesIterator operator++(int) { + const auto result = *this; + ++*this; + return result; + } + + protected: + Iterator iterator_; + PromPP::Primitives::TimeInterval interval_; + + void seek_to_first_sample() { + bool has_value{}; + + iterator_.template seek([&has_value, this](PromPP::Primitives::Timestamp timestamp) { + if (timestamp < interval_.min) { + return SeekResult::kNext; + } + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + has_value = true; + return SeekResult::kUpdateSampleNextAndStop; + }); + + if (!has_value) [[unlikely]] { + iterator_.invalidate_sample(); + } + } + + PROMPP_ALWAYS_INLINE void seek_to_next_sample() { + bool has_changes{}; + double previous_value = iterator_->value; + + iterator_.template seek([&has_changes, &previous_value, this](PromPP::Primitives::Timestamp timestamp, double value) { + using BareBones::Encoding::Gorilla::isstalenan; + + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + if (value != previous_value) { + has_changes = true; + previous_value = value; + return SeekResult::kUpdateSampleNextAndStop; + } + + return SeekResult::kNext; + }); + + if (!has_changes) [[unlikely]] { + iterator_.invalidate_sample(); + } + } +}; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/delta_iterator.h b/pp/series_data/decoder/decorator/delta_iterator.h new file mode 100644 index 0000000000..ce73f82b43 --- /dev/null +++ b/pp/series_data/decoder/decorator/delta_iterator.h @@ -0,0 +1,90 @@ +#pragma once + +#include "primitives/primitives.h" +#include "series_data/decoder/universal_decode_iterator.h" + +namespace series_data::decoder::decorator { + +template +class DeltaIterator { + public: + DECODE_ITERATOR_TYPE_TRAITS(); + + explicit DeltaIterator(PromPP::Primitives::TimeInterval interval) : interval_(interval) {} + explicit DeltaIterator(Iterator&& iterator, PromPP::Primitives::TimeInterval interval) : iterator_(std::move(iterator)), interval_(interval) { + seek_to_first_sample(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE const PromPP::Primitives::TimeInterval& interval() const noexcept { return interval_; } + PROMPP_ALWAYS_INLINE void set_interval(const PromPP::Primitives::TimeInterval& interval) { + interval_ = interval; + seek_to_first_sample(); + } + + PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const { return iterator_.operator*(); } + PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const { return iterator_.operator->(); } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel&) const { return iterator_->timestamp == kInvalidTimestamp; } + + PROMPP_ALWAYS_INLINE DeltaIterator& operator++() { + seek_to_next_sample(); + return *this; + } + + PROMPP_ALWAYS_INLINE DeltaIterator operator++(int) { + const auto result = *this; + ++*this; + return result; + } + + protected: + Iterator iterator_; + PromPP::Primitives::TimeInterval interval_; + + void seek_to_first_sample() { + bool has_value{}; + + iterator_.template seek([&has_value, this](PromPP::Primitives::Timestamp timestamp, double value) { + if (timestamp < interval_.min) { + return SeekResult::kNext; + } + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + if (BareBones::Encoding::Gorilla::isstalenan(value)) [[unlikely]] { + return SeekResult::kNext; + } + + has_value = true; + return SeekResult::kUpdateSampleNextAndStop; + }); + + if (!has_value) [[unlikely]] { + iterator_.invalidate_sample(); + } + } + + PROMPP_ALWAYS_INLINE void seek_to_next_sample() { + bool has_value{}; + + iterator_.template seek([&has_value, this](PromPP::Primitives::Timestamp timestamp, double value) { + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + if (BareBones::Encoding::Gorilla::isstalenan(value)) [[unlikely]] { + return SeekResult::kNext; + } + + has_value = true; + return SeekResult::kUpdateSample; + }); + + if (!has_value) [[unlikely]] { + iterator_.invalidate_sample(); + } + } +}; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/downsampling_decode_iterator.h b/pp/series_data/decoder/decorator/downsampling_decode_iterator.h new file mode 100644 index 0000000000..a7a074181b --- /dev/null +++ b/pp/series_data/decoder/decorator/downsampling_decode_iterator.h @@ -0,0 +1,83 @@ +#pragma once + +#include "series_data/decoder/traits.h" +#include "series_data/encoder/sample.h" + +namespace series_data::decoder::decorator { + +static constexpr PromPP::Primitives::Timestamp kNoDownsampling = 0; + +template +class DownsamplingDecodeIterator { + public: + using Timestamp = PromPP::Primitives::Timestamp; + + DECODE_ITERATOR_TYPE_TRAITS(); + + explicit DownsamplingDecodeIterator(Timestamp interval) : DownsamplingDecodeIterator(DecodeIterator{}, interval) {} + DownsamplingDecodeIterator(DecodeIterator&& iterator, Timestamp interval) : iterator_(std::move(iterator)), interval_(interval) { + advance_to_last_sample_in_interval(); + } + + PROMPP_ALWAYS_INLINE DownsamplingDecodeIterator& operator=(DecodeIterator&& iterator) noexcept { + iterator_ = std::move(iterator); + advance_to_last_sample_in_interval(); + return *this; + } + + PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const noexcept { return iterator_.operator*(); } + PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const noexcept { return iterator_.operator->(); } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel&) const noexcept { return (*this)->timestamp == kInvalidTimestamp; } + + PROMPP_ALWAYS_INLINE DownsamplingDecodeIterator& operator++() noexcept { + advance_to_last_sample_in_interval(); + return *this; + } + + PROMPP_ALWAYS_INLINE DownsamplingDecodeIterator operator++(int) noexcept { + const auto result = *this; + ++*this; + return result; + } + + template + PROMPP_ALWAYS_INLINE void seek(SeekHandler&& handler) { + for (; *this != DecodeIteratorSentinel{}; ++*this) { + if (handler(iterator_->timestamp, iterator_->value) == SeekResult::kStop) [[unlikely]] { + return; + } + } + } + + private: + DecodeIterator iterator_; + Timestamp interval_; + + PROMPP_ALWAYS_INLINE static Timestamp round_up_to_step(Timestamp timestamp, Timestamp step) noexcept { + const auto result = timestamp + step - 1; + return result - result % step; + } + + PROMPP_ALWAYS_INLINE void advance_to_last_sample_in_interval() noexcept { + Timestamp sample_timestamp = kInvalidTimestamp; + + iterator_.template seek([this, &sample_timestamp](Timestamp timestamp) noexcept { + if (timestamp > sample_timestamp) { + if (sample_timestamp != kInvalidTimestamp) [[likely]] { + return SeekResult::kStop; + } + + sample_timestamp = round_up_to_step(timestamp, interval_); + } + + return SeekResult::kUpdateSample; + }); + + if (sample_timestamp == kInvalidTimestamp) [[unlikely]] { + iterator_.invalidate_sample(); + } + } +}; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/interval_decode_iterator.h b/pp/series_data/decoder/decorator/interval_decode_iterator.h deleted file mode 100644 index cd52308b5d..0000000000 --- a/pp/series_data/decoder/decorator/interval_decode_iterator.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once - -#include "series_data/decoder/traits.h" -#include "series_data/encoder/sample.h" - -namespace series_data::decoder::decorator { - -template -class IntervalDecodeIterator { - public: - DECODE_ITERATOR_TYPE_TRAITS(); - - using Timestamp = PromPP::Primitives::Timestamp; - - IntervalDecodeIterator(DecodeIterator&& iterator, DecodeIteratorSentinel&& end, Timestamp interval, Timestamp lookback) - : iterator_(std::move(iterator)), iterator_end_(std::move(end)), interval_(std::max(interval, kMinInterval)), lookback_(lookback) { - if (iterator_ != iterator_end_) { - timestamp_ = round_up_to_step(iterator_->timestamp, interval_); - advance_to_next_sample(); - } - } - - PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const noexcept { return sample_; } - PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const noexcept { return &sample_; } - - PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel&) const noexcept { return iterator_ == iterator_end_ && sample_.timestamp == kNoSample; } - - PROMPP_ALWAYS_INLINE IntervalDecodeIterator& operator++() noexcept { - advance_to_next_sample(); - return *this; - } - - PROMPP_ALWAYS_INLINE IntervalDecodeIterator operator++(int) noexcept { - const auto result = *this; - ++*this; - return result; - } - - private: - static constexpr auto kNoSample = std::numeric_limits::min(); - static constexpr Timestamp kMinInterval = 1; - - DecodeIterator iterator_; - [[no_unique_address]] DecodeIteratorSentinel iterator_end_; - Timestamp interval_; - Timestamp lookback_; - Timestamp timestamp_{}; - encoder::Sample sample_{.timestamp = kNoSample}; - - PROMPP_ALWAYS_INLINE static Timestamp round_up_to_step(Timestamp timestamp, Timestamp step) noexcept { - const auto result = timestamp + step - 1; - return result - result % step; - } - - PROMPP_ALWAYS_INLINE void advance_to_next_sample() noexcept { - if (iterator_ == iterator_end_) [[unlikely]] { - sample_.timestamp = kNoSample; - return; - } - - Timestamp previous_timestamp; - do { - advance_to_last_sample_in_interval(); - previous_timestamp = std::exchange(timestamp_, timestamp_ + interval_); - } while (!in_lookback_interval(sample_.timestamp, previous_timestamp) && iterator_ != iterator_end_); - } - - PROMPP_ALWAYS_INLINE void advance_to_last_sample_in_interval() noexcept { - for (; iterator_ != iterator_end_ && timestamp_ >= iterator_->timestamp; ++iterator_) { - if (in_lookback_interval(iterator_->timestamp, timestamp_)) [[likely]] { - decode_sample(); - } - } - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool in_lookback_interval(Timestamp timestamp, Timestamp deadline) const noexcept { - return deadline <= lookback_ + timestamp; - } - - PROMPP_ALWAYS_INLINE void decode_sample() noexcept { - if (!BareBones::Encoding::Gorilla::isstalenan(iterator_->value)) [[likely]] { - sample_ = *iterator_; - } else { - sample_.timestamp = kNoSample; - } - } -}; - -} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/irate_iterator.h b/pp/series_data/decoder/decorator/irate_iterator.h new file mode 100644 index 0000000000..7c63d71a0c --- /dev/null +++ b/pp/series_data/decoder/decorator/irate_iterator.h @@ -0,0 +1,73 @@ +#pragma once + +#include "primitives/primitives.h" +#include "series_data/decoder/universal_decode_iterator.h" + +namespace series_data::decoder::decorator { + +template +class IRateIterator { + public: + DECODE_ITERATOR_TYPE_TRAITS(); + + explicit IRateIterator(PromPP::Primitives::TimeInterval interval) : interval_(interval) {} + explicit IRateIterator(Iterator&& iterator, PromPP::Primitives::TimeInterval interval) : iterator_(std::move(iterator)), interval_(interval) { + find_last_2samples(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE const PromPP::Primitives::TimeInterval& interval() const noexcept { return interval_; } + PROMPP_ALWAYS_INLINE void set_interval(const PromPP::Primitives::TimeInterval& interval) { + interval_ = interval; + find_last_2samples(); + } + + PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const { return sample_; } + PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const { return &sample_; } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel&) const { return sample_.timestamp == kInvalidTimestamp; } + + PROMPP_ALWAYS_INLINE IRateIterator& operator++() { + sample_ = *iterator_; + iterator_.invalidate_sample(); + return *this; + } + + PROMPP_ALWAYS_INLINE IRateIterator operator++(int) { + const auto result = *this; + ++*this; + return result; + } + + protected: + static constexpr encoder::Sample kInvalidSample = {.timestamp = kInvalidTimestamp, .value = BareBones::Encoding::Gorilla::STALE_NAN}; + + encoder::Sample sample_{kInvalidSample}; + Iterator iterator_; + PromPP::Primitives::TimeInterval interval_; + + void find_last_2samples() { + sample_ = kInvalidSample; + + iterator_.template seek([this](PromPP::Primitives::Timestamp timestamp, double value) { + if (timestamp < interval_.min) { + return SeekResult::kNext; + } + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + if (BareBones::Encoding::Gorilla::isstalenan(value)) [[unlikely]] { + return SeekResult::kNext; + } + + sample_ = *iterator_; + return SeekResult::kUpdateSample; + }); + + if (sample_.timestamp == iterator_->timestamp) { + iterator_.invalidate_sample(); + } + } +}; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/last_over_step.h b/pp/series_data/decoder/decorator/last_over_step.h new file mode 100644 index 0000000000..32bbe27e55 --- /dev/null +++ b/pp/series_data/decoder/decorator/last_over_step.h @@ -0,0 +1,27 @@ +#pragma once + +#include "over_time_func_iterator.h" + +namespace series_data::decoder::decorator { + +class LastOverStep { + public: + explicit LastOverStep(encoder::Sample& sample, const PromPP::Primitives::TimeInterval& interval) : sample_(sample), interval_(interval) {} + + PROMPP_ALWAYS_INLINE void operator()(PromPP::Primitives::Timestamp, double value) const noexcept { sample_.value = value; } + + ~LastOverStep() { + if (!BareBones::Encoding::Gorilla::isstalenan(sample_.value)) [[likely]] { + sample_.timestamp = interval_.max; + } + } + + private: + encoder::Sample& sample_; + const PromPP::Primitives::TimeInterval& interval_; +}; + +template +using LastOverStepIterator = OverTimeFuncIterator; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/last_over_time.h b/pp/series_data/decoder/decorator/last_over_time.h new file mode 100644 index 0000000000..b1d591ecca --- /dev/null +++ b/pp/series_data/decoder/decorator/last_over_time.h @@ -0,0 +1,23 @@ +#pragma once + +#include "over_time_func_iterator.h" + +namespace series_data::decoder::decorator { + +class LastOverTime { + public: + explicit LastOverTime(encoder::Sample& sample, const PromPP::Primitives::TimeInterval&) : sample_(sample) {} + + PROMPP_ALWAYS_INLINE void operator()(PromPP::Primitives::Timestamp timestamp, double value) const noexcept { + sample_.value = value; + sample_.timestamp = timestamp; + } + + private: + encoder::Sample& sample_; +}; + +template +using LastOverTimeIterator = OverTimeFuncIterator; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/max_over_time.h b/pp/series_data/decoder/decorator/max_over_time.h new file mode 100644 index 0000000000..9d2b29643b --- /dev/null +++ b/pp/series_data/decoder/decorator/max_over_time.h @@ -0,0 +1,25 @@ +#pragma once + +#include "over_time_func_iterator.h" + +namespace series_data::decoder::decorator { + +class FindMaxElement { + public: + explicit FindMaxElement(encoder::Sample& sample, const PromPP::Primitives::TimeInterval&) : sample_{sample} {} + + PROMPP_ALWAYS_INLINE void operator()(PromPP::Primitives::Timestamp timestamp, double value) const noexcept { + if (BareBones::Encoding::Gorilla::isstalenan(sample_.value) || value > sample_.value) { + sample_.value = value; + sample_.timestamp = timestamp; + } + } + + private: + encoder::Sample& sample_; +}; + +template +using MaxOverTimeIterator = OverTimeFuncIterator; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/min_over_time.h b/pp/series_data/decoder/decorator/min_over_time.h new file mode 100644 index 0000000000..dae567f73c --- /dev/null +++ b/pp/series_data/decoder/decorator/min_over_time.h @@ -0,0 +1,25 @@ +#pragma once + +#include "over_time_func_iterator.h" + +namespace series_data::decoder::decorator { + +class FindMinElement { + public: + explicit FindMinElement(encoder::Sample& sample, const PromPP::Primitives::TimeInterval&) : sample_(sample) {} + + PROMPP_ALWAYS_INLINE void operator()(PromPP::Primitives::Timestamp timestamp, double value) const noexcept { + if (BareBones::Encoding::Gorilla::isstalenan(sample_.value) || value < sample_.value) { + sample_.value = value; + sample_.timestamp = timestamp; + } + } + + private: + encoder::Sample& sample_; +}; + +template +using MinOverTimeIterator = OverTimeFuncIterator; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/over_time_func_iterator.h b/pp/series_data/decoder/decorator/over_time_func_iterator.h new file mode 100644 index 0000000000..cfff00044f --- /dev/null +++ b/pp/series_data/decoder/decorator/over_time_func_iterator.h @@ -0,0 +1,67 @@ +#pragma once + +#include "primitives/primitives.h" +#include "series_data/decoder/universal_decode_iterator.h" + +namespace series_data::decoder::decorator { + +template +class OverTimeFuncIterator { + public: + DECODE_ITERATOR_TYPE_TRAITS(); + + explicit OverTimeFuncIterator(const PromPP::Primitives::TimeInterval& interval) : interval_(interval) {} + explicit OverTimeFuncIterator(Iterator&& iterator, PromPP::Primitives::TimeInterval interval) : iterator_(std::move(iterator)), interval_(interval) { + find_element(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE const PromPP::Primitives::TimeInterval& interval() const noexcept { return interval_; } + PROMPP_ALWAYS_INLINE void set_interval(const PromPP::Primitives::TimeInterval& interval) { + interval_ = interval; + find_element(); + } + + PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const { return sample_; } + PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const { return &sample_; } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel&) const { return sample_.timestamp == kInvalidTimestamp; } + + PROMPP_ALWAYS_INLINE OverTimeFuncIterator& operator++() { + sample_.timestamp = kInvalidTimestamp; + return *this; + } + + PROMPP_ALWAYS_INLINE OverTimeFuncIterator operator++(int) { + const auto result = *this; + ++*this; + return result; + } + + protected: + encoder::Sample sample_; + Iterator iterator_; + PromPP::Primitives::TimeInterval interval_; + + void find_element() { + sample_ = encoder::Sample{.timestamp = kInvalidTimestamp, .value = BareBones::Encoding::Gorilla::STALE_NAN}; + SampleHandler handler{sample_, interval_}; + + iterator_.template seek([&handler, this](PromPP::Primitives::Timestamp timestamp, double value) { + if (timestamp < interval_.min) [[unlikely]] { + return SeekResult::kNext; + } + + if (timestamp > interval_.max) [[unlikely]] { + return SeekResult::kStop; + } + + if (!BareBones::Encoding::Gorilla::isstalenan(value)) [[likely]] { + handler(timestamp, value); + } + + return SeekResult::kNext; + }); + } +}; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/rate_iterator.h b/pp/series_data/decoder/decorator/rate_iterator.h new file mode 100644 index 0000000000..38530fedc0 --- /dev/null +++ b/pp/series_data/decoder/decorator/rate_iterator.h @@ -0,0 +1,102 @@ +#pragma once + +#include "primitives/primitives.h" +#include "series_data/decoder/universal_decode_iterator.h" + +namespace series_data::decoder::decorator { + +template +class RateIterator { + public: + DECODE_ITERATOR_TYPE_TRAITS(); + + explicit RateIterator(PromPP::Primitives::TimeInterval interval) : interval_(interval) {} + explicit RateIterator(Iterator&& iterator, PromPP::Primitives::TimeInterval interval) : iterator_(std::move(iterator)), interval_(interval) { + seek_to_first_sample(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE const PromPP::Primitives::TimeInterval& interval() const noexcept { return interval_; } + PROMPP_ALWAYS_INLINE void set_interval(const PromPP::Primitives::TimeInterval& interval) { + interval_ = interval; + counter_reset_ = false; + sample_ = kInvalidSample; + seek_to_first_sample(); + } + + PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const { return sample_; } + PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const { return &sample_; } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel&) const { return sample_.timestamp == kInvalidTimestamp; } + + PROMPP_ALWAYS_INLINE RateIterator& operator++() { + seek_to_next_sample(); + return *this; + } + + PROMPP_ALWAYS_INLINE RateIterator operator++(int) { + const auto result = *this; + ++*this; + return result; + } + + protected: + static constexpr encoder::Sample kInvalidSample = {.timestamp = kInvalidTimestamp, .value = BareBones::Encoding::Gorilla::STALE_NAN}; + + encoder::Sample sample_{kInvalidSample}; + Iterator iterator_; + PromPP::Primitives::TimeInterval interval_; + bool counter_reset_{}; + + void seek_to_first_sample() { + iterator_.template seek([this](PromPP::Primitives::Timestamp timestamp, double value) { + if (timestamp < interval_.min) { + return SeekResult::kNext; + } + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + if (BareBones::Encoding::Gorilla::isstalenan(value)) [[unlikely]] { + return SeekResult::kNext; + } + + sample_ = encoder::Sample{.timestamp = timestamp, .value = value}; + return SeekResult::kNextAndStop; + }); + } + + PROMPP_ALWAYS_INLINE void seek_to_next_sample() { + sample_.timestamp = kInvalidTimestamp; + + iterator_.template seek([this](PromPP::Primitives::Timestamp timestamp, double value) { + if (counter_reset_) [[unlikely]] { + counter_reset_ = false; + sample_ = encoder::Sample{.timestamp = timestamp, .value = value}; + return SeekResult::kNextAndStop; + } + + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + if (BareBones::Encoding::Gorilla::isstalenan(value)) [[unlikely]] { + return SeekResult::kNext; + } + + if (value < sample_.value) [[unlikely]] { + counter_reset_ = true; + if (sample_.timestamp != kInvalidTimestamp) { + return SeekResult::kStop; + } + + sample_ = encoder::Sample{.timestamp = timestamp, .value = value}; + return SeekResult::kNextAndStop; + } + + sample_ = encoder::Sample{.timestamp = timestamp, .value = value}; + return SeekResult::kNext; + }); + } +}; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/resets_iterator.h b/pp/series_data/decoder/decorator/resets_iterator.h new file mode 100644 index 0000000000..7488205c40 --- /dev/null +++ b/pp/series_data/decoder/decorator/resets_iterator.h @@ -0,0 +1,97 @@ +#pragma once + +#include "primitives/primitives.h" +#include "series_data/decoder/universal_decode_iterator.h" + +namespace series_data::decoder::decorator { + +template +class ResetsIterator { + public: + DECODE_ITERATOR_TYPE_TRAITS(); + + explicit ResetsIterator(PromPP::Primitives::TimeInterval interval) : interval_(interval) {} + explicit ResetsIterator(Iterator&& iterator, PromPP::Primitives::TimeInterval interval) : iterator_(std::move(iterator)), interval_(interval) { + seek_to_first_sample(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE const PromPP::Primitives::TimeInterval& interval() const noexcept { return interval_; } + PROMPP_ALWAYS_INLINE void set_interval(const PromPP::Primitives::TimeInterval& interval) { + interval_ = interval; + seek_to_first_sample(); + } + + PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const { return iterator_.operator*(); } + PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const { return iterator_.operator->(); } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel&) const { return iterator_->timestamp == kInvalidTimestamp; } + + PROMPP_ALWAYS_INLINE ResetsIterator& operator++() { + seek_to_next_sample(); + return *this; + } + + PROMPP_ALWAYS_INLINE ResetsIterator operator++(int) { + const auto result = *this; + ++*this; + return result; + } + + protected: + Iterator iterator_; + PromPP::Primitives::TimeInterval interval_; + + void seek_to_first_sample() { + bool has_value{}; + + iterator_.template seek([&has_value, this](PromPP::Primitives::Timestamp timestamp, double value) { + if (timestamp < interval_.min) { + return SeekResult::kNext; + } + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + if (BareBones::Encoding::Gorilla::isstalenan(value)) [[unlikely]] { + return SeekResult::kNext; + } + + has_value = true; + return SeekResult::kUpdateSampleNextAndStop; + }); + + if (!has_value) [[unlikely]] { + iterator_.invalidate_sample(); + } + } + + PROMPP_ALWAYS_INLINE void seek_to_next_sample() { + bool has_resets{}; + double previous_value = iterator_->value; + + iterator_.template seek([&has_resets, &previous_value, this](PromPP::Primitives::Timestamp timestamp, double value) { + if (timestamp > interval_.max) { + return SeekResult::kStop; + } + + if (BareBones::Encoding::Gorilla::isstalenan(value)) [[unlikely]] { + return SeekResult::kNext; + } + + if (value < previous_value) [[unlikely]] { + has_resets = true; + previous_value = value; + return SeekResult::kUpdateSampleNextAndStop; + } + + previous_value = value; + return SeekResult::kNext; + }); + + if (!has_resets) [[unlikely]] { + iterator_.invalidate_sample(); + } + } +}; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/sum_over_time.h b/pp/series_data/decoder/decorator/sum_over_time.h new file mode 100644 index 0000000000..f3866ccf34 --- /dev/null +++ b/pp/series_data/decoder/decorator/sum_over_time.h @@ -0,0 +1,52 @@ +#pragma once + +#include "over_time_func_iterator.h" + +namespace series_data::decoder::decorator { + +PROMPP_ALWAYS_INLINE void kahan_sum_inc(double inc, double& sum, double& c) noexcept { + const auto t = sum + inc; + if (std::isinf(t)) { + c = 0; + } else if (std::abs(sum) >= std::abs(inc)) { + c += sum - t + inc; + } else { + c += inc - t + sum; + } + + sum = t; +} + +class SumOfElements { + public: + explicit SumOfElements(encoder::Sample& sum, const PromPP::Primitives::TimeInterval& interval) : sum_(sum), interval_(interval) {} + + PROMPP_ALWAYS_INLINE void operator()(PromPP::Primitives::Timestamp timestamp, double value) noexcept { + if (BareBones::Encoding::Gorilla::isstalenan(sum_.value)) [[unlikely]] { + sum_.value = 0.0; + } + + kahan_sum_inc(value, sum_.value, c_); + sum_.timestamp = timestamp; + } + + ~SumOfElements() { + if (sum_.timestamp != kInvalidTimestamp) [[likely]] { + if (!std::isinf(sum_.value)) [[likely]] { + sum_.value += c_; + } + + sum_.timestamp = interval_.max; + } + } + + private: + encoder::Sample& sum_; + const PromPP::Primitives::TimeInterval& interval_; + double c_{}; +}; + +template +using SumOverTimeIterator = OverTimeFuncIterator; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/window_boundary_calculator.h b/pp/series_data/decoder/decorator/window_boundary_calculator.h new file mode 100644 index 0000000000..2fd06027d1 --- /dev/null +++ b/pp/series_data/decoder/decorator/window_boundary_calculator.h @@ -0,0 +1,104 @@ +#pragma once + +#include "primitives/primitives.h" + +namespace series_data::decoder::decorator { + +struct WindowFunctionParameters { + PromPP::Primitives::TimeInterval interval; + PromPP::Primitives::Timestamp step; + PromPP::Primitives::Timestamp range{}; + PromPP::Primitives::Timestamp lookback_delta{}; +}; + +template +concept WindowBoundaryCalculatorInterface = + requires(WindowCalculator calculator, const PromPP::Primitives::TimeInterval& current_window, const WindowFunctionParameters& parameters) { + { WindowCalculator::initial_window(parameters) } -> std::same_as; + { WindowCalculator::next_window(current_window, parameters) } -> std::same_as; + }; + +struct StepRangeWindowCalculator { + using Timestamp = PromPP::Primitives::Timestamp; + using TimeInterval = PromPP::Primitives::TimeInterval; + + [[nodiscard]] PROMPP_ALWAYS_INLINE static TimeInterval initial_window(const WindowFunctionParameters& parameters) noexcept { + TimeInterval result{.min = parameters.interval.min + 1}; + + if (parameters.step == 0) { + result.max = parameters.interval.max; + } else if (has_gap_between_windows(parameters)) { + result.max = std::min(parameters.interval.min + parameters.range, parameters.interval.max); + } else { + result.max = std::min(next_interval_boundary(parameters.interval.min, parameters), parameters.interval.max); + } + + return result; + } + + [[nodiscard]] static PROMPP_ALWAYS_INLINE TimeInterval next_window(const TimeInterval& current_window, const WindowFunctionParameters& parameters) noexcept { + TimeInterval interval{}; + if (has_gap_between_windows(parameters)) { + interval.min = current_window.min + parameters.step; + interval.max = interval.min + parameters.range - 1; + } else { + interval.min = current_window.max + 1; + interval.max = next_interval_boundary(current_window.max, parameters); + } + + interval.max = std::min(interval.max, parameters.interval.max); + return interval; + } + + private: + [[nodiscard]] PROMPP_ALWAYS_INLINE static Timestamp next_interval_boundary(Timestamp start, const WindowFunctionParameters& parameters) noexcept { + if (parameters.step != 0 && (parameters.range % parameters.step) == 0) { + return start + parameters.step; + } + + if (parameters.step == 0 || parameters.range < parameters.step) { + return start + parameters.range; + } + + const auto eval_start = parameters.interval.min + parameters.range; + const auto remainder = parameters.range % parameters.step; + return std::min(next_aligned_boundary(start, eval_start, parameters.step), next_aligned_boundary(start, eval_start - remainder, parameters.step)); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE static bool has_gap_between_windows(const WindowFunctionParameters& parameters) noexcept { + return parameters.range > 0 && parameters.range < parameters.step; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE static Timestamp next_aligned_boundary(Timestamp start, Timestamp anchor, Timestamp step) noexcept { + const auto quotient = floor_div(start - anchor, step) + 1; + return anchor + quotient * step; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE static Timestamp floor_div(Timestamp numerator, Timestamp denominator) noexcept { + auto quotient = numerator / denominator; + if (numerator % denominator < 0) { + --quotient; + } + + return quotient; + } +}; + +static_assert(WindowBoundaryCalculatorInterface); + +struct StepLookbackDeltaWindowCalculator { + using Timestamp = PromPP::Primitives::Timestamp; + using TimeInterval = PromPP::Primitives::TimeInterval; + + [[nodiscard]] PROMPP_ALWAYS_INLINE static TimeInterval initial_window(const WindowFunctionParameters& parameters) noexcept { + return {.min = parameters.interval.min + 1, .max = std::min(parameters.interval.min + parameters.lookback_delta, parameters.interval.max)}; + } + + [[nodiscard]] static PROMPP_ALWAYS_INLINE TimeInterval next_window(const TimeInterval& current_window, const WindowFunctionParameters& parameters) noexcept { + return {.min = current_window.max + 1, .max = std::min(current_window.max + parameters.step, parameters.interval.max)}; + } +}; + +static_assert(WindowBoundaryCalculatorInterface); + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/decorator/window_function_iterator.h b/pp/series_data/decoder/decorator/window_function_iterator.h new file mode 100644 index 0000000000..12ff090b13 --- /dev/null +++ b/pp/series_data/decoder/decorator/window_function_iterator.h @@ -0,0 +1,74 @@ +#pragma once + +#include "primitives/primitives.h" +#include "series_data/decoder/universal_decode_iterator.h" +#include "window_boundary_calculator.h" + +namespace series_data::decoder::decorator { + +template +concept WindowFunctionIteratorInterface = requires(Iterator iterator, const Iterator const_iterator) { + { Iterator(PromPP::Primitives::TimeInterval()) }; + + { const_iterator == DecodeIteratorSentinel() }; + { ++iterator } -> std::same_as; + { *iterator } -> std::same_as; + + { const_iterator.interval() } -> std::same_as; + + { iterator.set_interval(PromPP::Primitives::TimeInterval()) }; +}; + +template +class WindowFunctionIterator { + public: + using Timestamp = PromPP::Primitives::Timestamp; + using TimeInterval = PromPP::Primitives::TimeInterval; + + DECODE_ITERATOR_TYPE_TRAITS(); + + explicit WindowFunctionIterator(const WindowFunctionParameters& parameters) noexcept + : iterator_{WindowBoundaryCalculator::initial_window(parameters)}, parameters_(¶meters) { + advance(); + } + + template + explicit WindowFunctionIterator(Iterator&& iterator, const WindowFunctionParameters& parameters) + : iterator_(std::forward(iterator), WindowBoundaryCalculator::initial_window(parameters)), parameters_(¶meters) { + advance(); + } + + PROMPP_ALWAYS_INLINE const encoder::Sample& operator*() const { return iterator_.operator*(); } + PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const { return iterator_.operator->(); } + + PROMPP_ALWAYS_INLINE bool operator==(const DecodeIteratorSentinel& sentinel) const noexcept { return iterator_ == sentinel; } + + PROMPP_ALWAYS_INLINE WindowFunctionIterator& operator++() { + ++iterator_; + advance(); + return *this; + } + + PROMPP_ALWAYS_INLINE WindowFunctionIterator operator++(int) { + const auto result = *this; + ++*this; + return result; + } + + protected: + FunctionIterator iterator_; + const WindowFunctionParameters* parameters_; + + void advance() { + while (iterator_ == DecodeIteratorSentinel{}) { + if (const auto interval = WindowBoundaryCalculator::next_window(iterator_.interval(), *parameters_); interval.min < parameters_->interval.max) + [[likely]] { + iterator_.set_interval(interval); + } else { + break; + } + } + } +}; + +} // namespace series_data::decoder::decorator \ No newline at end of file diff --git a/pp/series_data/decoder/traits.h b/pp/series_data/decoder/traits.h index 7171401f6e..6e2b78a5f6 100644 --- a/pp/series_data/decoder/traits.h +++ b/pp/series_data/decoder/traits.h @@ -1,7 +1,7 @@ #pragma once +#include "bare_bones/algorithm.h" #include "bare_bones/preprocess.h" -#include "series_data/encoder/bit_sequence.h" #include "series_data/encoder/sample.h" #include "series_data/encoder/timestamp/encoder.h" @@ -20,15 +20,19 @@ class DecodeIteratorSentinel {}; enum class SeekResult : uint8_t { kUpdateSample = 1, - kNext, - kStop, - kUpdateSampleNextAndStop, + kNext = 2, + kStop = 4, + kNextAndStop = 8, + kUpdateSampleNextAndStop = 16, }; enum class SeekKind : uint8_t { - kNextStop = static_cast(SeekResult::kNext) | static_cast(SeekResult::kStop), - kAll = static_cast(SeekResult::kUpdateSample) | static_cast(SeekResult::kNext) | static_cast(SeekResult::kStop) | - static_cast(SeekResult::kUpdateSampleNextAndStop), + kUpdateSample_Stop = BareBones::build_bitmask(), + kNext_Stop = BareBones::build_bitmask(), + kNext_Stop_NextAndStop = BareBones::build_bitmask(), + kUpdateSample_Next_Stop = BareBones::build_bitmask(), + kAll = BareBones:: + build_bitmask(), }; template @@ -94,6 +98,9 @@ class DecodeIteratorTrait { derived()->update_sample(); } else if (has_kind(SeekResult::kStop) && result == SeekResult::kStop) { break; + } else if (has_kind(SeekResult::kNextAndStop) && result == SeekResult::kNextAndStop) { + derived()->decode(); + break; } else if (has_kind(SeekResult::kUpdateSampleNextAndStop) && result == SeekResult::kUpdateSampleNextAndStop) { derived()->update_sample(); derived()->decode(); diff --git a/pp/series_data/serialization/serialized_data.h b/pp/series_data/serialization/serialized_data.h index 8459fadffb..5b73e315ba 100644 --- a/pp/series_data/serialization/serialized_data.h +++ b/pp/series_data/serialization/serialized_data.h @@ -1,4 +1,5 @@ #pragma once + #include "bare_bones/memory.h" #include "series_data/chunk/serialized_chunk.h" #include "series_data/data_storage.h" @@ -258,11 +259,6 @@ class DataSerializer { const DataStorage& storage_; }; -template -concept AssignableFromUniversalDecodeIterator = requires(DecodeIterator iterator, decoder::UniversalDecodeIterator universal_decode_iterator) { - { iterator = universal_decode_iterator }; -}; - class SerializedDataView { public: using series_id_inner_chunk_id_t = std::pair; @@ -272,8 +268,8 @@ class SerializedDataView { public: DECODE_ITERATOR_TYPE_TRAITS(); - SeriesIterator(std::span buffer, chunk::SerializedChunkSpan chunks, uint32_t chunk_id) - : decode_iter_(create_decode_iterator(buffer, *(chunks.begin() + chunk_id))), + SeriesIterator(std::span buffer, chunk::SerializedChunkSpan chunks, uint32_t chunk_id) + : decode_iter_(create_decode_iterator(buffer, chunks.begin() + chunk_id)), chunk_iter_(chunks.begin() + chunk_id), chunk_iter_end_(chunks.end()), buffer_(buffer.data()), @@ -284,7 +280,7 @@ class SerializedDataView { [[nodiscard]] PROMPP_ALWAYS_INLINE const encoder::Sample* operator->() const noexcept { return decode_iter_.operator->(); } PROMPP_ALWAYS_INLINE SeriesIterator& operator++() noexcept { - const auto iterator_is_end = decode_iter_.visit([](auto& iterator) { return ++iterator == decoder::DecodeIteratorSentinel{}; }); + const auto iterator_is_end = decode_iter_.visit([](auto& iterator) PROMPP_LAMBDA_INLINE { return ++iterator == decoder::DecodeIteratorSentinel{}; }); if (iterator_is_end) [[unlikely]] { advance_to_next_series_chunk(); } @@ -310,6 +306,36 @@ class SerializedDataView { reset_decode_iterator(); } + template + PROMPP_ALWAYS_INLINE void seek(SeekHandler&& handler) { + using enum decoder::SeekResult; + + if (*this == decoder::DecodeIteratorSentinel{}) [[unlikely]] { + return; + } + + do { + if (decode_iter_ == decoder::DecodeIteratorSentinel{}) [[unlikely]] { + if (!advance_to_next_series_chunk()) { + return; + } + } + + auto handler_result = kUpdateSample; + if constexpr (decoder::SampleSeekHandler) { + decode_iter_.seek([&](PromPP::Primitives::Timestamp timestamp, double value) { return handler_result = handler(timestamp, value); }); + } else { + decode_iter_.seek([&](PromPP::Primitives::Timestamp timestamp) { return handler_result = handler(timestamp); }); + } + + if (!BareBones::is_in(handler_result, kUpdateSample, kNext)) { + break; + } + } while (true); + } + + PROMPP_ALWAYS_INLINE void invalidate_sample() noexcept { decode_iter_.invalidate_sample(); } + void seek_to(PromPP::Primitives::Timestamp timestamp) noexcept { if (*this == decoder::DecodeIteratorSentinel{}) [[unlikely]] { return; @@ -346,11 +372,11 @@ class SerializedDataView { return false; } - PROMPP_ALWAYS_INLINE void reset_decode_iterator() { decode_iter_ = create_decode_iterator({buffer_, buffer_size_}, *chunk_iter_); } + PROMPP_ALWAYS_INLINE void reset_decode_iterator() noexcept { decode_iter_ = create_decode_iterator({buffer_, buffer_size_}, chunk_iter_); } - PROMPP_ALWAYS_INLINE static decoder::UniversalDecodeIterator create_decode_iterator(std::span buffer, - const chunk::SerializedChunk& chunk) noexcept { - return Decoder::create_decode_iterator(buffer, chunk, [&](Iterator&& begin, auto&&) { + PROMPP_ALWAYS_INLINE decoder::UniversalDecodeIterator create_decode_iterator(std::span buffer, + chunk::SerializedChunkSpan::const_iterator chunk_iter) noexcept { + return Decoder::create_decode_iterator(buffer, *chunk_iter, [&](Iterator&& begin, auto&&) { return decoder::UniversalDecodeIterator{std::in_place_type, std::forward(begin)}; }); } diff --git a/pp/series_data/tests/decoder/decorator/changes_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/changes_iterator_tests.cpp new file mode 100644 index 0000000000..5018d4ee71 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/changes_iterator_tests.cpp @@ -0,0 +1,97 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/changes_iterator.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::ChangesIterator; +using series_data::encoder::Sample; + +struct ChangesIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class ChangesIteratorFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(ChangesIteratorFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(ChangesIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P( + OneSample, + ChangesIteratorFixture, + testing::Values(ChangesIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + ChangesIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + ChangesIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(StaleNan, + ChangesIteratorFixture, + testing::Values(ChangesIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.0}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 5, .value = STALE_NAN}, Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, Sample{.timestamp = 30, .value = 1.0}}}, + ChangesIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = STALE_NAN}}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + ChangesIteratorFixture, + testing::Values(ChangesIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.0}, + Sample{.timestamp = 100, .value = 1.1}, + Sample{.timestamp = 200, .value = 1.1}, + Sample{.timestamp = 201, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.1}}}, + ChangesIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.1}, + Sample{.timestamp = 120, .value = 1.1}, + Sample{.timestamp = 150, .value = 1.2}, + Sample{.timestamp = 180, .value = 1.2}, + Sample{.timestamp = 200, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.1}, Sample{.timestamp = 150, .value = 1.2}, + Sample{.timestamp = 200, .value = 1.0}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/delta_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/delta_iterator_tests.cpp new file mode 100644 index 0000000000..3dc50f5c67 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/delta_iterator_tests.cpp @@ -0,0 +1,91 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/delta_iterator.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::DeltaIterator; +using series_data::encoder::Sample; + +struct DeltaIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class DeltaIteratorFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(DeltaIteratorFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(DeltaIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P(OneSample, + DeltaIteratorFixture, + testing::Values(DeltaIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + DeltaIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + DeltaIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + DeltaIteratorFixture, + testing::Values(DeltaIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.1}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 10, .value = 1.0}, Sample{.timestamp = 30, .value = 1.1}}}, + DeltaIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + DeltaIteratorFixture, + testing::Values(DeltaIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.0}, + Sample{.timestamp = 100, .value = 1.1}, + Sample{.timestamp = 200, .value = 1.1}, + Sample{.timestamp = 201, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.1}, Sample{.timestamp = 200, .value = 1.1}}}, + DeltaIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.1}, + Sample{.timestamp = 150, .value = 1.2}, + Sample{.timestamp = 200, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.1}, Sample{.timestamp = 200, .value = 1.0}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/downsampling_decode_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/downsampling_decode_iterator_tests.cpp new file mode 100644 index 0000000000..d7dceb7dfd --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/downsampling_decode_iterator_tests.cpp @@ -0,0 +1,168 @@ +#include + +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/downsampling_decode_iterator.h" +#include "series_data/decoder/universal_decode_iterator.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::DownsamplingDecodeIterator; +using series_data::encoder::Sample; + +struct DownsamplingDecodeIteratorCase { + std::vector samples; + PromPP::Primitives::Timestamp interval; + std::vector expected{}; +}; + +class DownsamplingDecodeIteratorFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(DownsamplingDecodeIteratorFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(DownsamplingDecodeIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +TEST_P(DownsamplingDecodeIteratorFixture, TestReset) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + Iterator begin_at_start = begin; + DownsamplingDecodeIterator iterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval); + std::advance(iterator, GetParam().samples.size()); + + iterator = UniversalDecodeIterator{std::in_place_type, std::forward(begin_at_start)}; + std::ranges::copy(iterator, DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P( + OneSample, + DownsamplingDecodeIteratorFixture, + testing::Values( + DownsamplingDecodeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval = 100, .expected{Sample{.timestamp = 100, .value = 1.0}}}, + DownsamplingDecodeIteratorCase{.samples{Sample{.timestamp = 300, .value = 1.0}}, .interval = 400, .expected{Sample{.timestamp = 300, .value = 1.0}}})); +INSTANTIATE_TEST_SUITE_P(ManySamples, + DownsamplingDecodeIteratorFixture, + testing::Values(DownsamplingDecodeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + Sample{.timestamp = 300, .value = 1.0}, + }, + .interval = 100, + .expected{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + Sample{.timestamp = 300, .value = 1.0}, + }}, + DownsamplingDecodeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 150, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + }, + .interval = 100, + .expected{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + }}, + DownsamplingDecodeIteratorCase{.samples{ + Sample{.timestamp = 123, .value = 1.0}, + Sample{.timestamp = 152, .value = 1.0}, + Sample{.timestamp = 180, .value = 1.0}, + Sample{.timestamp = 215, .value = 1.0}, + Sample{.timestamp = 242, .value = 1.0}, + Sample{.timestamp = 275, .value = 1.0}, + Sample{.timestamp = 303, .value = 1.0}, + }, + .interval = 100, + .expected{ + Sample{.timestamp = 180, .value = 1.0}, + Sample{.timestamp = 275, .value = 1.0}, + Sample{.timestamp = 303, .value = 1.0}, + }}, + DownsamplingDecodeIteratorCase{.samples{ + Sample{.timestamp = 180, .value = 1.0}, + Sample{.timestamp = 275, .value = 1.0}, + Sample{.timestamp = 503, .value = 1.0}, + Sample{.timestamp = 603, .value = 1.0}, + Sample{.timestamp = 604, .value = 1.0}, + }, + .interval = 100, + .expected{ + Sample{.timestamp = 180, .value = 1.0}, + Sample{.timestamp = 275, .value = 1.0}, + Sample{.timestamp = 503, .value = 1.0}, + Sample{.timestamp = 604, .value = 1.0}, + }})); +INSTANTIATE_TEST_SUITE_P(StaleNan, + DownsamplingDecodeIteratorFixture, + testing::Values(DownsamplingDecodeIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, + .interval = 100, + .expected{ + Sample{.timestamp = 100, .value = STALE_NAN}, + }}, + DownsamplingDecodeIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.0}, + Sample{.timestamp = 100, .value = STALE_NAN}, + }, + .interval = 100, + .expected{ + Sample{.timestamp = 100, .value = STALE_NAN}, + }}, + DownsamplingDecodeIteratorCase{.samples{ + Sample{.timestamp = 98, .value = 1.0}, + Sample{.timestamp = 99, .value = STALE_NAN}, + Sample{.timestamp = 100, .value = 1.0}, + }, + .interval = 100, + .expected{ + Sample{.timestamp = 100, .value = 1.0}, + }}, + DownsamplingDecodeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = STALE_NAN}, + Sample{.timestamp = 101, .value = 1.0}, + Sample{.timestamp = 200, .value = STALE_NAN}, + Sample{.timestamp = 201, .value = 1.0}, + Sample{.timestamp = 300, .value = STALE_NAN}, + Sample{.timestamp = 400, .value = 1.0}, + }, + .interval = 100, + .expected{ + Sample{.timestamp = 100, .value = STALE_NAN}, + Sample{.timestamp = 200, .value = STALE_NAN}, + Sample{.timestamp = 300, .value = STALE_NAN}, + Sample{.timestamp = 400, .value = 1.0}, + }})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/interval_decode_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/interval_decode_iterator_tests.cpp deleted file mode 100644 index a2c94c3c53..0000000000 --- a/pp/series_data/tests/decoder/decorator/interval_decode_iterator_tests.cpp +++ /dev/null @@ -1,183 +0,0 @@ -#include - -#include "series_data/decoder/decorator/interval_decode_iterator.h" - -namespace { - -using BareBones::Encoding::Gorilla::STALE_NAN; -using series_data::decoder::decorator::IntervalDecodeIterator; -using series_data::encoder::Sample; - -struct IntervalDecodeIteratorCase { - std::vector samples; - PromPP::Primitives::Timestamp interval; - PromPP::Primitives::Timestamp lookback{1000}; - std::vector expected{}; -}; - -class IntervalDecodeIteratorFixture : public ::testing::TestWithParam {}; - -TEST_P(IntervalDecodeIteratorFixture, Test) { - // Arrange - std::vector actual_samples; - - // Act - std::ranges::copy(IntervalDecodeIterator(GetParam().samples.begin(), GetParam().samples.end(), GetParam().interval, GetParam().lookback), - GetParam().samples.end(), std::back_inserter(actual_samples)); - - // Assert - EXPECT_EQ(GetParam().expected, actual_samples); -} - -INSTANTIATE_TEST_SUITE_P(Empty, IntervalDecodeIteratorFixture, testing::Values(IntervalDecodeIteratorCase{})); -INSTANTIATE_TEST_SUITE_P( - OneSample, - IntervalDecodeIteratorFixture, - testing::Values( - IntervalDecodeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval = 100, .expected{Sample{.timestamp = 100, .value = 1.0}}}, - IntervalDecodeIteratorCase{.samples{Sample{.timestamp = 300, .value = 1.0}}, .interval = 400, .expected{Sample{.timestamp = 300, .value = 1.0}}})); -INSTANTIATE_TEST_SUITE_P(ManySamples, - IntervalDecodeIteratorFixture, - testing::Values(IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 100, .value = 1.0}, - Sample{.timestamp = 200, .value = 1.0}, - Sample{.timestamp = 300, .value = 1.0}, - }, - .interval = 100, - .expected{ - Sample{.timestamp = 100, .value = 1.0}, - Sample{.timestamp = 200, .value = 1.0}, - Sample{.timestamp = 300, .value = 1.0}, - }}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 100, .value = 1.0}, - Sample{.timestamp = 150, .value = 1.0}, - Sample{.timestamp = 200, .value = 1.0}, - }, - .interval = 100, - .expected{ - Sample{.timestamp = 100, .value = 1.0}, - Sample{.timestamp = 200, .value = 1.0}, - }}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 123, .value = 1.0}, - Sample{.timestamp = 152, .value = 1.0}, - Sample{.timestamp = 180, .value = 1.0}, - Sample{.timestamp = 215, .value = 1.0}, - Sample{.timestamp = 242, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 303, .value = 1.0}, - }, - .interval = 100, - .expected{ - Sample{.timestamp = 180, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 303, .value = 1.0}, - }}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 180, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 503, .value = 1.0}, - Sample{.timestamp = 603, .value = 1.0}, - Sample{.timestamp = 604, .value = 1.0}, - }, - .interval = 100, - .expected{ - Sample{.timestamp = 180, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 503, .value = 1.0}, - Sample{.timestamp = 604, .value = 1.0}, - }})); -INSTANTIATE_TEST_SUITE_P(UseMinInterval, - IntervalDecodeIteratorFixture, - testing::Values(IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 0, .value = 1.0}, - Sample{.timestamp = 1, .value = 1.0}, - Sample{.timestamp = 2, .value = 1.0}, - }, - .interval = 0, - .expected{ - Sample{.timestamp = 0, .value = 1.0}, - Sample{.timestamp = 1, .value = 1.0}, - Sample{.timestamp = 2, .value = 1.0}, - }})); -INSTANTIATE_TEST_SUITE_P(LookbackDelta, - IntervalDecodeIteratorFixture, - testing::Values(IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 180, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 503, .value = 1.0}, - Sample{.timestamp = 603, .value = 1.0}, - }, - .interval = 100, - .lookback = 125, - .expected{ - Sample{.timestamp = 180, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 503, .value = 1.0}, - Sample{.timestamp = 603, .value = 1.0}, - }}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 180, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 503, .value = 1.0}, - Sample{.timestamp = 603, .value = 1.0}, - }, - .interval = 100, - .lookback = 124, - .expected{ - Sample{.timestamp = 180, .value = 1.0}, - Sample{.timestamp = 275, .value = 1.0}, - Sample{.timestamp = 503, .value = 1.0}, - Sample{.timestamp = 603, .value = 1.0}, - }}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 1, .value = 1.0}, - }, - .interval = 101, - .lookback = 100, - .expected{ - Sample{.timestamp = 1, .value = 1.0}, - }})); -INSTANTIATE_TEST_SUITE_P(NoSamples, - IntervalDecodeIteratorFixture, - testing::Values(IntervalDecodeIteratorCase{.samples{Sample{.timestamp = 1, .value = 1.0}}, .interval = 100, .lookback = 98}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 1, .value = 1.0}, - Sample{.timestamp = 2, .value = 1.0}, - Sample{.timestamp = 3, .value = 1.0}, - }, - .interval = 100, - .lookback = 96})); -INSTANTIATE_TEST_SUITE_P(StaleNan, - IntervalDecodeIteratorFixture, - testing::Values(IntervalDecodeIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval = 100}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 99, .value = 1.0}, - Sample{.timestamp = 100, .value = STALE_NAN}, - }, - .interval = 100}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 98, .value = 1.0}, - Sample{.timestamp = 99, .value = STALE_NAN}, - Sample{.timestamp = 100, .value = 1.0}, - }, - .interval = 100, - .expected{ - Sample{.timestamp = 100, .value = 1.0}, - }}, - IntervalDecodeIteratorCase{.samples{ - Sample{.timestamp = 100, .value = STALE_NAN}, - Sample{.timestamp = 200, .value = STALE_NAN}, - Sample{.timestamp = 300, .value = STALE_NAN}, - Sample{.timestamp = 400, .value = 1.0}, - }, - .interval = 100, - .expected{ - Sample{.timestamp = 400, .value = 1.0}, - }})); - -} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/irate_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/irate_iterator_tests.cpp new file mode 100644 index 0000000000..9729eebfad --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/irate_iterator_tests.cpp @@ -0,0 +1,93 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/irate_iterator.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::IRateIterator; +using series_data::encoder::Sample; + +struct IRateIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class IRateIteratorFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(IRateIteratorFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(IRateIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P(OneSample, + IRateIteratorFixture, + testing::Values(IRateIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + IRateIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + IRateIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + IRateIteratorFixture, + testing::Values(IRateIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 15, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.1}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 15, .value = 1.0}, Sample{.timestamp = 30, .value = 1.1}}}, + IRateIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + IRateIteratorFixture, + testing::Values(IRateIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.0}, + Sample{.timestamp = 100, .value = 1.1}, + Sample{.timestamp = 200, .value = 1.1}, + Sample{.timestamp = 201, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.1}, Sample{.timestamp = 200, .value = 1.1}}}, + IRateIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.1}, + Sample{.timestamp = 150, .value = 1.2}, + Sample{.timestamp = 180, .value = 1.2}, + Sample{.timestamp = 200, .value = 1.3}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 180, .value = 1.2}, Sample{.timestamp = 200, .value = 1.3}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/iterator_over_series_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/iterator_over_series_iterator_tests.cpp new file mode 100644 index 0000000000..0b0c27b6df --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/iterator_over_series_iterator_tests.cpp @@ -0,0 +1,104 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/changes_iterator.h" +#include "series_data/decoder/decorator/rate_iterator.h" +#include "series_data/decoder/decorator/resets_iterator.h" +#include "series_data/encoder.h" +#include "series_data/serialization/deserializer.h" +#include "series_data/serialization/serialized_data.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using PromPP::Primitives::TimeInterval; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::decorator::ChangesIterator; +using series_data::decoder::decorator::RateIterator; +using series_data::decoder::decorator::ResetsIterator; +using series_data::encoder::Sample; +using series_data::serialization::DataSerializer; +using series_data::serialization::SerializedDataView; + +class IteratorOverSeriesIteratorFixture : public ::testing::Test { + protected: + DataStorage storage_; + Encoder<> encoder_{storage_}; +}; + +TEST_F(IteratorOverSeriesIteratorFixture, RateIteratorWith2Chunks) { + // Arrange + encoder_.encode(0, 100, 1.0); + encoder_.encode(0, 101, 2.0); + encoder_.encode(0, 102, 3.0); + series_data::ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 103, 0.0); + + const auto serialized_data = DataSerializer{storage_}.serialize(); + SerializedDataView serialized_view(serialized_data); + + auto [series_id, chunk_id] = serialized_view.next_series(); + + std::vector actual_samples; + + // Act + std::ranges::copy(RateIterator(serialized_view.create_series_iterator(chunk_id), TimeInterval{.min = 100, .max = 103}), DecodeIteratorSentinel{}, + std::back_inserter(actual_samples)); + + // Assert + EXPECT_EQ((std::vector{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 102, .value = 3.0}, Sample{.timestamp = 103, .value = 0.0}}), + actual_samples); +} + +TEST_F(IteratorOverSeriesIteratorFixture, ChangesIteratorWith2Chunks) { + // Arrange + encoder_.encode(0, 100, 1.0); + encoder_.encode(0, 101, 1.0); + encoder_.encode(0, 102, 1.0); + series_data::ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 103, 0.0); + + const auto serialized_data = DataSerializer{storage_}.serialize(); + SerializedDataView serialized_view(serialized_data); + + auto [series_id, chunk_id] = serialized_view.next_series(); + + std::vector actual_samples; + + // Act + std::ranges::copy(ChangesIterator(serialized_view.create_series_iterator(chunk_id), TimeInterval{.min = 100, .max = 103}), DecodeIteratorSentinel{}, + std::back_inserter(actual_samples)); + + // Assert + EXPECT_EQ((std::vector{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 103, .value = 0.0}}), actual_samples); +} + +TEST_F(IteratorOverSeriesIteratorFixture, ResetsIteratorWith2Chunks) { + // Arrange + encoder_.encode(0, 100, 1.0); + encoder_.encode(0, 101, 1.0); + encoder_.encode(0, 102, 1.0); + series_data::ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 103, 0.0); + + const auto serialized_data = DataSerializer{storage_}.serialize(); + SerializedDataView serialized_view(serialized_data); + + auto [series_id, chunk_id] = serialized_view.next_series(); + + std::vector actual_samples; + + // Act + std::ranges::copy(ResetsIterator(serialized_view.create_series_iterator(chunk_id), TimeInterval{.min = 100, .max = 103}), DecodeIteratorSentinel{}, + std::back_inserter(actual_samples)); + + // Assert + EXPECT_EQ((std::vector{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 103, .value = 0.0}}), actual_samples); +} + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/last_over_step_tests.cpp b/pp/series_data/tests/decoder/decorator/last_over_step_tests.cpp new file mode 100644 index 0000000000..75cb6976d8 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/last_over_step_tests.cpp @@ -0,0 +1,100 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/last_over_step.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::LastOverStepIterator; +using series_data::encoder::Sample; + +struct LastOverStepIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class LastOverStepFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(LastOverStepFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(LastOverStepIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P( + OneSample, + LastOverStepFixture, + testing::Values(LastOverStepIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + LastOverStepIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + LastOverStepIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + LastOverStepFixture, + testing::Values(LastOverStepIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.1}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.1}}}, + LastOverStepIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + LastOverStepFixture, + testing::Values(LastOverStepIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.1}, + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + Sample{.timestamp = 201, .value = 1.1}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 1.0}}}, + LastOverStepIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 150, .value = 1.1}, + Sample{.timestamp = 200, .value = 1.2}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 1.2}}}, + LastOverStepIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 150, .value = 1.1}, + Sample{.timestamp = 180, .value = 1.2}, + Sample{.timestamp = 200, .value = STALE_NAN}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 1.2}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/last_over_time_tests.cpp b/pp/series_data/tests/decoder/decorator/last_over_time_tests.cpp new file mode 100644 index 0000000000..88eaa87a83 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/last_over_time_tests.cpp @@ -0,0 +1,100 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/last_over_time.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::LastOverTimeIterator; +using series_data::encoder::Sample; + +struct LastOverTimeIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class LastOverTimeFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(LastOverTimeFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(LastOverTimeIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P( + OneSample, + LastOverTimeFixture, + testing::Values(LastOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + LastOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + LastOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + LastOverTimeFixture, + testing::Values(LastOverTimeIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.1}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 30, .value = 1.1}}}, + LastOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + LastOverTimeFixture, + testing::Values(LastOverTimeIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.1}, + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + Sample{.timestamp = 201, .value = 1.1}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 1.0}}}, + LastOverTimeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 150, .value = 1.1}, + Sample{.timestamp = 200, .value = 1.2}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 1.2}}}, + LastOverTimeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 150, .value = 1.1}, + Sample{.timestamp = 180, .value = 1.2}, + Sample{.timestamp = 200, .value = STALE_NAN}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 180, .value = 1.2}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/max_over_time_tests.cpp b/pp/series_data/tests/decoder/decorator/max_over_time_tests.cpp new file mode 100644 index 0000000000..8883a11999 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/max_over_time_tests.cpp @@ -0,0 +1,92 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/max_over_time.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::MaxOverTimeIterator; +using series_data::encoder::Sample; + +struct MaxOverTimeIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class MaxOverTimeFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(MaxOverTimeFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(MaxOverTimeIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P( + OneSample, + MaxOverTimeFixture, + testing::Values(MaxOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + MaxOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + MaxOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + MaxOverTimeFixture, + testing::Values(MaxOverTimeIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.1}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 30, .value = 1.1}}}, + MaxOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + MaxOverTimeFixture, + testing::Values(MaxOverTimeIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.1}, + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + Sample{.timestamp = 201, .value = 1.1}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + MaxOverTimeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 150, .value = 1.1}, + Sample{.timestamp = 200, .value = 1.2}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 1.2}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/min_over_time_tests.cpp b/pp/series_data/tests/decoder/decorator/min_over_time_tests.cpp new file mode 100644 index 0000000000..1507f4fd50 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/min_over_time_tests.cpp @@ -0,0 +1,92 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/min_over_time.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::MinOverTimeIterator; +using series_data::encoder::Sample; + +struct MinOverTimeIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class MinOverTimeFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(MinOverTimeFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(MinOverTimeIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P( + OneSample, + MinOverTimeFixture, + testing::Values(MinOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + MinOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + MinOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + MinOverTimeFixture, + testing::Values(MinOverTimeIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.1}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 10, .value = 1.0}}}, + MinOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + MinOverTimeFixture, + testing::Values(MinOverTimeIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.0}, + Sample{.timestamp = 100, .value = 1.1}, + Sample{.timestamp = 200, .value = 1.1}, + Sample{.timestamp = 201, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.1}}}, + MinOverTimeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.1}, + Sample{.timestamp = 150, .value = 1.2}, + Sample{.timestamp = 200, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 1.0}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/rate_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/rate_iterator_tests.cpp new file mode 100644 index 0000000000..1a526063b4 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/rate_iterator_tests.cpp @@ -0,0 +1,154 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/rate_iterator.h" +#include "series_data/encoder.h" +#include "series_data/serialization/deserializer.h" +#include "series_data/serialization/serialized_data.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using PromPP::Primitives::TimeInterval; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::RateIterator; +using series_data::encoder::Sample; +using series_data::serialization::DataSerializer; +using series_data::serialization::SerializedDataView; + +struct RateIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class RateIteratorFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + Encoder<> encoder_{storage_}; +}; + +TEST_P(RateIteratorFixture, Test) { + // Arrange + std::ranges::for_each(GetParam().samples, [this](const auto& sample) { encoder_.encode(0, sample.timestamp, sample.value); }); + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(RateIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P(OneSample, + RateIteratorFixture, + testing::Values(RateIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + RateIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + RateIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + RateIteratorFixture, + testing::Values(RateIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 2.0}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 10, .value = 1.0}, Sample{.timestamp = 30, .value = 2.0}}}, + RateIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + RateIteratorFixture, + testing::Values(RateIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.0}, + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + Sample{.timestamp = 201, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 200, .value = 1.0}}})); + +INSTANTIATE_TEST_SUITE_P(FirstAndLastValueInInterval, + RateIteratorFixture, + testing::Values(RateIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 120, .value = 2.0}, + Sample{.timestamp = 160, .value = 3.0}, + Sample{.timestamp = 200, .value = 4.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 200, .value = 4.0}}})); + +INSTANTIATE_TEST_SUITE_P( + CounterResetting, + RateIteratorFixture, + testing::Values( + RateIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 5.0}, + Sample{.timestamp = 200, .value = 1.0}, + }, + .interval{.min = 100, .max = 300}, + .expected{Sample{.timestamp = 100, .value = 5.0}, Sample{.timestamp = 200, .value = 1.0}}}, + RateIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 5.0}, + Sample{.timestamp = 200, .value = 4.0}, + Sample{.timestamp = 300, .value = 3.0}, + Sample{.timestamp = 400, .value = 2.0}, + Sample{.timestamp = 500, .value = 1.0}, + }, + .interval{.min = 100, .max = 500}, + .expected{Sample{.timestamp = 100, .value = 5.0}, Sample{.timestamp = 200, .value = 4.0}, Sample{.timestamp = 300, .value = 3.0}, + Sample{.timestamp = 400, .value = 2.0}, Sample{.timestamp = 500, .value = 1.0}}}, + RateIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 5.0}, + Sample{.timestamp = 200, .value = 4.0}, + Sample{.timestamp = 250, .value = 5.0}, + Sample{.timestamp = 300, .value = 3.0}, + Sample{.timestamp = 400, .value = 2.0}, + Sample{.timestamp = 500, .value = 1.0}, + }, + .interval{.min = 100, .max = 500}, + .expected{Sample{.timestamp = 100, .value = 5.0}, Sample{.timestamp = 200, .value = 4.0}, Sample{.timestamp = 250, .value = 5.0}, + Sample{.timestamp = 300, .value = 3.0}, Sample{.timestamp = 400, .value = 2.0}, Sample{.timestamp = 500, .value = 1.0}}}, + RateIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 120, .value = 2.0}, + Sample{.timestamp = 160, .value = 0.0}, + Sample{.timestamp = 200, .value = 2.0}, + Sample{.timestamp = 250, .value = 3.0}, + Sample{.timestamp = 300, .value = 4.0}, + }, + .interval{.min = 100, .max = 300}, + .expected{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 120, .value = 2.0}, Sample{.timestamp = 160, .value = 0.0}, + Sample{.timestamp = 300, .value = 4.0}}}, + RateIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 120, .value = 2.0}, + Sample{.timestamp = 160, .value = 0.0}, + Sample{.timestamp = 201, .value = 2.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 120, .value = 2.0}, Sample{.timestamp = 160, .value = 0.0}}}, + RateIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 120, .value = 2.0}, + Sample{.timestamp = 160, .value = 3.0}, + Sample{.timestamp = 200, .value = 0.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 160, .value = 3.0}, Sample{.timestamp = 200, .value = 0.0}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/resets_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/resets_iterator_tests.cpp new file mode 100644 index 0000000000..de99200b96 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/resets_iterator_tests.cpp @@ -0,0 +1,95 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/resets_iterator.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::ResetsIterator; +using series_data::encoder::Sample; + +struct ResetsIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class ResetsIteratorFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(ResetsIteratorFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(ResetsIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P(OneSample, + ResetsIteratorFixture, + testing::Values(ResetsIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + ResetsIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + ResetsIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + ResetsIteratorFixture, + testing::Values(ResetsIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.0}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 10, .value = 1.0}}}, + ResetsIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + ResetsIteratorFixture, + testing::Values(ResetsIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 4.0}, + Sample{.timestamp = 150, .value = 3.0}, + Sample{.timestamp = 180, .value = 2.0}, + Sample{.timestamp = 200, .value = 1.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 4.0}, Sample{.timestamp = 150, .value = 3.0}, + Sample{.timestamp = 180, .value = 2.0}, Sample{.timestamp = 200, .value = 1.0}}}, + ResetsIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 3.0}, + Sample{.timestamp = 120, .value = 0.0}, + Sample{.timestamp = 150, .value = 0.0}, + Sample{.timestamp = 180, .value = 1.0}, + Sample{.timestamp = 200, .value = 0.0}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 100, .value = 3.0}, Sample{.timestamp = 120, .value = 0.0}, + Sample{.timestamp = 200, .value = 0.0}}})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/sum_over_time_tests.cpp b/pp/series_data/tests/decoder/decorator/sum_over_time_tests.cpp new file mode 100644 index 0000000000..5d79c15a90 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/sum_over_time_tests.cpp @@ -0,0 +1,134 @@ +#include + +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder.h" +#include "series_data/decoder/decorator/sum_over_time.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::SumOverTimeIterator; +using series_data::encoder::Sample; + +struct SumOverTimeIteratorCase { + std::vector samples; + PromPP::Primitives::TimeInterval interval; + std::vector expected{}; +}; + +class SumOverTimeFixture : public ::testing::TestWithParam { + protected: + DataStorage storage_; + + void SetUp() override { + Encoder encoder(storage_); + for (const auto& sample : GetParam().samples) { + encoder.encode(0, sample.timestamp, sample.value); + } + } +}; + +TEST_P(SumOverTimeFixture, Test) { + // Arrange + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(SumOverTimeIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().interval), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); +} + +INSTANTIATE_TEST_SUITE_P( + OneSample, + SumOverTimeFixture, + testing::Values(SumOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + SumOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 0, .max = 99}, .expected{}}, + SumOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, .interval{.min = 101, .max = 200}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P( + StaleNan, + SumOverTimeFixture, + testing::Values(SumOverTimeIteratorCase{.samples{ + Sample{.timestamp = 5, .value = STALE_NAN}, + Sample{.timestamp = 10, .value = 1.0}, + Sample{.timestamp = 20, .value = STALE_NAN}, + Sample{.timestamp = 30, .value = 1.1}, + }, + .interval{.min = 0, .max = 100}, + .expected{Sample{.timestamp = 100, .value = 2.1}}}, + SumOverTimeIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}}, .interval{.min = 0, .max = 100}, .expected{}})); + +INSTANTIATE_TEST_SUITE_P(TimeInterval, + SumOverTimeFixture, + testing::Values(SumOverTimeIteratorCase{.samples{ + Sample{.timestamp = 99, .value = 1.1}, + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 200, .value = 1.0}, + Sample{.timestamp = 201, .value = 1.1}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 2.0}}}, + SumOverTimeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 150, .value = 1.1}, + Sample{.timestamp = 200, .value = 1.2}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 3.3}}}, + SumOverTimeIteratorCase{.samples{ + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 150, .value = 1.1}, + Sample{.timestamp = 180, .value = 1.2}, + Sample{.timestamp = 200, .value = STALE_NAN}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 3.3}}})); + +INSTANTIATE_TEST_SUITE_P(KahanSummation, + SumOverTimeFixture, + testing::Values(SumOverTimeIteratorCase{ + .samples{ + Sample{.timestamp = 100, .value = 1e16}, + Sample{.timestamp = 110, .value = 1.0}, + Sample{.timestamp = 120, .value = -1e16}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = 1.0}}, + })); + +INSTANTIATE_TEST_SUITE_P(SumOverflowsToInfinity, + SumOverTimeFixture, + testing::Values( + SumOverTimeIteratorCase{ + .samples{ + Sample{.timestamp = 100, .value = std::numeric_limits::max()}, + Sample{.timestamp = 110, .value = std::numeric_limits::max()}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = std::numeric_limits::infinity()}}, + }, + SumOverTimeIteratorCase{ + .samples{ + Sample{.timestamp = 100, .value = -std::numeric_limits::max()}, + Sample{.timestamp = 110, .value = -std::numeric_limits::max()}, + }, + .interval{.min = 100, .max = 200}, + .expected{Sample{.timestamp = 200, .value = -std::numeric_limits::infinity()}}, + })); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/window_boundary_calculator_tests.cpp b/pp/series_data/tests/decoder/decorator/window_boundary_calculator_tests.cpp new file mode 100644 index 0000000000..e74faea143 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/window_boundary_calculator_tests.cpp @@ -0,0 +1,487 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder/decorator/window_function_iterator.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using PromPP::Primitives::TimeInterval; +using PromPP::Primitives::Timestamp; +using series_data::decoder::kInvalidTimestamp; +using series_data::decoder::decorator::StepLookbackDeltaWindowCalculator; +using series_data::decoder::decorator::StepRangeWindowCalculator; +using series_data::decoder::decorator::WindowBoundaryCalculatorInterface; +using series_data::decoder::decorator::WindowFunctionIteratorInterface; +using series_data::decoder::decorator::WindowFunctionParameters; +using series_data::encoder::Sample; + +struct IntervalCalculatorCase { + WindowFunctionParameters parameters; + std::vector expected{}; +}; + +template +class WindowBoundaryCalculatorFixture : public testing::TestWithParam { + protected: + void test() { + // Arrange + std::vector actual; + + // Act + actual.emplace_back(Calculator::initial_window(GetParam().parameters)); + std::generate_n(std::back_inserter(actual), GetParam().expected.size() - 1, + [&actual] { return Calculator::next_window(actual.back(), GetParam().parameters); }); + + // Assert + EXPECT_EQ(GetParam().expected, actual); + } +}; + +using StepRangeWindowCalculatorFixture = WindowBoundaryCalculatorFixture; + +TEST_P(StepRangeWindowCalculatorFixture, Test) { + test(); +} + +INSTANTIATE_TEST_SUITE_P(NoInterval, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 0}, + .step = 0, + .range = 100, + }, + .expected{TimeInterval{.min = 1, .max = 0}}})); + +INSTANTIATE_TEST_SUITE_P(ZeroStep, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 1000}, + .step = 0, + .range = 100, + }, + .expected{TimeInterval{.min = 1, .max = 1000}}})); + +INSTANTIATE_TEST_SUITE_P(ZeroStepAndRange, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 1000}, + .step = 0, + .range = 0, + }, + .expected{TimeInterval{.min = 1, .max = 1000}}})); + +INSTANTIATE_TEST_SUITE_P(RangeEqualsStep, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 300}, + .step = 100, + .range = 100, + }, + .expected{ + TimeInterval{.min = 1, .max = 100}, + TimeInterval{.min = 101, .max = 200}, + TimeInterval{.min = 201, .max = 300}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeGreaterThanInterval, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 50}, + .step = 30, + .range = 200, + }, + .expected{ + TimeInterval{.min = 1, .max = 20}, + TimeInterval{.min = 21, .max = 30}, + TimeInterval{.min = 31, .max = 50}, + }})); + +INSTANTIATE_TEST_SUITE_P(StepGreaterThanInterval, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 50}, + .step = 60, + .range = 0, + }, + .expected{ + TimeInterval{.min = 1, .max = 50}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeAndStepGreaterThanInterval, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 50}, + .step = 60, + .range = 70, + }, + .expected{ + TimeInterval{.min = 1, .max = 10}, + TimeInterval{.min = 11, .max = 50}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeGreaterThanStepAndDivisibleIntervalCalculation, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 300}, + .step = 60, + .range = 120, + }, + .expected{ + TimeInterval{.min = 1, .max = 60}, + TimeInterval{.min = 61, .max = 120}, + TimeInterval{.min = 121, .max = 180}, + TimeInterval{.min = 181, .max = 240}, + TimeInterval{.min = 241, .max = 300}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeGreaterThanStepAndNotDivisible, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 80, .max = 310}, + .step = 60, + .range = 70, + }, + .expected{ + TimeInterval{.min = 81, .max = 90}, + TimeInterval{.min = 91, .max = 140}, + TimeInterval{.min = 141, .max = 150}, + TimeInterval{.min = 151, .max = 200}, + TimeInterval{.min = 201, .max = 210}, + TimeInterval{.min = 211, .max = 260}, + TimeInterval{.min = 261, .max = 270}, + TimeInterval{.min = 271, .max = 310}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeLessThanStepIntervalCalculation, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 300}, + .step = 100, + .range = 60, + }, + .expected{ + TimeInterval{.min = 1, .max = 60}, + TimeInterval{.min = 101, .max = 160}, + TimeInterval{.min = 201, .max = 260}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeEqualsStepTruncatedLastChunk, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 120}, + .step = 50, + .range = 50, + }, + .expected{ + TimeInterval{.min = 1, .max = 50}, + TimeInterval{.min = 51, .max = 100}, + TimeInterval{.min = 101, .max = 120}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeEqualsStepSingleChunk, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 50}, + .step = 50, + .range = 50, + }, + .expected{ + TimeInterval{.min = 1, .max = 50}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeZeroStepAlignedChunks, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 1000}, + .step = 100, + .range = 0, + }, + .expected{ + TimeInterval{.min = 1, .max = 100}, + TimeInterval{.min = 101, .max = 200}, + TimeInterval{.min = 201, .max = 300}, + TimeInterval{.min = 301, .max = 400}, + TimeInterval{.min = 401, .max = 500}, + TimeInterval{.min = 501, .max = 600}, + TimeInterval{.min = 601, .max = 700}, + TimeInterval{.min = 701, .max = 800}, + TimeInterval{.min = 801, .max = 900}, + TimeInterval{.min = 901, .max = 1000}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeGreaterThanStepDivisibleWithNonZeroCHints, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 10, .max = 130}, + .step = 40, + .range = 120, + }, + .expected{ + TimeInterval{.min = 11, .max = 50}, + TimeInterval{.min = 51, .max = 90}, + TimeInterval{.min = 91, .max = 130}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeGreaterThanStepDivisibleTruncatedByEnd, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 250}, + .step = 40, + .range = 120, + }, + .expected{ + TimeInterval{.min = 1, .max = 40}, + TimeInterval{.min = 41, .max = 80}, + TimeInterval{.min = 81, .max = 120}, + TimeInterval{.min = 121, .max = 160}, + TimeInterval{.min = 161, .max = 200}, + TimeInterval{.min = 201, .max = 240}, + TimeInterval{.min = 241, .max = 250}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeGreaterThanStepNotDivisibleFromZero, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 50}, + .step = 10, + .range = 15, + }, + .expected{ + TimeInterval{.min = 1, .max = 5}, + TimeInterval{.min = 6, .max = 10}, + TimeInterval{.min = 11, .max = 15}, + TimeInterval{.min = 16, .max = 20}, + TimeInterval{.min = 21, .max = 25}, + TimeInterval{.min = 26, .max = 30}, + TimeInterval{.min = 31, .max = 35}, + TimeInterval{.min = 36, .max = 40}, + TimeInterval{.min = 41, .max = 45}, + TimeInterval{.min = 46, .max = 50}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeGreaterThanStepNotDivisibleWithNonZeroCHints, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 100, .max = 200}, + .step = 30, + .range = 50, + }, + .expected{ + TimeInterval{.min = 101, .max = 120}, + TimeInterval{.min = 121, .max = 130}, + TimeInterval{.min = 131, .max = 150}, + TimeInterval{.min = 151, .max = 160}, + TimeInterval{.min = 161, .max = 180}, + TimeInterval{.min = 181, .max = 190}, + TimeInterval{.min = 191, .max = 200}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeLessThanStepWithNonZeroCHints, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 50, .max = 350}, + .step = 100, + .range = 60, + }, + .expected{ + TimeInterval{.min = 51, .max = 110}, + TimeInterval{.min = 151, .max = 210}, + TimeInterval{.min = 251, .max = 310}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeLessThanStepTruncatedLastWindow, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 250}, + .step = 100, + .range = 60, + }, + .expected{ + TimeInterval{.min = 1, .max = 60}, + TimeInterval{.min = 101, .max = 160}, + TimeInterval{.min = 201, .max = 250}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeLessThanStepSinglePartialWindow, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 40}, + .step = 100, + .range = 60, + }, + .expected{ + TimeInterval{.min = 1, .max = 40}, + }})); + +INSTANTIATE_TEST_SUITE_P(PointQueryIntervalWithWinLessThanStep, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 10, .max = 10}, + .step = 100, + .range = 60, + }, + .expected{ + TimeInterval{.min = 11, .max = 10}, + }})); + +INSTANTIATE_TEST_SUITE_P(RangeLargerThanStepNotDivisibleNoInvertedWindows, + StepRangeWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 20}, + .step = 4, + .range = 9, + }, + .expected{ + TimeInterval{.min = 1, .max = 1}, + TimeInterval{.min = 2, .max = 4}, + TimeInterval{.min = 5, .max = 5}, + TimeInterval{.min = 6, .max = 8}, + TimeInterval{.min = 9, .max = 9}, + TimeInterval{.min = 10, .max = 12}, + TimeInterval{.min = 13, .max = 13}, + TimeInterval{.min = 14, .max = 16}, + TimeInterval{.min = 17, .max = 17}, + TimeInterval{.min = 18, .max = 20}, + }}, + IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 32}, + .step = 10, + .range = 12, + }, + .expected{ + TimeInterval{.min = 1, .max = 2}, + TimeInterval{.min = 3, .max = 10}, + TimeInterval{.min = 11, .max = 12}, + TimeInterval{.min = 13, .max = 20}, + TimeInterval{.min = 21, .max = 22}, + TimeInterval{.min = 23, .max = 30}, + TimeInterval{.min = 31, .max = 32}, + }})); + +using StepLookbackDeltaWindowCalculatorFixture = WindowBoundaryCalculatorFixture; + +TEST_P(StepLookbackDeltaWindowCalculatorFixture, Test) { + test(); +} + +INSTANTIATE_TEST_SUITE_P(NoInterval, + StepLookbackDeltaWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 0}, + .step = 100, + .lookback_delta = 50, + }, + .expected{ + TimeInterval{.min = 1, .max = 0}, + }})); + +INSTANTIATE_TEST_SUITE_P(LookbackEqualsStep, + StepLookbackDeltaWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 200}, + .step = 100, + .lookback_delta = 100, + }, + .expected{ + TimeInterval{.min = 1, .max = 100}, + TimeInterval{.min = 101, .max = 200}, + }})); + +INSTANTIATE_TEST_SUITE_P(LookbackGreaterThanStep, + StepLookbackDeltaWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 250}, + .step = 60, + .lookback_delta = 120, + }, + .expected{ + TimeInterval{.min = 1, .max = 120}, + TimeInterval{.min = 121, .max = 180}, + TimeInterval{.min = 181, .max = 240}, + TimeInterval{.min = 241, .max = 250}, + }})); + +INSTANTIATE_TEST_SUITE_P(LookbackLessThanStep, + StepLookbackDeltaWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 300}, + .step = 100, + .lookback_delta = 60, + }, + .expected{ + TimeInterval{.min = 1, .max = 60}, + TimeInterval{.min = 61, .max = 160}, + TimeInterval{.min = 161, .max = 260}, + TimeInterval{.min = 261, .max = 300}, + }})); + +INSTANTIATE_TEST_SUITE_P(NonZeroIntervalMin, + StepLookbackDeltaWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 100, .max = 500}, + .step = 80, + .lookback_delta = 30, + }, + .expected{ + TimeInterval{.min = 101, .max = 130}, + TimeInterval{.min = 131, .max = 210}, + TimeInterval{.min = 211, .max = 290}, + TimeInterval{.min = 291, .max = 370}, + TimeInterval{.min = 371, .max = 450}, + }})); + +INSTANTIATE_TEST_SUITE_P(PointQueryInterval, + StepLookbackDeltaWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 10, .max = 10}, + .step = 100, + .lookback_delta = 60, + }, + .expected{ + TimeInterval{.min = 11, .max = 10}, + }})); + +INSTANTIATE_TEST_SUITE_P(SingleWindow, + StepLookbackDeltaWindowCalculatorFixture, + testing::Values(IntervalCalculatorCase{.parameters = + { + .interval{.min = 0, .max = 40}, + .step = 100, + .lookback_delta = 60, + }, + .expected{ + TimeInterval{.min = 1, .max = 40}, + }})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/decoder/decorator/window_function_iterator_tests.cpp b/pp/series_data/tests/decoder/decorator/window_function_iterator_tests.cpp new file mode 100644 index 0000000000..bb13d16d76 --- /dev/null +++ b/pp/series_data/tests/decoder/decorator/window_function_iterator_tests.cpp @@ -0,0 +1,242 @@ +#include + +#include "series_data/data_storage.h" +#include "series_data/decoder/decorator/max_over_time.h" +#include "series_data/decoder/decorator/sum_over_time.h" +#include "series_data/decoder/decorator/window_function_iterator.h" +#include "series_data/encoder.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using PromPP::Primitives::TimeInterval; +using PromPP::Primitives::Timestamp; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::decoder::kInvalidTimestamp; +using series_data::decoder::UniversalDecodeIterator; +using series_data::decoder::decorator::MaxOverTimeIterator; +using series_data::decoder::decorator::StepRangeWindowCalculator; +using series_data::decoder::decorator::SumOverTimeIterator; +using series_data::decoder::decorator::WindowBoundaryCalculatorInterface; +using series_data::decoder::decorator::WindowFunctionIteratorInterface; +using series_data::decoder::decorator::WindowFunctionParameters; +using series_data::encoder::Sample; + +struct WindowFunctionIteratorCase { + std::vector samples; + WindowFunctionParameters parameters; + std::vector expected{}; +}; + +template +class WindowFunctionIteratorFixture : public testing::TestWithParam { + protected: + using WindowFunctionIterator = series_data::decoder::decorator::WindowFunctionIterator; + + DataStorage storage_; + Encoder<> encoder_{storage_}; + + void encode_samples() { + for (const auto& sample : GetParam().samples) { + encoder_.encode(0, sample.timestamp, sample.value); + } + } + + void test() { + // Arrange + encode_samples(); + std::vector actual_samples; + + // Act + Decoder::create_decode_iterator(storage_, storage_.open_chunks[0], [&actual_samples](Iterator&& begin, auto&&) { + std::ranges::copy(WindowFunctionIterator(UniversalDecodeIterator{std::in_place_type, std::forward(begin)}, GetParam().parameters), + DecodeIteratorSentinel{}, std::back_inserter(actual_samples)); + }); + + // Assert + EXPECT_EQ(GetParam().expected, actual_samples); + } +}; + +using MaxOverTimeWindowFunctionIteratorFixture = WindowFunctionIteratorFixture, StepRangeWindowCalculator>; + +TEST_P(MaxOverTimeWindowFunctionIteratorFixture, Test) { + test(); +} + +INSTANTIATE_TEST_SUITE_P(NoStep, + MaxOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}}, + .parameters = + { + .interval{.min = 0, .max = 1000}, + .step = 0, + .range = 100, + }, + .expected{Sample{.timestamp = 100, .value = 1.0}}}, + WindowFunctionIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 1000, .value = 2.0}}, + .parameters = + { + .interval{.min = 0, .max = 1000}, + .step = 0, + .range = 100, + }, + .expected{Sample{.timestamp = 1000, .value = 2.0}}}, + WindowFunctionIteratorCase{.samples{Sample{.timestamp = 100, .value = 1.0}, Sample{.timestamp = 500, .value = 2.0}, + Sample{.timestamp = 1000, .value = 1.0}}, + .parameters = + { + .interval{.min = 0, .max = 1000}, + .step = 0, + .range = 100, + }, + .expected{Sample{.timestamp = 500, .value = 2.0}}})); + +INSTANTIATE_TEST_SUITE_P(NoRange, + MaxOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{ + .samples{Sample{.timestamp = 100, .value = 3.0}, Sample{.timestamp = 1000, .value = 2.0}}, + .parameters = + { + .interval{.min = 0, .max = 1000}, + .step = 100, + .range = 0, + }, + .expected{Sample{.timestamp = 100, .value = 3.0}, Sample{.timestamp = 1000, .value = 2.0}}})); + +INSTANTIATE_TEST_SUITE_P(StepGreaterThanRange, + MaxOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{ + .samples{Sample{.timestamp = 130, .value = 1.0}, Sample{.timestamp = 160, .value = 2.0}, Sample{.timestamp = 190, .value = 3.0}, + Sample{.timestamp = 220, .value = 4.0}, Sample{.timestamp = 250, .value = 5.0}, Sample{.timestamp = 280, .value = 6.0}, + Sample{.timestamp = 310, .value = 7.0}}, + .parameters = + { + .interval{.min = 0, .max = 1000}, + .step = 70, + .range = 60, + }, + .expected{Sample{.timestamp = 130, .value = 1.0}, Sample{.timestamp = 190, .value = 3.0}, Sample{.timestamp = 250, .value = 5.0}, + Sample{.timestamp = 310, .value = 7.0}}})); + +INSTANTIATE_TEST_SUITE_P(StepLessThanRange, + MaxOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{ + .samples{Sample{.timestamp = 130, .value = 1.0}, Sample{.timestamp = 160, .value = 2.0}, Sample{.timestamp = 190, .value = 3.0}, + Sample{.timestamp = 220, .value = 4.0}, Sample{.timestamp = 250, .value = 5.0}, Sample{.timestamp = 280, .value = 5.0}, + Sample{.timestamp = 310, .value = 6.0}}, + .parameters = + { + .interval{.min = 80, .max = 310}, + .step = 60, + .range = 70, + }, + .expected{Sample{.timestamp = 130, .value = 1.0}, Sample{.timestamp = 190, .value = 3.0}, Sample{.timestamp = 250, .value = 5.0}, + Sample{.timestamp = 310, .value = 6.0}}})); + +INSTANTIATE_TEST_SUITE_P(IntervalBoundaries, + MaxOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{.samples{Sample{.timestamp = 79, .value = 5.0}, Sample{.timestamp = 310, .value = 6.0}}, + .parameters = + { + .interval{.min = 80, .max = 309}, + .step = 60, + .range = 70, + }, + .expected{}}, + WindowFunctionIteratorCase{.samples{Sample{.timestamp = 79, .value = 5.0}, Sample{.timestamp = 310, .value = 6.0}}, + .parameters = + { + .interval{.min = 0, .max = 79}, + .step = 60, + .range = 70, + }, + .expected{Sample{.timestamp = 79, .value = 5.0}}}, + WindowFunctionIteratorCase{.samples{Sample{.timestamp = 79, .value = 5.0}, Sample{.timestamp = 310, .value = 6.0}}, + .parameters = + { + .interval{.min = 310, .max = 310}, + .step = 60, + .range = 70, + }, + .expected{}}, + WindowFunctionIteratorCase{.samples{Sample{.timestamp = 55, .value = 9.0}}, + .parameters = + { + .interval{.min = 0, .max = 50}, + .step = 60, + .range = 70, + }, + .expected{}})); + +INSTANTIATE_TEST_SUITE_P(StepEqualsRange, + MaxOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{ + .samples{Sample{.timestamp = 50, .value = 1.0}, Sample{.timestamp = 150, .value = 4.0}, Sample{.timestamp = 250, .value = 2.0}}, + .parameters = + { + .interval{.min = 0, .max = 300}, + .step = 100, + .range = 100, + }, + .expected{Sample{.timestamp = 50, .value = 1.0}, Sample{.timestamp = 150, .value = 4.0}, + Sample{.timestamp = 250, .value = 2.0}}})); + +INSTANTIATE_TEST_SUITE_P(StaleNanSkipped, + MaxOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{.samples{Sample{.timestamp = 100, .value = STALE_NAN}, + Sample{.timestamp = 150, .value = 2.0}, Sample{.timestamp = 200, .value = 1.0}}, + .parameters = + { + .interval{.min = 0, .max = 400}, + .step = 100, + .range = 100, + }, + .expected{Sample{.timestamp = 150, .value = 2.0}}})); + +using SumOverTimeWindowFunctionIteratorFixture = WindowFunctionIteratorFixture, StepRangeWindowCalculator>; + +TEST_P(SumOverTimeWindowFunctionIteratorFixture, Test) { + test(); +} + +INSTANTIATE_TEST_SUITE_P(DoesNotDoubleCountBoundarySample, + SumOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{ + .samples{ + Sample{.timestamp = 100, .value = 10.0}, + Sample{.timestamp = 150, .value = 1.0}, + Sample{.timestamp = 200, .value = 2.0}, + }, + .parameters = + { + .interval{.min = 0, .max = 200}, + .step = 100, + .range = 1000, + }, + .expected{Sample{.timestamp = 100, .value = 10.0}, Sample{.timestamp = 200, .value = 3.0}}})); + +INSTANTIATE_TEST_SUITE_P(EmptyWindows, + SumOverTimeWindowFunctionIteratorFixture, + testing::Values(WindowFunctionIteratorCase{.samples{ + Sample{.timestamp = 90, .value = 1.0}, + Sample{.timestamp = 100, .value = 1.0}, + Sample{.timestamp = 151, .value = 1.0}, + Sample{.timestamp = 190, .value = 1.0}, + }, + .parameters = + { + .interval{.min = 0, .max = 200}, + .step = 50, + .range = 0, + }, + .expected{ + Sample{.timestamp = 100, .value = 2.0}, + Sample{.timestamp = 200, .value = 2.0}, + }})); + +} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/serialization/serializer_deserializer_tests.cpp b/pp/series_data/tests/serialization/serializer_deserializer_tests.cpp index db1090c2ae..853219bf0d 100644 --- a/pp/series_data/tests/serialization/serializer_deserializer_tests.cpp +++ b/pp/series_data/tests/serialization/serializer_deserializer_tests.cpp @@ -10,6 +10,7 @@ namespace { using BareBones::Encoding::Gorilla::STALE_NAN; +using PromPP::Primitives::Timestamp; using series_data::ChunkFinalizer; using series_data::DataStorage; using series_data::Encoder; @@ -24,6 +25,9 @@ using series_data::serialization::DataSerializer; using series_data::serialization::SerializedData; using series_data::serialization::SerializedDataView; +using series_data::decoder::SeekKind; +using series_data::decoder::SeekResult; + class SerializerDeserializerTrait { protected: DataStorage storage_; @@ -1201,4 +1205,244 @@ TEST_F(SerializedDataIterSeekToFixture, SeekToOnEndIterator) { EXPECT_EQ(iterator, DecodeIteratorSentinel{}); } +class SerializedDataIterSeekFixture : public SerializerDeserializerTrait, public testing::Test { + protected: + SerializedData serialized_data_; + + SerializedDataView::SeriesIterator create_iterator() noexcept { + serialized_data_ = serialize(); + SerializedDataView serialized_view(serialized_data_); + + auto [series_id, chunk_id] = serialized_view.next_series(); + return serialized_view.create_series_iterator(chunk_id); + } +}; + +TEST_F(SerializedDataIterSeekFixture, SeekToLastSampleThroughtChunks) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 1.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 1.0); + encoder_.encode(0, 4, 1.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 5, 1.0); + encoder_.encode(0, 6, 1.0); + + auto iterator = create_iterator(); + + // Act + iterator.seek([](Timestamp) { return SeekResult::kUpdateSample; }); + + // Assert + EXPECT_EQ((Sample{.timestamp = 6, .value = 1.0}), *iterator); +} + +TEST_F(SerializedDataIterSeekFixture, SeekToLastSampleInChunk) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 1.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 1.0); + encoder_.encode(0, 4, 1.0); + + auto iterator = create_iterator(); + + // Act + iterator.seek([](Timestamp ts) { return ts == 2 ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kUpdateSample; }); + + // Assert + EXPECT_EQ((Sample{.timestamp = 2, .value = 1.0}), *iterator); +} + +TEST_F(SerializedDataIterSeekFixture, SeekToSampleInSecondChunk) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 1.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 2.0); + encoder_.encode(0, 4, 2.0); + + auto iterator = create_iterator(); + + // Act + iterator.seek([](Timestamp ts) { return ts == 3 ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kUpdateSample; }); + + // Assert + EXPECT_EQ((Sample{.timestamp = 3, .value = 2.0}), *iterator); +} + +TEST_F(SerializedDataIterSeekFixture, SeekWithSampleHandler) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + + auto iterator = create_iterator(); + + // Act + iterator.seek([](Timestamp, double value) { return value == 3.0 ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kUpdateSample; }); + + // Assert + EXPECT_EQ((Sample{.timestamp = 3, .value = 3.0}), *iterator); +} + +TEST_F(SerializedDataIterSeekFixture, SeekWithNextAcrossChunks) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 1.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 1.0); + encoder_.encode(0, 4, 1.0); + + auto iterator = create_iterator(); + + // Act + iterator.seek([](Timestamp ts) { return ts == 3 ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kNext; }); + + // Assert + EXPECT_EQ((Sample{.timestamp = 3, .value = 1.0}), *iterator); +} + +TEST_F(SerializedDataIterSeekFixture, SeekWithStop) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + + auto iterator = create_iterator(); + + // Act + iterator.seek([](Timestamp ts) { return ts == 2 ? SeekResult::kStop : SeekResult::kUpdateSample; }); + + // Assert + EXPECT_EQ((Sample{.timestamp = 1, .value = 1.0}), *iterator); +} + +TEST_F(SerializedDataIterSeekFixture, ConsecutiveSeekWithUpdateSampleNextAndStop) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 5, 5.0); + encoder_.encode(0, 6, 6.0); + + auto iterator = create_iterator(); + + const auto seek_and_stop_at = [&iterator](Timestamp target) { + iterator.seek([target](Timestamp ts) { return ts == target ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kUpdateSample; }); + }; + + // Act + seek_and_stop_at(2); + const auto sample_after_first_seek = *iterator; + seek_and_stop_at(4); + const auto sample_after_second_seek = *iterator; + seek_and_stop_at(6); + const auto sample_after_third_seek = *iterator; + + // Assert + EXPECT_EQ((Sample{.timestamp = 2, .value = 2.0}), sample_after_first_seek); + EXPECT_EQ((Sample{.timestamp = 4, .value = 4.0}), sample_after_second_seek); + EXPECT_EQ((Sample{.timestamp = 6, .value = 6.0}), sample_after_third_seek); +} + +TEST_F(SerializedDataIterSeekFixture, ConsecutiveSeekResumesFromCurrentPosition) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 5, 5.0); + encoder_.encode(0, 6, 6.0); + + auto iterator = create_iterator(); + + std::vector seen_on_second_seek; + std::vector seen_on_third_seek; + + // Act + iterator.seek([](Timestamp ts) { return ts == 2 ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kUpdateSample; }); + + iterator.seek([&seen_on_second_seek](Timestamp ts) { + seen_on_second_seek.push_back(ts); + return ts == 4 ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kUpdateSample; + }); + + iterator.seek([&seen_on_third_seek](Timestamp ts) { + seen_on_third_seek.push_back(ts); + return ts == 6 ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kUpdateSample; + }); + + // Assert + EXPECT_THAT(seen_on_second_seek, testing::ElementsAre(3, 4)); + EXPECT_THAT(seen_on_third_seek, testing::ElementsAre(5, 6)); + EXPECT_EQ((Sample{.timestamp = 6, .value = 6.0}), *iterator); +} + +TEST_F(SerializedDataIterSeekFixture, ConsecutiveSeekWithNext) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 5, 5.0); + encoder_.encode(0, 6, 6.0); + + auto iterator = create_iterator(); + + const auto seek_with_next_and_stop_at = [&iterator](Timestamp target) { + iterator.seek([target](Timestamp ts) { return ts == target ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kNext; }); + }; + + // Act + seek_with_next_and_stop_at(3); + const auto sample_after_first_seek = *iterator; + seek_with_next_and_stop_at(5); + const auto sample_after_second_seek = *iterator; + + // Assert + EXPECT_EQ((Sample{.timestamp = 3, .value = 3.0}), sample_after_first_seek); + EXPECT_EQ((Sample{.timestamp = 5, .value = 5.0}), sample_after_second_seek); +} + +TEST_F(SerializedDataIterSeekFixture, ConsecutiveSeekAfterStop) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + + auto iterator = create_iterator(); + + std::vector seen_on_second_seek; + + // Act + iterator.seek([](Timestamp ts) { return ts == 2 ? SeekResult::kStop : SeekResult::kUpdateSample; }); + + const auto sample_after_stop = *iterator; + + iterator.seek([&seen_on_second_seek](Timestamp ts) { + seen_on_second_seek.push_back(ts); + return ts == 4 ? SeekResult::kUpdateSampleNextAndStop : SeekResult::kUpdateSample; + }); + + // Assert + EXPECT_EQ((Sample{.timestamp = 1, .value = 1.0}), sample_after_stop); + EXPECT_THAT(seen_on_second_seek, testing::ElementsAre(2, 3, 4)); + EXPECT_EQ((Sample{.timestamp = 4, .value = 4.0}), *iterator); +} + } // namespace diff --git a/pp/third_party/patches/quasis_crypto/0001-md5.hh.patch b/pp/third_party/patches/quasis_crypto/0001-md5.hh.patch index d26b5c774c..bb6952863b 100644 --- a/pp/third_party/patches/quasis_crypto/0001-md5.hh.patch +++ b/pp/third_party/patches/quasis_crypto/0001-md5.hh.patch @@ -14,31 +14,31 @@ template @@ -154,22 +159,22 @@ } - + template constexpr MD5& - update(const input_type *input, size_type count) noexcept requires (__is_trivially_copyable(input_type)) { + update(const input_type *input, size_type count) noexcept requires (std::is_trivially_copyable_v) { return update(reinterpret_cast(input), sizeof(input_type) * count); } - + template constexpr MD5& - update(const input_type (&input)[count]) noexcept requires (__is_trivially_copyable(input_type)) { + update(const input_type (&input)[count]) noexcept requires (std::is_trivially_copyable_v) { return update(reinterpret_cast(input), sizeof(input_type) * count); } - + template constexpr MD5& - update(const input_type *begin, const input_type *end) noexcept requires (__is_trivially_copyable(input_type)) { + update(const input_type *begin, const input_type *end) noexcept requires (std::is_trivially_copyable_v) { return update(reinterpret_cast(begin), sizeof(input_type) * (end - begin)); } - + template constexpr MD5& - update(const input_type &input) noexcept requires (__is_trivially_copyable(input_type)) { + update(const input_type &input) noexcept requires (std::is_trivially_copyable_v) { return update(reinterpret_cast(&input), sizeof(input_type)); } - + @@ -180,7 +185,7 @@ template constexpr MD5& @@ -50,13 +50,13 @@ constexpr output_type @@ -194,7 +199,10 @@ auto factor = (length + sizeof(block_type) - 1) / sizeof(block_type); - + hasher.update(factor * sizeof(block_type) - length, uint8_type{0x00}); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" hasher.update(m_count << 3); +#pragma GCC diagnostic pop - + return *reinterpret_cast(&hasher.m_state); } @@ -207,7 +215,7 @@ diff --git a/promql/engine.go b/promql/engine.go index 4a51c4e393..d323d2323f 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -979,11 +979,13 @@ func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s interval = s.Interval } hints := &storage.SelectHints{ - Start: start, - End: end, - Step: durationMilliseconds(interval), - Range: durationMilliseconds(evalRange), - Func: extractFuncFromPath(path), + Start: start, + End: end, + Step: durationMilliseconds(interval), + Range: durationMilliseconds(evalRange), + Func: extractFuncFromPath(path), + LookbackDelta: durationMilliseconds(s.LookbackDelta), + IsSubquery: containsSubquery(path), } evalRange = 0 hints.By, hints.Grouping = extractGroupsFromPath(path) @@ -1002,6 +1004,7 @@ func extractFuncFromPath(p []parser.Node) string { if len(p) == 0 { return "" } + switch n := p[len(p)-1].(type) { case *parser.AggregateExpr: return n.Op.String() @@ -1015,6 +1018,17 @@ func extractFuncFromPath(p []parser.Node) string { return extractFuncFromPath(p[:len(p)-1]) } +// containsSubquery checks if the path contains a subquery. +func containsSubquery(p []parser.Node) bool { + for _, n := range p { + if _, ok := n.(*parser.SubqueryExpr); ok { + return true + } + } + + return false +} + // extractGroupsFromPath parses vector outer function and extracts grouping information if by or without was used. func extractGroupsFromPath(p []parser.Node) (bool, []string) { if len(p) == 0 { diff --git a/promql/engine_test.go b/promql/engine_test.go index e743a08e9e..28736d2b93 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -299,12 +299,13 @@ func (h *hintRecordingQuerier) Select(ctx context.Context, sortSeries bool, hint } func TestSelectHintsSetCorrectly(t *testing.T) { + lbdMs := int64(5000) opts := promql.EngineOpts{ Logger: nil, Reg: nil, MaxSamples: 10, Timeout: 10 * time.Second, - LookbackDelta: 5 * time.Second, + LookbackDelta: time.Duration(lbdMs) * time.Millisecond, EnableAtModifier: true, } @@ -321,271 +322,271 @@ func TestSelectHintsSetCorrectly(t *testing.T) { { query: "foo", start: 10000, expected: []*storage.SelectHints{ - {Start: 5001, End: 10000}, + {Start: 5001, End: 10000, LookbackDelta: lbdMs}, }, }, { query: "foo @ 15", start: 10000, expected: []*storage.SelectHints{ - {Start: 10001, End: 15000}, + {Start: 10001, End: 15000, LookbackDelta: lbdMs}, }, }, { query: "foo @ 1", start: 10000, expected: []*storage.SelectHints{ - {Start: -3999, End: 1000}, + {Start: -3999, End: 1000, LookbackDelta: lbdMs}, }, }, { query: "foo[2m]", start: 200000, expected: []*storage.SelectHints{ - {Start: 80001, End: 200000, Range: 120000}, + {Start: 80001, End: 200000, Range: 120000, LookbackDelta: lbdMs}, }, }, { query: "foo[2m] @ 180", start: 200000, expected: []*storage.SelectHints{ - {Start: 60001, End: 180000, Range: 120000}, + {Start: 60001, End: 180000, Range: 120000, LookbackDelta: lbdMs}, }, }, { query: "foo[2m] @ 300", start: 200000, expected: []*storage.SelectHints{ - {Start: 180001, End: 300000, Range: 120000}, + {Start: 180001, End: 300000, Range: 120000, LookbackDelta: lbdMs}, }, }, { query: "foo[2m] @ 60", start: 200000, expected: []*storage.SelectHints{ - {Start: -59999, End: 60000, Range: 120000}, + {Start: -59999, End: 60000, Range: 120000, LookbackDelta: lbdMs}, }, }, { query: "foo[2m] offset 2m", start: 300000, expected: []*storage.SelectHints{ - {Start: 60001, End: 180000, Range: 120000}, + {Start: 60001, End: 180000, Range: 120000, LookbackDelta: lbdMs}, }, }, { query: "foo[2m] @ 200 offset 2m", start: 300000, expected: []*storage.SelectHints{ - {Start: -39999, End: 80000, Range: 120000}, + {Start: -39999, End: 80000, Range: 120000, LookbackDelta: lbdMs}, }, }, { query: "foo[2m:1s]", start: 300000, expected: []*storage.SelectHints{ - {Start: 175001, End: 300000, Step: 1000}, + {Start: 175001, End: 300000, Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s])", start: 300000, expected: []*storage.SelectHints{ - {Start: 175001, End: 300000, Func: "count_over_time", Step: 1000}, + {Start: 175001, End: 300000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s] @ 300)", start: 200000, expected: []*storage.SelectHints{ - {Start: 175001, End: 300000, Func: "count_over_time", Step: 1000}, + {Start: 175001, End: 300000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s] @ 200)", start: 200000, expected: []*storage.SelectHints{ - {Start: 75001, End: 200000, Func: "count_over_time", Step: 1000}, + {Start: 75001, End: 200000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s] @ 100)", start: 200000, expected: []*storage.SelectHints{ - {Start: -24999, End: 100000, Func: "count_over_time", Step: 1000}, + {Start: -24999, End: 100000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s] offset 10s)", start: 300000, expected: []*storage.SelectHints{ - {Start: 165001, End: 290000, Func: "count_over_time", Step: 1000}, + {Start: 165001, End: 290000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time((foo offset 10s)[2m:1s] offset 10s)", start: 300000, expected: []*storage.SelectHints{ - {Start: 155001, End: 280000, Func: "count_over_time", Step: 1000}, + {Start: 155001, End: 280000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { // When the @ is on the vector selector, the enclosing subquery parameters // don't affect the hint ranges. query: "count_over_time((foo @ 200 offset 10s)[2m:1s] offset 10s)", start: 300000, expected: []*storage.SelectHints{ - {Start: 185001, End: 190000, Func: "count_over_time", Step: 1000}, + {Start: 185001, End: 190000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { // When the @ is on the vector selector, the enclosing subquery parameters // don't affect the hint ranges. query: "count_over_time((foo @ 200 offset 10s)[2m:1s] @ 100 offset 10s)", start: 300000, expected: []*storage.SelectHints{ - {Start: 185001, End: 190000, Func: "count_over_time", Step: 1000}, + {Start: 185001, End: 190000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time((foo offset 10s)[2m:1s] @ 100 offset 10s)", start: 300000, expected: []*storage.SelectHints{ - {Start: -44999, End: 80000, Func: "count_over_time", Step: 1000}, + {Start: -44999, End: 80000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "foo", start: 10000, end: 20000, expected: []*storage.SelectHints{ - {Start: 5001, End: 20000, Step: 1000}, + {Start: 5001, End: 20000, Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "foo @ 15", start: 10000, end: 20000, expected: []*storage.SelectHints{ - {Start: 10001, End: 15000, Step: 1000}, + {Start: 10001, End: 15000, Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "foo @ 1", start: 10000, end: 20000, expected: []*storage.SelectHints{ - {Start: -3999, End: 1000, Step: 1000}, + {Start: -3999, End: 1000, Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2m] @ 180)", start: 200000, end: 500000, expected: []*storage.SelectHints{ - {Start: 60001, End: 180000, Range: 120000, Func: "rate", Step: 1000}, + {Start: 60001, End: 180000, Range: 120000, Func: "rate", Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2m] @ 300)", start: 200000, end: 500000, expected: []*storage.SelectHints{ - {Start: 180001, End: 300000, Range: 120000, Func: "rate", Step: 1000}, + {Start: 180001, End: 300000, Range: 120000, Func: "rate", Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2m] @ 60)", start: 200000, end: 500000, expected: []*storage.SelectHints{ - {Start: -59999, End: 60000, Range: 120000, Func: "rate", Step: 1000}, + {Start: -59999, End: 60000, Range: 120000, Func: "rate", Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2m])", start: 200000, end: 500000, expected: []*storage.SelectHints{ - {Start: 80001, End: 500000, Range: 120000, Func: "rate", Step: 1000}, + {Start: 80001, End: 500000, Range: 120000, Func: "rate", Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2m] offset 2m)", start: 300000, end: 500000, expected: []*storage.SelectHints{ - {Start: 60001, End: 380000, Range: 120000, Func: "rate", Step: 1000}, + {Start: 60001, End: 380000, Range: 120000, Func: "rate", Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2m:1s])", start: 300000, end: 500000, expected: []*storage.SelectHints{ - {Start: 175001, End: 500000, Func: "rate", Step: 1000}, + {Start: 175001, End: 500000, Func: "rate", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s])", start: 300000, end: 500000, expected: []*storage.SelectHints{ - {Start: 175001, End: 500000, Func: "count_over_time", Step: 1000}, + {Start: 175001, End: 500000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s] offset 10s)", start: 300000, end: 500000, expected: []*storage.SelectHints{ - {Start: 165001, End: 490000, Func: "count_over_time", Step: 1000}, + {Start: 165001, End: 490000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s] @ 300)", start: 200000, end: 500000, expected: []*storage.SelectHints{ - {Start: 175001, End: 300000, Func: "count_over_time", Step: 1000}, + {Start: 175001, End: 300000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s] @ 200)", start: 200000, end: 500000, expected: []*storage.SelectHints{ - {Start: 75001, End: 200000, Func: "count_over_time", Step: 1000}, + {Start: 75001, End: 200000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time(foo[2m:1s] @ 100)", start: 200000, end: 500000, expected: []*storage.SelectHints{ - {Start: -24999, End: 100000, Func: "count_over_time", Step: 1000}, + {Start: -24999, End: 100000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time((foo offset 10s)[2m:1s] offset 10s)", start: 300000, end: 500000, expected: []*storage.SelectHints{ - {Start: 155001, End: 480000, Func: "count_over_time", Step: 1000}, + {Start: 155001, End: 480000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { // When the @ is on the vector selector, the enclosing subquery parameters // don't affect the hint ranges. query: "count_over_time((foo @ 200 offset 10s)[2m:1s] offset 10s)", start: 300000, end: 500000, expected: []*storage.SelectHints{ - {Start: 185001, End: 190000, Func: "count_over_time", Step: 1000}, + {Start: 185001, End: 190000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { // When the @ is on the vector selector, the enclosing subquery parameters // don't affect the hint ranges. query: "count_over_time((foo @ 200 offset 10s)[2m:1s] @ 100 offset 10s)", start: 300000, end: 500000, expected: []*storage.SelectHints{ - {Start: 185001, End: 190000, Func: "count_over_time", Step: 1000}, + {Start: 185001, End: 190000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "count_over_time((foo offset 10s)[2m:1s] @ 100 offset 10s)", start: 300000, end: 500000, expected: []*storage.SelectHints{ - {Start: -44999, End: 80000, Func: "count_over_time", Step: 1000}, + {Start: -44999, End: 80000, Func: "count_over_time", Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "sum by (dim1) (foo)", start: 10000, expected: []*storage.SelectHints{ - {Start: 5001, End: 10000, Func: "sum", By: true, Grouping: []string{"dim1"}}, + {Start: 5001, End: 10000, Func: "sum", By: true, Grouping: []string{"dim1"}, LookbackDelta: lbdMs}, }, }, { query: "sum without (dim1) (foo)", start: 10000, expected: []*storage.SelectHints{ - {Start: 5001, End: 10000, Func: "sum", Grouping: []string{"dim1"}}, + {Start: 5001, End: 10000, Func: "sum", Grouping: []string{"dim1"}, LookbackDelta: lbdMs}, }, }, { query: "sum by (dim1) (avg_over_time(foo[1s]))", start: 10000, expected: []*storage.SelectHints{ - {Start: 9001, End: 10000, Func: "avg_over_time", Range: 1000}, + {Start: 9001, End: 10000, Func: "avg_over_time", Range: 1000, LookbackDelta: lbdMs}, }, }, { query: "sum by (dim1) (max by (dim2) (foo))", start: 10000, expected: []*storage.SelectHints{ - {Start: 5001, End: 10000, Func: "max", By: true, Grouping: []string{"dim2"}}, + {Start: 5001, End: 10000, Func: "max", By: true, Grouping: []string{"dim2"}, LookbackDelta: lbdMs}, }, }, { query: "(max by (dim1) (foo))[5s:1s]", start: 10000, expected: []*storage.SelectHints{ - {Start: 1, End: 10000, Func: "max", By: true, Grouping: []string{"dim1"}, Step: 1000}, + {Start: 1, End: 10000, Func: "max", By: true, Grouping: []string{"dim1"}, Step: 1000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "(sum(http_requests{group=~\"p.*\"})+max(http_requests{group=~\"c.*\"}))[20s:5s]", start: 120000, expected: []*storage.SelectHints{ - {Start: 95001, End: 120000, Func: "sum", By: true, Step: 5000}, - {Start: 95001, End: 120000, Func: "max", By: true, Step: 5000}, + {Start: 95001, End: 120000, Func: "sum", By: true, Step: 5000, LookbackDelta: lbdMs, IsSubquery: true}, + {Start: 95001, End: 120000, Func: "max", By: true, Step: 5000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { query: "foo @ 50 + bar @ 250 + baz @ 900", start: 100000, end: 500000, expected: []*storage.SelectHints{ - {Start: 45001, End: 50000, Step: 1000}, - {Start: 245001, End: 250000, Step: 1000}, - {Start: 895001, End: 900000, Step: 1000}, + {Start: 45001, End: 50000, Step: 1000, LookbackDelta: lbdMs}, + {Start: 245001, End: 250000, Step: 1000, LookbackDelta: lbdMs}, + {Start: 895001, End: 900000, Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "foo @ 50 + bar + baz @ 900", start: 100000, end: 500000, expected: []*storage.SelectHints{ - {Start: 45001, End: 50000, Step: 1000}, - {Start: 95001, End: 500000, Step: 1000}, - {Start: 895001, End: 900000, Step: 1000}, + {Start: 45001, End: 50000, Step: 1000, LookbackDelta: lbdMs}, + {Start: 95001, End: 500000, Step: 1000, LookbackDelta: lbdMs}, + {Start: 895001, End: 900000, Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2s] @ 50) + bar @ 250 + baz @ 900", start: 100000, end: 500000, expected: []*storage.SelectHints{ - {Start: 48001, End: 50000, Step: 1000, Func: "rate", Range: 2000}, - {Start: 245001, End: 250000, Step: 1000}, - {Start: 895001, End: 900000, Step: 1000}, + {Start: 48001, End: 50000, Step: 1000, Func: "rate", Range: 2000, LookbackDelta: lbdMs}, + {Start: 245001, End: 250000, Step: 1000, LookbackDelta: lbdMs}, + {Start: 895001, End: 900000, Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2s:1s] @ 50) + bar + baz", start: 100000, end: 500000, expected: []*storage.SelectHints{ - {Start: 43001, End: 50000, Step: 1000, Func: "rate"}, - {Start: 95001, End: 500000, Step: 1000}, - {Start: 95001, End: 500000, Step: 1000}, + {Start: 43001, End: 50000, Step: 1000, Func: "rate", LookbackDelta: lbdMs, IsSubquery: true}, + {Start: 95001, End: 500000, Step: 1000, LookbackDelta: lbdMs}, + {Start: 95001, End: 500000, Step: 1000, LookbackDelta: lbdMs}, }, }, { query: "rate(foo[2s:1s] @ 50) + bar + rate(baz[2m:1s] @ 900 offset 2m) ", start: 100000, end: 500000, expected: []*storage.SelectHints{ - {Start: 43001, End: 50000, Step: 1000, Func: "rate"}, - {Start: 95001, End: 500000, Step: 1000}, - {Start: 655001, End: 780000, Step: 1000, Func: "rate"}, + {Start: 43001, End: 50000, Step: 1000, Func: "rate", LookbackDelta: lbdMs, IsSubquery: true}, + {Start: 95001, End: 500000, Step: 1000, LookbackDelta: lbdMs}, + {Start: 655001, End: 780000, Step: 1000, Func: "rate", LookbackDelta: lbdMs, IsSubquery: true}, }, }, { // Hints are based on the inner most subquery timestamp. query: `sum_over_time(sum_over_time(metric{job="1"}[100s])[100s:25s] @ 50)[3s:1s] @ 3000`, start: 100000, expected: []*storage.SelectHints{ - {Start: -149999, End: 50000, Range: 100000, Func: "sum_over_time", Step: 25000}, + {Start: -149999, End: 50000, Range: 100000, Func: "sum_over_time", Step: 25000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, { // Hints are based on the inner most subquery timestamp. query: `sum_over_time(sum_over_time(metric{job="1"}[100s])[100s:25s] @ 3000)[3s:1s] @ 50`, expected: []*storage.SelectHints{ - {Start: 2800001, End: 3000000, Range: 100000, Func: "sum_over_time", Step: 25000}, + {Start: 2800001, End: 3000000, Range: 100000, Func: "sum_over_time", Step: 25000, LookbackDelta: lbdMs, IsSubquery: true}, }, }, } { diff --git a/storage/interface.go b/storage/interface.go index 2f125e5902..3a9f5279ef 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -197,8 +197,8 @@ type SelectHints struct { Func string // String representation of surrounding function or aggregation. Grouping []string // List of label names used in aggregation. - By bool // Indicate whether it is without or by. - Range int64 // Range vector selector range in milliseconds. + + Range int64 // Range vector selector range in milliseconds. // ShardCount is the total number of shards that series should be split into // at query time. Then, only series in the ShardIndex shard will be returned @@ -214,10 +214,19 @@ type SelectHints struct { // Series are sharded by "labels stable hash" mod "ShardCount". ShardIndex uint64 + // LookbackDelta duration for the query in milliseconds. + LookbackDelta int64 + // DisableTrimming allows to disable trimming of matching series chunks based on query Start and End time. // When disabled, the result may contain samples outside the queried time range but Select() performances // may be improved. DisableTrimming bool + + By bool // Indicate whether it is without or by. + + // IsSubquery indicates whether the query is a subquery. + // For subquery, we need to disable the optimization of the query + IsSubquery bool } // LabelHints specifies hints passed for label reads.