From 1e6a3c9b0c9f0b3ccebee10143ca959e5a8da12f Mon Sep 17 00:00:00 2001 From: Kevin Nobel Date: Fri, 7 Nov 2025 15:35:01 +0000 Subject: [PATCH 1/2] MUL-311: Remove `de-accumulate` operation Replaced by difference operation --- .../action/statistics-mtg2/CMakeLists.txt | 1 - .../action/statistics-mtg2/Operations.h | 8 --- .../statistics-mtg2/operations/DeAccumulate.h | 60 ------------------- .../statistics-mtg2/operations/CMakeLists.txt | 7 --- .../operations/DeAccumulate.cc | 36 ----------- 5 files changed, 112 deletions(-) delete mode 100644 src/multio/action/statistics-mtg2/operations/DeAccumulate.h delete mode 100644 tests/multio/action/statistics-mtg2/operations/DeAccumulate.cc diff --git a/src/multio/action/statistics-mtg2/CMakeLists.txt b/src/multio/action/statistics-mtg2/CMakeLists.txt index 3dcfcf434..aa6957cd3 100644 --- a/src/multio/action/statistics-mtg2/CMakeLists.txt +++ b/src/multio/action/statistics-mtg2/CMakeLists.txt @@ -31,7 +31,6 @@ list( APPEND _statistics_sources operations/Instant.h operations/Minimum.h operations/Maximum.h - operations/DeAccumulate.h operations/FixedWindowFluxAverage.h operations/StdDev.h mappings/StatisticsParamMapping.cc diff --git a/src/multio/action/statistics-mtg2/Operations.h b/src/multio/action/statistics-mtg2/Operations.h index 9d9fc2899..a7314bd34 100644 --- a/src/multio/action/statistics-mtg2/Operations.h +++ b/src/multio/action/statistics-mtg2/Operations.h @@ -28,7 +28,6 @@ #include "multio/action/statistics-mtg2/operations/Minimum.h" #include "multio/action/statistics-mtg2/operations/StdDev.h" -#include "multio/action/statistics-mtg2/operations/DeAccumulate.h" #include "multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.h" namespace multio::action::statistics_mtg2 { @@ -61,9 +60,6 @@ std::unique_ptr make_operation(const std::string& opname, long sz, st if (opname == "accumulate") { return std::make_unique>(opname, sz, win, cfg); } - if (opname == "de-accumulate") { - return std::make_unique>(opname, sz, win, cfg); - } if (opname == "fixed-window-flux-average") { return std::make_unique>(opname, sz, win, cfg); } @@ -116,10 +112,6 @@ std::unique_ptr load_operation(const std::string& opname, std::shared found = true; ret = std::make_unique>(opname, win, IOmanager, opt); } - if (opname == "de-accumulate") { - found = true; - ret = std::make_unique>(opname, win, IOmanager, opt); - } if (opname == "fixed-window-flux-average") { found = true; ret = std::make_unique>(opname, win, IOmanager, opt); diff --git a/src/multio/action/statistics-mtg2/operations/DeAccumulate.h b/src/multio/action/statistics-mtg2/operations/DeAccumulate.h deleted file mode 100644 index 21b08b4a5..000000000 --- a/src/multio/action/statistics-mtg2/operations/DeAccumulate.h +++ /dev/null @@ -1,60 +0,0 @@ - -#pragma once - -#include "multio/LibMultio.h" -#include "multio/action/statistics-mtg2/operations/OperationWithDeaccumulatedData.h" - -namespace multio::action::statistics_mtg2 { - -template >> -class DeAccumulate final : public OperationWithDeaccumulatedData { -public: - using OperationWithDeaccumulatedData::name_; - using OperationWithDeaccumulatedData::logHeader_; - using OperationWithDeaccumulatedData::initValues_; - using OperationWithDeaccumulatedData::values_; - using OperationWithDeaccumulatedData::win_; - using OperationWithDeaccumulatedData::checkSize; - - - DeAccumulate(const std::string& name, long sz, const OperationWindow& win, const StatisticsConfiguration& cfg) : - OperationWithDeaccumulatedData{name, "accumulate", sz, true, win, cfg} {} - - DeAccumulate(const std::string& name, const OperationWindow& win, std::shared_ptr& IOmanager, - const StatisticsOptions& opt) : - OperationWithDeaccumulatedData{name, "accumulate", true, win, IOmanager, opt} {}; - - void compute(eckit::Buffer& buf, const StatisticsConfiguration& cfg) override { - LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".compute().count=" << win_.count() << std::endl; - auto val = static_cast(buf.data()); - cfg.bitmapPresent() ? computeWithMissing(val, cfg) : computeWithoutMissing(val, cfg); - std::copy(values_.begin(), values_.end(), initValues_.begin()); - return; - } - - void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz, cfg); - LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; - const T* val = static_cast(data); - std::copy(val, val + (sz / sizeof(T)), values_.begin()); - return; - } - -private: - void computeWithoutMissing(T* val, const StatisticsConfiguration& cfg) { - std::transform(values_.begin(), values_.end(), initValues_.begin(), val, - [](T v1, T v2) { return static_cast(v1 - v2); }); - return; - } - - void computeWithMissing(T* val, const StatisticsConfiguration& cfg) { - double m = cfg.missingValue(); - std::transform(values_.begin(), values_.end(), initValues_.begin(), val, - [m](T v1, T v2) { return static_cast(m == v1 || m == v2 ? m : v1 - v2); }); - return; - } - - void print(std::ostream& os) const override { os << logHeader_; } -}; - -} // namespace multio::action::statistics_mtg2 diff --git a/tests/multio/action/statistics-mtg2/operations/CMakeLists.txt b/tests/multio/action/statistics-mtg2/operations/CMakeLists.txt index 1835e6ed9..5e2f40542 100644 --- a/tests/multio/action/statistics-mtg2/operations/CMakeLists.txt +++ b/tests/multio/action/statistics-mtg2/operations/CMakeLists.txt @@ -43,13 +43,6 @@ ecbuild_add_test( LIBS multio-action-debug-sink multio-action-statistics-mtg2 ) -ecbuild_add_test( - TARGET ${PREFIX}_deaccumulate - SOURCES DeAccumulate.cc - NO_AS_NEEDED - LIBS multio-action-debug-sink multio-action-statistics-mtg2 -) - ecbuild_add_test( TARGET ${PREFIX}_flux_average SOURCES FluxAverage.cc diff --git a/tests/multio/action/statistics-mtg2/operations/DeAccumulate.cc b/tests/multio/action/statistics-mtg2/operations/DeAccumulate.cc deleted file mode 100644 index 4c421e35d..000000000 --- a/tests/multio/action/statistics-mtg2/operations/DeAccumulate.cc +++ /dev/null @@ -1,36 +0,0 @@ - -#include "eckit/testing/Test.h" - -#include "Operation.h" - -namespace multio::test::statistics_mtg2 { - -template -class DeAccumulateTest : public StatisticsOperationTest { -public: - using typename StatisticsOperationTest::SinglePointOverTime; - - DeAccumulateTest() : StatisticsOperationTest("de-accumulate") {} - - ElemType reference(const SinglePointOverTime &input, const ElemType init) override { - EXPECT_NOT_EQUAL(input.size(), 0); - return input[input.size()-1] - init; - } - -}; - -auto testFloat = DeAccumulateTest(); -auto testDouble = DeAccumulateTest(); - -CASE("single test float") { testFloat.runSingle(); } -CASE("single test double") { testDouble.runSingle(); } -CASE("multiple unaligned test float") { testFloat.runMultipleUnaligned(); } -CASE("multiple unaligned test double") { testDouble.runMultipleUnaligned(); } -CASE("multiple aligned test float") { testFloat.runMultipleAligned(); } -CASE("multiple aligned test double") { testDouble.runMultipleAligned(); } - -} // multio::test::statistics_mtg2 - -int main(int argc, char** argv) { - return eckit::testing::run_tests(argc, argv); -} From 30a13998c071141ed94ae5311d13e6c0d4b2ab48 Mon Sep 17 00:00:00 2001 From: Kevin Nobel Date: Fri, 7 Nov 2025 15:41:47 +0000 Subject: [PATCH 2/2] MUL-311: Remove `flux-average` and `fixed-window-flux-average` operations Replaced by average-rate action --- .../action/statistics-mtg2/CMakeLists.txt | 2 - .../action/statistics-mtg2/Operations.h | 17 ----- .../operations/FixedWindowFluxAverage.h | 62 ------------------- .../statistics-mtg2/operations/FluxAverage.h | 57 ----------------- .../statistics-mtg2/operations/CMakeLists.txt | 14 ----- .../operations/FixedWindowFluxAverage.cc | 38 ------------ .../statistics-mtg2/operations/FluxAverage.cc | 38 ------------ 7 files changed, 228 deletions(-) delete mode 100644 src/multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.h delete mode 100644 src/multio/action/statistics-mtg2/operations/FluxAverage.h delete mode 100644 tests/multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.cc delete mode 100644 tests/multio/action/statistics-mtg2/operations/FluxAverage.cc diff --git a/src/multio/action/statistics-mtg2/CMakeLists.txt b/src/multio/action/statistics-mtg2/CMakeLists.txt index aa6957cd3..88d5de53b 100644 --- a/src/multio/action/statistics-mtg2/CMakeLists.txt +++ b/src/multio/action/statistics-mtg2/CMakeLists.txt @@ -27,11 +27,9 @@ list( APPEND _statistics_sources operations/Average.h operations/Difference.h operations/InverseDifference.h - operations/FluxAverage.h operations/Instant.h operations/Minimum.h operations/Maximum.h - operations/FixedWindowFluxAverage.h operations/StdDev.h mappings/StatisticsParamMapping.cc mappings/StatisticsParamMapping.h diff --git a/src/multio/action/statistics-mtg2/Operations.h b/src/multio/action/statistics-mtg2/Operations.h index a7314bd34..98f132f32 100644 --- a/src/multio/action/statistics-mtg2/Operations.h +++ b/src/multio/action/statistics-mtg2/Operations.h @@ -22,14 +22,11 @@ #include "multio/action/statistics-mtg2/operations/Average.h" #include "multio/action/statistics-mtg2/operations/Difference.h" #include "multio/action/statistics-mtg2/operations/InverseDifference.h" -#include "multio/action/statistics-mtg2/operations/FluxAverage.h" #include "multio/action/statistics-mtg2/operations/Instant.h" #include "multio/action/statistics-mtg2/operations/Maximum.h" #include "multio/action/statistics-mtg2/operations/Minimum.h" #include "multio/action/statistics-mtg2/operations/StdDev.h" -#include "multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.h" - namespace multio::action::statistics_mtg2 { template @@ -48,9 +45,6 @@ std::unique_ptr make_operation(const std::string& opname, long sz, st if (opname == "inverse-difference") { return std::make_unique>(opname, sz, win, cfg); } - if (opname == "flux-average") { - return std::make_unique>(opname, sz, win, cfg); - } if (opname == "minimum") { return std::make_unique>(opname, sz, win, cfg); } @@ -60,9 +54,6 @@ std::unique_ptr make_operation(const std::string& opname, long sz, st if (opname == "accumulate") { return std::make_unique>(opname, sz, win, cfg); } - if (opname == "fixed-window-flux-average") { - return std::make_unique>(opname, sz, win, cfg); - } if (opname == "stddev") { return std::make_unique>(opname, sz, win, cfg); } @@ -96,10 +87,6 @@ std::unique_ptr load_operation(const std::string& opname, std::shared found = true; ret = std::make_unique>(opname, win, IOmanager, opt); } - if (opname == "flux-average") { - found = true; - ret = std::make_unique>(opname, win, IOmanager, opt); - } if (opname == "minimum") { found = true; ret = std::make_unique>(opname, win, IOmanager, opt); @@ -112,10 +99,6 @@ std::unique_ptr load_operation(const std::string& opname, std::shared found = true; ret = std::make_unique>(opname, win, IOmanager, opt); } - if (opname == "fixed-window-flux-average") { - found = true; - ret = std::make_unique>(opname, win, IOmanager, opt); - } if (opname == "stddev") { throw eckit::NotImplemented(Here()); } diff --git a/src/multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.h b/src/multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.h deleted file mode 100644 index 2d03ab656..000000000 --- a/src/multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.h +++ /dev/null @@ -1,62 +0,0 @@ - -#pragma once - -#include "multio/LibMultio.h" -#include "multio/action/statistics-mtg2/operations/OperationWithDeaccumulatedData.h" - -namespace multio::action::statistics_mtg2 { - -template >> -class FixedWindowFluxAverage final : public OperationWithDeaccumulatedData { -public: - using OperationWithDeaccumulatedData::name_; - using OperationWithDeaccumulatedData::logHeader_; - using OperationWithDeaccumulatedData::initValues_; - using OperationWithDeaccumulatedData::values_; - using OperationWithDeaccumulatedData::win_; - using OperationWithDeaccumulatedData::checkSize; - - FixedWindowFluxAverage(const std::string& name, long sz, const OperationWindow& win, - const StatisticsConfiguration& cfg) : - OperationWithDeaccumulatedData{name, "average", sz, true, win, cfg} {} - - FixedWindowFluxAverage(const std::string& name, const OperationWindow& win, - std::shared_ptr& IOmanager, const StatisticsOptions& opt) : - OperationWithDeaccumulatedData{name, "average", true, win, IOmanager, opt} {}; - - void compute(eckit::Buffer& buf, const StatisticsConfiguration& cfg) override { - LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".compute().count=" << win_.count() << std::endl; - auto val = static_cast(buf.data()); - cfg.bitmapPresent() ? computeWithMissing(val, cfg) : computeWithoutMissing(val, cfg); - std::copy(values_.begin(), values_.end(), initValues_.begin()); - return; - } - - void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz, cfg); - LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; - const T* val = static_cast(data); - std::copy(val, val + (sz / sizeof(T)), values_.begin()); - return; - } - -private: - void computeWithMissing(T* buf, const StatisticsConfiguration& cfg) { - const double m = cfg.missingValue(); - const double c = static_cast(1.0) / static_cast(win_.timeSpanInSeconds()); - std::transform(values_.begin(), values_.end(), initValues_.begin(), buf, - [c, m](T v1, T v2) { return static_cast(m == v1 || m == v2 ? m : (v1 - v2) * c); }); - return; - } - - void computeWithoutMissing(T* buf, const StatisticsConfiguration& cfg) { - const double c = static_cast(1.0) / static_cast(win_.currPointInSeconds() - win_.creationPointInSeconds()); - std::transform(values_.begin(), values_.end(), initValues_.begin(), buf, - [c](T v1, T v2) { return static_cast((v1 - v2) * c); }); - return; - } - - void print(std::ostream& os) const override { os << logHeader_; } -}; - -} // namespace multio::action::statistics_mtg2 diff --git a/src/multio/action/statistics-mtg2/operations/FluxAverage.h b/src/multio/action/statistics-mtg2/operations/FluxAverage.h deleted file mode 100644 index b23fe87ef..000000000 --- a/src/multio/action/statistics-mtg2/operations/FluxAverage.h +++ /dev/null @@ -1,57 +0,0 @@ - -#pragma once - -#include "multio/LibMultio.h" -#include "multio/action/statistics-mtg2/operations/OperationWithData.h" - -namespace multio::action::statistics_mtg2 { - -template >> -class FluxAverage final : public OperationWithData { -public: - using OperationWithData::name_; - using OperationWithData::logHeader_; - using OperationWithData::values_; - using OperationWithData::win_; - using OperationWithData::checkSize; - - FluxAverage(const std::string& name, long sz, const OperationWindow& win, const StatisticsConfiguration& cfg) : - OperationWithData{name, "average", sz, true, win, cfg} {} - - FluxAverage(const std::string& name, const OperationWindow& win, std::shared_ptr& IOmanager, - const StatisticsOptions& opt) : - OperationWithData{name, "average", true, win, IOmanager, opt} {}; - - void compute(eckit::Buffer& buf, const StatisticsConfiguration& cfg) override { - LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".compute().count=" << win_.count() << std::endl; - auto val = static_cast(buf.data()); - cfg.bitmapPresent() ? computeWithMissing(val, cfg) : computeWithoutMissing(val, cfg); - return; - } - - void updateData(const void* data, long sz, const StatisticsConfiguration& cfg) override { - checkSize(sz, cfg); - LOG_DEBUG_LIB(LibMultio) << logHeader_ << ".update().count=" << win_.count() << std::endl; - const T* val = static_cast(data); - std::copy(val, val + (sz / sizeof(T)), values_.begin()); - return; - } - -private: - void computeWithMissing(T* buf, const StatisticsConfiguration& cfg) { - const double m = cfg.missingValue(); - const double c = static_cast(1.0) / static_cast(win_.timeSpanInSeconds()); - std::transform(values_.begin(), values_.end(), buf, [c, m](T v) { return static_cast(m == v ? m : v * c); }); - return; - } - - void computeWithoutMissing(T* buf, const StatisticsConfiguration& cfg) { - const double c = static_cast(1.0) / static_cast(win_.currPointInSeconds() - win_.creationPointInSeconds()); - std::transform(values_.begin(), values_.end(), buf, [c](T v) { return static_cast(v * c); }); - return; - } - - void print(std::ostream& os) const override { os << logHeader_; } -}; - -} // namespace multio::action::statistics_mtg2 diff --git a/tests/multio/action/statistics-mtg2/operations/CMakeLists.txt b/tests/multio/action/statistics-mtg2/operations/CMakeLists.txt index 5e2f40542..1bc5b65c9 100644 --- a/tests/multio/action/statistics-mtg2/operations/CMakeLists.txt +++ b/tests/multio/action/statistics-mtg2/operations/CMakeLists.txt @@ -43,20 +43,6 @@ ecbuild_add_test( LIBS multio-action-debug-sink multio-action-statistics-mtg2 ) -ecbuild_add_test( - TARGET ${PREFIX}_flux_average - SOURCES FluxAverage.cc - NO_AS_NEEDED - LIBS multio-action-debug-sink multio-action-statistics-mtg2 -) - -ecbuild_add_test( - TARGET ${PREFIX}_fixed_window_flux_average - SOURCES FixedWindowFluxAverage.cc - NO_AS_NEEDED - LIBS multio-action-debug-sink multio-action-statistics-mtg2 -) - ecbuild_add_test( TARGET ${PREFIX}_instant SOURCES Instant.cc diff --git a/tests/multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.cc b/tests/multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.cc deleted file mode 100644 index bd116dac3..000000000 --- a/tests/multio/action/statistics-mtg2/operations/FixedWindowFluxAverage.cc +++ /dev/null @@ -1,38 +0,0 @@ - -#include "eckit/testing/Test.h" - -#include "Operation.h" - -inline constexpr std::size_t SECONDS_IN_24_HOURS = 86400; - -namespace multio::test::statistics_mtg2 { - -template -class FixedWindowFluxAverageTest : public StatisticsOperationTest { -public: - using typename StatisticsOperationTest::SinglePointOverTime; - - FixedWindowFluxAverageTest() : StatisticsOperationTest("fixed-window-flux-average") {} - - ElemType reference(const SinglePointOverTime &input, const ElemType init) override { - EXPECT_NOT_EQUAL(input.size(), 0); - return (input[input.size() - 1] - init) / (input.size() * SECONDS_IN_24_HOURS); - } - -}; - -auto testFloat = FixedWindowFluxAverageTest(); -auto testDouble = FixedWindowFluxAverageTest(); - -CASE("single test float") { testFloat.runSingle(); } -CASE("single test double") { testDouble.runSingle(); } -CASE("multiple unaligned test float") { testFloat.runMultipleUnaligned(); } -CASE("multiple unaligned test double") { testDouble.runMultipleUnaligned(); } -CASE("multiple aligned test float") { testFloat.runMultipleAligned(); } -CASE("multiple aligned test double") { testDouble.runMultipleAligned(); } - -} // multio::test::statistics_mtg2 - -int main(int argc, char** argv) { - return eckit::testing::run_tests(argc, argv); -} diff --git a/tests/multio/action/statistics-mtg2/operations/FluxAverage.cc b/tests/multio/action/statistics-mtg2/operations/FluxAverage.cc deleted file mode 100644 index cb4020222..000000000 --- a/tests/multio/action/statistics-mtg2/operations/FluxAverage.cc +++ /dev/null @@ -1,38 +0,0 @@ - -#include "eckit/testing/Test.h" - -#include "Operation.h" - -inline constexpr std::size_t SECONDS_IN_24_HOURS = 86400; - -namespace multio::test::statistics_mtg2 { - -template -class FluxAverageTest : public StatisticsOperationTest { -public: - using typename StatisticsOperationTest::SinglePointOverTime; - - FluxAverageTest() : StatisticsOperationTest("flux-average") {} - - ElemType reference(const SinglePointOverTime &input, const ElemType init) override { - EXPECT_NOT_EQUAL(input.size(), 0); - return input[input.size() - 1] / (input.size() * SECONDS_IN_24_HOURS); - } - -}; - -auto testFloat = FluxAverageTest(); -auto testDouble = FluxAverageTest(); - -CASE("single test float") { testFloat.runSingle(); } -CASE("single test double") { testDouble.runSingle(); } -CASE("multiple unaligned test float") { testFloat.runMultipleUnaligned(); } -CASE("multiple unaligned test double") { testDouble.runMultipleUnaligned(); } -CASE("multiple aligned test float") { testFloat.runMultipleAligned(); } -CASE("multiple aligned test double") { testDouble.runMultipleAligned(); } - -} // multio::test::statistics_mtg2 - -int main(int argc, char** argv) { - return eckit::testing::run_tests(argc, argv); -}