From eaa19f24f4e97bd00504b796bdd186cbc0aff8c9 Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:09:43 +0100 Subject: [PATCH 1/7] Add usage_billable to BaseRequest --- inference/core/entities/requests/inference.py | 1 + 1 file changed, 1 insertion(+) diff --git a/inference/core/entities/requests/inference.py b/inference/core/entities/requests/inference.py index 3a2107617..312a3baab 100644 --- a/inference/core/entities/requests/inference.py +++ b/inference/core/entities/requests/inference.py @@ -22,6 +22,7 @@ def __init__(self, **kwargs): model_config = ConfigDict(protected_namespaces=()) id: str api_key: Optional[str] = ApiKey + usage_billable: bool = True start: Optional[float] = None source: Optional[str] = None source_info: Optional[str] = None From ae48ec4a9791df1992fa0069b447b8d585b31505 Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:28:44 +0100 Subject: [PATCH 2/7] 0.26.1 --- inference/core/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/core/version.py b/inference/core/version.py index c14735062..4a6e856f0 100644 --- a/inference/core/version.py +++ b/inference/core/version.py @@ -1,4 +1,4 @@ -__version__ = "0.26.0" +__version__ = "0.26.1" if __name__ == "__main__": From 48a95262eb0fd0a83cb4759e981549874db8b0c7 Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:29:02 +0100 Subject: [PATCH 3/7] Handle malformed usage_fps --- inference/usage_tracking/collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/usage_tracking/collector.py b/inference/usage_tracking/collector.py index 5f5ff3e3e..d58036edd 100644 --- a/inference/usage_tracking/collector.py +++ b/inference/usage_tracking/collector.py @@ -616,7 +616,7 @@ def _extract_usage_params_from_func_kwargs( "resource_details": resource_details, "resource_id": resource_id, "inference_test_run": usage_inference_test_run, - "fps": usage_fps, + "fps": 0 if usage_fps is None else usage_fps, } def __call__(self, func: Callable[P, T]) -> Callable[P, T]: From a9f4ee894305313c368785b08324f599482e75f0 Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:52:48 +0100 Subject: [PATCH 4/7] Add test covering malformed usage payload --- inference/core/interfaces/http/http_api.py | 1 + inference/usage_tracking/collector.py | 12 +++++--- .../usage_tracking/test_collector.py | 28 +++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/inference/core/interfaces/http/http_api.py b/inference/core/interfaces/http/http_api.py index 04d0fd089..00b8b85e3 100644 --- a/inference/core/interfaces/http/http_api.py +++ b/inference/core/interfaces/http/http_api.py @@ -2270,6 +2270,7 @@ async def legacy_infer_from_request( raise MissingServiceSecretError( "Service secret is required to disable inference usage tracking" ) + logger.info("Not counting inference for usage") else: request_model_id = model_id logger.debug( diff --git a/inference/usage_tracking/collector.py b/inference/usage_tracking/collector.py index d58036edd..2c587c157 100644 --- a/inference/usage_tracking/collector.py +++ b/inference/usage_tracking/collector.py @@ -2,6 +2,7 @@ import atexit import json import mimetypes +import numbers import socket import sys import time @@ -315,6 +316,7 @@ def _update_usage_payload( fps: float = 0, ): source = str(source) if source else "" + frames = frames if isinstance(frames, numbers.Number) else 0 api_key_hash = self._calculate_api_key_hash(api_key=api_key) if not resource_id and resource_details: resource_id = UsageCollector._calculate_resource_hash(resource_details) @@ -332,7 +334,9 @@ def _update_usage_payload( source_usage["timestamp_start"] = time.time_ns() source_usage["timestamp_stop"] = time.time_ns() source_usage["processed_frames"] += frames if not inference_test_run else 0 - source_usage["fps"] = round(fps, 2) + source_usage["fps"] = ( + round(fps, 2) if isinstance(fps, numbers.Number) else 0 + ) source_usage["source_duration"] += ( frames / fps if fps and not inference_test_run else 0 ) @@ -355,7 +359,7 @@ def record_usage( resource_id: str = "", inference_test_run: bool = False, fps: float = 0, - ) -> DefaultDict[str, Any]: + ): if not api_key: return if self._settings.opt_out and not api_key: @@ -388,7 +392,7 @@ async def async_record_usage( resource_id: str = "", inference_test_run: bool = False, fps: float = 0, - ) -> DefaultDict[str, Any]: + ): if self._async_lock: async with self._async_lock: self.record_usage( @@ -616,7 +620,7 @@ def _extract_usage_params_from_func_kwargs( "resource_details": resource_details, "resource_id": resource_id, "inference_test_run": usage_inference_test_run, - "fps": 0 if usage_fps is None else usage_fps, + "fps": usage_fps, } def __call__(self, func: Callable[P, T]) -> Callable[P, T]: diff --git a/tests/inference/unit_tests/usage_tracking/test_collector.py b/tests/inference/unit_tests/usage_tracking/test_collector.py index 96f7aaa7f..bbbc027cb 100644 --- a/tests/inference/unit_tests/usage_tracking/test_collector.py +++ b/tests/inference/unit_tests/usage_tracking/test_collector.py @@ -855,3 +855,31 @@ def test_system_info_with_no_dedicated_deployment_id(): } for k, v in expected_system_info.items(): assert system_info[k] == v + + +def test_record_malformed_usage(): + # given + collector = UsageCollector() + + # when + collector.record_usage( + source=None, + category="model", + frames=None, + api_key="fake", + resource_details=None, + resource_id=None, + inference_test_run=None, + fps=None, + ) + + # then + assert "fake" in collector._usage + assert "model:None" in collector._usage["fake"] + assert collector._usage["fake"]["model:None"]["processed_frames"] == 0 + assert collector._usage["fake"]["model:None"]["fps"] == 0 + assert collector._usage["fake"]["model:None"]["source_duration"] == 0 + assert collector._usage["fake"]["model:None"]["category"] == "model" + assert collector._usage["fake"]["model:None"]["resource_id"] == None + assert collector._usage["fake"]["model:None"]["resource_details"] == "{}" + assert collector._usage["fake"]["model:None"]["api_key_hash"] == "fake" From b4ed9a363cd6f7dd98d59afa39a20f3ccc65071d Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:22:42 +0100 Subject: [PATCH 5/7] Extend line_coutner block outputs --- .../workflows/core_steps/analytics/line_counter/v2.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/inference/core/workflows/core_steps/analytics/line_counter/v2.py b/inference/core/workflows/core_steps/analytics/line_counter/v2.py index b9657842c..fa25094cb 100644 --- a/inference/core/workflows/core_steps/analytics/line_counter/v2.py +++ b/inference/core/workflows/core_steps/analytics/line_counter/v2.py @@ -26,6 +26,8 @@ OUTPUT_KEY_COUNT_IN: str = "count_in" OUTPUT_KEY_COUNT_OUT: str = "count_out" +OUTPUT_KEY_DETECTIONS_IN: str = "detections_in" +OUTPUT_KEY_DETECTIONS_OUT: str = "detections_out" IN: str = "in" OUT: str = "out" DETECTIONS_IN_OUT_PARAM: str = "in_out" @@ -136,9 +138,13 @@ def run( ) line_zone = self._batch_of_line_zones[metadata.video_identifier] - line_zone.trigger(detections=detections) + mask_in, mask_out = line_zone.trigger(detections=detections) + detections_in = detections[mask_in] + detections_out = detections[mask_out] return { OUTPUT_KEY_COUNT_IN: line_zone.in_count, OUTPUT_KEY_COUNT_OUT: line_zone.out_count, + OUTPUT_KEY_DETECTIONS_IN: detections_in, + OUTPUT_KEY_DETECTIONS_OUT: detections_out, } From 1df3383999ecc6f32465fc4dcf32ef842632ca74 Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:25:09 +0100 Subject: [PATCH 6/7] Outputs in manifest --- .../core_steps/analytics/line_counter/v2.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/inference/core/workflows/core_steps/analytics/line_counter/v2.py b/inference/core/workflows/core_steps/analytics/line_counter/v2.py index fa25094cb..978dc8017 100644 --- a/inference/core/workflows/core_steps/analytics/line_counter/v2.py +++ b/inference/core/workflows/core_steps/analytics/line_counter/v2.py @@ -88,6 +88,20 @@ def describe_outputs(cls) -> List[OutputDefinition]: name=OUTPUT_KEY_COUNT_OUT, kind=[INTEGER_KIND], ), + OutputDefinition( + name=OUTPUT_KEY_DETECTIONS_IN, + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + ], + ), + OutputDefinition( + name=OUTPUT_KEY_DETECTIONS_OUT, + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + ], + ), ] @classmethod From 75c88c1820a0867d8f3ff883f34a2337d5a7c150 Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:42:19 +0100 Subject: [PATCH 7/7] fix tests --- .../unit_tests/core_steps/analytics/test_line_counter_v2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/workflows/unit_tests/core_steps/analytics/test_line_counter_v2.py b/tests/workflows/unit_tests/core_steps/analytics/test_line_counter_v2.py index b9f7d4396..0ff8ddfb6 100644 --- a/tests/workflows/unit_tests/core_steps/analytics/test_line_counter_v2.py +++ b/tests/workflows/unit_tests/core_steps/analytics/test_line_counter_v2.py @@ -63,8 +63,8 @@ def test_line_counter() -> None: ) # then - assert frame1_result == {"count_in": 0, "count_out": 0} - assert frame2_result == {"count_in": 1, "count_out": 1} + assert frame1_result == {"count_in": 0, "count_out": 0, "detections_in": frame1_detections[[False, False, False, False]], "detections_out": frame1_detections[[False, False, False, False]]} + assert frame2_result == {"count_in": 1, "count_out": 1, "detections_in": frame2_detections[[True, False, False, False]], "detections_out": frame2_detections[[False, True, False, False]]} def test_line_counter_no_trackers() -> None: