Merge pull request #702 from Path-of-Modifiers/701-remove-all-items-a…

…bove-reasonable-prices

#701 Added filtering for too high priced items

src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py

@@ -72,6 +72,27 @@ def _create_item_table(
         item_df["createdHoursSinceLaunch"] = hours_since_launch
         return item_df
 
+    def _find_not_too_highly_priced_item_mask(
+        self, currency_df: pd.DataFrame, item_currency_merged_df: pd.DataFrame
+    ) -> pd.Series:
+        """
+        Some items are too highly priced to be legitimate. A boundary of the price equivelant to
+        >= 10 mirrors are seen to be too high priced.
+
+        1. Find currency type and amount equiveland in chaos
+        """
+
+        mirror_row = currency_df.loc[currency_df["tradeName"] == "mirror"]
+        mirror_value = mirror_row["valueInChaos"].get(0)
+
+        currency_too_high_mask = (
+            item_currency_merged_df["valueInChaos"]
+            * item_currency_merged_df["currencyAmount"].astype("float")
+            > mirror_value * 10
+        )
+
+        return ~currency_too_high_mask
+
     @sync_timing_tracker
     def _transform_item_table(
         self,
@@ -156,10 +177,19 @@ def transform_influences(row: pd.DataFrame, influence_columns: list[str]):
             suffixes=(None, "_y"),
         )
 
-        self.price_found_mask = ~item_df["tradeName"].isna()
+        price_found_mask = ~item_df["tradeName"].isna()
+
+        self.price_found_mask = price_found_mask
 
         item_df = item_df.loc[self.price_found_mask]
 
+        not_too_high_priced_item_mask = self._find_not_too_highly_priced_item_mask(
+            currency_df, item_df
+        )
+        self.items_not_too_high_priced_mask = not_too_high_priced_item_mask
+
+        item_df = item_df.loc[self.items_not_too_high_priced_mask]
+
         return item_df
 
     @property
@@ -269,7 +299,6 @@ def _create_item_modifier_table(
     def _transform_item_modifier_table(
         self,
         item_modifier_df: pd.DataFrame,
-        hours_since_launch: int,
     ) -> pd.DataFrame:
         """
         The `item_modifier` table heavily relies on what type of item the modifiers
@@ -348,9 +377,13 @@ def _create_item_modifier_table(
         relevant column contains a list and not a JSON-object
         """
         item_modifier_columns = ["name", "explicitMods"]
-
         item_modifier_df = df.loc[
-            self.price_found_mask, item_modifier_columns
+            self.price_found_mask,
+            item_modifier_columns,
+        ]
+
+        item_modifier_df = item_modifier_df.loc[
+            self.items_not_too_high_priced_mask
         ].reset_index()
 
         item_modifier_df["itemId"] = item_id

src/backend_data_retrieval/data_retrieval_app/tests/scripts/create_public_stashes_test_data/config.py

@@ -2,7 +2,7 @@
 
 
 class PoEPublicStashesTestDataSettings(BaseSettings):
-    N_OF_ITEMS_PER_MODIFIER_FILE: int = 20
+    N_OF_ITEMS_PER_MODIFIER_FILE: int = 10
     MODIFIER_CSV_FILES_TO_ITERATE: list[
         str
     ] = []  # For instance: ["AulsUprising.csv", "ThreadOfHope.csv"]
@@ -22,9 +22,8 @@ def dispersed_timing_enabled(self) -> bool:
 
     ITEM_NOTE_CURRENCY_TYPES: list[str] = [
         "chaos",
-        "divine",
-    ]  # "chaos", "divine", "mirror", etc.
-    MEAN_ITEM_PRICE: int = 30
+    ]  # , "divine", "mirror", etc.
+    MEAN_ITEM_PRICE: int = 200
 
 
 script_settings = PoEPublicStashesTestDataSettings()

src/backend_data_retrieval/data_retrieval_app/tests/scripts/create_public_stashes_test_data/utils/scrap_and_mock_poe_api_docs_objs.py

@@ -78,7 +78,7 @@ def _scrape_html_to_schema(
 
     def _check_obj_key_value_type(self, obj: dict, key: str, value_type: Any):
         if "id" == key:
-            obj[key] = str(uuid.uuid4())
+            obj[key] = str(uuid.uuid4()).replace("-", "")
         elif "league" == key:
             obj[key] = settings.CURRENT_SOFTCORE_LEAGUE
         elif "string" == value_type:

src/backend_data_retrieval/data_retrieval_app/tests/scripts/test_with_sim_api.py

@@ -58,7 +58,12 @@ def _create_item_modifier_table(
             item_modifier_columns.append("createdHoursSinceLaunch")
 
         item_modifier_df = df.loc[
-            self.price_found_mask, item_modifier_columns
+            self.price_found_mask,
+            item_modifier_columns,
+        ]
+
+        item_modifier_df = item_modifier_df.loc[
+            self.items_not_too_high_priced_mask
         ].reset_index()
 
         item_modifier_df["itemId"] = item_id