Merge pull request #159 from openstates/import-no-dupe-checks-in-import

Stop checking for duplicates in imports

Author: jessemortenson

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 6.20.14 - Jan 27, 2025
+
+* Allow duplicate items to be imported in import via new runtime flag --allow_duplicates
+
 ## 6.20.13 - Dec 27, 2024
 
 * Sanitize phone number for US people scrape.

openstates/cli/update.py

@@ -264,11 +264,11 @@ def do_import(juris: State, args: argparse.Namespace) -> dict[str, typing.Any]:
         logger.info("import jurisdictions...")
         report.update(juris_importer.import_directory(datadir))
         logger.info("import bills...")
-        report.update(bill_importer.import_directory(datadir))
+        report.update(bill_importer.import_directory(datadir, allow_duplicates=args.allow_duplicates))
         logger.info("import vote events...")
-        report.update(vote_event_importer.import_directory(datadir))
+        report.update(vote_event_importer.import_directory(datadir, allow_duplicates=args.allow_duplicates))
         logger.info("import events...")
-        report.update(event_importer.import_directory(datadir))
+        report.update(event_importer.import_directory(datadir, allow_duplicates=args.allow_duplicates))
         DatabaseJurisdiction.objects.filter(id=juris.jurisdiction_id).update(
             latest_bill_update=datetime.datetime.utcnow()
         )
@@ -520,6 +520,12 @@ def parse_args() -> tuple[argparse.Namespace, list[str]]:
         dest="strict",
         help="skip validation on save",
     )
+    parser.add_argument(
+        "--allow_duplicates",
+        action="store_true",
+        dest="allow_duplicates",
+        help="Skip throwing a DuplicateItemError, instead all import of duplicate items",
+    )
     parser.add_argument(
         "--fastmode", action="store_true", help="use cache and turn off throttling"
     )

openstates/importers/base.py

@@ -273,7 +273,7 @@ def resolve_json_id(
         except KeyError:
             raise UnresolvedIdError("cannot resolve id: {}".format(json_id))
 
-    def import_directory(self, datadir: str) -> typing.Dict[str, typing.Dict]:
+    def import_directory(self, datadir: str, allow_duplicates=False) -> typing.Dict[str, typing.Dict]:
         """import a JSON directory into the database"""
 
         def json_stream() -> typing.Iterator[_JsonDict]:
@@ -282,7 +282,7 @@ def json_stream() -> typing.Iterator[_JsonDict]:
                 with open(fname) as f:
                     yield json.load(f)
 
-        return self.import_data(json_stream())
+        return self.import_data(json_stream(), allow_duplicates)
 
     def _prepare_imports(
         self, dicts: typing.Iterable[_JsonDict]
@@ -309,7 +309,7 @@ def _prepare_imports(
                 self.duplicates[json_id] = seen_hashes[objhash]
 
     def import_data(
-        self, data_items: typing.Iterable[_JsonDict]
+        self, data_items: typing.Iterable[_JsonDict], allow_duplicates=False
     ) -> typing.Dict[str, typing.Dict]:
         """import a bunch of dicts together"""
         # keep counts of all actions
@@ -322,7 +322,7 @@ def import_data(
         }
 
         for json_id, data in self._prepare_imports(data_items):
-            obj_id, what = self.import_item(data)
+            obj_id, what = self.import_item(data, allow_duplicates)
             if not obj_id or not what:
                 "Skipping data because it did not have an associated ID or type"
                 continue
@@ -341,7 +341,7 @@ def import_data(
 
         return {self._type: record}
 
-    def import_item(self, data: _JsonDict) -> typing.Tuple[_ID, str]:
+    def import_item(self, data: _JsonDict, allow_duplicates=False) -> typing.Tuple[_ID, str]:
         """function used by import_data"""
         what = "noop"
 
@@ -369,8 +369,12 @@ def import_item(self, data: _JsonDict) -> typing.Tuple[_ID, str]:
 
         # obj existed, check if we need to do an update
         if obj:
-            if obj.id in self.json_to_db_id.values():
+            # If --allow_duplicates flag is set on client CLI command
+            # then we ignore duplicates instead of raising an exception
+            if not allow_duplicates and obj.id in self.json_to_db_id.values():
                 raise DuplicateItemError(data, obj, related.get("sources", []))
+            elif allow_duplicates and obj.id in self.json_to_db_id.values():
+                self.logger.warning(f"Ignored a DuplicateItemError for {obj.id}")
             # check base object for changes
             for key, value in data.items():
                 if getattr(obj, key) != value:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "openstates"
-version = "6.20.13"
+version = "6.20.14"
 description = "core infrastructure for the openstates project"
 authors = ["James Turk <dev@jamesturk.net>"]
 license = "MIT"