24593 Tombstone pipeline - implement conversion (exclude liquidation, invol dissolution, cont out)

data-tool/flows/corps_tombstone_flow.py

@@ -397,6 +397,12 @@ def tombstone_flow():
                     )
                 else:
                     skipped += 1
+                    processing_service.update_corp_status(
+                        flow_run_id,
+                        corp_num,
+                        ProcessingStatuses.FAILED,
+                        error="Migration failed - Skip due to data collection error"
+                    )
                     print(f'❗ Skip migrating {corp_num} due to data collection error.')
 
             wait(corp_futures)

data-tool/flows/tombstone/tombstone_base_data.py

@@ -189,7 +189,8 @@
         'source': 'COLIN',
         'colin_only': False,
         'deletion_locked': False,
-        # TODO: new column - hide_in_ledger
+        'hide_in_ledger': False, # TODO: double check when doing cleanup - dissolution (invol, admin)
+        # TODO: new columns for NoW
         # FK
         'business_id': None,
         'transaction_id': None,

data-tool/flows/tombstone/tombstone_mappings.py

@@ -35,7 +35,6 @@ class EventFilings(str, Enum):
     FILE_AMLRC = 'FILE_AMLRC'
     FILE_AMLVC = 'FILE_AMLVC'
 
-    CONVAMAL_NULL = 'CONVAMAL_NULL'  # TODO: re-map
 
     # Annual Report
     FILE_ANNBC = 'FILE_ANNBC'
@@ -62,9 +61,25 @@ class EventFilings(str, Enum):
     FILE_CONTU = 'FILE_CONTU'
     FILE_CONTC = 'FILE_CONTC'
 
-    # Conversion
+    # Conversion Ledger
     FILE_CONVL = 'FILE_CONVL'
 
+    # Conversion
+    CONVAMAL_NULL = 'CONVAMAL_NULL'
+    CONVCIN_NULL = 'CONVCIN_NULL'
+    CONVCOUT_NULL = 'CONVCOUT_NULL'
+    CONVDS_NULL = 'CONVDS_NULL'
+    CONVDSF_NULL = 'CONVDSF_NULL'
+    CONVDSL_NULL = 'CONVDSL_NULL'
+    CONVDSO_NULL = 'CONVDSO_NULL'
+    CONVICORP_NULL = 'CONVICORP_NULL'
+    CONVID1_NULL = 'CONVID1_NULL'
+    CONVID2_NULL = 'CONVID2_NULL'
+    CONVILIQ_NULL = 'CONVILIQ_NULL'
+    CONVLRSTR_NULL = 'CONVLRSTR_NULL'
+    CONVNC_NULL = 'CONVNC_NULL'
+    CONVRSTR_NULL = 'CONVRSTR_NULL'
+
     # Correction
     FILE_CO_AR = 'FILE_CO_AR'
     FILE_CO_BC = 'FILE_CO_BC'
@@ -103,9 +118,6 @@ class EventFilings(str, Enum):
     FILE_ICORP = 'FILE_ICORP'
     FILE_ICORU = 'FILE_ICORU'
     FILE_ICORC = 'FILE_ICORC'
-    CONVICORP_NULL = 'CONVICORP_NULL'  # TODO: re-map
-
-    # TODO: Ledger - unsupported
 
     # TODO: Legacy Other - unsupported
     FILE_AM_PF = 'FILE_AM_PF'
@@ -115,7 +127,7 @@ class EventFilings(str, Enum):
     # TODO: Liquidation - unsupported
     # FILE_ADCOL = 'FILE_ADCOL'
 
-    # Notice of Withdrawal
+    # TODO: Notice of Withdrawal - unsupported
     FILE_NWITH = 'FILE_NWITH'
 
     # Registrar's Notation
@@ -140,7 +152,7 @@ class EventFilings(str, Enum):
 
     # TODO:
     # Other COLIN events:
-    # CONV*, Adim Corp (ADCORP, BNUPD, ADMIN), XPRO filing
+    # Adim Corp (ADCORP, BNUPD, ADMIN), XPRO filing
     # SYSDL, SYST
     # more legacyOther filings
 
@@ -175,7 +187,6 @@ def has_value(cls, value):
     EventFilings.FILE_AMLHC: ['amalgamationApplication', 'horizontal'],
     EventFilings.FILE_AMLRC: ['amalgamationApplication', 'regular'],
     EventFilings.FILE_AMLVC: ['amalgamationApplication', 'vertical'],
-    EventFilings.CONVAMAL_NULL: ['amalgamationApplication', 'regular'],  # TODO: re-map
 
     EventFilings.FILE_ANNBC: 'annualReport',
 
@@ -197,6 +208,21 @@ def has_value(cls, value):
 
     EventFilings.FILE_CONVL: 'conversionLedger',
 
+    EventFilings.CONVAMAL_NULL: ['conversion', ('amalgamationApplication', 'unknown')],
+    EventFilings.CONVCIN_NULL: ['conversion', 'continuationIn'],
+    EventFilings.CONVCOUT_NULL: ['conversion', 'continuationOut'],  # TODO: continuation out
+    EventFilings.CONVDS_NULL: ['conversion', ('dissolution', 'voluntary')],
+    EventFilings.CONVDSF_NULL: ['conversion', ('dissolution', 'involuntary')],
+    EventFilings.CONVDSL_NULL: 'conversion',  # TODO: liquidation
+    EventFilings.CONVDSO_NULL: ['conversion', ('dissolution', 'unknown')],
+    EventFilings.CONVICORP_NULL: 'conversion',
+    EventFilings.CONVID1_NULL: 'conversion',  # TODO: related to invol dissolution
+    EventFilings.CONVID2_NULL: 'conversion',  # TODO: related to invol dissolution
+    EventFilings.CONVILIQ_NULL: 'conversion',  # TODO: liquidation
+    EventFilings.CONVLRSTR_NULL: ['conversion', ('restoration', 'limitedRestoration')],
+    EventFilings.CONVNC_NULL: ['conversion', 'changeOfName'],
+    EventFilings.CONVRSTR_NULL: ['conversion', ('restoration', 'fullRestoration')],
+
     EventFilings.FILE_CO_AR: 'correction',
     EventFilings.FILE_CO_BC: 'correction',
     EventFilings.FILE_CO_DI: 'correction',
@@ -228,9 +254,7 @@ def has_value(cls, value):
     EventFilings.FILE_ICORP: 'incorporationApplication',
     EventFilings.FILE_ICORU: 'incorporationApplication',
     EventFilings.FILE_ICORC: 'incorporationApplication',
-    EventFilings.CONVICORP_NULL: 'incorporationApplication',  # TODO: re-map
 
-    # TODO: Ledger - unsupported
     # TODO: Legacy Other - unsupported
     EventFilings.FILE_AM_PF: 'legacyOther',
     EventFilings.FILE_AM_PO: 'legacyOther',
@@ -285,7 +309,6 @@ def has_value(cls, value):
     EventFilings.FILE_AMLHC: 'Amalgamation Application Short Form (Horizontal) for a Community Contribution Company',
     EventFilings.FILE_AMLRC: 'Amalgamation Application (Regular) for a Community Contribution Company',
     EventFilings.FILE_AMLVC: 'Amalgamation Application Short Form (Vertical) for a Community Contribution Company',
-    EventFilings.CONVAMAL_NULL: None,  # TODO: re-map
 
     EventFilings.FILE_ANNBC: 'BC Annual Report',  # has suffix of date, dynamically add it during formatting
 
@@ -338,9 +361,7 @@ def has_value(cls, value):
     EventFilings.FILE_ICORP: 'Incorporation Application',
     EventFilings.FILE_ICORU: 'Incorporation Application for a BC Unlimited Liability Company',
     EventFilings.FILE_ICORC: 'Incorporation Application for a Community Contribution Company',
-    EventFilings.CONVICORP_NULL: None,  # TODO: re-map
 
-    # TODO: Ledger - unsupported
     # TODO: Legacy Other - unsupported
     EventFilings.FILE_AM_PF: 'Amendment - Put Back Off',
     EventFilings.FILE_AM_PO: 'Amendment - Put Back On',
@@ -393,7 +414,7 @@ def has_value(cls, value):
     'changeOfDirectors': ['last_cod_date'],
     'agmExtension': ['last_agm_date'],
     'amalgamationApplication': ['last_coa_date', 'last_cod_date'],
-    # TODO: 'dissolution_date' - Amalgamating business, continuation out
+    # TODO: 'dissolution_date' - continuation out
     # TODO: 'continuation_out_date' - continuation out
     'continuationIn': ['last_coa_date', 'last_cod_date'],
     'dissolution': ['dissolution_date'],

data-tool/flows/tombstone/tombstone_queries.py

@@ -605,17 +605,20 @@ def get_filings_query(corp_num):
             u.email_addr           as u_email_addr,
             u.role_typ_cd          as u_role_typ_cd,
             --- conversion ledger
-            cl.ledger_title_txt    as cl_ledger_title_txt
+            cl.ledger_title_txt    as cl_ledger_title_txt,
+            -- conv event
+            to_char(ce.effective_dt at time zone 'UTC', 'YYYY-MM-DD HH24:MI:SSTZH:TZM') as ce_effective_dt_str
         from event e
                  left outer join filing f on e.event_id = f.event_id
                  left outer join filing_user u on u.event_id = e.event_id
                  left outer join conv_ledger cl on cl.event_id = e.event_id
+                 left outer join conv_event ce on e.event_id = ce.event_id
         where 1 = 1
             and e.corp_num = '{corp_num}'
 --          and e.corp_num = 'BC0068889'
 --          and e.corp_num = 'BC0449924'  -- AR, ADCORP
 --        and e.trigger_dts is not null
-        order by e.event_timerstamp
+        order by e.event_timerstamp, e.event_id
         ;
     """
     return query

data-tool/flows/tombstone/tombstone_utils.py

@@ -248,8 +248,8 @@ def format_jurisdictions_data(data: dict, event_id: Decimal) -> dict:
     formatted_jurisdiction['country'] = None
     formatted_jurisdiction['region'] = None
 
-    can_jurisdiction_code = jurisdiction_info['j_can_jur_typ_cd']
-    other_jurisdiction_desc = jurisdiction_info['j_othr_juris_desc']
+    can_jurisdiction_code = jurisdiction_info['j_can_jur_typ_cd'] or ''
+    other_jurisdiction_desc = jurisdiction_info['j_othr_juris_desc'] or ''
 
     # when canadian jurisdiction, ignore othr_juris_desc
     if can_jurisdiction_code != 'OT':
@@ -278,19 +278,29 @@ def format_filings_data(data: dict) -> list[dict]:
     for x in filings_data:
         event_file_type = x['event_file_type']
         # TODO: build a new complete filing event mapper (WIP)
-        filing_type, filing_subtype = get_target_filing_type(event_file_type)
+        raw_filing_type, raw_filing_subtype = get_target_filing_type(event_file_type)
         # skip the unsupported ones
-        if not filing_type:
+        if not raw_filing_type:
             print(f'❗ Skip event filing type: {event_file_type}')
             unsupported_event_file_types.add(event_file_type)
             continue
+
+        # get converted filing_type and filing_subtype
+        if raw_filing_type == 'conversion':
+            if isinstance(raw_filing_subtype, tuple):
+                filing_type, filing_subtype = raw_filing_subtype
+            else:
+                filing_type = raw_filing_subtype
+                filing_subtype = None
+            raw_filing_subtype = None
+        else:
+            filing_type = raw_filing_type
+            filing_subtype = raw_filing_subtype
 
-        effective_date = x['f_effective_dt_str']
-        if not effective_date:
-            effective_date = x['e_event_dt_str']
+        effective_date = x['ce_effective_dt_str'] or x['f_effective_dt_str'] or x['e_event_dt_str']
         trigger_date = x['e_trigger_dt_str']
 
-        filing_json, meta_data = build_filing_json_meta_data(filing_type, filing_subtype,
+        filing_json, meta_data = build_filing_json_meta_data(raw_filing_type, filing_type, filing_subtype,
                                                              effective_date, x)
 
         filing_body = copy.deepcopy(FILING['filings'])
@@ -301,23 +311,31 @@ def format_filings_data(data: dict) -> list[dict]:
         if not (user_id := x['u_user_id']):
             user_id = x['u_full_name'] if x['u_full_name'] else None
 
+        if raw_filing_type == 'conversion' or raw_filing_subtype == 'involuntary':
+            hide_in_ledger = True
+        else:
+            hide_in_ledger = False
+
         filing_body = {
             **filing_body,
             'filing_date': effective_date,
-            'filing_type': filing_type,
-            'filing_sub_type': filing_subtype,
+            'filing_type': raw_filing_type,
+            'filing_sub_type': raw_filing_subtype,
             'completion_date': effective_date,
             'effective_date': effective_date,
             'filing_json': filing_json,
             'meta_data': meta_data,
+            'hide_in_ledger': hide_in_ledger,
             'submitter_id': user_id,  # will be updated to real user_id when loading data into db
         }
 
+        # conversion still need to populate create-new-business info
+        # based on converted filing type
         if filing_type == 'continuationIn':
             jurisdiction = format_jurisdictions_data(data, x['e_event_id'])
 
         if filing_type == 'amalgamationApplication':
-            amalgamation = format_amalgamations_data(data, x['e_event_id'])
+            amalgamation = format_amalgamations_data(data, x['e_event_id'], effective_date, filing_subtype)
 
         comments = format_filing_comments_data(data, x['e_event_id'])
 
@@ -348,7 +366,7 @@ def format_filings_data(data: dict) -> list[dict]:
     }
 
 
-def format_amalgamations_data(data: dict, event_id: Decimal) -> dict:
+def format_amalgamations_data(data: dict, event_id: Decimal, amalgamation_date: str, amalgamation_type: str) -> dict:
     amalgamations_data = data['amalgamations']
 
     matched_amalgamations = [
@@ -359,18 +377,12 @@ def format_amalgamations_data(data: dict, event_id: Decimal) -> dict:
         return None
 
     formatted_amalgmation = copy.deepcopy(AMALGAMATION)
-    amalgmation_info = matched_amalgamations[0]
+    amalgamation_info = matched_amalgamations[0]
 
-    amalgmation_date = amalgmation_info['f_effective_dt_str']
-    if not amalgmation_date:
-        amalgmation_date = amalgmation_info['e_event_dt_str']
-    formatted_amalgmation['amalgamations']['amalgamation_date'] = amalgmation_date
-    formatted_amalgmation['amalgamations']['court_approval'] = bool(amalgmation_info['f_court_approval'])
-
-    event_file_type = amalgmation_info['event_file_type']
-    _, filing_subtype = get_target_filing_type(event_file_type)
+    formatted_amalgmation['amalgamations']['amalgamation_date'] = amalgamation_date
+    formatted_amalgmation['amalgamations']['court_approval'] = bool(amalgamation_info['f_court_approval'])
 
-    formatted_amalgmation['amalgamations']['amalgamation_type'] = filing_subtype
+    formatted_amalgmation['amalgamations']['amalgamation_type'] = amalgamation_type
     formatted_tings = formatted_amalgmation['amalgamating_businesses']
     for ting in matched_amalgamations:
         formatted_tings.append(format_amalgamating_businesses(ting))
@@ -546,9 +558,11 @@ def get_business_update_value(key: str, effective_date: str, trigger_date: str,
     return value
 
 
-def build_filing_json_meta_data(filing_type: str, filing_subtype: str, effective_date: str, data: dict) -> tuple[dict, dict]:
+def build_filing_json_meta_data(raw_filing_type: str, filing_type: str, filing_subtype: str, effective_date: str, data: dict) -> tuple[dict, dict]:
     filing_json = copy.deepcopy(FILING_JSON)
-    filing_json['filing'][filing_type] = {}
+    filing_json['filing'][raw_filing_type] = {}
+    if raw_filing_type != filing_type:
+        filing_json['filing'][filing_type] = {}
 
     meta_data = {
         'colinFilingInfo': {
@@ -560,6 +574,24 @@ def build_filing_json_meta_data(filing_type: str, filing_subtype: str, effective
         'colinDisplayName': get_colin_display_name(data)
     }
 
+    if raw_filing_type == 'conversion':
+        # will populate state filing info for conversion in the following steps
+        # based on converted filing type and converted filing subtype
+        if filing_type in LEAR_STATE_FILINGS:
+            state_change = True
+        else:
+            state_change = False
+        filing_json['filing']['conversion'] = {
+            'convFilingType': filing_type,
+            'convFilingSubType': filing_subtype,
+            'stateChange': state_change
+        }
+        meta_data['conversion'] = {
+            'convFilingType': filing_type,
+            'convFilingSubType': filing_subtype,
+            'stateChange': state_change
+        }
+
     if filing_type == 'annualReport':
         meta_data['annualReport'] = {
             'annualReportFilingYear': int(effective_date[:4]),

legal-api/migrations/versions/d0b10576924c_alter_amalgamation_type_enum.py

@@ -0,0 +1,50 @@
+"""alter_amalgamation_type_enum
+
+Revision ID: d0b10576924c
+Revises: d9254d3cbbf4
+Create Date: 2025-02-03 21:47:05.061172
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'd0b10576924c'
+down_revision = 'd9254d3cbbf4'
+branch_labels = None
+depends_on = None
+
+
+amalgamation_type_old_enum = postgresql.ENUM('regular',
+                                         'vertical',
+                                         'horizontal',
+                                         name='amalgamation_type_old')
+
+
+def upgrade():
+    with op.get_context().autocommit_block():
+        op.execute("ALTER TYPE amalgamation_type ADD VALUE 'unknown'")
+
+
+def downgrade():
+    op.execute("UPDATE amalgamations SET amalgamation_type = 'regular' WHERE amalgamation_type = 'unknown'")
+    op.execute("UPDATE amalgamations_version SET amalgamation_type = 'regular' WHERE amalgamation_type = 'unknown'")
+
+    amalgamation_type_old_enum.create(op.get_bind(), checkfirst=True)
+
+    op.execute("""
+        ALTER TABLE amalgamations
+        ALTER COLUMN amalgamation_type
+        TYPE amalgamation_type_old
+        USING amalgamation_type::text::amalgamation_type_old
+    """)
+    op.execute("""
+        ALTER TABLE amalgamations_version
+        ALTER COLUMN amalgamation_type
+        TYPE amalgamation_type_old
+        USING amalgamation_type::text::amalgamation_type_old
+    """)
+
+    op.execute("DROP TYPE amalgamation_type")
+    op.execute("ALTER TYPE amalgamation_type_old RENAME TO amalgamation_type")

legal-api/src/legal_api/models/amalgamation.py

@@ -37,6 +37,7 @@ class AmalgamationTypes(BaseEnum):
         regular = auto()
         vertical = auto()
         horizontal = auto()
+        unknown = auto()
 
     __versioned__ = {}
     __tablename__ = 'amalgamations'
@@ -124,7 +125,8 @@ def get_revision_json(cls, transaction_id, business_id, tombstone=False):
         if tombstone:
             return {
                 'identifier': 'Not Available',
-                'legalName': 'Not Available'
+                'legalName': 'Not Available',
+                'amalgamationDate': 'Not Available'
             }
 
         amalgamation = Amalgamation.get_revision(transaction_id, business_id)