#1120 #1164 Change the database schema to make sure gene or protein with different timestamps can be added to the database when adding same data with different timestamps

Author: ntran18

database/network-database/database_services/populator.py

@@ -32,7 +32,7 @@ def process_file(self, conn, cursor, data_filepath, copy_statement):
         """
 
         # Determine if we need to drop the last column (PPI network type)
-        if self.network_mode == Constants.PPI_NETWORK_MODE and data_filepath == Constants.MISSING_PPI_GENE_DATA_FILEPATH:
+        if self.network_mode == Constants.PPI_NETWORK_MODE and data_filepath == Constants.GENE_DATA_FILEPATH:
             print("Dropping the regulator column from the input data...")
             processed_rows = []
 
@@ -72,12 +72,11 @@ class GeneDataPopulator(DataPopulator):
     def __init__(self, db_url, network_mode):
         super().__init__(db_url)
         self.network_mode = network_mode
+        self.filepath = Constants.GENE_DATA_FILEPATH
         if network_mode == Constants.GRN_NETWORK_MODE:
             self.database_namespace = Constants.GRN_DATABASE_NAMESPACE
-            self.filepath = Constants.MISSING_GRN_GENE_DATA_FILEPATH
         elif network_mode == Constants.PPI_NETWORK_MODE:
             self.database_namespace = Constants.PPI_DATABASE_NAMESPACE
-            self.filepath = Constants.MISSING_PPI_GENE_DATA_FILEPATH
         else:
             raise ValueError(f"Unknown network type: {network_mode}")
 
@@ -92,7 +91,7 @@ def get_copy_statement(self):
 class ProteinDataPopulator(DataPopulator):
     def __init__(self, db_url):
         super().__init__(db_url)
-        self.filepath = Constants.MISSING_PROTEIN_DATA_FILEPATH
+        self.filepath = Constants.PROTEIN_DATA_FILEPATH
 
     def get_copy_statement(self):
         return f"COPY {Constants.PPI_DATABASE_NAMESPACE}.protein (standard_name, gene_systematic_name, length, molecular_weight, PI, taxon_id, time_stamp, source) FROM stdin WITH CSV DELIMITER E'\\t' HEADER;"

database/network-database/main.py

@@ -2,7 +2,6 @@
 from data_services.data_generator import *
 from data_services.save_service import *
 from database_services.filter import *
-from database_services.updater import *
 from database_services.populator import *
 import argparse
 from datetime import datetime, timezone, timedelta
@@ -13,6 +12,7 @@ def load_data(network_option):
     print("Generating data.................................................")
     time_stamp = datetime.now(timezone(timedelta(hours=-8)))
     formatted_time_stamp = time_stamp.strftime("%Y-%m-%d %H:%M:%S%z")
+    print("Formatted_time_stamp: ", formatted_time_stamp)
     if network_option in ['all', Constants.GRN_NETWORK_MODE]:
         grnDataGenerator = GeneRegulatoryNetworkDataGenerator(GeneRegulatoryNetworkFetcherService(), GeneRegulatoryNetworkProcessor(formatted_time_stamp), save_service)
 
@@ -27,40 +27,26 @@ def load_data(network_option):
 
     SourceDataGenerator(SourceProcessor(formatted_time_stamp), save_service)
 
-def filter_data(network_option, db_url):
-    print("Filtering data.................................................")
-    if network_option in ['all', Constants.GRN_NETWORK_MODE]:
-        GeneFilter(db_url, save_service, network_mode="grn").filter_data()
-    
-    if network_option in ['all', Constants.PPI_NETWORK_MODE]:
-        GeneFilter(db_url, save_service, network_mode="ppi").filter_data()
-        ProteinFilter(db_url, save_service).filter_data()
-
 def adding_data_to_databse(network_option, db_url):
     print("Adding data to database.................................................")
     if network_option in ['all', Constants.GRN_NETWORK_MODE]:
         network_mode = Constants.GRN_NETWORK_MODE
         SourceDataPopulator(db_url, network_mode).populate_data()
         GeneDataPopulator(db_url, network_mode).populate_data()
-        GeneUpdater(db_url, network_mode).update_data()
         GeneRegulatoryNetworkDataPopulator(db_url).populate_data()
 
     if network_option in ['all', Constants.PPI_NETWORK_MODE]:
         network_mode = Constants.PPI_NETWORK_MODE
         SourceDataPopulator(db_url, network_mode).populate_data()
 
         GeneDataPopulator(db_url, network_mode).populate_data()
-        GeneUpdater(db_url, network_mode).update_data()
 
         ProteinDataPopulator(db_url).populate_data()
-        ProteinProteinInteractionsUpdater(db_url).update_data()
-        ProteinUpdater(db_url).update_data()
 
         ProteinProteinInteractionsDataPopulator(db_url).populate_data()
 
 def main(network_option, db_url):
     load_data(network_option)
-    filter_data(network_option, db_url)
     adding_data_to_databse(network_option, db_url)
 
 if __name__ == "__main__":

database/schema/gene_regulatory_network_schema.sql

@@ -15,7 +15,7 @@ CREATE TABLE gene_regulatory_network_with_timestamp.gene (
   regulator BOOLEAN,
   time_stamp TIMESTAMP WITH TIME ZONE,
   source VARCHAR,
-  PRIMARY KEY(gene_id, taxon_id),
+  PRIMARY KEY(gene_id, taxon_id, time_stamp, source),
   FOREIGN KEY (time_stamp, source) REFERENCES gene_regulatory_network_with_timestamp.source(time_stamp, source)
 );
 
@@ -26,8 +26,7 @@ CREATE TABLE gene_regulatory_network_with_timestamp.network (
   annotation_type VARCHAR,
   time_stamp TIMESTAMP WITH TIME ZONE,
   source VARCHAR,
-  FOREIGN KEY (regulator_gene_id, taxon_id) REFERENCES gene_regulatory_network_with_timestamp.gene(gene_id, taxon_id),
-  FOREIGN KEY (target_gene_id, taxon_id) REFERENCES gene_regulatory_network_with_timestamp.gene(gene_id, taxon_id),
-  FOREIGN KEY (time_stamp, source) REFERENCES gene_regulatory_network_with_timestamp.source(time_stamp, source),
+  FOREIGN KEY (regulator_gene_id, taxon_id, time_stamp, source) REFERENCES gene_regulatory_network_with_timestamp.gene(gene_id, taxon_id, time_stamp, source),
+  FOREIGN KEY (target_gene_id, taxon_id, time_stamp, source) REFERENCES gene_regulatory_network_with_timestamp.gene(gene_id, taxon_id, time_stamp, source),
   CONSTRAINT unique_network UNIQUE (regulator_gene_id, target_gene_id, taxon_id, time_stamp, source, annotation_type)
 ); 

database/schema/protein_protein_interactions_schema.sql

@@ -14,35 +14,32 @@ CREATE TABLE protein_protein_interactions_with_timestamp.gene (
   taxon_id VARCHAR,
   time_stamp TIMESTAMP WITH TIME ZONE,
   source VARCHAR,
-  PRIMARY KEY(gene_id, taxon_id),
+  PRIMARY KEY(gene_id, taxon_id, time_stamp, source),
   FOREIGN KEY (time_stamp, source) REFERENCES protein_protein_interactions_with_timestamp.source(time_stamp, source)
 ); 
 
 CREATE TABLE protein_protein_interactions_with_timestamp.protein (
-  standard_name VARCHAR PRIMARY KEY,
+  standard_name VARCHAR,
   gene_systematic_name VARCHAR,
   length FLOAT,
   molecular_weight FLOAT,
   PI FLOAT,
   taxon_id VARCHAR,
   time_stamp TIMESTAMP WITH TIME ZONE,
   source VARCHAR,
-  FOREIGN KEY (gene_systematic_name, taxon_id) REFERENCES protein_protein_interactions_with_timestamp.gene(gene_id, taxon_id),
-  FOREIGN KEY (time_stamp, source) REFERENCES protein_protein_interactions_with_timestamp.source(time_stamp, source)
+  PRIMARY KEY(standard_name, time_stamp, source),
+  FOREIGN KEY (gene_systematic_name, taxon_id, time_stamp, source) REFERENCES protein_protein_interactions_with_timestamp.gene(gene_id, taxon_id, time_stamp, source)
 );
 
   CREATE TABLE protein_protein_interactions_with_timestamp.physical_interactions (
     protein1 VARCHAR,
     protein2 VARCHAR,
-    gene_systematic_name1 VARCHAR,
-    gene_systematic_name2 VARCHAR,
     interaction_detection_methods_identifier VARCHAR,
     annotation_type VARCHAR,
     experiment_name VARCHAR,
     time_stamp TIMESTAMP WITH TIME ZONE,
     source VARCHAR,
-    FOREIGN KEY (protein1) REFERENCES protein_protein_interactions_with_timestamp.protein(standard_name),
-    FOREIGN KEY (protein2) REFERENCES protein_protein_interactions_with_timestamp.protein(standard_name),
-    FOREIGN KEY (time_stamp, source) REFERENCES protein_protein_interactions_with_timestamp.source(time_stamp, source),
+    FOREIGN KEY (protein1, time_stamp, source) REFERENCES protein_protein_interactions_with_timestamp.protein(standard_name, time_stamp, source),
+    FOREIGN KEY (protein2, time_stamp, source) REFERENCES protein_protein_interactions_with_timestamp.protein(standard_name, time_stamp, source),
     CONSTRAINT unique_physical_interaction UNIQUE (protein1, protein2, interaction_detection_methods_identifier, annotation_type, experiment_name, time_stamp, source)
   );