relations between tables defined on the database level (foreign keys …

…enabled explicitly for SQLite to make this work)

GO and InterPro import minor change to allow Whooshee to index things properly

planet/extensions.py

@@ -6,11 +6,25 @@
 from flask_sqlalchemy import SQLAlchemy
 from flask_whooshee import Whooshee
 
+from sqlalchemy.engine import Engine
+from sqlalchemy import event
+
 from planet.flask_blast import BlastThread
 
 __all__ = ['db', 'login_manager', 'cache', 'htmlmin', 'blast_thread', 'compress', 'whooshee']
 
 db = SQLAlchemy()
+
+
+@event.listens_for(Engine, "connect")
+def set_sqlite_pragma(dbapi_connection, connection_record):
+    """
+        Will force sqlite contraint foreign keys
+    """
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
 login_manager = LoginManager()
 toolbar = DebugToolbarExtension()
 cache = Cache()

planet/models/condition_tissue.py

@@ -8,11 +8,13 @@
 class ConditionTissue(db.Model):
     __tablename__ = 'conditions_tissue'
     id = db.Column(db.Integer, primary_key=True)
-    species_id = db.Column(db.Integer, db.ForeignKey('species.id'))
+    species_id = db.Column(db.Integer, db.ForeignKey('species.id', ondelete='CASCADE'))
     data = db.Column(db.Text)
     description = db.Column(db.Text)
 
-    expression_specificity_method_id = db.Column(db.Integer, db.ForeignKey('expression_specificity_method.id'), index=True)
+    expression_specificity_method_id = db.Column(db.Integer,
+                                                 db.ForeignKey('expression_specificity_method.id', ondelete='CASCADE'),
+                                                 index=True)
 
     @staticmethod
     def add(species_id, data, order, colors, expression_specificity_method_id, description=''):

planet/models/expression/coexpression_clusters.py

@@ -27,14 +27,15 @@
 class CoexpressionClusteringMethod(db.Model):
     __tablename__ = 'coexpression_clustering_methods'
     id = db.Column(db.Integer, primary_key=True)
-    network_method_id = db.Column(db.Integer, db.ForeignKey('expression_network_methods.id'), index=True)
+    network_method_id = db.Column(db.Integer, db.ForeignKey('expression_network_methods.id', ondelete='CASCADE'), index=True)
     method = db.Column(db.Text)
     cluster_count = db.Column(db.Integer)
 
     clusters = db.relationship('CoexpressionCluster',
                                backref=db.backref('method', lazy='joined'),
                                lazy='dynamic',
-                               cascade='all, delete-orphan')
+                               cascade="all, delete-orphan",
+                               passive_deletes=True)
 
     @staticmethod
     def update_counts():
@@ -334,13 +335,14 @@ def add_lstrap_coexpression_clusters(cluster_file, description, network_id, pref
 class CoexpressionCluster(db.Model):
     __tablename__ = 'coexpression_clusters'
     id = db.Column(db.Integer, primary_key=True)
-    method_id = db.Column(db.Integer, db.ForeignKey('coexpression_clustering_methods.id'))
+    method_id = db.Column(db.Integer, db.ForeignKey('coexpression_clustering_methods.id', ondelete='CASCADE'))
     name = db.Column(db.String(50), index=True)
 
     trees = db.relationship('Tree',
                             backref=db.backref('cluster', lazy='joined'),
                             lazy='dynamic',
-                            cascade='all, delete-orphan')
+                            cascade="all, delete-orphan",
+                            passive_deletes=True)
 
     # Other properties
     # sequences defined in Sequence

planet/models/expression/networks.py

@@ -30,12 +30,14 @@ class ExpressionNetworkMethod(db.Model):
     probes = db.relationship('ExpressionNetwork',
                              backref=db.backref('method', lazy='joined'),
                              lazy='dynamic',
-                             cascade='all, delete-orphan')
+                             cascade="all, delete-orphan",
+                             passive_deletes=True)
 
     clustering_methods = db.relationship('CoexpressionClusteringMethod',
                                          backref='network_method',
                                          lazy='dynamic',
-                                         cascade='all, delete-orphan')
+                                         cascade='all, delete-orphan',
+                                         passive_deletes=True)
 
     def __init__(self, species_id, description, edge_type="rank"):
         self.species_id = species_id
@@ -251,9 +253,9 @@ class ExpressionNetwork(db.Model):
     __tablename__ = 'expression_networks'
     id = db.Column(db.Integer, primary_key=True)
     probe = db.Column(db.String(50, collation=SQL_COLLATION), index=True)
-    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id'), index=True)
+    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id', ondelete='CASCADE'), index=True)
     network = db.Column(db.Text)
-    method_id = db.Column(db.Integer, db.ForeignKey('expression_network_methods.id'), index=True)
+    method_id = db.Column(db.Integer, db.ForeignKey('expression_network_methods.id', ondelete='CASCADE'), index=True)
 
     def __init__(self, probe, sequence_id, network, method_id):
         self.probe = probe

planet/models/expression/profiles.py

@@ -16,15 +16,16 @@
 class ExpressionProfile(db.Model):
     __tablename__ = 'expression_profiles'
     id = db.Column(db.Integer, primary_key=True)
-    species_id = db.Column(db.Integer, db.ForeignKey('species.id'), index=True)
+    species_id = db.Column(db.Integer, db.ForeignKey('species.id', ondelete='CASCADE'), index=True)
     probe = db.Column(db.String(50, collation=SQL_COLLATION), index=True)
-    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id'), index=True)
+    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id', ondelete='CASCADE'), index=True)
     profile = db.deferred(db.Column(db.Text))
 
     specificities = db.relationship('ExpressionSpecificity',
                                     backref=db.backref('profile', lazy='joined'),
                                     lazy='dynamic',
-                                    cascade='all, delete-orphan')
+                                    cascade="all, delete-orphan",
+                                    passive_deletes=True)
 
     def __init__(self, probe, sequence_id, profile):
         self.probe = probe

planet/models/expression/specificity.py

@@ -15,15 +15,17 @@ class ExpressionSpecificityMethod(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     description = db.Column(db.Text)
     conditions = db.Column(db.Text)
-    species_id = db.Column(db.Integer, db.ForeignKey('species.id'), index=True)
+    species_id = db.Column(db.Integer, db.ForeignKey('species.id', ondelete='CASCADE'), index=True)
 
     specificities = db.relationship('ExpressionSpecificity',
                                     backref='method',
                                     lazy='dynamic',
-                                    cascade='all, delete-orphan')
+                                    cascade="all, delete-orphan",
+                                    passive_deletes=True)
 
     condition_tissue = db.relationship('ConditionTissue', backref='expression_specificity_method', lazy='joined',
-                                        cascade='all, delete-orphan', uselist=False)
+                                       cascade="all, delete-orphan",
+                                       passive_deletes=True, uselist=False)
 
     menu_order = db.Column(db.Integer)
 

planet/models/gene_families.py

@@ -24,7 +24,8 @@ class GeneFamilyMethod(db.Model):
 
     families = db.relationship('GeneFamily', backref=db.backref('method', lazy='joined'),
                                lazy='dynamic',
-                               cascade='all, delete-orphan')
+                               cascade="all, delete-orphan",
+                               passive_deletes=True)
 
     def __init__(self, method):
         self.method = method
@@ -63,9 +64,9 @@ def add(description):
 class GeneFamily(db.Model):
     __tablename__ = 'gene_families'
     id = db.Column(db.Integer, primary_key=True)
-    method_id = db.Column(db.Integer, db.ForeignKey('gene_family_methods.id'), index=True)
+    method_id = db.Column(db.Integer, db.ForeignKey('gene_family_methods.id', ondelete='CASCADE'), index=True)
     name = db.Column(db.String(50, collation=SQL_COLLATION), unique=True, index=True)
-    clade_id = db.Column(db.Integer, db.ForeignKey('clades.id'), index=True)
+    clade_id = db.Column(db.Integer, db.ForeignKey('clades.id', ondelete='SET NULL'), index=True)
 
     # Original name is used to keep track of the original ID from OrthoFinder (required to link back to trees)
     original_name = db.Column(db.String(50, collation=SQL_COLLATION), index=True, default=None)

planet/models/go.py

@@ -180,12 +180,19 @@ def add_from_obo(filename, empty=True, compressed=False):
 
         obo_parser.extend_go()
 
-        for term in obo_parser.terms:
+        for i, term in enumerate(obo_parser.terms):
             go = GO(term.id, term.name, term.namespace, term.definition, term.is_obsolete, ";".join(term.is_a),
                     ";".join(term.extended_go))
 
             db.session.add(go)
 
+            if i % 40 == 0:
+                # commit to the db frequently to allow WHOOSHEE's indexing function to work without timing out
+                try:
+                    db.session.commit()
+                except Exception as e:
+                    db.session.rollback()
+                    print(e)
         try:
             db.session.commit()
         except Exception as e:

planet/models/interpro.py

@@ -19,7 +19,7 @@ class Interpro(db.Model):
     label = db.Column(db.String(50, collation=SQL_COLLATION), unique=True, index=True)
     description = db.Column(db.Text)
 
-    clade_id = db.Column(db.Integer, db.ForeignKey('clades.id'), index=True)
+    clade_id = db.Column(db.Integer, db.ForeignKey('clades.id', ondelete='SET NULL'), index=True)
 
     sequences = db.relationship('Sequence', secondary=sequence_interpro, lazy='dynamic')
 
@@ -141,10 +141,19 @@ def add_from_xml(filename, empty=True):
 
         interpro_parser.readfile(filename)
 
-        for domain in interpro_parser.domains:
+        for i, domain in enumerate(interpro_parser.domains):
             interpro = Interpro(domain.label, domain.description)
 
             db.session.add(interpro)
+
+            if i % 40 == 0:
+                # commit to the db frequently to allow WHOOSHEE's indexing function to work without timing out
+                try:
+                    db.session.commit()
+                except Exception as e:
+                    db.session.rollback()
+                    print(e)
+
         try:
             db.session.commit()
         except Exception as e:

planet/models/relationships/cluster_go.py

@@ -6,17 +6,17 @@ class ClusterGOEnrichment(db.Model):
     __table_args__ = {'extend_existing': True}
 
     id = db.Column(db.Integer, primary_key=True)
-    cluster_id = db.Column(db.Integer, db.ForeignKey('coexpression_clusters.id'))
-    go_id = db.Column(db.Integer, db.ForeignKey('go.id'))
+    cluster_id = db.Column(db.Integer, db.ForeignKey('coexpression_clusters.id', ondelete='CASCADE'))
+    go_id = db.Column(db.Integer, db.ForeignKey('go.id', ondelete='CASCADE'))
 
     cluster = db.relationship('CoexpressionCluster', backref=db.backref('go_enrichment',
                                                                         lazy='dynamic',
-                                                                        cascade='all, delete-orphan'),
+                                                                        passive_deletes=True),
                               lazy='joined')
 
     go = db.relationship('GO', backref=db.backref('enriched_clusters',
                                                   lazy='dynamic',
-                                                  cascade='all, delete-orphan'),
+                                                  passive_deletes=True),
                          lazy='joined')
 
     """

planet/models/relationships/cluster_similarity.py

@@ -6,10 +6,10 @@ class CoexpressionClusterSimilarity(db.Model):
     __table_args__ = {'extend_existing': True}
 
     id = db.Column(db.Integer, primary_key=True)
-    source_id = db.Column(db.Integer, db.ForeignKey('coexpression_clusters.id'))
-    target_id = db.Column(db.Integer, db.ForeignKey('coexpression_clusters.id'))
+    source_id = db.Column(db.Integer, db.ForeignKey('coexpression_clusters.id', ondelete='CASCADE'))
+    target_id = db.Column(db.Integer, db.ForeignKey('coexpression_clusters.id', ondelete='CASCADE'))
 
-    gene_family_method_id = db.Column('gene_family_method_id', db.Integer, db.ForeignKey('gene_family_methods.id'),
+    gene_family_method_id = db.Column('gene_family_method_id', db.Integer, db.ForeignKey('gene_family_methods.id', ondelete='CASCADE'),
                                       index=True)
 
     jaccard_index = db.Column(db.Float, index=True)
@@ -18,15 +18,17 @@ class CoexpressionClusterSimilarity(db.Model):
 
     source = db.relationship('CoexpressionCluster', backref=db.backref('similarity_sources',
                                                                        lazy='dynamic',
-                                                                       cascade='all, delete-orphan'),
+                                                                       passive_deletes=True),
                              lazy='joined', foreign_keys=[source_id])
 
     target = db.relationship('CoexpressionCluster', backref=db.backref('similarity_targets',
                                                                        lazy='dynamic',
-                                                                       cascade='all, delete-orphan')
+                                                                       passive_deletes=True)
                              , lazy='joined', foreign_keys=[target_id])
 
-    gene_family_method = db.relationship('GeneFamilyMethod', lazy='joined')
+    gene_family_method = db.relationship('GeneFamilyMethod',
+                                         backref=db.backref('CoexpressionClusterSimilarities', passive_deletes=True),
+                                         lazy='joined')
 
     @staticmethod
     def empty_table():

planet/models/relationships/family_xref.py

@@ -6,5 +6,5 @@ class FamilyXRefAssociation(db.Model):
     __table_args__ = {'extend_existing': True}
 
     id = db.Column(db.Integer, primary_key=True)
-    gene_family_id = db.Column(db.Integer, db.ForeignKey('gene_families.id'))
-    xref_id = db.Column(db.Integer, db.ForeignKey('xrefs.id'))
+    gene_family_id = db.Column(db.Integer, db.ForeignKey('gene_families.id', ondelete='CASCADE'))
+    xref_id = db.Column(db.Integer, db.ForeignKey('xrefs.id', ondelete='CASCADE'))

planet/models/relationships/sequence_cluster.py

@@ -7,15 +7,15 @@ class SequenceCoexpressionClusterAssociation(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     probe = db.Column(db.String(50), index=True)
-    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id'))
-    coexpression_cluster_id = db.Column(db.Integer, db.ForeignKey('coexpression_clusters.id'))
+    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id', ondelete='CASCADE'))
+    coexpression_cluster_id = db.Column(db.Integer, db.ForeignKey('coexpression_clusters.id', ondelete='CASCADE'))
 
     sequence = db.relationship('Sequence', backref=db.backref('coexpression_cluster_associations',
                                                               lazy='dynamic',
-                                                              cascade="all, delete-orphan"),
+                                                              passive_deletes=True),
                                lazy='joined')
     coexpression_cluster = db.relationship('CoexpressionCluster',
                                            backref=db.backref('sequence_associations',
                                                               lazy='dynamic',
-                                                              cascade="all, delete-orphan"),
+                                                              passive_deletes=True),
                                            lazy='joined')

planet/models/relationships/sequence_family.py

@@ -6,14 +6,14 @@ class SequenceFamilyAssociation(db.Model):
     __table_args__ = {'extend_existing': True}
 
     id = db.Column(db.Integer, primary_key=True)
-    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id'))
-    gene_family_id = db.Column(db.Integer, db.ForeignKey('gene_families.id'))
+    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id', ondelete='CASCADE'))
+    gene_family_id = db.Column(db.Integer, db.ForeignKey('gene_families.id', ondelete='CASCADE'))
 
     sequence = db.relationship('Sequence', backref=db.backref('family_associations',
                                                               lazy='dynamic',
-                                                              cascade='all, delete-orphan'), lazy='joined')
+                                                              passive_deletes=True), lazy='joined')
     family = db.relationship('GeneFamily', backref=db.backref('sequence_associations',
                                                               lazy='dynamic',
-                                                              cascade='all, delete-orphan'), lazy='joined')
+                                                              passive_deletes=True), lazy='joined')
 
 

planet/models/relationships/sequence_go.py

@@ -6,8 +6,8 @@ class SequenceGOAssociation(db.Model):
     __table_args__ = {'extend_existing': True}
 
     id = db.Column(db.Integer, primary_key=True)
-    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id'))
-    go_id = db.Column(db.Integer, db.ForeignKey('go.id'))
+    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id', ondelete='CASCADE'))
+    go_id = db.Column(db.Integer, db.ForeignKey('go.id', ondelete='CASCADE'))
 
     evidence = db.Column(db.Enum('EXP', 'IDA', 'IPI', 'IMP', 'IGI', 'IEP',
                                  'ISS', 'ISO', 'ISA', 'ISM', 'IGC', 'IBA', 'IBD', 'IKR', 'IRD', 'RCA',
@@ -16,8 +16,8 @@ class SequenceGOAssociation(db.Model):
 
     sequence = db.relationship('Sequence', backref=db.backref('go_associations',
                                                               lazy='dynamic',
-                                                              cascade='all, delete-orphan'), lazy='joined')
+                                                              passive_deletes=True), lazy='joined')
 
     go = db.relationship('GO', backref=db.backref('sequence_associations',
                                                   lazy='dynamic',
-                                                  cascade='all, delete-orphan'), lazy='joined')
+                                                  passive_deletes=True), lazy='joined')

planet/models/relationships/sequence_interpro.py

@@ -6,14 +6,14 @@ class SequenceInterproAssociation(db.Model):
     __table_args__ = {'extend_existing': True}
 
     id = db.Column(db.Integer, primary_key=True)
-    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id'))
-    interpro_id = db.Column(db.Integer, db.ForeignKey('interpro.id'))
+    sequence_id = db.Column(db.Integer, db.ForeignKey('sequences.id', ondelete='CASCADE'))
+    interpro_id = db.Column(db.Integer, db.ForeignKey('interpro.id', ondelete='CASCADE'))
     start = db.Column(db.Integer, default=None)
     stop = db.Column(db.Integer, default=None)
 
     sequence = db.relationship('Sequence', backref=db.backref('interpro_associations',
                                                               lazy='dynamic',
-                                                              cascade='all, delete-orphan'), lazy='joined')
+                                                              passive_deletes=True), lazy='joined')
 
     domain = db.relationship('Interpro', backref=db.backref('sequence_associations',
-                             lazy='dynamic', cascade='all, delete-orphan'), lazy='joined')
+                             lazy='dynamic', passive_deletes=True), lazy='joined')