Switch JBrowse data directories to be under data/ and create Docker mount

Increase Ensembl REST API timeout

Author: kevinlul

.gitignore

@@ -3,7 +3,7 @@
 bin/dicey
 bin/ncbi-blast-2.7.1+
 bin/primer3
-jbrowse/data.*
+jbrowse/data*
 jbrowse/Makefile
 jbrowse/setup.log
 jbrowse/jbrowse.conf

docker-compose.yaml

@@ -10,6 +10,7 @@ services:
       - ./bin:/var/www/html/bin
       - ./docs:/var/www/html/docs
       - ./src:/var/www/html/src
+      - ./jbrowse/data:/var/www/html/jbrowse/data
       - ./jbrowse/plugins/ColorByCDS:/var/www/html/jbrowse/plugins/ColorByCDS
       - ./jbrowse/plugins/ExportSequence:/var/www/html/jbrowse/plugins/ExportSequence
       - ./crispr-icon.png:/var/www/html/crispr-icon.png

src/guide-finder/GuideAdd.py

@@ -22,21 +22,21 @@
 class GuideAdd:
     def __init__(self, **kwargs):
         """ class for interfacing to the guide collection """
-        # check required inputs set 
+        # check required inputs set
         for parameter in ['batchID', 'guideID', 'label']:
             if parameter not in kwargs:
-                self.sendErrorHTML("'{parameter}' not set".format(**locals())) 
+                self.sendErrorHTML("'{parameter}' not set".format(**locals()))
         self.dbConnection = Config()
         self.batchID = kwargs['batchID']
         self.guideID = kwargs['guideID']
         self.label = kwargs['label']
         self.notes = kwargs['notes'] if 'notes' in kwargs else ''
-        
+
         # fetch the stored guide info
         self.metadata, self.guide = self.parseJSON()
         self.rgen = self.getRGEN(str(self.metadata['rgenID']))
         # redefine the connection to the database based on the genome
-        self.dbConnection = Config(self.metadata['genome']) 
+        self.dbConnection = Config(self.metadata['genome'])
         # determine if the guide already exists
         self.existingGuide = self.existingGuideInDatabase()
 
@@ -46,12 +46,12 @@ def __init__(self, **kwargs):
             self.rewriteGFF()
             print("Successfully Updated Guide")
         else:
-            # insert the guide into the database 
+            # insert the guide into the database
             self.insertGuide()
             self.rewriteGFF()
             print("Successfully Inserted Guide")
 
-    
+
     def rewriteGFF(self):
         """ rewrite the gff file """
         # intialize the strings
@@ -65,7 +65,7 @@ def rewriteGFF(self):
             sortedGuideCoordinates = sorted([int(x) for x in guideCoordinates.split('-')])
             # calculate the pam coordinates
             rgen = self.getRGEN(guideRecord['rgenID'])
-            pamLocation = rgen['PamLocation'] 
+            pamLocation = rgen['PamLocation']
             if (pamLocation == 'downstream' and strand == '+') or (pamLocation == 'upstream' and strand == '-'):
                 pamStart = sortedGuideCoordinates[-1]
                 pamEnd = int(pamStart) + len(guideRecord['pamSeq'])
@@ -107,9 +107,9 @@ def rewriteGFF(self):
 
             # increment the count
             idCounter += 1
-            
+
         #try:
-        guideGFF = os.path.join(self.dbConnection.ROOT_PATH, str('jbrowse/data.'+self.metadata['genome']+"/gRNA_CRISPR.gff"))
+        guideGFF = os.path.join(self.dbConnection.ROOT_PATH, str('jbrowse/data/'+self.metadata['genome']+"/gRNA_CRISPR.gff"))
         gffFile = open(guideGFF, 'wb')
         gffFile.write(featureStr.encode('utf-8'))
         gffFile.write(pamStr.encode('utf-8'))
@@ -118,7 +118,7 @@ def rewriteGFF(self):
         #except Exception as e:
         #    self.sendErrorHTML(str(e))
 
-            
+
     def getRGEN(self, rgenID):
         # fetch the correct rgen record using the rgenID attribute
         rgenCollection = self.dbConnection.rgenCollection
@@ -133,16 +133,16 @@ def combineSequence(self, rgen, guideSeq, pamSeq):
         if rgen['PamLocation'] == 'downstream':
             return guideSeq.upper() + ", " + pamSeq
         elif rgen['PamLocation'] == 'upstream':
-            return pamSeq.upper() + ", " + guideSeq.upper() 
+            return pamSeq.upper() + ", " + guideSeq.upper()
         else:
             sendErrorHMTL("Unrecognized PAM Location for RGEN: " + str(self.rgen['PamLocation']))
-    
+
     def insertGuide(self):
         """ create a new record in the database for the guide """
         newGuideRecord = {
             'batchName': self.metadata['gene'],
             'status': 'Accepted',
-            'guideScore': self.guide['MIT'] if 'MIT' in self.guide else '', 
+            'guideScore': self.guide['MIT'] if 'MIT' in self.guide else '',
             'guideSeq': self.guide['guide_seq'],
             'Notes': self.notes,
             'inputSearchCoordinates': self.metadata['inputSearchCoordinates'],
@@ -163,7 +163,7 @@ def insertGuide(self):
         result = self.dbConnection.guideCollection.insert_one(newGuideRecord)
         if not result:
             self.sendErrorHTML("Problem inserting record into Database")
-    
+
     def updateGuide(self):
         """ find the existing guide and update its notes and label"""
         existing_id = self.existingGuide['_id']
@@ -176,7 +176,7 @@ def existingGuideInDatabase(self):
         searchQuery = {
             "guideSeq": self.guide['guide_seq'],
             "pamSeq": self.guide['pam_seq'],
-            "guideLocation": self.guide['guideLocation'] 
+            "guideLocation": self.guide['guideLocation']
         }
         if self.dbConnection.guideCollection.find(searchQuery).count() > 0:
             existingGuide = self.dbConnection.guideCollection.find_one(searchQuery)
@@ -198,7 +198,7 @@ def parseJSON(self):
             self.sendErrorHTML("Batch JSON file is misconfigured; no metadata found")
 
     def sendErrorHTML(self, errorString):
-        """ write error and exit the program """ 
+        """ write error and exit the program """
         print(errorString)
         sys.exit()
 
@@ -221,12 +221,12 @@ def main():
                 print(traceback.format_exc())
     else:
         #parameters = {'guideID': '92+', 'label': 'TestLabel', 'batchID': 'dda41408f7b56fcb3b', 'notes': 'nospecialcharacters'}
-        #parameters = {'notes': "Le'sfjkd%20fsj#%20s", 'batchID': 'bfd5911343bde9a7cd', 'label': 'Testing%20Update', 'guideID': '6-'} 
+        #parameters = {'notes': "Le'sfjkd%20fsj#%20s", 'batchID': 'bfd5911343bde9a7cd', 'label': 'Testing%20Update', 'guideID': '6-'}
         #{'label': 'testinglabel', 'guideID': '6-', 'notes': 'testing notes', 'batchID': '6aa1c0912c773384c7'}
         try:
             GuideAdd(**parameters)
         except Exception as e:
             print(str(e))
 
 if __name__ == "__main__":
-    main()
+    main()

src/guide-finder/core/categorize_offtargets.py

@@ -38,14 +38,14 @@ def categorizeOffTargets(guideDict, rgenID, genome, batchID):
 	dbConnection = Config(genome)
 	rgen = getRgenRecord(rgenID, dbConnection)
 	# construct bed intersect command
-	segmentsFile = os.path.join(dbConnection.ROOT_PATH, "jbrowse/data."+genome, "downloads", genome+".segments.bed")
+	segmentsFile = os.path.join(dbConnection.ROOT_PATH, "jbrowse/data/"+genome, "downloads", genome+".segments.bed")
 	extendedBed = os.path.join(dbConnection.ROOT_PATH, "src/guide-finder/tempfiles", str(batchID)+"_extended.bed")
 	bedCommand = ["bedtools", "intersect", "-a", extendedBed, "-b", segmentsFile, "-wb"]
 	p = Popen(bedCommand, stdin=PIPE, stdout=PIPE, stderr=PIPE)
 	out, err = p.communicate()
 	if err:
 		sys.exit(err)
-	
+
 	categorized = []
 	for line in out.splitlines():
 		# assign every intersection to its off-target
@@ -59,7 +59,7 @@ def categorizeOffTargets(guideDict, rgenID, genome, batchID):
 				contextString = str(intersect[-1])
 				# start the context string or append to the existing one
 				offTarget['context'] = formatContext('', contextString) if 'context' not in offTarget else formatContext(offTarget['context'], contextString)
-	
+
 	return guideDict
 
 
@@ -76,7 +76,7 @@ def formatContext(existingContext, newIntersection):
 		if 'intronic' in existingContext:
 			return existingContext
 		elif 'exonic' in existingContext:
-			return existingContext.replace('exonic', 'exonic/intronic') # give more specific categorization 
+			return existingContext.replace('exonic', 'exonic/intronic') # give more specific categorization
 		else:
 			return newIntersection + " | " + existingContext
 	elif newIntersection == 'intergenic':