Mercurial > repos > yating-l > jbrowsearchivecreator

comparison TrackHub.py @ 6:237707a6b74d draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/Yating-L/jbrowse-archive-creator.git commit a500f7ab2119cc5faaf80393bd87428389d06880-dirty
author yating-l
date Thu, 15 Feb 2018 17:05:05 -0500
parents 7e471cdd9e71
children 5d5fdcb798da
comparison
equal deleted inserted replaced
5:e762f4b9e4bd 6:237707a6b74d
3 import os 3 import os
4 import subprocess 4 import subprocess
5 import shutil 5 import shutil
6 import zipfile 6 import zipfile
7 import json 7 import json
8 import utils 8 import tempfile
9 import logging
10 from mako.lookup import TemplateLookup
11
12 from datatypes.Datatype import Datatype
13 from tracks.TrackStyles import TrackStyles
14 from util import subtools
15 from util import santitizer
9 16
10 17
11 class TrackHub: 18 class TrackHub:
12 def __init__(self, inputFiles, reference, outputDirect, tool_dir, genome, extra_files_path, metaData, jbrowse_host): 19 def __init__(self, inputFastaFile, outputFile, extra_files_path, tool_directory, trackType):
13 self.input_files = inputFiles.tracks 20
14 self.outfile = outputDirect 21 self.rootAssemblyHub = None
15 self.outfolder = extra_files_path 22
16 self.out_path = os.path.join(extra_files_path, 'myHub') 23 self.mySpecieFolderPath = None
17 self.reference = reference 24
18 self.tool_dir = tool_dir 25 # Store intermediate files, will be removed if not in debug mode
19 self.metaData = metaData 26 self.myTracksFolderPath = None
20 self.raw = os.path.join(self.out_path, 'raw') 27
21 self.json = os.path.join(self.out_path, 'json') 28 # Store binary files: Bam, BigWig
22 self.jbrowse_host = jbrowse_host 29 self.myBinaryFolderPath = None
23 try: 30
24 if os.path.exists(self.json): 31 self.tool_directory = tool_directory
25 shutil.rmtree(self.json) 32 self.trackType = trackType
26 os.makedirs(self.json) 33 self.reference_genome = inputFastaFile
27 except OSError as e: 34 self.genome_name = inputFastaFile.assembly_id
28 print "Cannot create json folder error({0}): {1}".format(e.errno, e.strerror) 35 self.extra_files_path = extra_files_path
29 else: 36 self.outputFile = outputFile
30 print "Create jbrowse folder {}".format(self.out_path) 37 self.chromSizesFile = None
38
39
40 # Set all the missing variables of this class, and create physically the folders/files
41 self.rootAssemblyHub = self.__createAssemblyHub__(extra_files_path=extra_files_path)
42 # Init the Datatype
43 Datatype.pre_init(self.reference_genome, self.chromSizesFile,
44 self.extra_files_path, self.tool_directory,
45 self.mySpecieFolderPath, self.myTracksFolderPath, self.myBinaryFolderPath, self.trackType)
46
47 self._prepareRefseq()
48 self.trackList = os.path.join(self.mySpecieFolderPath, "trackList.json")
49 self._createTrackList()
50
51 self.myTrackStyle = TrackStyles(self.tool_directory, self.mySpecieFolderPath, self.trackList)
52 #self.cssFolderPath = os.path.join(self.mySpecieFolderPath, 'css')
53 #self.cssFilePath = os.path.join(self.cssFolderPath, 'custom_track_styles.css')
54 self.logger = logging.getLogger(__name__)
55
31 56
32 def createHub(self): 57
33 self.prepareRefseq() 58 def addTrack(self, trackDbObject):
34 for input_file in self.input_files: 59 if trackDbObject['dataType'].lower() == 'bam':
35 self.addTrack(input_file) 60 #new_track = subprocess.Popen(['echo', trackDbObject['options']], stdout=subprocess.PIPE)
36 self.indexName() 61 #subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
37 slink = self.makeArchive() 62 subtools.add_track_json(self.trackList, trackDbObject['options'])
38 self.outHtml(slink) 63 #subtools.add_track_json(self.trackList, trackDbObject['track_json'])
64 elif trackDbObject['dataType'].lower() == 'bigwig':
65 subtools.add_track_json(self.trackList, trackDbObject['options'])
66 else:
67 if trackDbObject['trackType'] == 'HTMLFeatures':
68 self._customizeHTMLFeature(trackDbObject)
69 subtools.flatfile_to_json(trackDbObject['trackDataURL'], trackDbObject['dataType'], trackDbObject['trackType'], trackDbObject['trackLabel'], self.mySpecieFolderPath, trackDbObject['options'])
70
71
72 def terminate(self, debug=False):
73 """ Write html file """
74 self._indexName()
75 if not debug:
76 self._removeRaw()
77 #self._makeArchive()
78 self._outHtml()
39 print "Success!\n" 79 print "Success!\n"
40 80
41 def prepareRefseq(self): 81
42 try: 82 def _customizeHTMLFeature(self, trackDbObject):
83 if trackDbObject['options']:
84 subfeatures = trackDbObject['options'].get('subfeatureClasses')
85 feature_color = trackDbObject['options']['feature_color']
86 if subfeatures:
87 for key, value in subfeatures.items():
88 self.myTrackStyle.addCustomColor(value, feature_color)
89 else:
90 customizedFeature = santitizer.sanitize_name(trackDbObject['trackLabel'])
91 clientConfig = json.loads(trackDbObject['options']['clientConfig'])
92 clientConfig['renderClassName'] = customizedFeature
93 trackDbObject['options']['clientConfig'] = json.dumps(clientConfig)
94 self.myTrackStyle.addCustomColor(customizedFeature, feature_color)
95
96 def _removeRaw(self):
97 if os.path.exists(self.myTracksFolderPath):
98 shutil.rmtree(self.myTracksFolderPath)
99
100 def _createTrackList(self):
101 if not os.path.exists(self.trackList):
102 os.mknod(self.trackList)
103
104 def _prepareRefseq(self):
105 subtools.prepare_refseqs(self.reference_genome.false_path, self.mySpecieFolderPath)
106 #try:
43 #print os.path.join(self.tool_dir, 'prepare-refseqs.pl') + ", '--fasta', " + self.reference +", '--out', self.json])" 107 #print os.path.join(self.tool_dir, 'prepare-refseqs.pl') + ", '--fasta', " + self.reference +", '--out', self.json])"
44 subprocess.call(['prepare-refseqs.pl', '--fasta', self.reference, '--out', self.json]) 108 #subprocess.call(['prepare-refseqs.pl', '--fasta', self.reference_genome.false_path, '--out', self.mySpecieFolderPath])
45 except OSError as e: 109 #except OSError as e:
46 print "Cannot prepare reference error({0}): {1}".format(e.errno, e.strerror) 110 #print "Cannot prepare reference error({0}): {1}".format(e.errno, e.strerror)
47 #TODO: hard coded the bam and bigwig tracks. Need to allow users to customize the settings 111
48 def addTrack(self, track): 112 def _indexName(self):
49 #print "false_path" , track['false_path'] 113 #subprocess.call(['generate-names.pl', '-v', '--out', self.mySpecieFolderPath])
50 if track['false_path'] in self.metaData.keys(): 114 subtools.generate_names(self.mySpecieFolderPath)
51 metadata = self.metaData[track['false_path']]
52 else:
53 metadata = {}
54 self.SetMetadata(track, metadata)
55 if track['dataType'] == 'bam':
56 self.Bam(track, metadata)
57 # print "add bam track\n"
58 elif track['dataType'] == 'bigwig':
59 self.BigWig(track, metadata)
60 else:
61 flat_file = os.path.join(self.raw, track['fileName'])
62 if track['dataType'] == 'bed':
63 subprocess.call(['flatfile-to-json.pl', '--bed', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
64 elif track['dataType'] == 'bedSpliceJunctions' or track['dataType'] == 'gtf' or track['dataType'] == 'blastxml':
65 subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"glyph": "JBrowse/View/FeatureGlyph/Segments", "category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
66 elif track['dataType'] == 'gff3_transcript':
67 subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"transcriptType": "transcript", "category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
68 else:
69 subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
70
71 def indexName(self):
72 subprocess.call(['generate-names.pl', '-v', '--out', self.json])
73 print "finished name index \n" 115 print "finished name index \n"
74 116
75 def makeArchive(self): 117 def _outHtml(self):
76 file_dir = os.path.abspath(self.outfile) 118 mylookup = TemplateLookup(directories=[os.path.join(self.tool_directory, 'templates')],
77 source_dir = os.path.dirname(file_dir) 119 output_encoding='utf-8', encoding_errors='replace')
78 folder_name = os.path.basename(self.outfolder) 120 htmlTemplate = mylookup.get_template("display.txt")
79 source_name = os.path.basename(self.out_path)
80 source = os.path.join(source_dir, folder_name, source_name)
81 slink = source.replace('/', '_')
82 slink = os.path.join('/var/www/html/JBrowse-1.12.1/data', slink)
83 try:
84 if os.path.islink(slink):
85 os.unlink(slink)
86 except OSError as oserror:
87 print "Cannot create symlink to the data({0}): {1}".format(oserror.errno, oserror.strerror)
88 os.symlink(source, slink)
89 return slink
90
91 def outHtml(self, slink):
92 with open(self.outfile, 'w') as htmlfile:
93 htmlstr = 'The JBrowse Hub is created: <br>'
94 url = self.jbrowse_host + "/JBrowse-1.12.1/index.html?data=%s"
95 jbrowse_hub = '<li><a href = "%s" target="_blank">View JBrowse Hub</a></li>' % url
96 link_name = os.path.basename(slink)
97 relative_path = os.path.join('data', link_name + '/json')
98 htmlstr += jbrowse_hub % relative_path
99 htmlfile.write(htmlstr)
100 121
101 def createTrackList(self): 122 with open(self.outputFile, 'w') as htmlfile:
102 trackList = os.path.join(self.json, "trackList.json") 123 htmlMakoRendered = htmlTemplate.render(
103 if not os.path.exists(trackList): 124 species_folder = os.path.relpath(self.mySpecieFolderPath, self.extra_files_path),
104 os.mknod(trackList) 125 trackList = os.path.relpath(self.trackList, self.extra_files_path)
105 126 )
106 def Bam(self, track, metadata): 127 htmlfile.write(htmlMakoRendered)
107 #create trackList.json if not exist 128 #with open(self.outputFile, 'w') as htmlfile:
108 self.createTrackList() 129 # htmlstr = 'The new Organism "%s" is created on Apollo: <br>' % self.genome_name
109 json_file = os.path.join(self.json, "trackList.json") 130 # jbrowse_hub = '<li><a href = "%s" target="_blank">View JBrowse Hub on Apollo</a></li>' % host_name
110 bam_track = dict() 131 # htmlstr += jbrowse_hub
111 bam_track['type'] = 'JBrowse/View/Track/Alignments2' 132 # htmlfile.write(htmlstr)
112 bam_track['storeClass'] = 'JBrowse/Store/SeqFeature/BAM' 133
113 bam_track['urlTemplate'] = os.path.join('../raw', track['fileName'])
114 bam_track['baiUrlTemplate'] = os.path.join('../raw', track['index'])
115 bam_track['label'] = metadata['label']
116 bam_track['category'] = metadata['category']
117 bam_track = json.dumps(bam_track)
118 #Use add-track-json.pl to add bam track to json file
119 new_track = subprocess.Popen(['echo', bam_track], stdout=subprocess.PIPE)
120 subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
121
122 def BigWig(self, track, metadata):
123 #create trackList.json if not exist
124 self.createTrackList()
125 json_file = os.path.join(self.json, "trackList.json")
126 bigwig_track = dict()
127 bigwig_track['urlTemplate'] = os.path.join('../raw', track['fileName'])
128 bigwig_track['type'] = 'JBrowse/View/Track/Wiggle/XYPlot'
129 bigwig_track['storeClass'] = 'JBrowse/Store/SeqFeature/BigWig'
130 bigwig_track['label'] = metadata['label']
131 bigwig_track['style'] = metadata['style']
132 bigwig_track['category'] = metadata['category']
133 bigwig_track = json.dumps(bigwig_track)
134 #Use add-track-json.pl to add bigwig track to json file
135 new_track = subprocess.Popen(['echo', bigwig_track], stdout=subprocess.PIPE)
136 #output = new_track.communicate()[0]
137 subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
138
139 #If the metadata is not set, use the default value
140 def SetMetadata(self, track, metadata):
141 if 'label' not in metadata.keys() or metadata['label'] == '':
142 metadata['label'] = track['fileName']
143 if 'color' not in metadata.keys() or metadata['color'] == '':
144 metadata['color'] = "#daa520"
145 if track['dataType'] == 'bigwig':
146 if 'style' not in metadata.keys():
147 metadata['style'] = {}
148 if 'pos_color' not in metadata['style'] or metadata['style']['pos_color'] == '':
149 metadata['style']['pos_color'] = "#FFA600"
150 if 'neg_color' not in metadata['style'] or metadata['style']['neg_color'] == '':
151 metadata['style']['neg_color'] = "#005EFF"
152 if 'category' not in metadata.keys() or metadata['category'] == '':
153 metadata['category'] = "Default group"
154 if track['dataType'] == 'blastxml':
155 metadata['type'] = "G-OnRamp_plugin/BlastAlignment"
156 elif track['dataType'] == 'bigpsl':
157 metadata['type'] = "G-OnRamp_plugin/BlatAlignment"
158 elif track['dataType'] == 'gff3_transcript' or track['dataType'] == 'gff3_mrna':
159 metadata['type'] = "G-OnRamp_plugin/GenePred"
160 else:
161 metadata['type'] = "CanvasFeatures"
162 134
163 135
136 def __createAssemblyHub__(self, extra_files_path):
137 # Get all necessaries infos first
138 # 2bit file creation from input fasta
164 139
165 140 # baseNameFasta = os.path.basename(fasta_file_name)
141 # suffixTwoBit, extensionTwoBit = os.path.splitext(baseNameFasta)
142 # nameTwoBit = suffixTwoBit + '.2bit'
143 twoBitFile = tempfile.NamedTemporaryFile(bufsize=0)
144 subtools.faToTwoBit(self.reference_genome.false_path, twoBitFile.name)
166 145
146 # Generate the twoBitInfo
147 twoBitInfoFile = tempfile.NamedTemporaryFile(bufsize=0)
148 subtools.twoBitInfo(twoBitFile.name, twoBitInfoFile.name)
167 149
150 # Then we get the output to generate the chromSizes
151 self.chromSizesFile = tempfile.NamedTemporaryFile(bufsize=0, suffix=".chrom.sizes")
152 subtools.sortChromSizes(twoBitInfoFile.name, self.chromSizesFile.name)
168 153
154 # We can get the biggest scaffold here, with chromSizesFile
155 with open(self.chromSizesFile.name, 'r') as chrom_sizes:
156 # TODO: Check if exists
157 self.default_pos = chrom_sizes.readline().split()[0]
158
159 # TODO: Manage to put every fill Function in a file dedicated for reading reasons
160 # Create the root directory
161 myHubPath = os.path.join(extra_files_path, "myHub")
162 if not os.path.exists(myHubPath):
163 os.makedirs(myHubPath)
164
165 # Create the specie folder
166 # TODO: Generate the name depending on the specie
167 mySpecieFolderPath = os.path.join(myHubPath, self.genome_name)
168 if not os.path.exists(mySpecieFolderPath):
169 os.makedirs(mySpecieFolderPath)
170 self.mySpecieFolderPath = mySpecieFolderPath
171
172 # We create the 2bit file while we just created the specie folder
173 #self.twoBitName = self.genome_name + ".2bit"
174 #self.two_bit_final_path = os.path.join(self.mySpecieFolderPath, self.twoBitName)
175 #shutil.copyfile(twoBitFile.name, self.two_bit_final_path)
176
177 # Create the folder tracks into the specie folder
178 tracksFolderPath = os.path.join(mySpecieFolderPath, "raw")
179 if not os.path.exists(tracksFolderPath):
180 os.makedirs(tracksFolderPath)
181 self.myTracksFolderPath = tracksFolderPath
182
183 myBinaryFolderPath = os.path.join(mySpecieFolderPath, 'bbi')
184 if not os.path.exists(myBinaryFolderPath):
185 os.makedirs(myBinaryFolderPath)
186 self.myBinaryFolderPath = myBinaryFolderPath
187
188 return myHubPath