# HG changeset patch # User iuc # Date 1421936090 18000 # Node ID 9ac823a8b328ff17c49ec8e984934a8fa313a58a Uploaded diff -r 000000000000 -r 9ac823a8b328 data_manager/data_manager_snpEff_databases.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_snpEff_databases.py Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +import sys +import os +import re +import tempfile +import subprocess +import fileinput +import shutil +import optparse +import urllib2 +from ftplib import FTP +import tarfile + +from galaxy.util.json import from_json_string, to_json_string + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit(1) + +def fetch_databases(data_manager_dict, target_directory, jar_path): + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + if not os.path.exists(target_directory): + os.makedirs(target_directory) + databases_path = os.path.join( target_directory, 'databases.out' ) + databases_output = open(databases_path,'w') + args = [ 'java','-jar', ] + args.append( snpEff_jar ) + args.append( 'databases' ) + # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) + # databases_output = open(databases_path) + # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() ) + return_code = proc.wait() + if return_code: + sys.exit( return_code ) + databases_output.close() + try: + data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) + data_manager_dict['data_tables']['snpeff4_databases'] = data_manager_dict['data_tables'].get( 'snpeff4_databases', [] ) + data_table_entries = [] + fh = open(databases_path,'r') + for i,line in enumerate(fh): + fields = line.split('\t') + if len(fields) >= 2: + genome_version = fields[0].strip() + if genome_version.startswith("Genome") or genome_version.startswith("-"): + continue + #snpeff test genome + if genome_version == '30c2c903' or fields[1].strip() == 'TestCase' or fields[1].strip().startswith('Test_'): + continue + description = fields[1].strip() + ' : ' + genome_version + data_table_entries.append(dict(value=genome_version, name=description)) + data_manager_dict['data_tables']['snpeff4_databases'] = data_table_entries + except Exception, e: + stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) + else: + fh.close() + return data_manager_dict + +def main(): + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) + (options, args) = parser.parse_args() + + filename = args[0] + + params = from_json_string( open( filename ).read() ) + target_directory = params[ 'output_data' ][0]['extra_files_path'] + os.mkdir( target_directory ) + data_manager_dict = {} + + + #Create Defuse Reference Data + data_manager_dict = fetch_databases( data_manager_dict, target_directory, options.jar_path) + + #save info to json file + open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) + +if __name__ == "__main__": main() + diff -r 000000000000 -r 9ac823a8b328 data_manager/data_manager_snpEff_databases.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_snpEff_databases.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,41 @@ + + Read the list of available snpEff databases + + snpEff + + + data_manager_snpEff_databases.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar "$out_file" + + + + + + + + + + + + + + + + + + + + + + +This tool updatess the list of SnpEff databases for the SnpEff Download data manager. +It should only need to be run once for a snpEff version, +since it populates the SnpEff Download data manager from the snpEff config file. + +For information about snpEff: http://snpEff.sourceforge.net + +Please cite: +"A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process] + + + + diff -r 000000000000 -r 9ac823a8b328 data_manager/data_manager_snpEff_download.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_snpEff_download.py Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,205 @@ +#!/usr/bin/env python + +import sys +import os +import re +import tempfile +import subprocess +import fileinput +import shutil +import optparse +import urllib2 +import gzip +from ftplib import FTP +import tarfile + +from galaxy.util.json import from_json_string, to_json_string + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit(1) + + +def fetch_databases(jar_path,genome_list=None): + snpDBs = dict() + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + databases_path = 'databases.out' + databases_output = open(databases_path,'w') + args = [ 'java','-jar', ] + args.append( snpEff_jar ) + args.append( 'databases' ) + # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) + # databases_output = open(databases_path) + # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() ) + return_code = proc.wait() + if return_code: + sys.exit( return_code ) + databases_output.close() + try: + fh = open(databases_path,'r') + for i,line in enumerate(fh): + fields = line.split('\t') + if len(fields) >= 2: + genome_version = fields[0].strip() + if genome_list and genome_version not in genome_list: + continue + if genome_version.startswith("Genome") or genome_version.startswith("-"): + continue + description = fields[1].strip() + snpDBs[genome_version] = description; + except Exception, e: + stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) + else: + fh.close() + return snpDBs + +def getOrganismNames(jar_path,genomes,organisms) : + genome_list = genomes.split(',') + organism_list = organisms.split(',') if organisms else [] + if len(genome_list) != len(organism_list): + descriptions = [] + snpDBdict = fetch_databases(jar_path,genome_list=genome_list); + for genome in snpDBdict: + descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome) + return ','.join(descriptions) + return organisms + +def getSnpeffVersion(jar_path): + snpeff_version = 'SnpEff ?.?' + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + stderr_path = 'snpeff.err' + stderr_fh = open(stderr_path,'w') + args = [ 'java','-jar', ] + args.append( snpEff_jar ) + args.append( '-h' ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stderr=stderr_fh.fileno() ) + return_code = proc.wait() + if return_code != 255: + sys.exit( return_code ) + stderr_fh.close() + fh = open(stderr_path,'r') + for line in fh: + m = re.match('^[Ss]npEff version (SnpEff)\s*(\d+\.\d+).*$',line) + if m: + snpeff_version = m.groups()[0] + m.groups()[1] + break + fh.close() + return snpeff_version + +# Starting with SnpEff 4.1 the .bin files contain the SnpEff version: +# Example - the first 3 line of GRCh37.75/snpEffectPredictor.bin (uncompressed): +""" +SnpEff 4.1 +CHROMOSOME 2 1 0 179197 GL000219.1 false +CHROMOSOME 3 1 0 81347269 HSCHR17_1 false +""" +def getSnpeffVersionFromFile(path): + snpeff_version = None + try: + fh = gzip.open(path, 'rb') + buf = fh.read(100) + lines = buf.splitlines() + m = re.match('^(SnpEff)\s+(\d+\.\d+).*$',lines[0].strip()) + if m: + snpeff_version = m.groups()[0] + m.groups()[1] + fh.close() + except Exception, e: + stop_err( 'Error parsing SnpEff version from: %s %s\n' % (path,str( e )) ) + return snpeff_version + +""" +# Download human database 'hg19' +java -jar snpEff.jar download -v hg19 + + java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile + +snpEffectPredictor.bin +regulation_HeLa-S3.bin +regulation_pattern = 'regulation_(.+).bin' +""" +def download_database(data_manager_dict, target_directory, jar_path, config, genome_version, organism): + ## get data_dir from config + ##--- + ## Databases are stored here + ## E.g.: Information for 'hg19' is stored in data_dir/hg19/ + ## + ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory + ##--- + #data_dir = ~/snpEff/data/ + data_dir = target_directory + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + args = [ 'java','-jar' ] + args.append( jar_path ) + args.append( 'download' ) + args.append( '-c' ) + args.append( config ) + args.append( '-dataDir' ) + args.append( data_dir ) + args.append( '-v' ) + args.append( genome_version ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir ) + return_code = proc.wait() + if return_code: + sys.exit( return_code ) + ## search data_dir/genome_version for files + regulation_pattern = 'regulation_(.+).bin' + # annotation files that are included in snpEff by a flag + annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'} + genome_path = os.path.join(data_dir,genome_version) + snpeff_version = getSnpeffVersion(jar_path) + key = snpeff_version + '_' + genome_version + if os.path.isdir(genome_path): + for root, dirs, files in os.walk(genome_path): + for fname in files: + if fname.startswith('snpEffectPredictor'): + # if snpEffectPredictor.bin download succeeded + name = genome_version + (' : ' + organism if organism else '') + # version = getSnpeffVersionFromFile(os.path.join(root,fname)) + data_table_entry = dict(key=key,version=snpeff_version,value=genome_version, name=name, path=data_dir) + _add_data_table_entry( data_manager_dict, 'snpeffv_genomedb', data_table_entry ) + else: + m = re.match(regulation_pattern,fname) + if m: + name = m.groups()[0] + data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=name, name=name) + _add_data_table_entry( data_manager_dict, 'snpeffv_regulationdb', data_table_entry ) + elif fname in annotations_dict: + value = annotations_dict[fname] + name = value.lstrip('-') + data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=value, name=name) + _add_data_table_entry( data_manager_dict, 'snpeffv_annotations', data_table_entry ) + return data_manager_dict + +def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ): + data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) + data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] ) + data_manager_dict['data_tables'][data_table].append( data_table_entry ) + return data_manager_dict + +def main(): + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) + parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' ) + parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' ) + parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' ) + (options, args) = parser.parse_args() + + filename = args[0] + + params = from_json_string( open( filename ).read() ) + target_directory = params[ 'output_data' ][0]['extra_files_path'] + os.mkdir( target_directory ) + data_manager_dict = {} + + + #Create SnpEff Reference Data + for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')): + download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism ) + + #save info to json file + open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) + +if __name__ == "__main__": main() + diff -r 000000000000 -r 9ac823a8b328 data_manager/data_manager_snpEff_download.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_snpEff_download.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,53 @@ + + Download a new database + + snpEff + + + data_manager_snpEff_download.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar --config \$SNPEFF_JAR_PATH/snpEff.config + --genome_version "$genome_version" "$out_file" + + + + https://snpeff-data.galaxyproject.org/databases/ + \S+ + + + + + + + + + + + + + + + + + + + + + + + + + +This tool downloads a SnpEff database and populates data tables: snpeffv_genomedb, snpeffv_regulationdb, and snpeffv_annotations. + +To see the list of available SnpEff genomes run the "SnpEff Databases" data manager which records the available genome databases in data table: snpeff4_databases + +The SnpEff genome databases are at: http://sourceforge.net/projects/snpeff/files/databases/ + + +For details about this tool, please go to http://snpEff.sourceforge.net + +Please cite: +"A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process] + + + + diff -r 000000000000 -r 9ac823a8b328 data_manager_conf.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + snpEff/v4_0/data + + ${GALAXY_DATA_MANAGER_DATA_PATH}/snpEff/v4_0/data + abspath + + + + + + + + + + + + + + + + + + + + + + + + + diff -r 000000000000 -r 9ac823a8b328 repository_dependencies.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/repository_dependencies.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,4 @@ + + + + diff -r 000000000000 -r 9ac823a8b328 tool-data/snpeffv_annotations.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/snpeffv_annotations.loc.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,5 @@ +## Regulation Databases for SnpEff +## These are from the list on: http://snpeff.sourceforge.net/download.html +#key snpeff_version genome annotation_name description +#SnpEff4.0_GRCh37.75 SnpEff4.0 GRCh37.75 nextprot nextprot +#SnpEff4.0_GRCh38.76 SnpEff4.1 GRCh38.76 motif motif diff -r 000000000000 -r 9ac823a8b328 tool-data/snpeffv_databases.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/snpeffv_databases.loc.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,5 @@ +## Available Databases for SnpEff +## These are from the list on: http://snpeff.sourceforge.net/download.html +## the Description field in this sample is "Genome : Version" +#key snpeff_version Version Description +#SnpEff4.0_GRCh37.75 SnpEff4.0 GRCh37.75 Homo sapiens : GRCh37.75 diff -r 000000000000 -r 9ac823a8b328 tool-data/snpeffv_genomedb.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/snpeffv_genomedb.loc.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,6 @@ +## Downloaded Databases for SnpEff +## These are from the list on: http://snpeff.sourceforge.net/download.html +## the Description field in this sample is "Genome : Version" +#Key snpeff_version Version Description data_dir path +#SnpEff4.0_GRCh37.74 SnpEff4.0 GRCh37.74 Homo sapiens : GRCh37.74 /home/galaxy/snpEff/v4_0/data +#SnpEff4.1_GRCh38.76 SnpEff4.1 GRCh38.76 Homo sapiens : GRCh38.76 /home/galaxy/snpEff/v4_1/data diff -r 000000000000 -r 9ac823a8b328 tool-data/snpeffv_regulationdb.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/snpeffv_regulationdb.loc.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,5 @@ +## Regulation Databases for SnpEff +## These are from the list on: http://snpeff.sourceforge.net/download.html +#Key snpeff_version genome regulation_name description +#SnpEff4.0_GRCh37.74 SnpEff4.0 GRCh37.74 CD4 CD4 +#SnpEff4.1_GRCh38.76 SnpEff4.1 GRCh38.76 CD4 CD4 diff -r 000000000000 -r 9ac823a8b328 tool_data_table_conf.xml.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,19 @@ + + + key, version, value, name, path + +
+ + key, version, genome, value, name + +
+ + key, version, genome, value, name + +
+ + key, version, value, name + +
+
+ diff -r 000000000000 -r 9ac823a8b328 tool_dependencies.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,6 @@ + + + + + +