Mercurial > repos > iuc > data_manager_snpeff
changeset 0:9ac823a8b328 draft
Uploaded
author | iuc |
---|---|
date | Thu, 22 Jan 2015 09:14:50 -0500 |
parents | |
children | 85a23e2dd92b |
files | data_manager/data_manager_snpEff_databases.py data_manager/data_manager_snpEff_databases.xml data_manager/data_manager_snpEff_download.py data_manager/data_manager_snpEff_download.xml data_manager_conf.xml repository_dependencies.xml tool-data/snpeffv_annotations.loc.sample tool-data/snpeffv_databases.loc.sample tool-data/snpeffv_genomedb.loc.sample tool-data/snpeffv_regulationdb.loc.sample tool_data_table_conf.xml.sample tool_dependencies.xml |
diffstat | 12 files changed, 481 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_snpEff_databases.py Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +import sys +import os +import re +import tempfile +import subprocess +import fileinput +import shutil +import optparse +import urllib2 +from ftplib import FTP +import tarfile + +from galaxy.util.json import from_json_string, to_json_string + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit(1) + +def fetch_databases(data_manager_dict, target_directory, jar_path): + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + if not os.path.exists(target_directory): + os.makedirs(target_directory) + databases_path = os.path.join( target_directory, 'databases.out' ) + databases_output = open(databases_path,'w') + args = [ 'java','-jar', ] + args.append( snpEff_jar ) + args.append( 'databases' ) + # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) + # databases_output = open(databases_path) + # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() ) + return_code = proc.wait() + if return_code: + sys.exit( return_code ) + databases_output.close() + try: + data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) + data_manager_dict['data_tables']['snpeff4_databases'] = data_manager_dict['data_tables'].get( 'snpeff4_databases', [] ) + data_table_entries = [] + fh = open(databases_path,'r') + for i,line in enumerate(fh): + fields = line.split('\t') + if len(fields) >= 2: + genome_version = fields[0].strip() + if genome_version.startswith("Genome") or genome_version.startswith("-"): + continue + #snpeff test genome + if genome_version == '30c2c903' or fields[1].strip() == 'TestCase' or fields[1].strip().startswith('Test_'): + continue + description = fields[1].strip() + ' : ' + genome_version + data_table_entries.append(dict(value=genome_version, name=description)) + data_manager_dict['data_tables']['snpeff4_databases'] = data_table_entries + except Exception, e: + stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) + else: + fh.close() + return data_manager_dict + +def main(): + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) + (options, args) = parser.parse_args() + + filename = args[0] + + params = from_json_string( open( filename ).read() ) + target_directory = params[ 'output_data' ][0]['extra_files_path'] + os.mkdir( target_directory ) + data_manager_dict = {} + + + #Create Defuse Reference Data + data_manager_dict = fetch_databases( data_manager_dict, target_directory, options.jar_path) + + #save info to json file + open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) + +if __name__ == "__main__": main() +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_snpEff_databases.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,41 @@ +<tool id="data_manager_snpeff_databases" name="SnpEff Databases" version="4.0.0" tool_type="manage_data"> + <description>Read the list of available snpEff databases</description> + <requirements> + <requirement type="package" version="4.0">snpEff</requirement> + </requirements> + <command interpreter="python"> + data_manager_snpEff_databases.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar "$out_file" + </command> + <inputs> + </inputs> + <outputs> + <data name="out_file" format="data_manager_json"/> + </outputs> + <stdio> + <exit_code range=":-1" level="fatal" description="Error: Cannot open file" /> + <exit_code range="1:" level="fatal" description="Error" /> + </stdio> + <tests> + <test> + <output name="out_file"> + <assert_contents> + <!-- Check that a genome was added --> + <has_text text="GRCh38.76" /> + </assert_contents> + </output> + </test> + </tests> + <help> + +This tool updatess the list of SnpEff databases for the SnpEff Download data manager. +It should only need to be run once for a snpEff version, +since it populates the SnpEff Download data manager from the snpEff config file. + +For information about snpEff: http://snpEff.sourceforge.net + +Please cite: +"A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process] + + </help> +</tool> +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_snpEff_download.py Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,205 @@ +#!/usr/bin/env python + +import sys +import os +import re +import tempfile +import subprocess +import fileinput +import shutil +import optparse +import urllib2 +import gzip +from ftplib import FTP +import tarfile + +from galaxy.util.json import from_json_string, to_json_string + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit(1) + + +def fetch_databases(jar_path,genome_list=None): + snpDBs = dict() + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + databases_path = 'databases.out' + databases_output = open(databases_path,'w') + args = [ 'java','-jar', ] + args.append( snpEff_jar ) + args.append( 'databases' ) + # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) + # databases_output = open(databases_path) + # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() ) + return_code = proc.wait() + if return_code: + sys.exit( return_code ) + databases_output.close() + try: + fh = open(databases_path,'r') + for i,line in enumerate(fh): + fields = line.split('\t') + if len(fields) >= 2: + genome_version = fields[0].strip() + if genome_list and genome_version not in genome_list: + continue + if genome_version.startswith("Genome") or genome_version.startswith("-"): + continue + description = fields[1].strip() + snpDBs[genome_version] = description; + except Exception, e: + stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) + else: + fh.close() + return snpDBs + +def getOrganismNames(jar_path,genomes,organisms) : + genome_list = genomes.split(',') + organism_list = organisms.split(',') if organisms else [] + if len(genome_list) != len(organism_list): + descriptions = [] + snpDBdict = fetch_databases(jar_path,genome_list=genome_list); + for genome in snpDBdict: + descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome) + return ','.join(descriptions) + return organisms + +def getSnpeffVersion(jar_path): + snpeff_version = 'SnpEff ?.?' + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + stderr_path = 'snpeff.err' + stderr_fh = open(stderr_path,'w') + args = [ 'java','-jar', ] + args.append( snpEff_jar ) + args.append( '-h' ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stderr=stderr_fh.fileno() ) + return_code = proc.wait() + if return_code != 255: + sys.exit( return_code ) + stderr_fh.close() + fh = open(stderr_path,'r') + for line in fh: + m = re.match('^[Ss]npEff version (SnpEff)\s*(\d+\.\d+).*$',line) + if m: + snpeff_version = m.groups()[0] + m.groups()[1] + break + fh.close() + return snpeff_version + +# Starting with SnpEff 4.1 the .bin files contain the SnpEff version: +# Example - the first 3 line of GRCh37.75/snpEffectPredictor.bin (uncompressed): +""" +SnpEff 4.1 +CHROMOSOME 2 1 0 179197 GL000219.1 false +CHROMOSOME 3 1 0 81347269 HSCHR17_1 false +""" +def getSnpeffVersionFromFile(path): + snpeff_version = None + try: + fh = gzip.open(path, 'rb') + buf = fh.read(100) + lines = buf.splitlines() + m = re.match('^(SnpEff)\s+(\d+\.\d+).*$',lines[0].strip()) + if m: + snpeff_version = m.groups()[0] + m.groups()[1] + fh.close() + except Exception, e: + stop_err( 'Error parsing SnpEff version from: %s %s\n' % (path,str( e )) ) + return snpeff_version + +""" +# Download human database 'hg19' +java -jar snpEff.jar download -v hg19 + + <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command> + +snpEffectPredictor.bin +regulation_HeLa-S3.bin +regulation_pattern = 'regulation_(.+).bin' +""" +def download_database(data_manager_dict, target_directory, jar_path, config, genome_version, organism): + ## get data_dir from config + ##--- + ## Databases are stored here + ## E.g.: Information for 'hg19' is stored in data_dir/hg19/ + ## + ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory + ##--- + #data_dir = ~/snpEff/data/ + data_dir = target_directory + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + args = [ 'java','-jar' ] + args.append( jar_path ) + args.append( 'download' ) + args.append( '-c' ) + args.append( config ) + args.append( '-dataDir' ) + args.append( data_dir ) + args.append( '-v' ) + args.append( genome_version ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir ) + return_code = proc.wait() + if return_code: + sys.exit( return_code ) + ## search data_dir/genome_version for files + regulation_pattern = 'regulation_(.+).bin' + # annotation files that are included in snpEff by a flag + annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'} + genome_path = os.path.join(data_dir,genome_version) + snpeff_version = getSnpeffVersion(jar_path) + key = snpeff_version + '_' + genome_version + if os.path.isdir(genome_path): + for root, dirs, files in os.walk(genome_path): + for fname in files: + if fname.startswith('snpEffectPredictor'): + # if snpEffectPredictor.bin download succeeded + name = genome_version + (' : ' + organism if organism else '') + # version = getSnpeffVersionFromFile(os.path.join(root,fname)) + data_table_entry = dict(key=key,version=snpeff_version,value=genome_version, name=name, path=data_dir) + _add_data_table_entry( data_manager_dict, 'snpeffv_genomedb', data_table_entry ) + else: + m = re.match(regulation_pattern,fname) + if m: + name = m.groups()[0] + data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=name, name=name) + _add_data_table_entry( data_manager_dict, 'snpeffv_regulationdb', data_table_entry ) + elif fname in annotations_dict: + value = annotations_dict[fname] + name = value.lstrip('-') + data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=value, name=name) + _add_data_table_entry( data_manager_dict, 'snpeffv_annotations', data_table_entry ) + return data_manager_dict + +def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ): + data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) + data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] ) + data_manager_dict['data_tables'][data_table].append( data_table_entry ) + return data_manager_dict + +def main(): + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) + parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' ) + parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' ) + parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' ) + (options, args) = parser.parse_args() + + filename = args[0] + + params = from_json_string( open( filename ).read() ) + target_directory = params[ 'output_data' ][0]['extra_files_path'] + os.mkdir( target_directory ) + data_manager_dict = {} + + + #Create SnpEff Reference Data + for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')): + download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism ) + + #save info to json file + open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) + +if __name__ == "__main__": main() +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_snpEff_download.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,53 @@ +<tool id="data_manager_snpeff_download" name="SnpEff Download" version="4.0.0" tool_type="manage_data"> + <description>Download a new database</description> + <requirements> + <requirement type="package" version="4.0">snpEff</requirement> + </requirements> + <command interpreter="python"> + data_manager_snpEff_download.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar --config \$SNPEFF_JAR_PATH/snpEff.config + --genome_version "$genome_version" "$out_file" + </command> + <inputs> + <param name="genome_version" type="text" size="40" value="" label="Snpff Genome Version Name (e.g. GRCh38.76)"> + <help>https://snpeff-data.galaxyproject.org/databases/</help> + <validator type="regex" message="A genome version name is required">\S+</validator> + </param> + </inputs> + + <outputs> + <data name="out_file" format="data_manager_json" label="${tool.name}"/> + </outputs> + <stdio> + <exit_code range=":-1" level="fatal" description="Error: Cannot open file" /> + <exit_code range="1:" level="fatal" description="Error" /> + </stdio> + <tests> + <test> + <param name="genome_version" value="GRCh38.76"/> + <output name="out_file"> + <assert_contents> + <!-- Check that a genome was added --> + <has_text text="GRCh38.76" /> + <has_text text="snpeffv_regulationdb" /> + <has_text text="snpeffv_annotations" /> + </assert_contents> + </output> + </test> + </tests> + <help> + +This tool downloads a SnpEff database and populates data tables: snpeffv_genomedb, snpeffv_regulationdb, and snpeffv_annotations. + +To see the list of available SnpEff genomes run the "SnpEff Databases" data manager which records the available genome databases in data table: snpeff4_databases + +The SnpEff genome databases are at: http://sourceforge.net/projects/snpeff/files/databases/ + + +For details about this tool, please go to http://snpEff.sourceforge.net + +Please cite: +"A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process] + + </help> +</tool> +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,50 @@ +<?xml version="1.0"?> +<data_managers> + <data_manager tool_file="data_manager/data_manager_snpEff_databases.xml" id="data_manager_snpeff_databases" > + <data_table name="snpeffv_databases"> <!-- Defines a Data Table to be modified. --> + <output> <!-- Handle the output of the Data Manager Tool --> + <column name="key" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="version" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="name" /> <!-- columns that are going to be specified by the Data Manager Tool --> + </output> + </data_table> + </data_manager> + <data_manager tool_file="data_manager/data_manager_snpEff_download.xml" id="data_manager_snpeff_download" > + <data_table name="snpeffv_genomedb"> <!-- Defines a Data Table to be modified. --> + <output> <!-- Handle the output of the Data Manager Tool --> + <column name="key" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="version" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="name" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="path" output_ref="out_file" > + <move type="directory" relativize_symlinks="True"> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">snpEff/v4_0/data</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/snpEff/v4_0/data</value_translation> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + <data_table name="snpeffv_regulationdb"> <!-- Defines a Data Table to be modified. --> + <output> <!-- Handle the output of the Data Manager Tool --> + <column name="key" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="version" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="genome" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="name" /> <!-- columns that are going to be specified by the Data Manager Tool --> + </output> + </data_table> + <data_table name="snpeffv_annotations"> <!-- Defines a Data Table to be modified. --> + <output> <!-- Handle the output of the Data Manager Tool --> + <column name="key" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="version" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="genome" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="name" /> <!-- columns that are going to be specified by the Data Manager Tool --> + </output> + </data_table> + </data_manager> +</data_managers> + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/repository_dependencies.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,4 @@ +<?xml version="1.0"?> +<repositories description="This requires the SnpEff datatype definitions."> + <repository changeset_revision="d78b2b2a3388" name="snpeff_datatypes" owner="iuc" toolshed="https://toolshed.g2.bx.psu.edu" /> +</repositories>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/snpeffv_annotations.loc.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,5 @@ +## Regulation Databases for SnpEff +## These are from the list on: http://snpeff.sourceforge.net/download.html +#key snpeff_version genome annotation_name description +#SnpEff4.0_GRCh37.75 SnpEff4.0 GRCh37.75 nextprot nextprot +#SnpEff4.0_GRCh38.76 SnpEff4.1 GRCh38.76 motif motif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/snpeffv_databases.loc.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,5 @@ +## Available Databases for SnpEff +## These are from the list on: http://snpeff.sourceforge.net/download.html +## the Description field in this sample is "Genome : Version" +#key snpeff_version Version Description +#SnpEff4.0_GRCh37.75 SnpEff4.0 GRCh37.75 Homo sapiens : GRCh37.75
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/snpeffv_genomedb.loc.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,6 @@ +## Downloaded Databases for SnpEff +## These are from the list on: http://snpeff.sourceforge.net/download.html +## the Description field in this sample is "Genome : Version" +#Key snpeff_version Version Description data_dir path +#SnpEff4.0_GRCh37.74 SnpEff4.0 GRCh37.74 Homo sapiens : GRCh37.74 /home/galaxy/snpEff/v4_0/data +#SnpEff4.1_GRCh38.76 SnpEff4.1 GRCh38.76 Homo sapiens : GRCh38.76 /home/galaxy/snpEff/v4_1/data
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/snpeffv_regulationdb.loc.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,5 @@ +## Regulation Databases for SnpEff +## These are from the list on: http://snpeff.sourceforge.net/download.html +#Key snpeff_version genome regulation_name description +#SnpEff4.0_GRCh37.74 SnpEff4.0 GRCh37.74 CD4 CD4 +#SnpEff4.1_GRCh38.76 SnpEff4.1 GRCh38.76 CD4 CD4
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,19 @@ +<tables> + <table name="snpeffv_genomedb" comment_char="#"> + <columns>key, version, value, name, path</columns> + <file path="tool-data/snpeffv_genomedb.loc" /> + </table> + <table name="snpeffv_regulationdb" comment_char="#"> + <columns>key, version, genome, value, name</columns> + <file path="tool-data/snpeffv_regulationdb.loc" /> + </table> + <table name="snpeffv_annotations" comment_char="#"> + <columns>key, version, genome, value, name</columns> + <file path="tool-data/snpeffv_annotations.loc" /> + </table> + <table name="snpeffv_databases" comment_char="#"> + <columns>key, version, value, name</columns> + <file path="tool-data/snpeffv_databases.loc" /> + </table> +</tables> +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Thu Jan 22 09:14:50 2015 -0500 @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<tool_dependency> + <package name="snpEff" version="4.0"> + <repository changeset_revision="6bc55957927b" name="package_snpeff_4_0" owner="iuc" toolshed="https://toolshed.g2.bx.psu.edu" /> + </package> +</tool_dependency>