Mercurial > repos > iuc > data_manager_snpeff
changeset 1:85a23e2dd92b draft
planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_snpeff commit 88c982c5adcd32b11d98428fc554a4fdfcc19584
| author | iuc |
|---|---|
| date | Tue, 07 Jun 2016 10:11:50 -0400 |
| parents | 9ac823a8b328 |
| children | 984733fcab49 |
| files | data_manager/data_manager_snpEff_databases.py data_manager/data_manager_snpEff_databases.xml data_manager/data_manager_snpEff_download.py data_manager/data_manager_snpEff_download.xml repository_dependencies.xml tool_dependencies.xml |
| diffstat | 6 files changed, 117 insertions(+), 127 deletions(-) [+] |
line wrap: on
line diff
--- a/data_manager/data_manager_snpEff_databases.py Thu Jan 22 09:14:50 2015 -0500 +++ b/data_manager/data_manager_snpEff_databases.py Tue Jun 07 10:11:50 2016 -0400 @@ -1,30 +1,23 @@ #!/usr/bin/env python - -import sys +import json +import optparse import os -import re -import tempfile import subprocess -import fileinput -import shutil -import optparse -import urllib2 -from ftplib import FTP -import tarfile +import sys -from galaxy.util.json import from_json_string, to_json_string def stop_err(msg): sys.stderr.write(msg) sys.exit(1) + def fetch_databases(data_manager_dict, target_directory, jar_path): - (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + (snpEff_dir, snpEff_jar) = os.path.split(jar_path) if not os.path.exists(target_directory): os.makedirs(target_directory) databases_path = os.path.join( target_directory, 'databases.out' ) - databases_output = open(databases_path,'w') - args = [ 'java','-jar', ] + databases_output = open(databases_path, 'w') + args = [ 'java', '-jar' ] args.append( snpEff_jar ) args.append( 'databases' ) # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) @@ -35,48 +28,48 @@ if return_code: sys.exit( return_code ) databases_output.close() + data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) + data_manager_dict['data_tables']['snpeff4_databases'] = data_manager_dict['data_tables'].get( 'snpeff4_databases', [] ) + data_table_entries = [] try: - data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) - data_manager_dict['data_tables']['snpeff4_databases'] = data_manager_dict['data_tables'].get( 'snpeff4_databases', [] ) - data_table_entries = [] - fh = open(databases_path,'r') - for i,line in enumerate(fh): + fh = open(databases_path, 'r') + for i, line in enumerate(fh): fields = line.split('\t') if len(fields) >= 2: genome_version = fields[0].strip() if genome_version.startswith("Genome") or genome_version.startswith("-"): continue - #snpeff test genome + # snpeff test genome if genome_version == '30c2c903' or fields[1].strip() == 'TestCase' or fields[1].strip().startswith('Test_'): continue description = fields[1].strip() + ' : ' + genome_version data_table_entries.append(dict(value=genome_version, name=description)) data_manager_dict['data_tables']['snpeff4_databases'] = data_table_entries - except Exception, e: - stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) + except Exception as e: + stop_err( 'Error parsing %s %s\n' % (databases_path, str( e )) ) else: fh.close() return data_manager_dict + def main(): - #Parse Command Line + # Parse Command Line parser = optparse.OptionParser() parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) (options, args) = parser.parse_args() filename = args[0] - params = from_json_string( open( filename ).read() ) + params = json.loads( open( filename ).read() ) target_directory = params[ 'output_data' ][0]['extra_files_path'] os.mkdir( target_directory ) data_manager_dict = {} - - #Create Defuse Reference Data + # Create Defuse Reference Data data_manager_dict = fetch_databases( data_manager_dict, target_directory, options.jar_path) - #save info to json file - open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) + # save info to json file + open( filename, 'wb' ).write( json.dumps( data_manager_dict ) ) -if __name__ == "__main__": main() - +if __name__ == "__main__": + main()
--- a/data_manager/data_manager_snpEff_databases.xml Thu Jan 22 09:14:50 2015 -0500 +++ b/data_manager/data_manager_snpEff_databases.xml Tue Jun 07 10:11:50 2016 -0400 @@ -1,20 +1,20 @@ -<tool id="data_manager_snpeff_databases" name="SnpEff Databases" version="4.0.0" tool_type="manage_data"> +<tool id="data_manager_snpeff_databases" name="SnpEff Databases" version="4.1.0" tool_type="manage_data"> <description>Read the list of available snpEff databases</description> <requirements> - <requirement type="package" version="4.0">snpEff</requirement> + <requirement type="package" version="4.1">snpEff</requirement> </requirements> + <stdio> + <exit_code range=":-1" level="fatal" description="Error: Cannot open file" /> + <exit_code range="1:" level="fatal" description="Error" /> + </stdio> <command interpreter="python"> - data_manager_snpEff_databases.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar "$out_file" + data_manager_snpEff_databases.py --jar_path "\$SNPEFF_JAR_PATH/snpEff.jar" "$out_file" </command> <inputs> </inputs> <outputs> <data name="out_file" format="data_manager_json"/> </outputs> - <stdio> - <exit_code range=":-1" level="fatal" description="Error: Cannot open file" /> - <exit_code range="1:" level="fatal" description="Error" /> - </stdio> <tests> <test> <output name="out_file"> @@ -37,5 +37,8 @@ "A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process] </help> + <citations> + <citation type="doi">10.4161/fly.19695</citation> + </citations> </tool>
--- a/data_manager/data_manager_snpEff_download.py Thu Jan 22 09:14:50 2015 -0500 +++ b/data_manager/data_manager_snpEff_download.py Tue Jun 07 10:11:50 2016 -0400 @@ -1,31 +1,24 @@ #!/usr/bin/env python - -import sys +import gzip +import json +import optparse import os import re -import tempfile import subprocess -import fileinput -import shutil -import optparse -import urllib2 -import gzip -from ftplib import FTP -import tarfile +import sys -from galaxy.util.json import from_json_string, to_json_string def stop_err(msg): sys.stderr.write(msg) sys.exit(1) -def fetch_databases(jar_path,genome_list=None): +def fetch_databases(jar_path, genome_list=None): snpDBs = dict() - (snpEff_dir,snpEff_jar) = os.path.split(jar_path) - databases_path = 'databases.out' - databases_output = open(databases_path,'w') - args = [ 'java','-jar', ] + (snpEff_dir, snpEff_jar) = os.path.split(jar_path) + databases_path = 'databases.out' + databases_output = open(databases_path, 'w') + args = [ 'java', '-jar' ] args.append( snpEff_jar ) args.append( 'databases' ) # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) @@ -37,8 +30,8 @@ sys.exit( return_code ) databases_output.close() try: - fh = open(databases_path,'r') - for i,line in enumerate(fh): + fh = open(databases_path, 'r') + for i, line in enumerate(fh): fields = line.split('\t') if len(fields) >= 2: genome_version = fields[0].strip() @@ -47,30 +40,32 @@ if genome_version.startswith("Genome") or genome_version.startswith("-"): continue description = fields[1].strip() - snpDBs[genome_version] = description; - except Exception, e: - stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) + snpDBs[genome_version] = description + except Exception as e: + stop_err( 'Error parsing %s %s\n' % (databases_path, str( e )) ) else: fh.close() return snpDBs -def getOrganismNames(jar_path,genomes,organisms) : + +def getOrganismNames(jar_path, genomes, organisms): genome_list = genomes.split(',') organism_list = organisms.split(',') if organisms else [] if len(genome_list) != len(organism_list): descriptions = [] - snpDBdict = fetch_databases(jar_path,genome_list=genome_list); + snpDBdict = fetch_databases(jar_path, genome_list=genome_list) for genome in snpDBdict: descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome) return ','.join(descriptions) - return organisms + return organisms + def getSnpeffVersion(jar_path): snpeff_version = 'SnpEff ?.?' - (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + (snpEff_dir, snpEff_jar) = os.path.split(jar_path) stderr_path = 'snpeff.err' - stderr_fh = open(stderr_path,'w') - args = [ 'java','-jar', ] + stderr_fh = open(stderr_path, 'w') + args = [ 'java', '-jar' ] args.append( snpEff_jar ) args.append( '-h' ) proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stderr=stderr_fh.fileno() ) @@ -78,58 +73,57 @@ if return_code != 255: sys.exit( return_code ) stderr_fh.close() - fh = open(stderr_path,'r') + fh = open(stderr_path, 'r') for line in fh: - m = re.match('^[Ss]npEff version (SnpEff)\s*(\d+\.\d+).*$',line) + m = re.match('^[Ss]npEff version (SnpEff)\s*(\d+\.\d+).*$', line) if m: snpeff_version = m.groups()[0] + m.groups()[1] break fh.close() return snpeff_version + # Starting with SnpEff 4.1 the .bin files contain the SnpEff version: # Example - the first 3 line of GRCh37.75/snpEffectPredictor.bin (uncompressed): -""" -SnpEff 4.1 -CHROMOSOME 2 1 0 179197 GL000219.1 false -CHROMOSOME 3 1 0 81347269 HSCHR17_1 false -""" +# +# SnpEff 4.1 +# CHROMOSOME 2 1 0 179197 GL000219.1 false +# CHROMOSOME 3 1 0 81347269 HSCHR17_1 false def getSnpeffVersionFromFile(path): snpeff_version = None try: fh = gzip.open(path, 'rb') buf = fh.read(100) lines = buf.splitlines() - m = re.match('^(SnpEff)\s+(\d+\.\d+).*$',lines[0].strip()) + m = re.match('^(SnpEff)\s+(\d+\.\d+).*$', lines[0].strip()) if m: snpeff_version = m.groups()[0] + m.groups()[1] fh.close() - except Exception, e: - stop_err( 'Error parsing SnpEff version from: %s %s\n' % (path,str( e )) ) - return snpeff_version + except Exception as e: + stop_err( 'Error parsing SnpEff version from: %s %s\n' % (path, str( e )) ) + return snpeff_version -""" -# Download human database 'hg19' -java -jar snpEff.jar download -v hg19 - - <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command> -snpEffectPredictor.bin -regulation_HeLa-S3.bin -regulation_pattern = 'regulation_(.+).bin' -""" +# Download human database 'hg19' +# java -jar snpEff.jar download -v hg19 +# +# <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command> +# +# snpEffectPredictor.bin +# regulation_HeLa-S3.bin +# regulation_pattern = 'regulation_(.+).bin' def download_database(data_manager_dict, target_directory, jar_path, config, genome_version, organism): - ## get data_dir from config - ##--- - ## Databases are stored here - ## E.g.: Information for 'hg19' is stored in data_dir/hg19/ - ## - ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory - ##--- - #data_dir = ~/snpEff/data/ + # get data_dir from config + # --- + # Databases are stored here + # E.g.: Information for 'hg19' is stored in data_dir/hg19/ + # + # Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory + # --- + # data_dir = ~/snpEff/data/ data_dir = target_directory - (snpEff_dir,snpEff_jar) = os.path.split(jar_path) - args = [ 'java','-jar' ] + (snpEff_dir, snpEff_jar) = os.path.split(jar_path) + args = [ 'java', '-jar' ] args.append( jar_path ) args.append( 'download' ) args.append( '-c' ) @@ -142,43 +136,45 @@ return_code = proc.wait() if return_code: sys.exit( return_code ) - ## search data_dir/genome_version for files + # search data_dir/genome_version for files regulation_pattern = 'regulation_(.+).bin' # annotation files that are included in snpEff by a flag - annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'} - genome_path = os.path.join(data_dir,genome_version) + annotations_dict = {'nextProt.bin': '-nextprot', 'motif.bin': '-motif'} + genome_path = os.path.join(data_dir, genome_version) snpeff_version = getSnpeffVersion(jar_path) - key = snpeff_version + '_' + genome_version + key = snpeff_version + '_' + genome_version if os.path.isdir(genome_path): for root, dirs, files in os.walk(genome_path): for fname in files: if fname.startswith('snpEffectPredictor'): # if snpEffectPredictor.bin download succeeded - name = genome_version + (' : ' + organism if organism else '') + name = genome_version + (' : ' + organism if organism else '') # version = getSnpeffVersionFromFile(os.path.join(root,fname)) - data_table_entry = dict(key=key,version=snpeff_version,value=genome_version, name=name, path=data_dir) + data_table_entry = dict(key=key, version=snpeff_version, value=genome_version, name=name, path=data_dir) _add_data_table_entry( data_manager_dict, 'snpeffv_genomedb', data_table_entry ) else: - m = re.match(regulation_pattern,fname) + m = re.match(regulation_pattern, fname) if m: name = m.groups()[0] - data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=name, name=name) + data_table_entry = dict(key=key, version=snpeff_version, genome=genome_version, value=name, name=name) _add_data_table_entry( data_manager_dict, 'snpeffv_regulationdb', data_table_entry ) elif fname in annotations_dict: value = annotations_dict[fname] name = value.lstrip('-') - data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=value, name=name) + data_table_entry = dict(key=key, version=snpeff_version, genome=genome_version, value=value, name=name) _add_data_table_entry( data_manager_dict, 'snpeffv_annotations', data_table_entry ) return data_manager_dict + def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ): data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] ) data_manager_dict['data_tables'][data_table].append( data_table_entry ) return data_manager_dict + def main(): - #Parse Command Line + # Parse Command Line parser = optparse.OptionParser() parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' ) @@ -188,18 +184,17 @@ filename = args[0] - params = from_json_string( open( filename ).read() ) + params = json.loads( open( filename ).read() ) target_directory = params[ 'output_data' ][0]['extra_files_path'] os.mkdir( target_directory ) data_manager_dict = {} - - #Create SnpEff Reference Data - for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')): + # Create SnpEff Reference Data + for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path, options.genome_version, options.organism).split(',')): download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism ) - #save info to json file - open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) + # save info to json file + open( filename, 'wb' ).write( json.dumps( data_manager_dict ) ) -if __name__ == "__main__": main() - +if __name__ == "__main__": + main()
--- a/data_manager/data_manager_snpEff_download.xml Thu Jan 22 09:14:50 2015 -0500 +++ b/data_manager/data_manager_snpEff_download.xml Tue Jun 07 10:11:50 2016 -0400 @@ -1,14 +1,18 @@ -<tool id="data_manager_snpeff_download" name="SnpEff Download" version="4.0.0" tool_type="manage_data"> +<tool id="data_manager_snpeff_download" name="SnpEff Download" version="4.1.0" tool_type="manage_data"> <description>Download a new database</description> <requirements> - <requirement type="package" version="4.0">snpEff</requirement> + <requirement type="package" version="4.1">snpEff</requirement> </requirements> + <stdio> + <exit_code range=":-1" level="fatal" description="Error: Cannot open file" /> + <exit_code range="1:" level="fatal" description="Error" /> + </stdio> <command interpreter="python"> - data_manager_snpEff_download.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar --config \$SNPEFF_JAR_PATH/snpEff.config + data_manager_snpEff_download.py --jar_path "\$SNPEFF_JAR_PATH/snpEff.jar" --config "\$SNPEFF_JAR_PATH/snpEff.config" --genome_version "$genome_version" "$out_file" </command> <inputs> - <param name="genome_version" type="text" size="40" value="" label="Snpff Genome Version Name (e.g. GRCh38.76)"> + <param name="genome_version" type="text" value="" label="Snpff Genome Version Name (e.g. GRCh38.76)"> <help>https://snpeff-data.galaxyproject.org/databases/</help> <validator type="regex" message="A genome version name is required">\S+</validator> </param> @@ -17,10 +21,6 @@ <outputs> <data name="out_file" format="data_manager_json" label="${tool.name}"/> </outputs> - <stdio> - <exit_code range=":-1" level="fatal" description="Error: Cannot open file" /> - <exit_code range="1:" level="fatal" description="Error" /> - </stdio> <tests> <test> <param name="genome_version" value="GRCh38.76"/> @@ -38,7 +38,7 @@ This tool downloads a SnpEff database and populates data tables: snpeffv_genomedb, snpeffv_regulationdb, and snpeffv_annotations. -To see the list of available SnpEff genomes run the "SnpEff Databases" data manager which records the available genome databases in data table: snpeff4_databases +To see the list of available SnpEff genomes run the "SnpEff Databases" data manager which records the available genome databases in data table: snpeff4_databases The SnpEff genome databases are at: http://sourceforge.net/projects/snpeff/files/databases/ @@ -49,5 +49,8 @@ "A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process] </help> + <citations> + <citation type="doi">10.4161/fly.19695</citation> + </citations> </tool>
--- a/repository_dependencies.xml Thu Jan 22 09:14:50 2015 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,4 +0,0 @@ -<?xml version="1.0"?> -<repositories description="This requires the SnpEff datatype definitions."> - <repository changeset_revision="d78b2b2a3388" name="snpeff_datatypes" owner="iuc" toolshed="https://toolshed.g2.bx.psu.edu" /> -</repositories>
--- a/tool_dependencies.xml Thu Jan 22 09:14:50 2015 -0500 +++ b/tool_dependencies.xml Tue Jun 07 10:11:50 2016 -0400 @@ -1,6 +1,6 @@ <?xml version="1.0"?> <tool_dependency> - <package name="snpEff" version="4.0"> - <repository changeset_revision="6bc55957927b" name="package_snpeff_4_0" owner="iuc" toolshed="https://toolshed.g2.bx.psu.edu" /> + <package name="snpEff" version="4.1"> + <repository changeset_revision="374c7f8421fb" name="package_snpeff_4_1" owner="iuc" toolshed="https://toolshed.g2.bx.psu.edu" /> </package> </tool_dependency>