# HG changeset patch
# User iuc
# Date 1421936090 18000
# Node ID 9ac823a8b328ff17c49ec8e984934a8fa313a58a
Uploaded
diff -r 000000000000 -r 9ac823a8b328 data_manager/data_manager_snpEff_databases.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_snpEff_databases.py Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import tempfile
+import subprocess
+import fileinput
+import shutil
+import optparse
+import urllib2
+from ftplib import FTP
+import tarfile
+
+from galaxy.util.json import from_json_string, to_json_string
+
+def stop_err(msg):
+ sys.stderr.write(msg)
+ sys.exit(1)
+
+def fetch_databases(data_manager_dict, target_directory, jar_path):
+ (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
+ if not os.path.exists(target_directory):
+ os.makedirs(target_directory)
+ databases_path = os.path.join( target_directory, 'databases.out' )
+ databases_output = open(databases_path,'w')
+ args = [ 'java','-jar', ]
+ args.append( snpEff_jar )
+ args.append( 'databases' )
+ # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" )
+ # databases_output = open(databases_path)
+ # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() )
+ proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() )
+ return_code = proc.wait()
+ if return_code:
+ sys.exit( return_code )
+ databases_output.close()
+ try:
+ data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
+ data_manager_dict['data_tables']['snpeff4_databases'] = data_manager_dict['data_tables'].get( 'snpeff4_databases', [] )
+ data_table_entries = []
+ fh = open(databases_path,'r')
+ for i,line in enumerate(fh):
+ fields = line.split('\t')
+ if len(fields) >= 2:
+ genome_version = fields[0].strip()
+ if genome_version.startswith("Genome") or genome_version.startswith("-"):
+ continue
+ #snpeff test genome
+ if genome_version == '30c2c903' or fields[1].strip() == 'TestCase' or fields[1].strip().startswith('Test_'):
+ continue
+ description = fields[1].strip() + ' : ' + genome_version
+ data_table_entries.append(dict(value=genome_version, name=description))
+ data_manager_dict['data_tables']['snpeff4_databases'] = data_table_entries
+ except Exception, e:
+ stop_err( 'Error parsing %s %s\n' % (config,str( e )) )
+ else:
+ fh.close()
+ return data_manager_dict
+
+def main():
+ #Parse Command Line
+ parser = optparse.OptionParser()
+ parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' )
+ (options, args) = parser.parse_args()
+
+ filename = args[0]
+
+ params = from_json_string( open( filename ).read() )
+ target_directory = params[ 'output_data' ][0]['extra_files_path']
+ os.mkdir( target_directory )
+ data_manager_dict = {}
+
+
+ #Create Defuse Reference Data
+ data_manager_dict = fetch_databases( data_manager_dict, target_directory, options.jar_path)
+
+ #save info to json file
+ open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
+
+if __name__ == "__main__": main()
+
diff -r 000000000000 -r 9ac823a8b328 data_manager/data_manager_snpEff_databases.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_snpEff_databases.xml Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,41 @@
+
+ Read the list of available snpEff databases
+
+ snpEff
+
+
+ data_manager_snpEff_databases.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar "$out_file"
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+This tool updatess the list of SnpEff databases for the SnpEff Download data manager.
+It should only need to be run once for a snpEff version,
+since it populates the SnpEff Download data manager from the snpEff config file.
+
+For information about snpEff: http://snpEff.sourceforge.net
+
+Please cite:
+"A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process]
+
+
+
+
diff -r 000000000000 -r 9ac823a8b328 data_manager/data_manager_snpEff_download.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_snpEff_download.py Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import tempfile
+import subprocess
+import fileinput
+import shutil
+import optparse
+import urllib2
+import gzip
+from ftplib import FTP
+import tarfile
+
+from galaxy.util.json import from_json_string, to_json_string
+
+def stop_err(msg):
+ sys.stderr.write(msg)
+ sys.exit(1)
+
+
+def fetch_databases(jar_path,genome_list=None):
+ snpDBs = dict()
+ (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
+ databases_path = 'databases.out'
+ databases_output = open(databases_path,'w')
+ args = [ 'java','-jar', ]
+ args.append( snpEff_jar )
+ args.append( 'databases' )
+ # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" )
+ # databases_output = open(databases_path)
+ # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() )
+ proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() )
+ return_code = proc.wait()
+ if return_code:
+ sys.exit( return_code )
+ databases_output.close()
+ try:
+ fh = open(databases_path,'r')
+ for i,line in enumerate(fh):
+ fields = line.split('\t')
+ if len(fields) >= 2:
+ genome_version = fields[0].strip()
+ if genome_list and genome_version not in genome_list:
+ continue
+ if genome_version.startswith("Genome") or genome_version.startswith("-"):
+ continue
+ description = fields[1].strip()
+ snpDBs[genome_version] = description;
+ except Exception, e:
+ stop_err( 'Error parsing %s %s\n' % (config,str( e )) )
+ else:
+ fh.close()
+ return snpDBs
+
+def getOrganismNames(jar_path,genomes,organisms) :
+ genome_list = genomes.split(',')
+ organism_list = organisms.split(',') if organisms else []
+ if len(genome_list) != len(organism_list):
+ descriptions = []
+ snpDBdict = fetch_databases(jar_path,genome_list=genome_list);
+ for genome in snpDBdict:
+ descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome)
+ return ','.join(descriptions)
+ return organisms
+
+def getSnpeffVersion(jar_path):
+ snpeff_version = 'SnpEff ?.?'
+ (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
+ stderr_path = 'snpeff.err'
+ stderr_fh = open(stderr_path,'w')
+ args = [ 'java','-jar', ]
+ args.append( snpEff_jar )
+ args.append( '-h' )
+ proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stderr=stderr_fh.fileno() )
+ return_code = proc.wait()
+ if return_code != 255:
+ sys.exit( return_code )
+ stderr_fh.close()
+ fh = open(stderr_path,'r')
+ for line in fh:
+ m = re.match('^[Ss]npEff version (SnpEff)\s*(\d+\.\d+).*$',line)
+ if m:
+ snpeff_version = m.groups()[0] + m.groups()[1]
+ break
+ fh.close()
+ return snpeff_version
+
+# Starting with SnpEff 4.1 the .bin files contain the SnpEff version:
+# Example - the first 3 line of GRCh37.75/snpEffectPredictor.bin (uncompressed):
+"""
+SnpEff 4.1
+CHROMOSOME 2 1 0 179197 GL000219.1 false
+CHROMOSOME 3 1 0 81347269 HSCHR17_1 false
+"""
+def getSnpeffVersionFromFile(path):
+ snpeff_version = None
+ try:
+ fh = gzip.open(path, 'rb')
+ buf = fh.read(100)
+ lines = buf.splitlines()
+ m = re.match('^(SnpEff)\s+(\d+\.\d+).*$',lines[0].strip())
+ if m:
+ snpeff_version = m.groups()[0] + m.groups()[1]
+ fh.close()
+ except Exception, e:
+ stop_err( 'Error parsing SnpEff version from: %s %s\n' % (path,str( e )) )
+ return snpeff_version
+
+"""
+# Download human database 'hg19'
+java -jar snpEff.jar download -v hg19
+
+ java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile
+
+snpEffectPredictor.bin
+regulation_HeLa-S3.bin
+regulation_pattern = 'regulation_(.+).bin'
+"""
+def download_database(data_manager_dict, target_directory, jar_path, config, genome_version, organism):
+ ## get data_dir from config
+ ##---
+ ## Databases are stored here
+ ## E.g.: Information for 'hg19' is stored in data_dir/hg19/
+ ##
+ ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory
+ ##---
+ #data_dir = ~/snpEff/data/
+ data_dir = target_directory
+ (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
+ args = [ 'java','-jar' ]
+ args.append( jar_path )
+ args.append( 'download' )
+ args.append( '-c' )
+ args.append( config )
+ args.append( '-dataDir' )
+ args.append( data_dir )
+ args.append( '-v' )
+ args.append( genome_version )
+ proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir )
+ return_code = proc.wait()
+ if return_code:
+ sys.exit( return_code )
+ ## search data_dir/genome_version for files
+ regulation_pattern = 'regulation_(.+).bin'
+ # annotation files that are included in snpEff by a flag
+ annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'}
+ genome_path = os.path.join(data_dir,genome_version)
+ snpeff_version = getSnpeffVersion(jar_path)
+ key = snpeff_version + '_' + genome_version
+ if os.path.isdir(genome_path):
+ for root, dirs, files in os.walk(genome_path):
+ for fname in files:
+ if fname.startswith('snpEffectPredictor'):
+ # if snpEffectPredictor.bin download succeeded
+ name = genome_version + (' : ' + organism if organism else '')
+ # version = getSnpeffVersionFromFile(os.path.join(root,fname))
+ data_table_entry = dict(key=key,version=snpeff_version,value=genome_version, name=name, path=data_dir)
+ _add_data_table_entry( data_manager_dict, 'snpeffv_genomedb', data_table_entry )
+ else:
+ m = re.match(regulation_pattern,fname)
+ if m:
+ name = m.groups()[0]
+ data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=name, name=name)
+ _add_data_table_entry( data_manager_dict, 'snpeffv_regulationdb', data_table_entry )
+ elif fname in annotations_dict:
+ value = annotations_dict[fname]
+ name = value.lstrip('-')
+ data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=value, name=name)
+ _add_data_table_entry( data_manager_dict, 'snpeffv_annotations', data_table_entry )
+ return data_manager_dict
+
+def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ):
+ data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
+ data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] )
+ data_manager_dict['data_tables'][data_table].append( data_table_entry )
+ return data_manager_dict
+
+def main():
+ #Parse Command Line
+ parser = optparse.OptionParser()
+ parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' )
+ parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' )
+ parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' )
+ parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' )
+ (options, args) = parser.parse_args()
+
+ filename = args[0]
+
+ params = from_json_string( open( filename ).read() )
+ target_directory = params[ 'output_data' ][0]['extra_files_path']
+ os.mkdir( target_directory )
+ data_manager_dict = {}
+
+
+ #Create SnpEff Reference Data
+ for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')):
+ download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism )
+
+ #save info to json file
+ open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
+
+if __name__ == "__main__": main()
+
diff -r 000000000000 -r 9ac823a8b328 data_manager/data_manager_snpEff_download.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_snpEff_download.xml Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,53 @@
+
+ Download a new database
+
+ snpEff
+
+
+ data_manager_snpEff_download.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar --config \$SNPEFF_JAR_PATH/snpEff.config
+ --genome_version "$genome_version" "$out_file"
+
+
+
+ https://snpeff-data.galaxyproject.org/databases/
+ \S+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+This tool downloads a SnpEff database and populates data tables: snpeffv_genomedb, snpeffv_regulationdb, and snpeffv_annotations.
+
+To see the list of available SnpEff genomes run the "SnpEff Databases" data manager which records the available genome databases in data table: snpeff4_databases
+
+The SnpEff genome databases are at: http://sourceforge.net/projects/snpeff/files/databases/
+
+
+For details about this tool, please go to http://snpEff.sourceforge.net
+
+Please cite:
+"A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process]
+
+
+
+
diff -r 000000000000 -r 9ac823a8b328 data_manager_conf.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager_conf.xml Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,50 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff -r 000000000000 -r 9ac823a8b328 repository_dependencies.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/repository_dependencies.xml Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,4 @@
+
+
+
+
diff -r 000000000000 -r 9ac823a8b328 tool-data/snpeffv_annotations.loc.sample
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/snpeffv_annotations.loc.sample Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,5 @@
+## Regulation Databases for SnpEff
+## These are from the list on: http://snpeff.sourceforge.net/download.html
+#key snpeff_version genome annotation_name description
+#SnpEff4.0_GRCh37.75 SnpEff4.0 GRCh37.75 nextprot nextprot
+#SnpEff4.0_GRCh38.76 SnpEff4.1 GRCh38.76 motif motif
diff -r 000000000000 -r 9ac823a8b328 tool-data/snpeffv_databases.loc.sample
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/snpeffv_databases.loc.sample Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,5 @@
+## Available Databases for SnpEff
+## These are from the list on: http://snpeff.sourceforge.net/download.html
+## the Description field in this sample is "Genome : Version"
+#key snpeff_version Version Description
+#SnpEff4.0_GRCh37.75 SnpEff4.0 GRCh37.75 Homo sapiens : GRCh37.75
diff -r 000000000000 -r 9ac823a8b328 tool-data/snpeffv_genomedb.loc.sample
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/snpeffv_genomedb.loc.sample Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,6 @@
+## Downloaded Databases for SnpEff
+## These are from the list on: http://snpeff.sourceforge.net/download.html
+## the Description field in this sample is "Genome : Version"
+#Key snpeff_version Version Description data_dir path
+#SnpEff4.0_GRCh37.74 SnpEff4.0 GRCh37.74 Homo sapiens : GRCh37.74 /home/galaxy/snpEff/v4_0/data
+#SnpEff4.1_GRCh38.76 SnpEff4.1 GRCh38.76 Homo sapiens : GRCh38.76 /home/galaxy/snpEff/v4_1/data
diff -r 000000000000 -r 9ac823a8b328 tool-data/snpeffv_regulationdb.loc.sample
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/snpeffv_regulationdb.loc.sample Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,5 @@
+## Regulation Databases for SnpEff
+## These are from the list on: http://snpeff.sourceforge.net/download.html
+#Key snpeff_version genome regulation_name description
+#SnpEff4.0_GRCh37.74 SnpEff4.0 GRCh37.74 CD4 CD4
+#SnpEff4.1_GRCh38.76 SnpEff4.1 GRCh38.76 CD4 CD4
diff -r 000000000000 -r 9ac823a8b328 tool_data_table_conf.xml.sample
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.sample Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,19 @@
+
+
+ key, version, value, name, path
+
+
+
+ key, version, genome, value, name
+
+
+
+ key, version, genome, value, name
+
+
+
+ key, version, value, name
+
+
+
+
diff -r 000000000000 -r 9ac823a8b328 tool_dependencies.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_dependencies.xml Thu Jan 22 09:14:50 2015 -0500
@@ -0,0 +1,6 @@
+
+
+
+
+
+