Mercurial > repos > ieguinoa > data_manager_fetch_tx2gene
changeset 0:a4d4da89aae1 draft
Uploaded
| author | ieguinoa |
|---|---|
| date | Wed, 10 Oct 2018 05:44:36 -0400 |
| parents | |
| children | b193b5b8633e |
| files | README.md data_manager/data_manager_fetch_tx2gene.py data_manager/data_manager_fetch_tx2gene.xml data_manager_conf.xml tool-data/all_gff.loc.sample tool-data/representative_gff.loc.sample tool_data_table_conf.xml.sample |
| diffstat | 7 files changed, 407 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.md Wed Oct 10 05:44:36 2018 -0400 @@ -0,0 +1,2 @@ +# data_manager_fetch_tx2gene +Load entries in tx2gene table with transcript to gene tables or GTF/GFF file.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_fetch_tx2gene.py Wed Oct 10 05:44:36 2018 -0400 @@ -0,0 +1,307 @@ +#!/usr/bin/env python +#Dan Blankenberg + +import sys +import os +import tempfile +import shutil +import optparse +from ftplib import FTP +import tarfile +import zipfile +import gzip +import bz2 +try: + # For Python 3.0 and later + from urllib.request import urlopen + from io import BytesIO as StringIO + from io import UnsupportedOperation +except ImportError: + # Fall back to Python 2's urllib2 + from urllib2 import urlopen + from StringIO import StringIO + UnsupportedOperation = AttributeError +from json import loads, dumps + + +CHUNK_SIZE = 2**20 # 1mb + +DATA_TABLE_NAME = 'tx2gene' + +def cleanup_before_exit( tmp_dir ): + if tmp_dir and os.path.exists( tmp_dir ): + shutil.rmtree( tmp_dir ) + + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit(1) + + +def get_dbkey_dbname_id_name( params, dbkey_description=None ): +# dbkey = params['param_dict']['dbkey_source']['dbkey'] + #TODO: ensure sequence_id is unique and does not already appear in location file + sequence_id = params['param_dict']['sequence_id'] + if not sequence_id: + sequence_id = dbkey #uuid.uuid4() generate and use an uuid instead? + +# if params['param_dict']['dbkey_source']['dbkey_source_selector'] == 'new': +# dbkey_name = params['param_dict']['dbkey_source']['dbkey_name'] +# if not dbkey_name: +# dbkey_name = dbkey +# else: +# dbkey_name = None + dbkey = params['param_dict']['dbkey'] + dbkey_name = dbkey_description + sequence_name = params['param_dict']['sequence_name'] + if not sequence_name: + sequence_name = dbkey_description + if not sequence_name: + sequence_name = dbkey + return dbkey, dbkey_name, sequence_id, sequence_name + + +def _get_files_in_ftp_path( ftp, path ): + path_contents = [] + ftp.retrlines( 'MLSD %s' % ( path ), path_contents.append ) + return [ line.split( ';' )[ -1 ].lstrip() for line in path_contents ] + + +def _get_stream_readers_for_tar( fh, tmp_dir ): + fasta_tar = tarfile.open( fileobj=fh, mode='r:*' ) + return [x for x in [fasta_tar.extractfile(member) for member in fasta_tar.getmembers()] if x] + + +def _get_stream_readers_for_zip( fh, tmp_dir ): + """ + Unpacks all archived files in a zip file. + Individual files will be concatenated (in _stream_fasta_to_file) + """ + fasta_zip = zipfile.ZipFile( fh, 'r' ) + rval = [] + for member in fasta_zip.namelist(): + fasta_zip.extract( member, tmp_dir ) + rval.append( open( os.path.join( tmp_dir, member ), 'rb' ) ) + return rval + + +def _get_stream_readers_for_gzip( fh, tmp_dir ): + return [ gzip.GzipFile( fileobj=fh, mode='rb') ] + + +def _get_stream_readers_for_bz2( fh, tmp_dir ): + return [ bz2.BZ2File( fh.name, 'rb') ] + + +def convert_tx2gene( fasta_filename, file_type, params ): + if file_type is 'tx2gene': + return #no need to extract tx2gene table + #If the file is actually a GFF/GTF file then extract the tx2gene + gff_temp_filename = tempfile.NamedTemporaryFile().name + shutil.move(fasta_filename, gff_temp_filename) + args= ['Rscript'] + args.append(RSCRIPT_GFF_TO_TX2GENE) + args.append(gff_temp_filename) + args.append(fasta_filename) + + #assert sort_method in SORTING_METHODS, ValueError( "%s is not a valid sorting option." % sort_method ) + #return SORTING_METHODS[ sort_method ]( fasta_filename, params ) + +def _download_file(start, fh): + tmp = tempfile.NamedTemporaryFile() + tmp.write(start) + tmp.write(fh.read()) + tmp.flush() + tmp.seek(0) + return tmp + + +def get_stream_reader(fh, tmp_dir): + """ + Check if file is compressed and return correct stream reader. + If file has to be downloaded, do it now. + """ + magic_dict = { + b"\x1f\x8b\x08": _get_stream_readers_for_gzip, + b"\x42\x5a\x68": _get_stream_readers_for_bz2, + b"\x50\x4b\x03\x04": _get_stream_readers_for_zip, + } + start_of_file = fh.read(CHUNK_SIZE) + try: + fh.seek(0) + except UnsupportedOperation: # This is if fh has been created by urlopen + fh = _download_file(start_of_file, fh) + for k,v in magic_dict.items(): + if start_of_file.startswith(k): + return v(fh, tmp_dir) + try: # Check if file is tar file + if tarfile.open(fileobj=StringIO(start_of_file)): + return _get_stream_readers_for_tar(fh, tmp_dir) + except tarfile.ReadError: + pass + return fh + + + +def add_fasta_to_table(data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params): + for data_table_name, data_table_entry in _stream_fasta_to_file( fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params ): + if data_table_entry: + _add_data_table_entry( data_manager_dict, data_table_entry, data_table_name ) + + +def download_from_url( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ): + urls = filter( bool, map( lambda x: x.strip(), params['param_dict']['reference_source']['user_url'].split( '\n' ) ) ) + fasta_readers = [ get_stream_reader(urlopen( url ), tmp_dir) for url in urls ] + add_fasta_to_table(data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id,sequence_name, params) + + +def download_from_history( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ): + #TODO: allow multiple FASTA input files + input_filename = params['param_dict']['reference_source']['input_fasta'] + if isinstance( input_filename, list ): + fasta_readers = [ get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename ] + else: + fasta_readers = get_stream_reader(open(input_filename), tmp_dir) + add_fasta_to_table(data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params) + + +def copy_from_directory( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ): + input_filename = params['param_dict']['reference_source']['fasta_filename'] + create_symlink = params['param_dict']['reference_source']['create_symlink'] == 'create_symlink' + if create_symlink: + data_table_entries = _create_symlink( input_filename, target_directory, dbkey, dbkey_name, sequence_id, sequence_name ) + else: + if isinstance( input_filename, list ): + fasta_readers = [ get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename ] + else: + fasta_readers = get_stream_reader(open(input_filename), tmp_dir) + data_table_entries = _stream_fasta_to_file( fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params ) + for data_table_name, data_table_entry in data_table_entries: + if data_table_entry: + _add_data_table_entry( data_manager_dict, data_table_entry, data_table_name ) + + +def _add_data_table_entry( data_manager_dict, data_table_entry, data_table_name ): + data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) + data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get( DATA_TABLE_NAME, [] ) + data_manager_dict['data_tables'][data_table_name].append( data_table_entry ) + return data_manager_dict + + +def _stream_fasta_to_file( fasta_stream, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params, close_stream=True ): + fasta_base_filename = "%s_tx2gene.tab" % sequence_id + fasta_filename = os.path.join( target_directory, fasta_base_filename ) + with open( fasta_filename, 'wb+' ) as fasta_writer: + + if isinstance( fasta_stream, list ) and len( fasta_stream ) == 1: + fasta_stream = fasta_stream[0] + + if isinstance( fasta_stream, list ): + last_char = None + for fh in fasta_stream: + if last_char not in [ None, '\n', '\r', b'\n', b'\r' ]: + fasta_writer.write( b'\n' ) + while True: + data = fh.read( CHUNK_SIZE ) + if data: + fasta_writer.write( data ) + last_char = data[-1] + else: + break + if close_stream: + fh.close() + else: + while True: + data = fasta_stream.read( CHUNK_SIZE ) + if data: + fasta_writer.write( data ) + else: + break + if close_stream: + fasta_stream.close() + + convert_to_tx2gene( fasta_filename, params['param_dict']['file_type'], params ) + return [ ( DATA_TABLE_NAME, dict( value=sequence_id, dbkey=dbkey, name=sequence_name, path=fasta_base_filename ) ) ] + + +def compute_fasta_length( fasta_file, out_file, keep_first_word=False ): + + infile = fasta_file + out = open( out_file, 'w') + + fasta_title = '' + seq_len = 0 + + first_entry = True + + for line in open( infile ): + line = line.strip() + if not line or line.startswith( '#' ): + continue + if line[0] == '>': + if first_entry == False: + if keep_first_word: + fasta_title = fasta_title.split()[0] + out.write( "%s\t%d\n" % ( fasta_title[ 1: ], seq_len ) ) + else: + first_entry = False + fasta_title = line + seq_len = 0 + else: + seq_len += len(line) + + # last fasta-entry + if keep_first_word: + fasta_title = fasta_title.split()[0] + out.write( "%s\t%d\n" % ( fasta_title[ 1: ], seq_len ) ) + out.close() + + +def _create_symlink( input_filename, target_directory, dbkey, dbkey_name, sequence_id, sequence_name ): + fasta_base_filename = "%s.fa" % sequence_id + fasta_filename = os.path.join( target_directory, fasta_base_filename ) + os.symlink( input_filename, fasta_filename ) + return [ ( DATA_TABLE_NAME, dict( value=sequence_id, dbkey=dbkey, name=sequence_name, path=fasta_base_filename ) ) ] + + +REFERENCE_SOURCE_TO_DOWNLOAD = dict( url=download_from_url, history=download_from_history, directory=copy_from_directory ) + + +def main(): + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-d', '--dbkey_description', dest='dbkey_description', action='store', type="string", default=None, help='dbkey_description' ) + parser.add_option( '-t', '--type', dest='file_type', action='store', type='string', default=None, help='file_type') + (options, args) = parser.parse_args() + + filename = args[0] + #global DATA_TABLE_NAME + global RSCRIPT_GFF_TO_TX2GENE= os.path.join( options.base_dir, 'tximport.r') + + + if options.file_type == 'gff_gtf': + #DATA_TABLE_NAME= 'representative_gff' + else: #file_type='tx2gene' + + params = loads( open( filename ).read() ) + target_directory = params[ 'output_data' ][0]['extra_files_path'] + os.mkdir( target_directory ) + data_manager_dict = {} + + dbkey, dbkey_name, sequence_id, sequence_name = get_dbkey_dbname_id_name( params, dbkey_description=options.dbkey_description ) + + if dbkey in [ None, '', '?' ]: + raise Exception( '"%s" is not a valid dbkey. You must specify a valid dbkey.' % ( dbkey ) ) + + # Create a tmp_dir, in case a zip file needs to be uncompressed + tmp_dir = tempfile.mkdtemp() + #Fetch the input file + try: + REFERENCE_SOURCE_TO_DOWNLOAD[ params['param_dict']['reference_source']['reference_source_selector'] ]( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir) + finally: + cleanup_before_exit(tmp_dir) + #save info to json file + open( filename, 'wb' ).write( dumps( data_manager_dict ).encode() ) + +if __name__ == "__main__": + main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_fetch_tx2gene.xml Wed Oct 10 05:44:36 2018 -0400 @@ -0,0 +1,68 @@ +<tool id="data_manager_fetch_gff" name="Create entries in tx2gene data table" version="0.0.1" tool_type="manage_data"> + <description>fetching</description> + <command><![CDATA[ + python "$__tool_directory__"/data_manager_fetch_tx2gene.py "${out_file}" + --type $file_type + --dbkey_description ${ dbkey.get_display_text() } + --base_dir $__tool_directory__ + + ]]></command> + <inputs> + + <param name="dbkey" type="genomebuild" label="DBKEY to assign to data" /> + <param type="text" name="sequence_name" value="" label="Name of sequence" /> + <param type="text" name="sequence_id" value="" label="ID for sequence" /> + + <param name="file_type" type="select" label="Select input type: GFF/GTF file(features will be extracted to create tx2gene table) or transcript to gene table file(tab separated)"> + <option value="gff_gtf">GFF/GTF file</option> + <option value="tx2gene">tx2gene</option> + </param> + <conditional name="reference_source"> + <param name="reference_source_selector" type="select" label="Choose the source for the reference genome"> + <option value="url">URL</option> + <option value="history">History</option> + <option value="directory">Directory on Server</option> + </param> + <when value="url"> + <param type="text" area="True" name="user_url" value="http://" label="URLs" optional="False" /> + </when> + <when value="history"> + <param name="input_fasta" type="data" label="File from History" multiple="False" optional="False" /> + </when> + <when value="directory"> + <param type="text" name="filename" value="" label="Full path to File on disk" optional="False" /> + <param type="boolean" name="create_symlink" truevalue="create_symlink" falsevalue="copy_file" label="Create symlink to original data instead of copying" checked="False" /> + </when> + </conditional> + </inputs> + <outputs> + <data name="out_file" format="data_manager_json"/> + </outputs> + <tests> + <!-- TODO: need some way to test that new entry was added to data table --> + <test> + <param name="dbkey" value="anoGam1"/> + <param name="sequence_name" value=""/> + <param name="sequence_desc" value=""/> + <param name="sequence_id" value=""/> + <param name="reference_source_selector" value="history"/> + <param name="input_fasta" value="phiX174.fasta"/> + <param name="sort_selector" value="as_is"/> + <output name="out_file" file="phiX174.data_manager_json"/> + </test> + </tests> + <help> +**What it does** + +Fetches a gff file from various sources (URL, Galaxy History, or a server directory) and populates the "all_gff" data table. + +------ + + + +.. class:: infomark + +**Notice:** If you leave name, description, or id blank, it will be generated automatically. + + </help> +</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Wed Oct 10 05:44:36 2018 -0400 @@ -0,0 +1,20 @@ +<?xml version="1.0"?> +<data_managers> +<data_manager tool_file="data_manager/data_manager_fetch_tx2gene.xml" id="data_manager_fetch_tx2gene"> + <data_table name="all_gff"> + <output> + <column name="value" /> + <column name="dbkey" /> + <column name="name" /> + <column name="path" output_ref="out_file"> + <move type="file"> + <source>${path}</source> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">${dbkey}/tx2gene/${path}</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/tx2gene/${path}</value_translation> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + </data_manager> +</data_managers>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/all_gff.loc.sample Wed Oct 10 05:44:36 2018 -0400 @@ -0,0 +1,3 @@ +#The all_gff.loc file has this format: +# +#<unique_build_id> <dbkey> <display_name> <path_to_gff_file>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/representative_gff.loc.sample Wed Oct 10 05:44:36 2018 -0400 @@ -0,0 +1,3 @@ +#The representative_gff.loc file has this format: +# +#<unique_build_id> <dbkey> <display_name> <path_to_gff_file>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Wed Oct 10 05:44:36 2018 -0400 @@ -0,0 +1,4 @@ +<?xml version="1.0"?> +<tables> + <table name="tx2gene_table" comment_char="#" allow_duplicate_entries="False"><columns>value, dbkey, name, path</columns><file path="tool-data/tx2gene.loc" /></table> +</tables>