Mercurial > repos > bgruening > data_manager_diamond_database_builder
changeset 2:5558f74bd296 draft
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_diamond_database_builder commit 75abf7d4b23ed7ae8abce80609d81b20bc882863"
| author | iuc |
|---|---|
| date | Sun, 21 Mar 2021 12:07:34 +0000 |
| parents | 5a0d0bee4f8d |
| children | 431198ad8a09 |
| files | data_manager/data_manager_diamond_database_builder.py data_manager/data_manager_diamond_database_builder.xml data_manager_conf.xml test-data/diamond_data_manager_ncbi.json test-data/ncbi_accession2taxid.loc test-data/taxonomy/names.dmp test-data/taxonomy/nodes.dmp test-data/taxonomy/prot.accession2taxid tool_data_table_conf.xml.sample tool_data_table_conf.xml.test |
| diffstat | 10 files changed, 194 insertions(+), 83 deletions(-) [+] |
line wrap: on
line diff
--- a/data_manager/data_manager_diamond_database_builder.py Tue Dec 03 17:39:48 2019 -0500 +++ b/data_manager/data_manager_diamond_database_builder.py Sun Mar 21 12:07:34 2021 +0000 @@ -1,17 +1,19 @@ #!/usr/bin/env python +import bz2 +import gzip import json -import sys -import os -import tempfile -import shutil import optparse -import urllib2 +import os +import shutil import subprocess -from ftplib import FTP +import sys import tarfile +import tempfile +import urllib.error +import urllib.parse +import urllib.request import zipfile -import gzip -import bz2 +from ftplib import FTP CHUNK_SIZE = 2**20 # 1mb @@ -21,11 +23,6 @@ shutil.rmtree(tmp_dir) -def stop_err(msg): - sys.stderr.write(msg) - sys.exit(1) - - def _get_files_in_ftp_path(ftp, path): path_contents = [] ftp.retrlines('MLSD %s' % (path), path_contents.append) @@ -54,12 +51,17 @@ return [bz2.BZ2File(file_obj.name, 'rb')] -def download_from_ncbi(data_manager_dict, params, target_directory, database_id, database_name): +def download_from_ncbi(data_manager_dict, params, target_directory, + database_id, database_name): NCBI_FTP_SERVER = 'ftp.ncbi.nlm.nih.gov' NCBI_DOWNLOAD_PATH = '/blast/db/FASTA/' - COMPRESSED_EXTENSIONS = [('.tar.gz', _get_stream_readers_for_tar), ('.tar.bz2', _get_stream_readers_for_tar), ('.zip', _get_stream_readers_for_zip), ('.gz', _get_stream_readers_for_gzip), ('.bz2', _get_stream_readers_for_bz2)] + COMPRESSED_EXTENSIONS = [('.tar.gz', _get_stream_readers_for_tar), + ('.tar.bz2', _get_stream_readers_for_tar), + ('.zip', _get_stream_readers_for_zip), + ('.gz', _get_stream_readers_for_gzip), + ('.bz2', _get_stream_readers_for_bz2)] - ncbi_identifier = params['param_dict']['reference_source']['requested_identifier'] + ncbi_identifier = params['reference_source']['requested_identifier'] ftp = FTP(NCBI_FTP_SERVER) ftp.login() @@ -79,9 +81,9 @@ tmp_dir = tempfile.mkdtemp(prefix='tmp-data-manager-ncbi-') ncbi_fasta_filename = os.path.join(tmp_dir, "%s%s" % (ncbi_identifier, ext)) - fasta_base_filename = "%s.fa" % database_id - fasta_filename = os.path.join(target_directory, fasta_base_filename) - fasta_writer = open(fasta_filename, 'wb+') + # fasta_base_filename = "%s.fa" % database_id + # fasta_filename = os.path.join(target_directory, fasta_base_filename) + # fasta_writer = open(fasta_filename, 'wb+') tmp_extract_dir = os.path.join(tmp_dir, 'extracted_fasta') os.mkdir(tmp_extract_dir) @@ -106,8 +108,8 @@ def download_from_url(data_manager_dict, params, target_directory, database_id, database_name): # TODO: we should automatically do decompression here - urls = filter(bool, map(lambda x: x.strip(), params['param_dict']['reference_source']['user_url'].split('\n'))) - fasta_reader = [urllib2.urlopen(url) for url in urls] + urls = list(filter(bool, [x.strip() for x in params['reference_source']['user_url'].split('\n')])) + fasta_reader = [urllib.request.urlopen(url) for url in urls] data_table_entry = _stream_fasta_to_file(fasta_reader, target_directory, database_id, database_name, params) _add_data_table_entry(data_manager_dict, data_table_entry) @@ -115,19 +117,19 @@ def download_from_history(data_manager_dict, params, target_directory, database_id, database_name): # TODO: allow multiple FASTA input files - input_filename = params['param_dict']['reference_source']['input_fasta'] + input_filename = params['reference_source']['input_fasta'] if isinstance(input_filename, list): fasta_reader = [open(filename, 'rb') for filename in input_filename] else: - fasta_reader = open(input_filename) + fasta_reader = open(input_filename, 'rb') data_table_entry = _stream_fasta_to_file(fasta_reader, target_directory, database_id, database_name, params) _add_data_table_entry(data_manager_dict, data_table_entry) def copy_from_directory(data_manager_dict, params, target_directory, database_id, database_name): - input_filename = params['param_dict']['reference_source']['fasta_filename'] - create_symlink = params['param_dict']['reference_source']['create_symlink'] == 'create_symlink' + input_filename = params['reference_source']['fasta_filename'] + create_symlink = params['reference_source']['create_symlink'] == 'create_symlink' if create_symlink: data_table_entry = _create_symlink(input_filename, target_directory, database_id, database_name) else: @@ -146,7 +148,8 @@ return data_manager_dict -def _stream_fasta_to_file(fasta_stream, target_directory, database_id, database_name, params, close_stream=True): +def _stream_fasta_to_file(fasta_stream, target_directory, database_id, + database_name, params, close_stream=True): fasta_base_filename = "%s.fa" % database_id fasta_filename = os.path.join(target_directory, fasta_base_filename) @@ -154,53 +157,70 @@ temp_fasta.close() fasta_writer = open(temp_fasta.name, 'wb+') - if isinstance(fasta_stream, list) and len(fasta_stream) == 1: - fasta_stream = fasta_stream[0] + if not isinstance(fasta_stream, list): + fasta_stream = [fasta_stream] - if isinstance(fasta_stream, list): - last_char = None - for fh in fasta_stream: - if last_char not in [None, '\n', '\r']: - fasta_writer.write('\n') - while True: - data = fh.read(CHUNK_SIZE) - if data: - fasta_writer.write(data) - last_char = data[-1] - else: - break - if close_stream: - fh.close() - else: + last_char = None + for fh in fasta_stream: + if last_char not in [None, '\n', '\r']: + fasta_writer.write('\n') while True: - data = fasta_stream.read(CHUNK_SIZE) + data = fh.read(CHUNK_SIZE) if data: fasta_writer.write(data) + last_char = data[-1] else: break if close_stream: - fasta_stream.close() + fh.close() fasta_writer.close() - args = ['diamond', 'makedb', '--in', temp_fasta.name, '--db', fasta_filename] + args = ['diamond', 'makedb', + '--in', temp_fasta.name, + '--db', fasta_filename] + if params['tax_cond']['tax_select'] == "history": + for i in ["taxonmap", "taxonnodes", "taxonnames"]: + args.extend(['--' + i, params['tax_cond'][i]]) + elif params['tax_cond']['tax_select'] == "ncbi": + if os.path.isfile(os.path.join(params['tax_cond']['ncbi_tax'], 'prot.accession2taxid.FULL.gz')): + args.extend(['--taxonmap', + os.path.join(params['tax_cond']['ncbi_tax'], 'prot.accession2taxid.FULL.gz')]) + elif os.path.isfile(os.path.join(params['tax_cond']['ncbi_tax'], 'prot.accession2taxid.FULL')): + args.extend(['--taxonmap', + os.path.join(params['tax_cond']['ncbi_tax'], 'prot.accession2taxid.FULL')]) + elif os.path.isfile(os.path.join(params['tax_cond']['ncbi_tax'], 'prot.accession2taxid.gz')): + args.extend(['--taxonmap', + os.path.join(params['tax_cond']['ncbi_tax'], 'prot.accession2taxid.gz')]) + elif os.path.isfile(os.path.join(params['tax_cond']['ncbi_tax'], 'prot.accession2taxid')): + args.extend(['--taxonmap', + os.path.join(params['tax_cond']['ncbi_tax'], 'prot.accession2taxid')]) + else: + raise Exception('Unable to find prot.accession2taxid file in %s' % (params['tax_cond']['ncbi_tax'])) + + args.extend(['--taxonnodes', + os.path.join(params['tax_cond']['ncbi_tax'], 'nodes.dmp')]) + args.extend(['--taxonnames', + os.path.join(params['tax_cond']['ncbi_tax'], 'names.dmp')]) tmp_stderr = tempfile.NamedTemporaryFile(prefix="tmp-data-manager-diamond-database-builder-stderr") - proc = subprocess.Popen(args=args, shell=False, cwd=target_directory, stderr=tmp_stderr.fileno()) + proc = subprocess.Popen(args=args, shell=False, cwd=target_directory, + stderr=tmp_stderr.fileno()) return_code = proc.wait() if return_code: tmp_stderr.flush() tmp_stderr.seek(0) - print >> sys.stderr, "Error building diamond database:" + print("Error building diamond database:", file=sys.stderr) while True: chunk = tmp_stderr.read(CHUNK_SIZE) if not chunk: break - sys.stderr.write(chunk) + sys.stderr.write(chunk.decode('utf-8')) sys.exit(return_code) tmp_stderr.close() os.remove(temp_fasta.name) - return dict(value=database_id, name=database_name, db_path="%s.dmnd" % fasta_base_filename) + return dict(value=database_id, name=database_name, + db_path="%s.dmnd" % fasta_base_filename) def _create_symlink(input_filename, target_directory, database_id, database_name): @@ -210,27 +230,36 @@ return dict(value=database_id, name=database_name, db_path=fasta_base_filename) -REFERENCE_SOURCE_TO_DOWNLOAD = dict(ncbi=download_from_ncbi, url=download_from_url, history=download_from_history, directory=copy_from_directory) +REFERENCE_SOURCE_TO_DOWNLOAD = dict(ncbi=download_from_ncbi, + url=download_from_url, + history=download_from_history, + directory=copy_from_directory) def main(): # Parse Command Line parser = optparse.OptionParser() - parser.add_option('-d', '--dbkey_description', dest='dbkey_description', action='store', type="string", default=None, help='dbkey_description') + parser.add_option('-d', '--dbkey_description', dest='dbkey_description', + action='store', type="string", default=None, + help='dbkey_description') (options, args) = parser.parse_args() filename = args[0] - params = json.loads(open(filename).read()) + with open(filename) as fp: + params = json.load(fp) target_directory = params['output_data'][0]['extra_files_path'] os.mkdir(target_directory) data_manager_dict = {} - database_id = params['param_dict']['database_id'] - database_name = params['param_dict']['database_name'] + param_dict = params['param_dict'] + database_id = param_dict['database_id'] + database_name = param_dict['database_name'] + if param_dict['tax_cond']['tax_select'] == "ncbi": + param_dict['tax_cond']['ncbi_tax'] = args[1] # Fetch the FASTA - REFERENCE_SOURCE_TO_DOWNLOAD[params['param_dict']['reference_source']['reference_source_selector']](data_manager_dict, params, target_directory, database_id, database_name) + REFERENCE_SOURCE_TO_DOWNLOAD[param_dict['reference_source']['reference_source_selector']](data_manager_dict, param_dict, target_directory, database_id, database_name) # save info to json file open(filename, 'w').write(json.dumps(data_manager_dict, sort_keys=True))
--- a/data_manager/data_manager_diamond_database_builder.xml Tue Dec 03 17:39:48 2019 -0500 +++ b/data_manager/data_manager_diamond_database_builder.xml Sun Mar 21 12:07:34 2021 +0000 @@ -1,36 +1,60 @@ -<tool id="diamond_database_builder" name="Diamond" tool_type="manage_data" version="0.0.2"> +<tool id="diamond_database_builder" name="Diamond" tool_type="manage_data" version="0.0.3" profile="19.05"> <description> Database builder</description> <requirements> - <requirement type="package" version="0.9.29">diamond</requirement> - <requirement type="package" version="2.7">python</requirement> + <requirement type="package" version="2.0.8">diamond</requirement> + <requirement type="package" version="3.7">python</requirement> </requirements> - <command> + <command><![CDATA[ python '$__tool_directory__/data_manager_diamond_database_builder.py' '${out_file}' - </command> + #if $tax_cond.tax_select == "ncbi" + '$tax_cond.ncbi_tax.fields.path' + #end if + ]]></command> <inputs> <param type="text" name="database_name" value="" label="Database name or description" help="This will be shown to the user in the database selectbox."/> <param type="text" name="database_id" value="" label="Database ID" help="This needs to be unique. No whitespaces allowed. Use underscores." /> <conditional name="reference_source"> - <param name="reference_source_selector" type="select" label="Choose the source for the reference genome"> - <option value="ncbi">NCBI</option> - <option value="url">URL</option> - <option value="history" selected="True" help="Recommended to archive the original FASTA file as well.">History</option> - <option value="directory">Directory on Server</option> - </param> - <when value="ncbi"> - <param type="text" name="requested_identifier" value="" label="NCBI identifier" help="e.g. nr" optional="False" /> - </when> - <when value="url"> - <param type="text" area="True" name="user_url" value="http://" label="URLs" optional="False" /> - </when> - <when value="history"> - <param name="input_fasta" type="data" format="fasta" label="FASTA File" multiple="False" optional="False" /> - </when> - <when value="directory"> - <param type="text" name="fasta_filename" value="" label="Full path to FASTA File on disk" optional="False" /> - <param type="boolean" name="create_symlink" truevalue="create_symlink" falsevalue="copy_file" label="Create symlink to orignal data instead of copying" checked="False" /> - </when> + <param name="reference_source_selector" type="select" label="Choose the source for the reference genome"> + <option value="ncbi">NCBI</option> + <option value="url">URL</option> + <option value="history" selected="true" help="Recommended to archive the original FASTA file as well.">History</option> + <option value="directory">Directory on Server</option> + </param> + <when value="ncbi"> + <param type="text" name="requested_identifier" value="" label="NCBI identifier" help="e.g. nr" optional="false" /> + </when> + <when value="url"> + <param type="text" name="user_url" value="http://" label="URLs" optional="false" /> + </when> + <when value="history"> + <param name="input_fasta" type="data" format="fasta" label="FASTA File" multiple="false" optional="false" /> + </when> + <when value="directory"> + <param type="text" name="fasta_filename" value="" label="Full path to FASTA File on disk" optional="false" /> + <param type="boolean" name="create_symlink" truevalue="create_symlink" falsevalue="copy_file" label="Create symlink to orignal data instead of copying" checked="false" /> + </when> + </conditional> + <conditional name="tax_cond"> + <param name="tax_select" type="select" label="Add taxonomic information"> + <option value="no">Do not add taxonomic information</option> + <option value="ncbi">Use cached NCBI taxonomy</option> + <option value="history">Use data from history</option> + </param> + <when value="no"/> + <when value="ncbi"> + <param name="ncbi_tax" type="select" label="Select NCBI taxonomy"> + <options from_data_table="ncbi_accession2taxid"> + <filter type="sort_by" column="2"/> + <validator type="no_options" message="No taxonomy available"/> + </options> + </param> + </when> + <when value="history"> + <param argument="--taxonmap" type="data" format="tabular" label="protein accession to taxid mapping file" help="" /> + <param argument="--taxonnodes" type="data" format="tabular" label="taxonomy nodes.dmp from NCBI" help="" /> + <param argument="--taxonnames" type="data" format="tabular" label="taxonomy names.dmp from NCBI" help="" /> + </when> </conditional> </inputs> <outputs> @@ -44,16 +68,41 @@ <param name="reference_source_selector" value="history"/> <param name="input_fasta" value="phiX174.fasta"/> </conditional> + <conditional name="tax_cond"> + <param name="tax_select" value="history"/> + <param name="taxonmap" ftype="tabular" value="taxonomy/prot.accession2taxid" /> + <param name="taxonnodes" ftype="tabular" value="taxonomy/nodes.dmp" /> + <param name="taxonnames" ftype="tabular" value="taxonomy/names.dmp" /> + </conditional> <output name="out_file" value="diamond_data_manager.json"/> </test> + <test> + <param name="database_name" value="pdbaa"/> + <param name="database_id" value="pdbaa"/> + <conditional name="reference_source"> + <param name="reference_source_selector" value="ncbi"/> + <param name="requested_identifier" value="pdbaa"/> + </conditional> + <conditional name="tax_cond"> + <param name="tax_select" value="ncbi"/> + <param name="ncbi_tax" value="2021-03-19"/> + </conditional> + <output name="out_file" value="diamond_data_manager_ncbi.json"/> + </test> </tests> <help> .. class:: infomark -NCBI databases can be downded from ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/ +NCBI databases can be downloaded from ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/ For example the NR database can be downloaded from ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/nr.gz + +- taxonmap: Path to mapping file that maps NCBI protein accession numbers to taxon ids (gzip compressed). This parameter is optional and needs to be supplied in order to provide taxonomy features. The file can be downloaded from NCBI: ftp.ncbi.nlm.nih.gov/pub/taxonomy/accession2taxid/prot.accession2taxid.FULL.gz + +- taxonnames: Path to the names.dmp file from the NCBI taxonomy. This parameter is optional and needs to be supplied in order to provide taxonomy features. The file is contained within this archive downloadable at NCBI: ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdmp.zip + +- taxonnodes: Path to the nodes.dmp file from the NCBI taxonomy. This parameter is optional and needs to be supplied in order to provide taxonomy features. The file is contained within this archive downloadable at NCBI: ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdmp.zip </help> <citations> <citation type="doi">10.1038/nmeth.3176</citation>
--- a/data_manager_conf.xml Tue Dec 03 17:39:48 2019 -0500 +++ b/data_manager_conf.xml Sun Mar 21 12:07:34 2021 +0000 @@ -1,13 +1,12 @@ <?xml version="1.0"?> <data_managers> - <data_manager tool_file="data_manager/data_manager_diamond_database_builder.xml" id="diamond_database_builder" version="0.0.1"> + <data_manager tool_file="data_manager/data_manager_diamond_database_builder.xml" id="diamond_database_builder" version="0.0.3"> <data_table name="diamond_database"> <output> <column name="value" /> <column name="name" /> <column name="db_path" output_ref="out_file" > <move type="directory" relativize_symlinks="True"> - <!-- <source>${db_path}</source>--> <!-- out_file.extra_files_path is used as base by default --> <!-- if no source, eg for type=directory, then refers to base --> <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">diamond_database/${value}</target> </move> <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/diamond_database/${value}/${db_path}</value_translation>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/diamond_data_manager_ncbi.json Sun Mar 21 12:07:34 2021 +0000 @@ -0,0 +1,1 @@ +{"data_tables": {"diamond_database": [{"db_path": "pdbaa.fa.dmnd", "name": "pdbaa", "value": "pdbaa"}]}} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/ncbi_accession2taxid.loc Sun Mar 21 12:07:34 2021 +0000 @@ -0,0 +1,6 @@ +# Tab separated fields where +# value is unique key +# name is descriptive name +# path is path to directory containing accession2taxid files +#value name path +2021-03-19 2021-03-19 ${__HERE__}/taxonomy
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/taxonomy/names.dmp Sun Mar 21 12:07:34 2021 +0000 @@ -0,0 +1,11 @@ +1 | all | | synonym | +1 | root | | scientific name | +2 | Bacteria | Bacteria <bacteria> | scientific name | +2 | bacteria | | blast name | +2 | eubacteria | | genbank common name | +2 | Monera | Monera <bacteria> | in-part | +3 | Procaryotae | Procaryotae <bacteria> | in-part | +3 | Prokaryotae | Prokaryotae <bacteria> | in-part | +3 | Prokaryota | Prokaryota <bacteria> | in-part | +3 | prokaryote | prokaryote <bacteria> | in-part | +3 | prokaryotes | prokaryotes <bacteria> | in-part |
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/taxonomy/nodes.dmp Sun Mar 21 12:07:34 2021 +0000 @@ -0,0 +1,3 @@ +1 | 1 | no rank | | 8 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | | +2 | 1 | species | AC | 0 | 1 | 11 | 1 | 0 | 1 | 1 | 0 | | +3 | 1 | species | AC | 0 | 1 | 11 | 1 | 0 | 1 | 1 | 0 | |
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/taxonomy/prot.accession2taxid Sun Mar 21 12:07:34 2021 +0000 @@ -0,0 +1,4 @@ +accession accession.version taxid gi +AAD44166 AAD44166.1 2 5524211 +AAD44167 AAD44167.1 3 5524212 +
--- a/tool_data_table_conf.xml.sample Tue Dec 03 17:39:48 2019 -0500 +++ b/tool_data_table_conf.xml.sample Sun Mar 21 12:07:34 2021 +0000 @@ -4,4 +4,9 @@ <columns>value, name, db_path</columns> <file path="tool-data/diamond_database.loc" /> </table> + <!-- Locations of taxonomy data downloaded from NCBI --> + <table name="ncbi_accession2taxid" comment_char="#"> + <columns>value, name, path</columns> + <file path="tool-data/ncbi_accession2taxid.loc" /> + </table> </tables>
--- a/tool_data_table_conf.xml.test Tue Dec 03 17:39:48 2019 -0500 +++ b/tool_data_table_conf.xml.test Sun Mar 21 12:07:34 2021 +0000 @@ -4,4 +4,8 @@ <columns>value, name, db_path</columns> <file path="${__HERE__}/test-data/diamond_database.loc" /> </table> + <table name="ncbi_accession2taxid" comment_char="#"> + <columns>value, name, path</columns> + <file path="${__HERE__}/test-data/ncbi_accession2taxid.loc" /> + </table> </tables>