Mercurial > repos > jeremyjliu > region_motif_data_manager

changeset 0:cab6e1cb4c2a draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author jeremyjliu
date Sat, 16 May 2015 17:23:06 -0400
parents
children
files data_manager/data_manager_fetch_motifs.py data_manager/data_manager_fetch_motifs.xml data_manager_conf.xml tool-data/motif_databases.loc.sample tool_data_table_conf.xml.sample
diffstat 5 files changed, 179 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_fetch_motifs.py	Sat May 16 17:23:06 2015 -0400
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# Jeremy Liu
+# February 2015
+# Adapted from Dan Blackenburg's sample data manager
+
+import sys
+import os
+import tempfile
+import shutil
+import optparse
+import urllib2
+#import uuid
+from ftplib import FTP
+import tarfile
+import zipfile
+import gzip
+import bz2
+
+from galaxy.util.json import from_json_string, to_json_string
+
+CHUNK_SIZE = 2**20 #1mb
+
+def download_motif_databases( data_manager_dict, params, target_directory, motif_db ):
+
+    # Select download URL, file name, data table name, and path using motif_db selector variable
+    if motif_db == "encode":
+        BGZ = ['http://compbio.med.harvard.edu/motif-enrichment/pouya_motifs.bed.bgz',
+                "pouya_motifs.bed.bgz", "encode_bgz", "Encode Motifs (hg19) BGZ"]
+        TBI = ['http://compbio.med.harvard.edu/motif-enrichment/pouya_motifs.bed.bgz.tbi',
+                "pouya_motifs.bed.bgz.tbi", "encode_tbi", "Encode Motifs (hg19) TBI"]
+        PWM = ['http://compbio.med.harvard.edu/motif-enrichment/pwms/pouya.pwms.from.seq.meme.txt',
+                "pouya.pwms.from.seq.meme.txt", "encode_pwm", "Encode Motifs (hg19) PWM MEME"]        
+    elif motif_db == "jaspar":
+        BGZ = ['http://compbio.med.harvard.edu/motif-enrichment/jaspar_jolma_motifs.bed.bgz',
+                "jaspar_jolma_motifs.bed.bgz", "jaspar_bgz", "Jaspar and Jolma Motifs (hg19) BGZ"]
+        TBI = ['http://compbio.med.harvard.edu/motif-enrichment/jaspar_jolma_motifs.bed.bgz.tbi',
+                "jaspar_jolma_motifs.bed.bgz.tbi", "jaspar_tbi", "Jaspar and Jolma Motifs (hg19) TBI"]
+        PWM = ['http://compbio.med.harvard.edu/motif-enrichment/pwms/jaspar.jolma.pwms.from.seq.meme.txt',
+                "jaspar.jolma.pwms.from.seq.meme.txt", "jaspar_pwm", "Jaspar and Jolma Motifs (hg19) PWM MEME"]
+    elif motif_db == "mouse":
+        BGZ = ['http://compbio.med.harvard.edu/motif-enrichment/mm9_motifs_split.bed.bgz',
+                "mm9_motifs_split.bed.bgz", "mouse_bgz", "Mouse Motifs (mm9) BGZ"]
+        TBI = ['http://compbio.med.harvard.edu/motif-enrichment/mm9_motifs_split.bed.bgz.tbi',
+                "mm9_motifs_split.bed.bgz.tbi", "mouse_tbi", "Mouse Motifs (mm9) TBI"]
+        PWM = ['http://compbio.med.harvard.edu/motif-enrichment/pwms/mm9.pwms.from.seq.meme.txt',
+                "mm9.pwms.from.seq.meme.txt", "mouse_pwm", "Mouse Motifs (mm9) PWM MEME"]
+    else:
+        BGZ = ['http://compbio.med.harvard.edu/motif-enrichment/pouya_test_motifs.bed.bgz', 
+               "pouya_test_motifs.bed.bgz", "test_bgz", "Test Encode Motifs (hg19) BGZ"]
+        TBI = ['http://compbio.med.harvard.edu/motif-enrichment/pouya_test_motifs.bed.bgz.tbi',
+               "pouya_test_motifs.bed.bgz.tbi", "test_tbi", "Test Encode Motifs (hg19) TBI"]
+        PWM = ['http://compbio.med.harvard.edu/motif-enrichment/pwms/pouya.pwms.from.seq.meme.txt',
+                "pouya.pwms.from.seq.meme.txt", "test_pwm", "Test Encode Motifs (hg19) PWM MEME"] 
+
+
+    # Save and add motif bgz file to motif_databases data table
+    bgz_reader = urllib2.urlopen( BGZ[0] )
+    bgz_data_table_entry = _stream_fasta_to_file( bgz_reader, target_directory, params,
+                            BGZ[1], BGZ[2], BGZ[3] )
+    _add_data_table_entry( data_manager_dict, 'motif_databases', bgz_data_table_entry )
+
+    # Save and add motif tbi file to motif_databases data table
+    tbi_reader = urllib2.urlopen( TBI[0] )
+    tbi_data_table_entry = _stream_fasta_to_file( tbi_reader, target_directory, params,
+                            TBI[1], TBI[2], TBI[3] )
+    _add_data_table_entry( data_manager_dict, 'motif_databases', tbi_data_table_entry )
+
+    # Save and add motif pwm file to motif_databases data table
+    tbi_reader = urllib2.urlopen( PWM[0] )
+    tbi_data_table_entry = _stream_fasta_to_file( tbi_reader, target_directory, params,
+                            PWM[1], PWM[2], PWM[3] )
+    _add_data_table_entry( data_manager_dict, 'motif_databases', tbi_data_table_entry )
+
+def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ):
+    data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
+    data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] )
+    data_manager_dict['data_tables'][data_table].append( data_table_entry )
+    return data_manager_dict
+
+def _stream_fasta_to_file( fasta_stream, target_directory, params,
+                        fasta_base_filename, value, name, close_stream=True ):
+    fasta_filename = os.path.join( target_directory, fasta_base_filename )
+    fasta_writer = open( fasta_filename, 'wb+' )
+    
+    while True:
+        buffer = fasta_stream.read(CHUNK_SIZE)
+        if not buffer:
+            break
+
+        fasta_writer.write(buffer)
+
+    fasta_stream.close()
+    fasta_writer.close()
+    
+    return dict( value=value, name=name, path=fasta_base_filename )
+
+def main():
+    #Parse Command Line
+    parser = optparse.OptionParser()
+    parser.add_option( '-m', '--motif_db', dest='motif_db', action='store', type="string", default=None, help='motif_db' )
+    (options, args) = parser.parse_args()
+    
+    filename = args[0]
+    
+    params = from_json_string( open( filename ).read() )
+    target_directory = params[ 'output_data' ][0]['extra_files_path']
+    os.mkdir( target_directory )
+    data_manager_dict = {}
+    
+    #Fetch the Motif Database
+    download_motif_databases( data_manager_dict, params, target_directory, options.motif_db )
+    
+    #save info to json file
+    open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
+        
+if __name__ == "__main__": main()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_fetch_motifs.xml	Sat May 16 17:23:06 2015 -0400
@@ -0,0 +1,26 @@
+<tool id="data_manager_fetch_motifs" name="Install Motif Database" version="1.0.0" tool_type="manage_data">
+    <description>Downloads motif databases in tabix format for use with region_motif_enrichment</description>
+    <command interpreter="python">data_manager_fetch_motifs.py "${out_file}" --motif_db ${motif_db_selector}</command>
+    <inputs>
+        <param name="motif_db_selector" type="select" label="Choose the source for the motif db">
+            <option value="test" selected="True">Test Encode Motifs (hg19)</option>
+            <option value="encode">Encode Motifs (hg19)</option>
+            <option value="jaspar">Jaspar and Jolma Motifs (hg19)</option>
+            <option value="mouse">Mouse Motifs (mm9)</option>
+        </param>
+    </inputs>
+    <outputs>
+        <data name="out_file" format="data_manager_json"/>
+    </outputs>
+    <help>
+        **What it does**
+
+        Fetches a reference motif database (bgz and tbi files) from Harvard
+        CBMI source and populates the "motif_databases" data table.
+
+        **TO DO**
+        
+        Ability to add motif database from local file, as the last step in 
+        a MEME pipeline.
+    </help>
+</tool>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager_conf.xml	Sat May 16 17:23:06 2015 -0400
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<data_managers>
+    
+    <data_manager tool_file="data_manager/data_manager_fetch_motifs.xml" id="data_manager_fetch_motifs">
+        <data_table name="motif_databases">
+            <output>
+                <column name="value" />
+                <column name="name" />
+                <column name="path" output_ref="out_file" >
+                    <move type="file">
+                        <source>${path}</source>
+                        <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">motifs/${path}</target>
+                    </move>
+                    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/motifs/${path}</value_translation>
+                    <value_translation type="function">abspath</value_translation>
+                </column>
+            </output>
+        </data_table>
+    </data_manager>
+    
+</data_managers>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/motif_databases.loc.sample	Sat May 16 17:23:06 2015 -0400
@@ -0,0 +1,9 @@
+# This file lists the locations and values of all the motif databases bgz
+# and tbi files. This file has the format (white space characters are TAB
+# characters)
+# <value>		<display_name>	<file_path>
+#
+# So, motif_databases.loc could look something like this:
+#
+# test_bgz	  Test Encode Motifs (hg19) BGZ	  /<path_to_galaxy>/tool-data/motifs/pouya_test_motifs.bed.bgz
+# test_tbi    Test Encode motifs (hg19) TBI    /<path_to_galaxy>/tool-data/motifs/pouya_test_motifs.bed.bgz.tbi
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.sample	Sat May 16 17:23:06 2015 -0400
@@ -0,0 +1,7 @@
+<tables>
+    <!-- Locations of motif database files -->
+    <table name="motif_databases" comment_char="#">
+        <columns>value, name, path</columns>
+        <file path="tool-data/motif_databases.loc" />
+    </table>
+</tables>