Mercurial > repos > iuc > data_manager_manual

comparison data_manager/data_manager_manual.py @ 0:81ed42c0721a draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_manual commit 9f2cdfc1302dce000f9a86d13d4af204cf1dc89d
author iuc
date Sun, 14 Jan 2018 09:52:16 -0500
parents
children 6524e573d9c2
comparison
equal deleted inserted replaced
-1:000000000000 0:81ed42c0721a
1 #!/usr/bin/env python
2 # Dan Blankenberg
3
4 import json
5 import logging
6 import optparse
7 import os
8 import shutil
9 import tempfile
10 import urllib2
11
12 from xml.etree.ElementTree import tostring
13 try:
14 # For Python 3.0 and later
15 from shutil import unpack_archive
16 except ImportError:
17 # Fall back to Python 2 import
18 from setuptools.archive_util import unpack_archive
19
20 try:
21 # For Python 3.0 and later
22 from urllib.request import urlretrieve
23 except ImportError:
24 # Fall back to Python 2 imports
25 from urllib import urlretrieve
26
27 _log_name = __name__
28 if _log_name == '__builtin__':
29 _log_name = 'toolshed.installed.manual.data.manager'
30 log = logging.getLogger( _log_name )
31
32
33 # --- These methods are called by/within the Galaxy Application
34 def exec_before_job( app, inp_data, out_data, param_dict, tool=None, **kwd ):
35 # Look for any data tables that haven't been defined for this data manager before and dynamically add them to Galaxy
36 param_dict = dict( **param_dict )
37 data_tables_param = param_dict.get( 'data_tables', [] )
38 if not isinstance( data_tables_param, list ):
39 data_tables_param = [data_tables_param]
40 if tool:
41 tool_shed_repository = tool.tool_shed_repository
42 else:
43 tool_shed_repository = None
44 tdtm = None
45 data_manager = app.data_managers.get_manager( tool.data_manager_id, None )
46 for data_table_param in data_tables_param:
47 data_table_name = data_table_param.get( 'data_table_name', None )
48 if data_table_name:
49 # get data table managed by this data Manager
50 data_table = app.tool_data_tables.get_tables().get( str( data_table_name ), None )
51 if data_table:
52 data_table_filename = data_table.get_filename_for_source( data_manager, None )
53 if not data_table_filename:
54 if tdtm is None:
55 from tool_shed.tools import data_table_manager
56 tdtm = data_table_manager.ToolDataTableManager( app )
57 target_dir, tool_path, relative_target_dir = tdtm.get_target_install_dir( tool_shed_repository )
58 # Dynamically add this data table
59 log.debug( "Attempting to dynamically create a missing Tool Data Table named %s." % data_table_name )
60 repo_info = tdtm.generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=None )
61 if repo_info is not None:
62 repo_info = tostring( repo_info )
63 tmp_file = tempfile.NamedTemporaryFile()
64 tmp_file.write( __get_new_xml_definition( app, data_table, data_manager, repo_info, target_dir ) )
65 tmp_file.flush()
66 app.tool_data_tables.add_new_entries_from_config_file( tmp_file.name, None, app.config.shed_tool_data_table_config, persist=True )
67 tmp_file.close()
68
69
70 def __get_new_xml_definition( app, data_table, data_manager, repo_info=None, location_file_dir=None ):
71 sub_dict = { 'table_name': data_table.name, 'comment_char': '', 'columns': '', 'file_path': '' }
72 sub_dict.update( data_manager.get_tool_shed_repository_info_dict() )
73 if data_table.comment_char:
74 sub_dict['comment_char'] = 'comment_char="%s"' % ( data_table.comment_char )
75 for i, name in enumerate( data_table.get_column_name_list() ):
76 if name is not None:
77 sub_dict['columns'] = "%s\n%s" % ( sub_dict['columns'], '<column name="%s" index="%s" />' % ( name, i ) )
78 location_file_dir = location_file_dir or app.config.galaxy_data_manager_data_path
79 for filename in data_table.filenames.keys():
80 sub_dict['file_path'] = os.path.basename( filename )
81 sub_dict['file_path'] = os.path.join( location_file_dir, sub_dict['file_path'] ) # os.path.abspath?
82 if not os.path.exists( sub_dict['file_path'] ):
83 # Create empty file
84 log.debug( "Attempting to create a missing location file %s." % sub_dict['file_path'] )
85 open( sub_dict['file_path'], 'wb+' ).close()
86 break
87 sub_dict[ 'repo_info' ] = repo_info or ''
88 return """
89 <tables><table name="%(table_name)s" %(comment_char)s>
90 %(columns)s
91 <file path="%(file_path)s" />
92 %(repo_info)s
93 </table></tables>
94 """ % sub_dict
95
96
97 def galaxy_code_get_available_data_tables( trans ):
98 # list of data tables
99 return [ ( x, x, False ) for x in trans.app.tool_data_tables.get_tables().keys() ]
100
101
102 def galaxy_code_get_available_data_table_columns( trans, data_table_name ):
103 return [ ( x, x, True ) for x in trans.app.tool_data_tables.get( data_table_name ).get_column_name_list() ]
104 # --- End Galaxy called Methods ---
105
106
107 def get_data_table_entries( params, galaxy_data_manager_data_path ):
108 rval = {}
109 data_tables = params.get( 'data_tables', [] )
110 for data_table in data_tables:
111 entry_dict = {}
112 for column in data_table.get( 'columns', [] ):
113 value = column.get( 'data_table_column_value', '' )
114 if column.get( 'is_path', {} ).get( 'is_path_selector', None ) == 'yes' and column.get( 'is_path', {} ).get( 'abspath', None ) == 'abspath':
115 value = os.path.abspath( os.path.join( galaxy_data_manager_data_path, value ) )
116 entry_dict[ column.get( 'data_table_column_name', '' ) ] = value
117 data_table_name = data_table['data_table_name']
118 rval[ data_table_name ] = rval.get( data_table_name, [] )
119 rval[ data_table_name ].append( entry_dict )
120 return rval
121
122
123 def get_file_content( params, target_directory ):
124 directory_content = params.get( 'directory_content', [] )
125 for content in directory_content:
126 target_path = os.path.join( target_directory, content.get( 'subdir', '' ) )
127 try:
128 os.makedirs( target_path )
129 except OSError:
130 pass
131 if content.get( 'file_source', {}).get( 'file_source_selector', None ) == 'URL':
132 ( filename, headers ) = urlretrieve( content.get( 'file_source', {}).get( 'file_URL', None ) )
133 try:
134 bname = headers['Content-Disposition']
135 except KeyError:
136 bname = os.path.basename( urllib2.urlparse.urlsplit( content.get( 'file_source', {}).get( 'file_URL', None ) ).path )
137 else:
138 filename = content.get( 'file_source', {}).get( 'file_history', None )
139 bname = os.path.basename( filename )
140 file_action = content.get( 'file_action', {}).get( 'file_action_selector', None )
141 if file_action == 'unpack':
142 unpack_archive( filename, target_path )
143 else:
144 filename_override = content.get( 'file_action', {}).get( 'filename_override', None )
145 if filename_override:
146 target_path = os.path.join( target_path, filename_override )
147 else:
148 target_path = os.path.join( target_path, bname )
149 shutil.copyfile( filename, target_path )
150 return len( directory_content )
151
152
153 def main():
154 parser = optparse.OptionParser()
155 parser.add_option( '', '--galaxy_data_manager_data_path', dest='galaxy_data_manager_data_path', default='', help='Root path for galaxy_data_manager_data_path' )
156 (options, args) = parser.parse_args()
157
158 filename = args[0]
159
160 params = json.loads( open( filename ).read() )
161 target_directory = params[ 'output_data' ][0]['extra_files_path']
162
163 data_table_entries = get_data_table_entries( params['param_dict'], options.galaxy_data_manager_data_path )
164
165 # save info to json file
166 open( filename, 'wb' ).write( json.dumps( { "data_tables": data_table_entries} ) )
167
168 get_file_content( params['param_dict'], target_directory )
169
170
171 if __name__ == "__main__":
172 main()