|
0
|
1 #!/usr/bin/env python
|
|
|
2 # Retrieves data from external data source applications and stores in a dataset file.
|
|
|
3 # Data source application parameters are temporarily stored in the dataset file.
|
|
|
4 import socket, urllib, sys, os
|
|
|
5 from galaxy import eggs #eggs needs to be imported so that galaxy.util can find docutils egg...
|
|
|
6 from galaxy.util.json import from_json_string, to_json_string
|
|
|
7 import galaxy.model # need to import model before sniff to resolve a circular import dependency
|
|
|
8 from galaxy.datatypes import sniff
|
|
|
9 from galaxy.datatypes.registry import Registry
|
|
|
10 from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_FILE
|
|
|
11
|
|
|
12 assert sys.version_info[:2] >= ( 2, 4 )
|
|
|
13
|
|
|
14 def stop_err( msg ):
|
|
|
15 sys.stderr.write( msg )
|
|
|
16 sys.exit()
|
|
|
17
|
|
|
18 GALAXY_PARAM_PREFIX = 'GALAXY'
|
|
|
19 GALAXY_ROOT_DIR = os.path.realpath( os.path.join( os.path.split( os.path.realpath( __file__ ) )[0], '..', '..' ) )
|
|
|
20 GALAXY_DATATYPES_CONF_FILE = os.path.join( GALAXY_ROOT_DIR, 'datatypes_conf.xml' )
|
|
|
21
|
|
|
22 def load_input_parameters( filename, erase_file = True ):
|
|
|
23 datasource_params = {}
|
|
|
24 try:
|
|
|
25 json_params = from_json_string( open( filename, 'r' ).read() )
|
|
|
26 datasource_params = json_params.get( 'param_dict' )
|
|
|
27 except:
|
|
|
28 json_params = None
|
|
|
29 for line in open( filename, 'r' ):
|
|
|
30 try:
|
|
|
31 line = line.strip()
|
|
|
32 fields = line.split( '\t' )
|
|
|
33 datasource_params[ fields[0] ] = fields[1]
|
|
|
34 except:
|
|
|
35 continue
|
|
|
36 if erase_file:
|
|
|
37 open( filename, 'w' ).close() #open file for writing, then close, removes params from file
|
|
|
38 return json_params, datasource_params
|
|
|
39
|
|
|
40 def __main__():
|
|
|
41 filename = sys.argv[1]
|
|
|
42 try:
|
|
|
43 max_file_size = int( sys.argv[2] )
|
|
|
44 except:
|
|
|
45 max_file_size = 0
|
|
|
46
|
|
|
47 job_params, params = load_input_parameters( filename )
|
|
|
48 if job_params is None: #using an older tabular file
|
|
|
49 enhanced_handling = False
|
|
|
50 job_params = dict( param_dict = params )
|
|
|
51 job_params[ 'output_data' ] = [ dict( out_data_name = 'output',
|
|
|
52 ext = 'data',
|
|
|
53 file_name = filename,
|
|
|
54 extra_files_path = None ) ]
|
|
|
55 job_params[ 'job_config' ] = dict( GALAXY_ROOT_DIR=GALAXY_ROOT_DIR, GALAXY_DATATYPES_CONF_FILE=GALAXY_DATATYPES_CONF_FILE, TOOL_PROVIDED_JOB_METADATA_FILE = TOOL_PROVIDED_JOB_METADATA_FILE )
|
|
|
56 else:
|
|
|
57 enhanced_handling = True
|
|
|
58 json_file = open( job_params[ 'job_config' ][ 'TOOL_PROVIDED_JOB_METADATA_FILE' ], 'w' ) #specially named file for output junk to pass onto set metadata
|
|
|
59
|
|
|
60 datatypes_registry = Registry( root_dir = job_params[ 'job_config' ][ 'GALAXY_ROOT_DIR' ], config = job_params[ 'job_config' ][ 'GALAXY_DATATYPES_CONF_FILE' ] )
|
|
|
61
|
|
|
62 URL = params.get( 'URL', None ) #using exactly URL indicates that only one dataset is being downloaded
|
|
|
63 URL_method = params.get( 'URL_method', None )
|
|
|
64
|
|
|
65 # The Python support for fetching resources from the web is layered. urllib uses the httplib
|
|
|
66 # library, which in turn uses the socket library. As of Python 2.3 you can specify how long
|
|
|
67 # a socket should wait for a response before timing out. By default the socket module has no
|
|
|
68 # timeout and can hang. Currently, the socket timeout is not exposed at the httplib or urllib2
|
|
|
69 # levels. However, you can set the default timeout ( in seconds ) globally for all sockets by
|
|
|
70 # doing the following.
|
|
|
71 socket.setdefaulttimeout( 600 )
|
|
|
72
|
|
|
73 for data_dict in job_params[ 'output_data' ]:
|
|
|
74 cur_filename = data_dict.get( 'file_name', filename )
|
|
|
75 cur_URL = params.get( '%s|%s|URL' % ( GALAXY_PARAM_PREFIX, data_dict[ 'out_data_name' ] ), URL )
|
|
|
76 if not cur_URL:
|
|
|
77 open( cur_filename, 'w' ).write( "" )
|
|
|
78 stop_err( 'The remote data source application has not sent back a URL parameter in the request.' )
|
|
|
79
|
|
|
80 # The following calls to urllib.urlopen() will use the above default timeout
|
|
|
81 try:
|
|
|
82 if not URL_method or URL_method == 'get':
|
|
|
83 page = urllib.urlopen( cur_URL )
|
|
|
84 elif URL_method == 'post':
|
|
|
85 page = urllib.urlopen( cur_URL, urllib.urlencode( params ) )
|
|
|
86 except Exception, e:
|
|
|
87 stop_err( 'The remote data source application may be off line, please try again later. Error: %s' % str( e ) )
|
|
|
88 if max_file_size:
|
|
|
89 file_size = int( page.info().get( 'Content-Length', 0 ) )
|
|
|
90 if file_size > max_file_size:
|
|
|
91 stop_err( 'The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % ( file_size, max_file_size ) )
|
|
|
92 #do sniff stream for multi_byte
|
|
|
93 try:
|
|
|
94 cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename )
|
|
|
95 except Exception, e:
|
|
|
96 stop_err( 'Unable to fetch %s:\n%s' % ( cur_URL, e ) )
|
|
|
97
|
|
|
98 #here import checks that upload tool performs
|
|
|
99 if enhanced_handling:
|
|
|
100 try:
|
|
|
101 ext = sniff.handle_uploaded_dataset_file( filename, datatypes_registry, ext = data_dict[ 'ext' ], is_multi_byte = is_multi_byte )
|
|
|
102 except Exception, e:
|
|
|
103 stop_err( str( e ) )
|
|
|
104 info = dict( type = 'dataset',
|
|
|
105 dataset_id = data_dict[ 'dataset_id' ],
|
|
|
106 ext = ext)
|
|
|
107
|
|
|
108 json_file.write( "%s\n" % to_json_string( info ) )
|
|
|
109
|
|
|
110 if __name__ == "__main__": __main__()
|
