annotate json_data_source_mod.py @ 28:0ea269725abc draft

Uploaded 20170606
author fabio
date Tue, 06 Jun 2017 22:31:35 -0400
parents 8c0633c02758
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
1 #!/usr/bin/env python
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
2 import json
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
3 import optparse
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
4 import urllib
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
5 import os.path
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
6 import os
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
7 from operator import itemgetter
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
8 import tarfile
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
9
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
10 __version__ = "1.0.0"
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
11 CHUNK_SIZE = 2**20 #1mb
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
12 VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
13
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
14
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
15 def splitext(path):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
16 for ext in ['.tar.gz', '.tar.bz2']:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
17 if path.endswith(ext):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
18 path, ext = path[:-len(ext)], path[-len(ext):]
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
19 break
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
20 else:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
21 path, ext = os.path.splitext(path)
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
22 return path, ext[1:]
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
23
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
24
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
25 def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
26 source_method = getattr( source_stream, source_method )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
27 target_method = getattr( target_stream, target_method )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
28 while True:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
29 chunk = source_method( CHUNK_SIZE )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
30 if chunk:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
31 target_method( chunk )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
32 else:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
33 break
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
34
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
35
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
36 def deconstruct_multi_filename( multi_filename ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
37 keys = [ 'primary', 'id', 'name', 'visible', 'file_type' ]
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
38 return ( dict( zip( keys, multi_filename.split('_') ) ) )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
39
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
40
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
41 def construct_multi_filename( id, name, file_type ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
42 """ Implementation of *Number of Output datasets cannot be determined until tool run* from documentation_.
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
43 .. _documentation: http://wiki.galaxyproject.org/Admin/Tools/Multiple%20Output%20Files
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
44 """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
45 filename = "%s_%s_%s_%s_%s" % ( 'primary', id, name, 'visible', file_type )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
46 return filename
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
47
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
48
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
49 def download_from_query( query_data, target_output_filename ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
50 """ Download file from the json data and write it to target_output_filename.
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
51 """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
52 query_url = query_data.get( 'url' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
53 query_file_type = query_data.get( 'extension' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
54 query_stream = urllib.urlopen( query_url )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
55 output_stream = open( target_output_filename, 'wb' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
56 chunk_write( query_stream, output_stream )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
57 query_stream.close()
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
58 output_stream.close()
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
59
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
60 def store_file_from_archive( file_object, target_output_filename, isString=False ):
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
61 """ Store file after extracting from archive and organize them as a collection using the structure
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
62 (collection-name)_(file-name).ext as file name
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
63 """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
64 output_stream = open( target_output_filename, 'wb' )
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
65 #chunk_write( file_object.read(), output_stream )
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
66 if not isString:
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
67 output_stream.write(file_object.read())
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
68 else:
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
69 output_stream.write(file_object)
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
70 output_stream.close()
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
71
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
72
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
73 def download_extra_data( query_ext_data, base_path ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
74 """ Download any extra data defined in the JSON.
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
75 NOTE: the "path" value is a relative path to the file on our
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
76 file system. This is slightly dangerous and we should make every effort
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
77 to avoid a malicious absolute path to write the file elsewhere on the
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
78 filesystem.
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
79 """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
80 for ext_data in query_ext_data:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
81 if not os.path.exists( base_path ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
82 os.mkdir( base_path )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
83 query_stream = urllib.urlopen( ext_data.get( 'url' ) )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
84 ext_path = ext_data.get( 'path' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
85 os.makedirs( os.path.normpath( '/'.join( [ base_path, os.path.dirname( ext_path ) ] ) ) )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
86 output_stream = open( os.path.normpath( '/'.join( [ base_path, ext_path ] ) ), 'wb' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
87 chunk_write( query_stream, output_stream )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
88 query_stream.close()
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
89 output_stream.close()
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
90
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
91
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
92 def metadata_to_json( dataset_id, metadata, filename, ds_type='dataset', primary=False):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
93 """ Return line separated JSON """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
94 meta_dict = dict( type = ds_type,
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
95 ext = metadata.get( 'extension' ),
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
96 filename = filename,
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
97 name = metadata.get( 'name' ),
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
98 metadata = metadata.get( 'metadata', {} ) )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
99 if metadata.get( 'extra_data', None ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
100 meta_dict[ 'extra_files' ] = '_'.join( [ filename, 'files' ] )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
101 if primary:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
102 meta_dict[ 'base_dataset_id' ] = dataset_id
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
103 else:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
104 meta_dict[ 'dataset_id' ] = dataset_id
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
105 return "%s\n" % json.dumps( meta_dict )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
106
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
107
16
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
108 def walk_on_archive(target_output_filename, check_ext, archive_name, appdata_path, db_key="?"):
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
109 archive_name = archive_name.replace("_", "-").replace(".", "-")
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
110 with tarfile.open( target_output_filename, check_ext ) as tf:
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
111 for entry in tf:
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
112 if entry.isfile():
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
113 fileobj = tf.extractfile( entry )
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
114 # reserve the underscore for the collection searator
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
115 filename = os.path.basename( entry.name ).replace("_", "-")
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
116 extension = splitext( filename )[1]
16
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
117 # pattern: (?P<identifier_0>[^_]+)_(?P<identifier_1>[^_]+)_(?P<ext>[^_]+)_(?P<dbkey>[^_]+)
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
118 if (len(extension) > 0):
18
2c58e3cc1b30 Uploaded 20170601
fabio
parents: 16
diff changeset
119 filename = (filename[0:len(filename)-(len(extension)+1)]).replace(".", "-") + "." + extension + "_" + extension
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
120 else:
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
121 extension = "auto"
16
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
122 filename_with_collection_prefix = archive_name + "_" + filename + "_" + db_key
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
123 target_entry_output_filename = os.path.join(appdata_path, filename_with_collection_prefix)
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
124 store_file_from_archive( fileobj, target_entry_output_filename )
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
125 return True
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
126
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
127
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
128 def download_files_and_write_metadata(query_item, json_params, output_base_path, metadata_parameter_file, primary, appdata_path, options, args):
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
129 """ Main work function that operates on the JSON representation of
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
130 one dataset and its metadata. Returns True.
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
131 """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
132 dataset_url, output_filename, \
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
133 extra_files_path, file_name, \
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
134 ext, out_data_name, \
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
135 hda_id, dataset_id = set_up_config_values(json_params)
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
136 extension = query_item.get( 'extension' )
14
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
137 url = query_item.get( 'url' )
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
138 filename = query_item.get( 'name' )
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
139
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
140 check_ext = ""
14
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
141 if ( url.endswith( "gz" ) ):
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
142 check_ext = "r:gz"
14
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
143 elif ( url.endswith( "bz2" ) ):
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
144 check_ext = "r:bz2"
14
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
145 elif ( url.endswith( "tar" ) ):
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
146 check_ext = "r:"
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
147 isArchive = bool( check_ext and check_ext.strip() )
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
148
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
149 extra_data = query_item.get( 'extra_data', None )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
150 if primary:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
151 filename = ''.join( c in VALID_CHARS and c or '-' for c in filename )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
152 name = construct_multi_filename( hda_id, filename, extension )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
153 target_output_filename = os.path.normpath( '/'.join( [ output_base_path, name ] ) )
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
154 if isArchive is False:
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
155 metadata_parameter_file.write( metadata_to_json( dataset_id, query_item,
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
156 target_output_filename,
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
157 ds_type='new_primary_dataset',
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
158 primary=primary) )
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
159 else:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
160 target_output_filename = output_filename
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
161 if isArchive is False:
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
162 metadata_parameter_file.write( metadata_to_json( dataset_id, query_item,
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
163 target_output_filename,
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
164 ds_type='dataset',
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
165 primary=primary) )
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
166
14
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
167 if isArchive is False:
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
168 download_from_query( query_item, target_output_filename )
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
169 else:
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
170 target_output_path = os.path.join(appdata_path, filename)
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
171 download_from_query( query_item, target_output_path )
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
172 if extra_data:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
173 extra_files_path = ''.join( [ target_output_filename, 'files' ] )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
174 download_extra_data( extra_data, extra_files_path )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
175
10
c0be9583df97 Uploaded 20170525
fabio
parents: 6
diff changeset
176 """ the following code handles archives and decompress them in a collection """
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
177 if ( isArchive ):
16
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
178 db_key = "?"
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
179 archive_metadata = query_item.get( 'metadata', None )
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
180 if archive_metadata is not None:
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
181 try:
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
182 db_key = archive_metadata.get( 'db_key' )
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
183 except:
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
184 pass
3eabece82abb Uploaded 20170601
fabio
parents: 14
diff changeset
185 walk_on_archive(target_output_path, check_ext, filename, appdata_path, db_key)
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
186
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
187 return True
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
188
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
189
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
190 def set_up_config_values(json_params):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
191 """ Parse json_params file and return a tuple of necessary configuration
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
192 values.
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
193 """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
194 datasource_params = json_params.get( 'param_dict' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
195 dataset_url = datasource_params.get( 'URL' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
196 output_filename = datasource_params.get( 'output1', None )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
197 output_data = json_params.get( 'output_data' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
198 extra_files_path, file_name, ext, out_data_name, hda_id, dataset_id = \
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
199 itemgetter('extra_files_path', 'file_name', 'ext', 'out_data_name', 'hda_id', 'dataset_id')(output_data[0])
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
200 return (dataset_url, output_filename,
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
201 extra_files_path, file_name,
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
202 ext, out_data_name,
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
203 hda_id, dataset_id)
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
204
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
205
14
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
206 def download_from_json_data( options, args ):
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
207 """ Parse the returned JSON data and download files. Write metadata
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
208 to flat JSON file.
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
209 """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
210 output_base_path = options.path
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
211 appdata_path = options.appdata
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
212 if not os.path.exists(appdata_path):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
213 os.makedirs(appdata_path)
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
214
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
215 # read tool job configuration file and parse parameters we need
14
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
216 json_params = json.loads( open( options.json_param_file, 'r' ).read() )
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
217
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
218 dataset_url, output_filename, \
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
219 extra_files_path, file_name, \
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
220 ext, out_data_name, \
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
221 hda_id, dataset_id = set_up_config_values(json_params)
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
222 # line separated JSON file to contain all dataset metadata
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
223 metadata_parameter_file = open( json_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'wb' )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
224
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
225 # get JSON response from data source
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
226 # TODO: make sure response is not enormous
14
babc444d4bd0 Uploaded 20170530
fabio
parents: 12
diff changeset
227 query_params = json.loads(urllib.urlopen( dataset_url ).read())
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
228 # download and write files
20
8c0633c02758 Uploaded 20170602
fabio
parents: 18
diff changeset
229 primary = False
8c0633c02758 Uploaded 20170602
fabio
parents: 18
diff changeset
230 #primary = True
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
231 # query_item, hda_id, output_base_path, dataset_id
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
232 for query_item in query_params:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
233 if isinstance( query_item, list ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
234 # TODO: do something with the nested list as a collection
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
235 for query_subitem in query_item:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
236 primary = download_files_and_write_metadata(query_subitem, json_params, output_base_path,
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
237 metadata_parameter_file, primary, appdata_path, options, args)
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
238
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
239 elif isinstance( query_item, dict ):
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
240 primary = download_files_and_write_metadata(query_item, json_params, output_base_path,
12
80593f75d74a Uploaded
fabio
parents: 10
diff changeset
241 metadata_parameter_file, primary, appdata_path, options, args)
6
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
242 metadata_parameter_file.close()
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
243
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
244 def __main__():
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
245 """ Read the JSON return from a data source. Parse each line and request
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
246 the data, download to "newfilepath", and write metadata.
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
247
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
248 Schema
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
249 ------
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
250
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
251 [ {"url":"http://url_of_file",
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
252 "name":"encode WigData",
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
253 "extension":"wig",
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
254 "metadata":{"db_key":"hg19"},
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
255 "extra_data":[ {"url":"http://url_of_ext_file",
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
256 "path":"rel/path/to/ext_file"}
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
257 ]
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
258 }
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
259 ]
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
260
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
261 """
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
262 # Parse the command line options
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
263 usage = "Usage: json_data_source_mod.py max_size --json_param_file filename [options]"
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
264 parser = optparse.OptionParser(usage = usage)
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
265 parser.add_option("-j", "--json_param_file", type="string",
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
266 action="store", dest="json_param_file", help="json schema return data")
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
267 parser.add_option("-p", "--path", type="string",
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
268 action="store", dest="path", help="new file path")
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
269 parser.add_option("-a", "--appdata", type="string",
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
270 action="store", dest="appdata", help="appdata folder name")
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
271 parser.add_option("-v", "--version", action="store_true", dest="version",
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
272 default=False, help="display version and exit")
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
273
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
274 (options, args) = parser.parse_args()
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
275 if options.version:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
276 print __version__
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
277 else:
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
278 download_from_json_data( options, args )
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
279
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
280
7815152f70c6 Uploaded 20170525
fabio
parents:
diff changeset
281 if __name__ == "__main__": __main__()