Mercurial > repos > iuc > data_manager_mothur_toolsuite

comparison data_manager/fetch_mothur_reference_data.py @ 3:9d09724f2bf1 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_mothur_toolsuite/ commit 02d2967f77e3fa5a18aea63dc84aa9ab418dc165"
author iuc
date Sun, 22 Nov 2020 12:51:44 +0000
parents 0e532fc0a0a6
children 0db22932bc39
comparison
equal deleted inserted replaced
2:0e532fc0a0a6 3:9d09724f2bf1
158 NB the directory pointed to by 'extra_files_path' 158 NB the directory pointed to by 'extra_files_path'
159 doesn't exist initially, it is the job of the script 159 doesn't exist initially, it is the job of the script
160 to create it if necessary. 160 to create it if necessary.
161 161
162 """ 162 """
163 params = json.loads(open(jsonfile).read()) 163 with open(jsonfile) as fh:
164 params = json.load(fh)
164 return (params['param_dict'], 165 return (params['param_dict'],
165 params['output_data'][0]['extra_files_path']) 166 params['output_data'][0]['extra_files_path'])
166 167
167 168
168 # Utility functions for creating data table dictionaries 169 # Utility functions for creating data table dictionaries
170 # Example usage: 171 # Example usage:
171 # >>> d = create_data_tables_dict() 172 # >>> d = create_data_tables_dict()
172 # >>> add_data_table(d,'my_data') 173 # >>> add_data_table(d,'my_data')
173 # >>> add_data_table_entry(dict(dbkey='hg19',value='human')) 174 # >>> add_data_table_entry(dict(dbkey='hg19',value='human'))
174 # >>> add_data_table_entry(dict(dbkey='mm9',value='mouse')) 175 # >>> add_data_table_entry(dict(dbkey='mm9',value='mouse'))
175 # >>> print str(json.dumps(d)) 176 # >>> print(json.dumps(d))
176 def create_data_tables_dict(): 177 def create_data_tables_dict():
177 """Return a dictionary for storing data table information 178 """Return a dictionary for storing data table information
178 179
179 Returns a dictionary that can be used with 'add_data_table' 180 Returns a dictionary that can be used with 'add_data_table'
180 and 'add_data_table_entry' to store information about a 181 and 'add_data_table_entry' to store information about a
233 if not target: 234 if not target:
234 target = os.path.basename(url) 235 target = os.path.basename(url)
235 if wd: 236 if wd:
236 target = os.path.join(wd, target) 237 target = os.path.join(wd, target)
237 print("Saving to %s" % target) 238 print("Saving to %s" % target)
238 open(target, 'wb').write(urllib2.urlopen(url).read()) 239 with open(target, 'wb') as fh:
240 fh.write(urllib2.urlopen(url).read())
239 return target 241 return target
240 242
241 243
242 def unpack_zip_archive(filen, wd=None): 244 def unpack_zip_archive(filen, wd=None):
243 """Extract files from a ZIP archive 245 """Extract files from a ZIP archive
256 """ 258 """
257 if not zipfile.is_zipfile(filen): 259 if not zipfile.is_zipfile(filen):
258 print("%s: not ZIP formatted file") 260 print("%s: not ZIP formatted file")
259 return [filen] 261 return [filen]
260 file_list = [] 262 file_list = []
261 z = zipfile.ZipFile(filen) 263 with zipfile.ZipFile(filen) as z:
262 for name in z.namelist(): 264 for name in z.namelist():
263 if reduce(lambda x, y: x or name.startswith(y), IGNORE_PATHS, False): 265 if reduce(lambda x, y: x or name.startswith(y), IGNORE_PATHS, False):
264 print("Ignoring %s" % name) 266 print("Ignoring %s" % name)
265 continue 267 continue
266 if wd: 268 if wd:
267 target = os.path.join(wd, name) 269 target = os.path.join(wd, name)
268 else: 270 else:
269 target = name 271 target = name
270 if name.endswith('/'): 272 if name.endswith('/'):
271 # Make directory 273 # Make directory
272 print("Creating dir %s" % target) 274 print("Creating dir %s" % target)
273 try: 275 try:
274 os.makedirs(target) 276 os.makedirs(target)
275 except OSError: 277 except OSError:
276 pass 278 pass
277 else: 279 else:
278 # Extract file 280 # Extract file
279 print("Extracting %s" % name) 281 print("Extracting %s" % name)
280 try: 282 try:
281 os.makedirs(os.path.dirname(target)) 283 os.makedirs(os.path.dirname(target))
282 except OSError: 284 except OSError:
283 pass 285 pass
284 open(target, 'wb').write(z.read(name)) 286 with open(target, 'wb') as fh:
285 file_list.append(target) 287 fh.write(z.read(name))
288 file_list.append(target)
286 print("Removing %s" % filen) 289 print("Removing %s" % filen)
287 os.remove(filen) 290 os.remove(filen)
288 return file_list 291 return file_list
289 292
290 293
306 """ 309 """
307 file_list = [] 310 file_list = []
308 if not tarfile.is_tarfile(filen): 311 if not tarfile.is_tarfile(filen):
309 print("%s: not TAR file") 312 print("%s: not TAR file")
310 return [filen] 313 return [filen]
311 t = tarfile.open(filen) 314 with tarfile.open(filen) as t:
312 for name in t.getnames(): 315 for name in t.getnames():
313 # Check for unwanted files 316 # Check for unwanted files
314 if reduce(lambda x, y: x or name.startswith(y), IGNORE_PATHS, False): 317 if reduce(lambda x, y: x or name.startswith(y), IGNORE_PATHS, False):
315 print("Ignoring %s" % name) 318 print("Ignoring %s" % name)
316 continue 319 continue
317 # Extract file 320 # Extract file
318 print("Extracting %s" % name) 321 print("Extracting %s" % name)
319 t.extract(name, wd) 322 t.extract(name, wd)
320 if wd: 323 if wd:
321 target = os.path.join(wd, name) 324 target = os.path.join(wd, name)
322 else: 325 else:
323 target = name 326 target = name
324 file_list.append(target) 327 file_list.append(target)
325 print("Removing %s" % filen) 328 print("Removing %s" % filen)
326 os.remove(filen) 329 os.remove(filen)
327 return file_list 330 return file_list
328 331
329 332
554 # that might have been inserted by Galaxy) 557 # that might have been inserted by Galaxy)
555 paths = options.paths.replace('__cn__', '\n').replace('__cr__', '\r').split() 558 paths = options.paths.replace('__cn__', '\n').replace('__cr__', '\r').split()
556 import_from_server(data_tables, target_dir, paths, description, link_to_data=options.link_to_data) 559 import_from_server(data_tables, target_dir, paths, description, link_to_data=options.link_to_data)
557 # Write output JSON 560 # Write output JSON
558 print("Outputting JSON") 561 print("Outputting JSON")
559 print(json.dumps(data_tables)) 562 with open(jsonfile, 'w') as fh:
560 open(jsonfile, 'w').write(json.dumps(data_tables, sort_keys=True)) 563 json.dump(data_tables, fh, sort_keys=True)
561 print("Done.") 564 print("Done.")