gdcwebapp: json_collect_data_source.py comparison

comparison json_collect_data_source.py @ 34:d65de900967e draft

Uploaded 20170613

author	fabio
date	Tue, 13 Jun 2017 16:39:40 -0400
parents	1edc869cd008
children

comparison

equal deleted inserted replaced

-:228038cd0683
+:d65de900967e
 import urllib
 import os.path
 import os
 from operator import itemgetter
 import tarfile
+import zipfile
 __version__ = "1.0.0"
 CHUNK_SIZE = 2**20 #1mb
 VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
 def splitext(path):
+# extract the folder path and extension of a file from its path
 for ext in ['.tar.gz', '.tar.bz2']:
 if path.endswith(ext):
 path, ext = path[:-len(ext)], path[-len(ext):]
 break
 else:
 output_stream = open( target_output_filename, 'wb' )
 chunk_write( query_stream, output_stream )
 query_stream.close()
 output_stream.close()
-def store_file_from_archive( file_object, target_output_filename, isString=False ):
+def store_file_from_tarfile( file_object, target_output_filename, isString=False ):
-""" Store file after extracting from archive and organize them as a collection using the structure
+# store the file_object (from tarfile) on the filesystem
-(collection-name)_(file-name).ext as file name
-"""
 output_stream = open( target_output_filename, 'wb' )
-#chunk_write( file_object.read(), output_stream )
+output_stream.write(file_object.read())
-if not isString:
-output_stream.write(file_object.read())
-else:
-output_stream.write(file_object)
 output_stream.close()
 def download_extra_data( query_ext_data, base_path ):
 """ Download any extra data defined in the JSON.
 else:
 meta_dict[ 'dataset_id' ] = dataset_id
 return "%s\n" % json.dumps( meta_dict )
-def walk_on_archive(target_output_filename, check_ext, archive_name, appdata_path, db_key="?"):
+def walk_on_archive(target_output_filename, check_ext, archive_library, archive_name, appdata_path, db_key="?"):
+# fix archive name using valid chars only
+archive_name = ''.join(e for e in archive_name if e in VALID_CHARS)
 archive_name = archive_name.replace("_", "-").replace(".", "-")
-with tarfile.open( target_output_filename, check_ext ) as tf:
+if archive_library is "zipfile":
-for entry in tf:
+# iterate over entries inside the archive [zip]
-if entry.isfile():
+with zipfile.ZipFile( target_output_filename, check_ext ) as zf:
-fileobj = tf.extractfile( entry )
+for entry in zf.namelist():
-# reserve the underscore for the collection searator
+# if entry is file
-filename = os.path.basename( entry.name ).replace("_", "-")
+if entry.startswith("%s/" % entry.rstrip("/")) is False:
-extension = splitext( filename )[1]
+# retrieve file name
-# pattern: (?P<identifier_0>[^_]+)_(?P<identifier_1>[^_]+)_(?P<ext>[^_]+)_(?P<dbkey>[^_]+)
+# the underscore character is reserved
-if (len(extension) > 0):
+filename = os.path.basename( entry.split("/")[-1] ).replace("_", "-")
-filename = (filename[0:len(filename)-(len(extension)+1)]).replace(".", "-") + "." + extension + "_" + extension
+# retrieve file extension
-else:
+extension = splitext( filename )[1]
-extension = "auto"
+# if no extension use 'auto'
-filename_with_collection_prefix = archive_name + "_" + filename + "_" + db_key
+if (len(extension) == 0):
-target_entry_output_filename = os.path.join(appdata_path, filename_with_collection_prefix)
+extension = "auto"
-store_file_from_archive( fileobj, target_entry_output_filename )
+# pattern: (?P<identifier_0>[^_]+)_(?P<identifier_1>[^_]+)_(?P<ext>[^_]+)_(?P<dbkey>[^_]+)
+filename_with_collection_prefix = archive_name + "_" + filename + "_" + extension + "_" + db_key
+# store current entry on filesystem
+zf.extract( filename_with_collection_prefix, appdata_path )
+elif archive_library is "tarfile":
+# iterate over entries inside the archive [gz, bz2, tar]
+with tarfile.open( target_output_filename, check_ext ) as tf:
+for entry in tf:
+if entry.isfile():
+fileobj = tf.extractfile( entry )
+# retrieve file name
+# the underscore character is reserved
+filename = os.path.basename( (entry.name).split("/")[-1] ).replace("_", "-")
+# retrieve file extension
+extension = splitext( filename )[1]
+# if no extension use 'auto'
+if (len(extension) == 0):
+extension = "auto"
+# pattern: (?P<identifier_0>[^_]+)_(?P<identifier_1>[^_]+)_(?P<ext>[^_]+)_(?P<dbkey>[^_]+)
+filename_with_collection_prefix = archive_name + "_" + filename + "_" + extension + "_" + db_key
+target_entry_output_filename = os.path.join(appdata_path, filename_with_collection_prefix)
+# store current entry on filesystem
+store_file_from_tarfile( fileobj, target_entry_output_filename )
 return True
 def download_files_and_write_metadata(query_item, json_params, output_base_path, metadata_parameter_file, primary, appdata_path, options, args):
 """ Main work function that operates on the JSON representation of
 hda_id, dataset_id = set_up_config_values(json_params)
 extension = query_item.get( 'extension' )
 url = query_item.get( 'url' )
 filename = query_item.get( 'name' )
+# the organize parameter is considered for archives only
+organize = query_item.get( 'organize', None )
+if organize is None:
+organize = False
+else:
+if organize.lower() == "true":
+organize = True
+elif organize.lower() == "false":
+organize = False
+else:
+# if organize parameter is malformed -> set organize to False
+organize = False
+# check file extension
+# if the file is an archive -> do not write metadata and extract files
 check_ext = ""
+archive_library = None
 if ( url.endswith( "gz" ) ):
 check_ext = "r:gz"
+archive_library = "tarfile"
 elif ( url.endswith( "bz2" ) ):
 check_ext = "r:bz2"
+archive_library = "tarfile"
 elif ( url.endswith( "tar" ) ):
 check_ext = "r:"
+archive_library = "tarfile"
+elif ( url.endswith( "zip" ) ):
+check_ext = "r"
+archive_library = "zipfile"
 isArchive = bool( check_ext and check_ext.strip() )
 extra_data = query_item.get( 'extra_data', None )
 if primary:
 filename = ''.join( c in VALID_CHARS and c or '-' for c in filename )
 name = construct_multi_filename( hda_id, filename, extension )
 target_output_filename = os.path.normpath( '/'.join( [ output_base_path, name ] ) )
-if isArchive is False:
+if (isArchive is False) or ((isArchive is True) and (organize is False)):
 metadata_parameter_file.write( metadata_to_json( dataset_id, query_item,
 target_output_filename,
 ds_type='new_primary_dataset',
 primary=primary) )
 else:
 target_output_filename = output_filename
-if isArchive is False:
+if (isArchive is False) or ((isArchive is True) and (organize is False)):
 metadata_parameter_file.write( metadata_to_json( dataset_id, query_item,
 target_output_filename,
 ds_type='dataset',
 primary=primary) )
-if isArchive is False:
+if (isArchive is False) or ((isArchive is True) and (organize is False)):
 download_from_query( query_item, target_output_filename )
 else:
+# if the current entry is an archive download it inside appdata folder
 target_output_path = os.path.join(appdata_path, filename)
 download_from_query( query_item, target_output_path )
 if extra_data:
+# just download extra data
 extra_files_path = ''.join( [ target_output_filename, 'files' ] )
 download_extra_data( extra_data, extra_files_path )
-""" the following code handles archives and decompress them in a collection """
+# if the current file is an archive and want to organize the content
-if ( isArchive ):
+# -> decompress the archive and populate the collection (has to be defined in the tool xml schema)
+if isArchive and organize:
+# set the same db_key for each file inside the archive
+# use the db_key associated to the archive (if it exists)
 db_key = "?"
 archive_metadata = query_item.get( 'metadata', None )
 if archive_metadata is not None:
 try:
 db_key = archive_metadata.get( 'db_key' )
 except:
 pass
-walk_on_archive(target_output_path, check_ext, filename, appdata_path, db_key)
+archive_name = query_item.get( 'name', None )
+if archive_name is None:
+archive_name = filename
+# iterate over the archive content
+walk_on_archive(target_output_path, check_ext, archive_library, archive_name, appdata_path, db_key)
 return True
 def set_up_config_values(json_params):
 if not os.path.exists(appdata_path):
 os.makedirs(appdata_path)
 # read tool job configuration file and parse parameters we need
 json_params = json.loads( open( options.json_param_file, 'r' ).read() )
-print("json_params: "+str(json_params))
+#print("json_params: "+str(json_params))
 dataset_url, output_filename, \
 extra_files_path, file_name, \
 ext, out_data_name, \
 hda_id, dataset_id = set_up_config_values(json_params)
 Schema
 ------
 [ {"url":"http://url_of_file",
-"name":"encode WigData",
+"name":"My Archive",
-"extension":"wig",
+"extension":"tar.gz",
-"metadata":{"db_key":"hg19"},
+"organize":"true",
+"metadata":{"db_key":"hg38"},
 "extra_data":[ {"url":"http://url_of_ext_file",
 "path":"rel/path/to/ext_file"}
 ]
 }
 ]
 """
 # Parse the command line options
-usage = "Usage: json_data_source_mod.py max_size --json_param_file filename [options]"
+usage = "Usage: json_collect_data_source.py max_size --json_param_file filename [options]"
 parser = optparse.OptionParser(usage = usage)
 parser.add_option("-j", "--json_param_file", type="string",
 action="store", dest="json_param_file", help="json schema return data")
 parser.add_option("-p", "--path", type="string",
 action="store", dest="path", help="new file path")
+# set appdata: temporary directory in which the archives will be decompressed
 parser.add_option("-a", "--appdata", type="string",
 action="store", dest="appdata", help="appdata folder name")
 parser.add_option("-v", "--version", action="store_true", dest="version",
 default=False, help="display version and exit")

Mercurial > repos > fabio > gdcwebapp

comparison json_collect_data_source.py @ 34:d65de900967e draft