Mercurial > repos > scottx611x > data_manager_fetch_gene_annotation

comparison data_manager_gene_annotation/data_manager/data_manager.py @ 7:89ba3a52e764 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded new data_manager.py
author scottx611x
date Fri, 17 Jun 2016 16:20:54 -0400
parents 810ef478fc98
children 57ce598b7737
comparison
equal deleted inserted replaced
6:810ef478fc98 7:89ba3a52e764
1 import os
2 import sys
3 import uuid
4 import json
1 import argparse 5 import argparse
2 import datetime 6 import datetime
3 import json 7 import requests
4 import os 8 from requests.exceptions import ContentDecodingError
5 import shutil
6 import sys
7 import tarfile
8 import urllib2
9 import zipfile
10 9
11 parser = argparse.ArgumentParser(description='Create data manager json.') 10 parser = argparse.ArgumentParser(description='Create data manager json.')
12 parser.add_argument('--out', dest='output', action='store', help='JSON filename') 11 parser.add_argument('--out',
13 parser.add_argument('--name', dest='name', action='store', default=str(datetime.date.today()), help='Data table entry unique ID') 12 dest='output',
14 parser.add_argument('--url', dest='url', action='store', help='Download URL') 13 action='store',
14 help='JSON filename',
15 default="gene_annotation.json"
16 )
17 parser.add_argument('--name',
18 dest='name',
19 action='store',
20 default=uuid.uuid4(),
21 help='Data table entry unique ID'
22 )
23 parser.add_argument('--url',
24 dest='url',
25 action='store',
26 help='Download URL',
27 default="http://www.scott-ouellette.com/gene_annotations/chr1-hg19_genes.gtf")
15 28
16 args = parser.parse_args() 29 args = parser.parse_args()
17 30
18 def url_download(url, workdir): 31
32 def url_download(url, name, workdir):
33 # Create path if it doesn't exist
19 if not os.path.exists(workdir): 34 if not os.path.exists(workdir):
20 os.makedirs(workdir) 35 os.makedirs(workdir)
21 file_path = os.path.join(workdir, 'download.dat') 36
22 if not os.path.exists(workdir): 37 response = requests.get(url=url, stream=True)
23 os.makedirs(workdir) 38
24 src = None 39 # Create path that we will write the file to
25 dst = None 40 file_path = os.path.join(workdir, 'download_{}.dat'.format(name))
26 hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', 41
27 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 42 block_size = 10 * 1024 * 1024 # 10MB chunked download
28 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 43 with open(file_path, 'w+') as f:
29 'Accept-Encoding': 'none', 44 try:
30 'Accept-Language': 'en-US,en;q=0.8', 45 for buf in response.iter_content(block_size):
31 'Connection': 'keep-alive'} 46 f.write(buf)
32 try: 47 except (ContentDecodingError, IOError) as e:
33 req = urllib2.Request(url, headers=hdr) 48 sys.stderr.write("Error occured downloading reference file: %s"
34 src = urllib2.urlopen(req) 49 % e)
35 dst = open(file_path, 'wb') 50 os.remove(file_path)
36 while True: 51
37 chunk = src.read(2**10) 52 return file_path
38 if chunk:
39 dst.write(chunk)
40 else:
41 break
42 except Exception as e:
43 print e
44 finally:
45 if src:
46 src.close()
47 if dst:
48 dst.close()
49 if tarfile.is_tarfile(file_path):
50 fh = tarfile.open(file_path, 'r:*')
51 elif zipfile.is_zipfile(file_path):
52 fh = zipfile.ZipFile(file_path, 'r')
53 else:
54 return
55 fh.extractall(workdir)
56 os.remove(file_path)
57 53
58 54
59 def main(args): 55 def main(args):
60 workdir = os.path.join(os.getcwd(), 'gene_annotation') 56 workdir = os.path.join(os.getcwd(), 'gene_annotation')
61 url_download(args.url, workdir) 57
62 data_manager_entry = {} 58 # Attempt to download gene annotation file from given url
63 data_manager_entry['value'] = args.name.lower() 59 gene_annotation_file_path = url_download(args.url, args.name, workdir)
64 data_manager_entry['name'] = args.name 60
65 data_manager_entry['path'] = args.output 61 # Update Data Manager Json and write out
66 data_manager_json = dict(data_tables=dict(gene_annotation=data_manager_entry)) 62 data_manager_entry = {
67 params = json.loads(open(args.output).read()) 63 'data_tables': {
68 target_directory = params['output_data'][0]['extra_files_path'] 64 'gene_annotation': {
69 os.mkdir(target_directory) 65 'date': str(datetime.datetime.now()),
70 output_path = os.path.abspath(os.path.join(os.getcwd(), 'gene_annotation')) 66 'name': str(args.name),
71 for filename in os.listdir(workdir): 67 'path': gene_annotation_file_path
72 shutil.move(os.path.join(output_path, filename), target_directory) 68 }
73 file(args.output, 'w').write(json.dumps(data_manager_json)) 69 }
70 }
71
72 with open(os.path.join(workdir, args.output), "w+") as f:
73 f.write(json.dumps(data_manager_entry))
74 74
75 if __name__ == '__main__': 75 if __name__ == '__main__':
76 main(args) 76 main(args)