|
0
|
1 import argparse
|
|
|
2 import datetime
|
|
|
3 import json
|
|
|
4 import os
|
|
|
5 import shutil
|
|
|
6 import sys
|
|
|
7 import tarfile
|
|
|
8 import urllib2
|
|
|
9 import zipfile
|
|
|
10
|
|
|
11 parser = argparse.ArgumentParser(description='Create data manager json.')
|
|
|
12 parser.add_argument('--out', dest='output', action='store', help='JSON filename')
|
|
|
13 parser.add_argument('--name', dest='name', action='store', default=str(datetime.date.today()), help='Data table entry unique ID')
|
|
|
14 parser.add_argument('--url', dest='url', action='store', help='Download URL', default="http://www.scott-ouellette.com/gene_annotations/chr1-hg19_genes.gtf")
|
|
|
15
|
|
|
16 args = parser.parse_args()
|
|
|
17
|
|
|
18 def url_download(url, workdir):
|
|
|
19 if not os.path.exists(workdir):
|
|
|
20 os.makedirs(workdir)
|
|
|
21 file_path = os.path.join(workdir, 'download.dat')
|
|
|
22 if not os.path.exists(workdir):
|
|
|
23 os.makedirs(workdir)
|
|
|
24 src = None
|
|
|
25 dst = None
|
|
|
26 hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
|
|
|
27 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
|
28 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
|
|
|
29 'Accept-Encoding': 'none',
|
|
|
30 'Accept-Language': 'en-US,en;q=0.8',
|
|
|
31 'Connection': 'keep-alive'}
|
|
|
32 try:
|
|
|
33 req = urllib2.Request(url, headers=hdr)
|
|
|
34 src = urllib2.urlopen(req)
|
|
|
35 dst = open(file_path, 'wb')
|
|
|
36 while True:
|
|
|
37 chunk = src.read(2**10)
|
|
|
38 if chunk:
|
|
|
39 dst.write(chunk)
|
|
|
40 else:
|
|
|
41 break
|
|
|
42 except Exception as e:
|
|
|
43 print e, "FUCK"
|
|
|
44 finally:
|
|
|
45 if src:
|
|
|
46 src.close()
|
|
|
47 if dst:
|
|
|
48 dst.close()
|
|
|
49 if tarfile.is_tarfile(file_path):
|
|
|
50 fh = tarfile.open(file_path, 'r:*')
|
|
|
51 elif zipfile.is_zipfile(file_path):
|
|
|
52 fh = zipfile.ZipFile(file_path, 'r')
|
|
|
53 else:
|
|
|
54 return
|
|
|
55 fh.extractall(workdir)
|
|
|
56 os.remove(file_path)
|
|
|
57
|
|
|
58
|
|
|
59 def main(args):
|
|
|
60 workdir = os.path.join(os.getcwd(), 'gene_annotation')
|
|
|
61 url_download(args.url, workdir)
|
|
|
62 data_manager_entry = {}
|
|
|
63 data_manager_entry['value'] = args.name.lower()
|
|
|
64 data_manager_entry['name'] = args.name
|
|
|
65 data_manager_entry['path'] = args.output
|
|
|
66 data_manager_json = dict(data_tables=dict(gene_annotation=data_manager_entry))
|
|
|
67 params = json.loads(open(args.output).read())
|
|
|
68 target_directory = params['output_data'][0]['extra_files_path']
|
|
|
69 os.mkdir(target_directory)
|
|
|
70 output_path = os.path.abspath(os.path.join(os.getcwd(), 'gene_annotation'))
|
|
|
71 for filename in os.listdir(workdir):
|
|
|
72 shutil.move(os.path.join(output_path, filename), target_directory)
|
|
|
73 file(args.output, 'w').write(json.dumps(data_manager_json))
|
|
|
74
|
|
|
75 if __name__ == '__main__':
|
|
|
76 main(args)
|
