|
3
|
1 <tool id="bulk_download" name="Bulk Download" version="1.0">
|
|
|
2 <description>Bulk Downloader</description>
|
|
|
3 <command interpreter="python">$script_file $output $output.id $__new_file_path__</command>
|
|
|
4 <inputs>
|
|
|
5 <param name="urls_txt" type="text" area="True" size="5x35" label="URL Text" optional="True"/>
|
|
|
6 <param name="urls_file" type="data" label="URL File" optional="True"/>
|
|
|
7 <param name="decompress" type="boolean" label="Decompress" value="true"/>
|
|
|
8
|
|
|
9 </inputs>
|
|
|
10 <outputs>
|
|
|
11 <data name="output"/>
|
|
|
12 </outputs>
|
|
|
13 <configfiles>
|
|
|
14 <configfile name="script_file"><![CDATA[#!/usr/bin/env python
|
|
|
15 import os
|
|
|
16 import sys
|
|
|
17 import urllib
|
|
|
18 import tempfile
|
|
|
19
|
|
|
20 urls_txt = """${urls_txt}"""
|
|
|
21 urls_file = """${urls_file}"""
|
|
|
22 decompress = "${decompress}"
|
|
|
23
|
|
|
24 output = sys.argv[1]
|
|
|
25 output_id = sys.argv[2]
|
|
|
26 output_dir = sys.argv[3]
|
|
|
27
|
|
|
28
|
|
|
29 if len(urls_file) and urls_file != "None":
|
|
|
30 handle = open(urls_file)
|
|
|
31 else:
|
|
|
32 handle = StringIO(urls_txt)
|
|
|
33
|
|
|
34 #fix for multiple outputs
|
|
|
35 opath = output
|
|
|
36 for line in handle:
|
|
|
37 url = line.rstrip()
|
|
|
38 base = os.path.basename(url)
|
|
|
39 h, path = tempfile.mkstemp(dir="./")
|
|
|
40 os.close(h)
|
|
|
41 urllib.urlretrieve(url, path)
|
|
|
42 if decompress == "true":
|
|
|
43 if base.endswith(".gz"):
|
|
|
44 import gzip
|
|
|
45 ohandle = open(opath, "wb")
|
|
|
46 f = gzip.GzipFile(path, "rb")
|
|
|
47 for chunk in iter(lambda: f.read(8192), ''):
|
|
|
48 ohandle.write(chunk)
|
|
|
49 ohandle.close()
|
|
|
50 f.close()
|
|
|
51
|
|
|
52 if os.path.exists(path):
|
|
|
53 os.unlink(path)
|
|
|
54 else:
|
|
|
55 shutil.move(path, opath)
|
|
|
56
|
|
|
57 ]]></configfile>
|
|
|
58 </configfiles>
|
|
|
59 </tool>
|
