Mercurial > repos > iuc > data_manager_gemini_database_downloader

annotate data_manager/data_manager_gemini_download.py @ 5:b4b2b284230a draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
author iuc
date Wed, 19 Dec 2018 12:36:22 -0500
parents fe5a9a7d95b0
children f57426daa04d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
1 #!/usr/bin/env python
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
2
3
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
3 import datetime
0
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
4 import json
3
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
5 import os
0
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
6 import subprocess
3
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
7 import sys
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
8
4
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
9 import yaml
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
10
0
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
11
5
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
12 def write_gemini_config(config, config_file):
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
13 with open(config_file, 'w') as fo:
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
14 yaml.dump(config, fo, allow_unicode=False, default_flow_style=False)
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
15
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
16
0
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
17 def main():
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
18 today = datetime.date.today()
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
19 params = json.loads( open( sys.argv[1] ).read() )
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
20 target_directory = params[ 'output_data' ][0]['extra_files_path']
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
21 os.mkdir( target_directory )
5
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
22
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
23 # Generate a minimal configuration file for GEMINI update
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
24 # to instruct the tool to download the annotation data into a
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
25 # subfolder of the target directory.
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
26 config_file = os.path.join(target_directory, 'gemini-config.yaml')
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
27 anno_dir = os.path.join(target_directory, 'gemini/data')
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
28 gemini_bootstrap_config = {'annotation_dir': anno_dir}
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
29 write_gemini_config(gemini_bootstrap_config, config_file)
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
30
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
31 # Now gemini update can be called to download the data.
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
32 # The GEMINI_CONFIG environment variable lets the tool discover
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
33 # the configuration file we prepared for it.
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
34 # Note that the tool will rewrite the file turning it into a
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
35 # complete gemini configuration file.
4
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
36 gemini_env = os.environ.copy()
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
37 gemini_env['GEMINI_CONFIG'] = target_directory
5
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
38 cmd = "gemini update --dataonly %s %s" % (
4
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
39 params['param_dict']['gerp_bp'],
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
40 params['param_dict']['cadd']
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
41 )
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
42 subprocess.check_call( cmd, shell=True, env=gemini_env )
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
43
5
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
44 # GEMINI tool wrappers that need access to the annotation files
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
45 # are supposed to symlink them into a gemini/data subfolder of
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
46 # the job working directory. To have GEMINI discover them there,
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
47 # we need to set this location as the 'annotation_dir' in the
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
48 # configuration file.
4
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
49 with open(config_file) as fi:
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
50 config = yaml.load(fi)
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
51 config['annotation_dir'] = 'gemini/data'
5
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
52 write_gemini_config(config, config_file)
4
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
53
5
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
54 # Finally, we prepare the metadata for the new data table record ...
2
345412d58d75 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_gemini_database_downloader commit 4bbfca6f0e9cae9a8f263aad4eab7304c96358c4
iuc
parents: 0
diff changeset
55 data_manager_dict = {
3
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
56 'data_tables': {
4
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
57 'gemini_versioned_databases': [
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
58 {
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
59 'value': today.isoformat(),
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
60 'dbkey': 'hg19',
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
61 'version': params['param_dict']['gemini_db_version'],
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
62 'name':
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
63 'GEMINI annotations (%s snapshot)' % today.isoformat(),
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
64 'path': './%s' % today.isoformat()
fe5a9a7d95b0 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 283362494058ed64143b1f27afb447b8a1cb4313
iuc
parents: 3
diff changeset
65 }
3
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
66 ]
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
67 }
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
68 }
0
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
69
5
b4b2b284230a planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 9981ac1338c91a3ab46557ce6b821da3d5b4bc86
iuc
parents: 4
diff changeset
70 # ... and save it to the json results file
0
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
71 with open( sys.argv[1], 'wb' ) as out:
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
72 out.write( json.dumps( data_manager_dict ) )
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
73
3
172815da3d41 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gemini_database_downloader commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 2
diff changeset
74
0
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
75 if __name__ == "__main__":
fd9e9ac5ecb3 Imported from capsule None
iuc
parents:
diff changeset
76 main()