Mercurial > repos > devteam > data_manager_rsync_g2

annotate data_manager/data_manager_rsync.py @ 2:e0329ab30f6d draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
author iuc
date Thu, 28 Nov 2019 15:47:47 -0500
parents 8ff92bd7e2a3
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
1 #!/usr/bin/env python
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
2 # Dan Blankenberg
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
3 from __future__ import print_function
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
4
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
5 import base64
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
6 import datetime
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
7 import logging
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
8 import optparse
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
9 import os
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
10 import shutil
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
11 import subprocess
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
12 import tempfile
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
13 from json import (
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
14 dumps,
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
15 loads
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
16 )
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
17 from os.path import basename
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
18 from xml.etree.ElementTree import tostring
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
19 try:
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
20 # For Python 3.0 and later
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
21 from urllib.request import urlopen
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
22 except ImportError:
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
23 # Fall back to Python 2 imports
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
24 from urllib2 import urlopen
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
25
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
26 _log_name = __name__
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
27 if _log_name == '__builtin__':
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
28 _log_name = 'toolshed.installed.g2.rsync.data.manager'
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
29 log = logging.getLogger(_log_name)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
30
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
31 # Get the Data from the Galaxy Project rsync server
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
32 RSYNC_CMD = 'rsync'
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
33 RSYNC_SERVER = "rsync://datacache.g2.bx.psu.edu/"
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
34 LOCATION_DIR = "location"
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
35 INDEX_DIR = "indexes"
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
36
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
37 # Pull the Tool Data Table files from github
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
38 # FIXME: These files should be accessible from the rsync server directly.
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
39 TOOL_DATA_TABLE_CONF_XML_URLS = {'main': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/env/main/files/galaxy/config/tool_data_table_conf.xml",
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
40 'test': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/env/test/files/galaxy/config/tool_data_table_conf.xml"}
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
41
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
42 # Replace data table source entries with local temporary location
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
43 GALAXY_DATA_CANONICAL_PATH = "/galaxy/data/"
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
44 TOOL_DATA_TABLE_CONF_XML_REPLACE_SOURCE = '<file path="%slocation/' % (GALAXY_DATA_CANONICAL_PATH)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
45 TOOL_DATA_TABLE_CONF_XML_REPLACE_TARGET = '<file path="%s/'
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
46
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
47 # Some basic Caching, so we don't have to reload and download everything every time
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
48 CACHE_TIME = datetime.timedelta(minutes=10)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
49 TOOL_DATA_TABLES_LOADED_BY_URL = {}
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
50
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
51 # Entries will not be selected by default
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
52 DEFAULT_SELECTED = False
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
53
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
54 # Exclude data managers without 'path' column or that are in the manual exclude list
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
55 PATH_COLUMN_NAMES = ['path']
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
56 EXCLUDE_DATA_TABLES = []
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
57 # TODO: Make additional handler actions available for tables that can't fit into the the basic
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
58 # "take the value of path" as a dir and copy contents.
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
59 # e.g. mafs. Although this maf table is goofy and doesn't have path defined in <table> def,
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
60 # but it does exit in the .loc.
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
61
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
62
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
63 # --- These methods are called by/within the Galaxy Application
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
64 def exec_before_job(app, inp_data, out_data, param_dict, tool=None, **kwd):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
65 # Look for any data tables that haven't been defined for this data manager before and dynamically add them to Galaxy
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
66 param_dict = dict(**param_dict)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
67 param_dict['data_table_entries'] = param_dict.get('data_table_entries', [])
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
68 if not isinstance(param_dict['data_table_entries'], list):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
69 param_dict['data_table_entries'] = [param_dict['data_table_entries']]
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
70 param_dict['data_table_entries'] = ",".join(param_dict['data_table_entries'])
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
71 if tool:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
72 tool_shed_repository = tool.tool_shed_repository
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
73 else:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
74 tool_shed_repository = None
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
75 tdtm = None
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
76 data_manager = app.data_managers.get_manager(tool.data_manager_id, None)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
77 data_table_entries = get_data_table_entries(param_dict)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
78 data_tables = load_data_tables_from_url(data_table_class=app.tool_data_tables.__class__).get('data_tables')
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
79 for data_table_name, entries in data_table_entries.items():
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
80 # get data table managed by this data Manager
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
81 has_data_table = app.tool_data_tables.get_tables().get(str(data_table_name))
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
82 if has_data_table:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
83 has_data_table = bool(has_data_table.get_filename_for_source(data_manager, None))
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
84 if not has_data_table:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
85 if tdtm is None:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
86 from tool_shed.tools import data_table_manager
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
87 tdtm = data_table_manager.ToolDataTableManager(app)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
88 target_dir, tool_path, relative_target_dir = tdtm.get_target_install_dir(tool_shed_repository)
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
89 # Dynamically add this data table
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
90 log.debug("Attempting to dynamically create a missing Tool Data Table named %s." % data_table_name)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
91 data_table = data_tables[data_table_name]
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
92 repo_info = tdtm.generate_repository_info_elem_from_repository(tool_shed_repository, parent_elem=None)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
93 if repo_info is not None:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
94 repo_info = tostring(repo_info)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
95 tmp_file = tempfile.NamedTemporaryFile(mode="w")
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
96 tmp_file.write(get_new_xml_definition(app, data_table, data_manager, repo_info, target_dir))
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
97 tmp_file.flush()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
98 app.tool_data_tables.add_new_entries_from_config_file(tmp_file.name, None, app.config.shed_tool_data_table_config, persist=True)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
99 tmp_file.close()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
100
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
101
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
102 def galaxy_code_get_available_data_tables(trans):
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
103 # list of data tables
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
104 found_tables = get_available_tables(trans)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
105 rval = [(x, x, DEFAULT_SELECTED) for x in found_tables]
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
106 return rval
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
107
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
108
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
109 def galaxy_code_get_available_data_tables_entries(trans, dbkey, data_table_names):
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
110 # available entries, optionally filtered by dbkey and table names
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
111 if dbkey in [None, '', '?']:
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
112 dbkey = None
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
113 if data_table_names in [None, '', '?']:
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
114 data_table_names = None
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
115 found_tables = get_available_tables_for_dbkey(trans, dbkey, data_table_names)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
116 dbkey_text = '(%s) ' % (dbkey) if dbkey else ''
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
117 rval = [("%s%s" % (dbkey_text, x[0]), base64.b64encode(dumps(dict(name=x[0].split(': ')[0], entry=x[1]), sort_keys=True).rstrip().encode('utf-8')).decode('utf-8'), DEFAULT_SELECTED) for x in found_tables.items()]
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
118 return rval
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
119
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
120 # --- End Galaxy called Methods ---
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
121
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
122
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
123 def rsync_urljoin(base, url):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
124 # urlparse.urljoin doesn't work correctly for our use-case
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
125 # probably because it doesn't recognize rhe rsync scheme
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
126 base = base.rstrip('/')
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
127 url = url.lstrip('/')
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
128 return "%s/%s" % (base, url)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
129
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
130
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
131 def rsync_list_dir(server, dir=None, skip_names=[]):
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
132 # drwxr-xr-x 50 2014/05/16 20:58:11 .
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
133 if dir:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
134 dir = rsync_urljoin(server, dir)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
135 else:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
136 dir = server
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
137 rsync_response = tempfile.NamedTemporaryFile()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
138 rsync_stderr = tempfile.NamedTemporaryFile()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
139 rsync_cmd = [RSYNC_CMD, '--list-only', dir]
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
140 return_code = subprocess.call(rsync_cmd, stdout=rsync_response, stderr=rsync_stderr)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
141 rsync_response.flush()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
142 rsync_response.seek(0)
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
143 rsync_stderr.flush()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
144 rsync_stderr.seek(0)
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
145 if return_code:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
146 msg = "stdout:\n%s\nstderr:\n%s" % (rsync_response.read(), rsync_stderr.read())
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
147 rsync_response.close()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
148 rsync_stderr.close()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
149 raise Exception('Failed to execute rsync command (%s), returncode=%s. Rsync_output:\n%s' % (rsync_cmd, return_code, msg))
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
150 rsync_stderr.close()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
151 rval = {}
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
152 for line in rsync_response:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
153 perms, line = line.split(None, 1)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
154 line = line.strip()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
155 size, line = line.split(None, 1)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
156 line = line.strip()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
157 date, line = line.split(None, 1)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
158 line = line.strip()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
159 time, line = line.split(None, 1)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
160 name = line.strip()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
161 if name in skip_names:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
162 continue
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
163 size = line.strip()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
164 rval[name] = dict(name=name, permissions=perms, bytes=size, date=date, time=time)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
165 rsync_response.close()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
166 return rval
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
167
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
168
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
169 def rsync_sync_to_dir(source, target):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
170 rsync_response = tempfile.NamedTemporaryFile()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
171 rsync_stderr = tempfile.NamedTemporaryFile()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
172 rsync_cmd = [RSYNC_CMD, '-avzP', source, target]
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
173 return_code = subprocess.call(rsync_cmd, stdout=rsync_response, stderr=rsync_stderr)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
174 rsync_response.flush()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
175 rsync_response.seek(0)
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
176 rsync_stderr.flush()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
177 rsync_stderr.seek(0)
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
178 if return_code:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
179 msg = "stdout:\n%s\nstderr:\n%s" % (rsync_response.read(), rsync_stderr.read())
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
180 rsync_response.close()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
181 rsync_stderr.close()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
182 raise Exception('Failed to execute rsync command (%s), returncode=%s. Rsync_output:\n%s' % (rsync_cmd, return_code, msg))
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
183 rsync_response.close()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
184 rsync_stderr.close()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
185 return return_code
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
186
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
187
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
188 def data_table_needs_refresh(cached_data_table, url):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
189 if cached_data_table is None:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
190 return True, {}
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
191 if datetime.datetime.now() - cached_data_table.get('time_loaded') > CACHE_TIME:
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
192 data_table_text = urlopen(url).read().decode('utf-8')
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
193 if cached_data_table.get('data_table_text', None) != data_table_text:
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
194 return True, {'data_table_text': data_table_text}
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
195 loc_file_attrs = rsync_list_dir(RSYNC_SERVER, LOCATION_DIR)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
196 if cached_data_table.get('loc_file_attrs', None) != loc_file_attrs:
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
197 return True, {'loc_file_attrs': loc_file_attrs}
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
198 return False, {}
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
199
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
200
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
201 def load_data_tables_from_url(url=None, site='main', data_table_class=None):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
202 if not url:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
203 url = TOOL_DATA_TABLE_CONF_XML_URLS.get(site, None)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
204 assert url, ValueError('You must provide either a URL or a site=name.')
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
205
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
206 cached_data_table = TOOL_DATA_TABLES_LOADED_BY_URL.get(url, None)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
207 refresh, attribs = data_table_needs_refresh(cached_data_table, url)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
208 if refresh:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
209 data_table_text = attribs.get('data_table_text') or urlopen(url).read().decode('utf-8')
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
210 loc_file_attrs = attribs.get('loc_file_attrs') or rsync_list_dir(RSYNC_SERVER, LOCATION_DIR)
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
211
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
212 tmp_dir = tempfile.mkdtemp(prefix='rsync_g2_')
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
213 tmp_loc_dir = os.path.join(tmp_dir, 'location')
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
214 os.mkdir(tmp_loc_dir)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
215 rsync_sync_to_dir(rsync_urljoin(RSYNC_SERVER, LOCATION_DIR), os.path.abspath(tmp_loc_dir))
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
216
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
217 new_data_table_text = data_table_text.replace(TOOL_DATA_TABLE_CONF_XML_REPLACE_SOURCE, TOOL_DATA_TABLE_CONF_XML_REPLACE_TARGET % (tmp_loc_dir))
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
218 data_table_fh = tempfile.NamedTemporaryFile(dir=tmp_dir, prefix='rysnc_data_manager_data_table_conf_', mode="w")
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
219 data_table_fh.write(new_data_table_text)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
220 data_table_fh.flush()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
221 tmp_data_dir = os.path.join(tmp_dir, 'tool-data')
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
222 os.mkdir(tmp_data_dir)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
223 data_tables = data_table_class(tmp_data_dir, config_filename=data_table_fh.name)
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
224 for name, data_table in list(data_tables.data_tables.items()):
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
225 if name in EXCLUDE_DATA_TABLES or not data_table_has_path_column(data_table):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
226 log.debug('Removing data table "%s" because it is excluded by name or does not have a defined "path" column.', name)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
227 del data_tables.data_tables[name]
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
228 cached_data_table = {'data_tables': data_tables, 'tmp_dir': tmp_dir, 'data_table_text': data_table_text, 'tmp_loc_dir': tmp_loc_dir, 'loc_file_attrs': loc_file_attrs, 'time_loaded': datetime.datetime.now()}
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
229 TOOL_DATA_TABLES_LOADED_BY_URL[url] = cached_data_table
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
230 # delete the files
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
231 data_table_fh.close()
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
232 cleanup_before_exit(tmp_dir)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
233 return cached_data_table
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
234
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
235
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
236 def data_table_has_path_column(data_table):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
237 col_names = data_table.get_column_name_list()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
238 for name in PATH_COLUMN_NAMES:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
239 if name in col_names:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
240 return True
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
241 return False
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
242
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
243
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
244 def get_available_tables(trans):
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
245 # list of data tables
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
246 data_tables = load_data_tables_from_url(data_table_class=trans.app.tool_data_tables.__class__)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
247 return data_tables.get('data_tables').get_tables().keys()
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
248
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
249
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
250 def get_new_xml_definition(app, data_table, data_manager, repo_info=None, location_file_dir=None):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
251 sub_dict = {'table_name': data_table.name, 'comment_char': '', 'columns': '', 'file_path': ''}
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
252 sub_dict.update(data_manager.get_tool_shed_repository_info_dict())
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
253 if data_table.comment_char:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
254 sub_dict['comment_char'] = 'comment_char="%s"' % (data_table.comment_char)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
255 for i, name in enumerate(data_table.get_column_name_list()):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
256 if name is not None:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
257 sub_dict['columns'] = "%s\n%s" % (sub_dict['columns'], '<column name="%s" index="%s" />' % (name, i))
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
258 location_file_dir = location_file_dir or app.config.galaxy_data_manager_data_path
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
259 for filename in data_table.filenames.keys():
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
260 sub_dict['file_path'] = basename(filename)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
261 sub_dict['file_path'] = os.path.join(location_file_dir, sub_dict['file_path']) # os.path.abspath?
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
262 if not os.path.exists(sub_dict['file_path']):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
263 # Create empty file
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
264 open(sub_dict['file_path'], 'wb+').close()
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
265 break
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
266 sub_dict['repo_info'] = repo_info or ''
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
267 return """
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
268 <tables><table name="%(table_name)s" %(comment_char)s>
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
269 %(columns)s
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
270 <file path="%(file_path)s" />
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
271 %(repo_info)s
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
272 </table></tables>
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
273 """ % sub_dict
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
274
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
275
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
276 def get_available_tables_for_dbkey(trans, dbkey, data_table_names):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
277 data_tables = load_data_tables_from_url(data_table_class=trans.app.tool_data_tables.__class__)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
278 rval = {}
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
279 for name, data_table in data_tables.get('data_tables').get_tables().items():
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
280 if (not data_table_names or name in data_table_names):
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
281 # TODO: check that columns are similiar
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
282 if not dbkey:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
283 entry_getter = data_table.get_named_fields_list()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
284 else:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
285 entry_getter = data_table.get_entries('dbkey', dbkey, None, default=[])
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
286 for entry in entry_getter:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
287 name = "%s: %s" % (data_table.name, dumps(entry))
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
288 rval[name] = entry
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
289 return rval
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
290
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
291
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
292 def split_path_all(path):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
293 rval = []
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
294 path = path.rstrip('/')
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
295 while True:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
296 head, tail = os.path.split(path)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
297 if tail:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
298 rval.append(tail)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
299 path = head
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
300 elif head:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
301 rval.append(head)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
302 break
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
303 else:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
304 break
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
305 rval.reverse()
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
306 return rval
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
307
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
308
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
309 def get_data_for_path(path, data_root_dir):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
310 # We list dir with a /, but copy data without
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
311 # listing with / gives a . entry when its a dir
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
312 # cloning without the / will copy that whole directory into the target,
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
313 # instead of just that target's contents
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
314 if path.startswith(GALAXY_DATA_CANONICAL_PATH):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
315 path = path[len(GALAXY_DATA_CANONICAL_PATH):]
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
316 make_path = path
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
317 rsync_source = rsync_urljoin(rsync_urljoin(RSYNC_SERVER, INDEX_DIR), path)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
318 if rsync_source.endswith('/'):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
319 rsync_source = rsync_source[:-1]
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
320 try:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
321 dir_list = rsync_list_dir(rsync_source + "/")
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
322 except Exception:
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
323 dir_list = None
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
324 while not dir_list or '.' not in dir_list:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
325 head, tail = os.path.split(make_path)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
326 if not head:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
327 head = tail
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
328 make_path = head
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
329 rsync_source = rsync_urljoin(rsync_urljoin(RSYNC_SERVER, INDEX_DIR), head) # if we error here, likely due to a connection issue
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
330 if rsync_source.endswith('/'):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
331 rsync_source = rsync_source[:-1]
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
332 dir_list = rsync_list_dir(rsync_source + "/")
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
333 split_path = split_path_all(make_path)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
334 target_path = data_root_dir
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
335 for p in split_path[:-1]:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
336 target_path = os.path.join(target_path, p)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
337 if not os.path.exists(target_path):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
338 os.mkdir(target_path)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
339 rsync_sync_to_dir(rsync_source, target_path)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
340 return path
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
341
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
342
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
343 def get_data_and_munge_path(data_table_name, data_table_entry, data_root_dir):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
344 path_cols = []
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
345 for key, value in data_table_entry.items():
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
346 if key in PATH_COLUMN_NAMES:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
347 path_cols.append((key, value))
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
348 if path_cols:
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
349 for col_name, value in path_cols:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
350 if value.startswith(GALAXY_DATA_CANONICAL_PATH):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
351 data_table_entry[col_name] = get_data_for_path(value, data_root_dir)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
352 else:
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
353 print('unable to determine location of rsync data for', data_table_name, data_table_entry)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
354 return data_table_entry
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
355
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
356
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
357 def fulfill_data_table_entries(data_table_entries, data_manager_dict, data_root_dir):
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
358 for data_table_name, entries in data_table_entries.items():
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
359 for entry in entries:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
360 entry = get_data_and_munge_path(data_table_name, entry, data_root_dir)
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
361 _add_data_table_entry(data_manager_dict, data_table_name, entry)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
362 return data_manager_dict
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
363
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
364
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
365 def _add_data_table_entry(data_manager_dict, data_table_name, data_table_entry):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
366 data_manager_dict['data_tables'] = data_manager_dict.get('data_tables', {})
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
367 data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get(data_table_name, [])
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
368 data_manager_dict['data_tables'][data_table_name].append(data_table_entry)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
369 return data_manager_dict
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
370
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
371
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
372 def cleanup_before_exit(tmp_dir):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
373 if tmp_dir and os.path.exists(tmp_dir):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
374 shutil.rmtree(tmp_dir)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
375
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
376
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
377 def get_data_table_entries(params):
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
378 rval = {}
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
379 data_table_entries = params.get('data_table_entries', None)
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
380 if data_table_entries:
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
381 for entry_text in data_table_entries.split(','):
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
382 entry_text = base64.b64decode(entry_text.strip().encode('utf-8'))
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
383 entry_dict = loads(entry_text)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
384 data_table_name = entry_dict['name']
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
385 data_table_entry = entry_dict['entry']
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
386 rval[data_table_name] = rval.get(data_table_name, [])
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
387 rval[data_table_name].append(data_table_entry)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
388 return rval
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
389
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
390
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
391 def main():
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
392 parser = optparse.OptionParser()
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
393 (options, args) = parser.parse_args()
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
394
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
395 filename = args[0]
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
396
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
397 params = loads(open(filename).read())
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
398 target_directory = params['output_data'][0]['extra_files_path']
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
399 os.mkdir(target_directory)
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
400 data_manager_dict = {}
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
401
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
402 data_table_entries = get_data_table_entries(params['param_dict'])
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
403
0
0a3a6f862104 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 704060ebdf7399ecce9e0e8bd7262727fe750c27-dirty
devteam
parents:
diff changeset
404 # Populate the data Tables
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
405 data_manager_dict = fulfill_data_table_entries(data_table_entries, data_manager_dict, target_directory)
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
406
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
407 # save info to json file
2
e0329ab30f6d "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e"
iuc
parents: 1
diff changeset
408 open(filename, 'w').write(dumps(data_manager_dict, sort_keys=True))
1
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
409
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
410
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
411 if __name__ == "__main__":
8ff92bd7e2a3 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
412 main()