Mercurial > repos > shellac > guppy_basecaller
comparison env/lib/python3.7/site-packages/planemo/shed2tap/base.py @ 0:26e78fe6e8c4 draft
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author | shellac |
---|---|
date | Sat, 02 May 2020 07:14:21 -0400 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:26e78fe6e8c4 |
---|---|
1 from __future__ import print_function | |
2 | |
3 import os | |
4 import subprocess | |
5 import sys | |
6 import tarfile | |
7 import zipfile | |
8 from ftplib import all_errors as FTPErrors # tuple of exceptions | |
9 from xml.etree import ElementTree | |
10 | |
11 from galaxy.util import unicodify | |
12 from six import iteritems | |
13 from six import string_types | |
14 from six.moves import map as imap | |
15 from six.moves.urllib.error import URLError | |
16 from six.moves.urllib.request import urlretrieve | |
17 | |
18 | |
19 TOOLSHED_MAP = { | |
20 "toolshed": "https://toolshed.g2.bx.psu.edu", | |
21 "testtoolshed": "https://testtoolshed.g2.bx.psu.edu", | |
22 } | |
23 | |
24 | |
25 class Dependencies(object): | |
26 """ Base class for parsing Tool Shed dependency files. | |
27 """ | |
28 | |
29 def __init__( | |
30 self, | |
31 dependencies_file, | |
32 repo=None, | |
33 package_factory=None, | |
34 ): | |
35 if package_factory is None: | |
36 package_factory = BasePackage | |
37 self.repo = repo | |
38 self.root = ElementTree.parse(dependencies_file).getroot() | |
39 packages = [] | |
40 dependencies = [] | |
41 package_els = self.root.findall("package") | |
42 assert package_els is not None | |
43 for package_el in package_els: | |
44 install_els = package_el.findall("install") | |
45 readme_els = package_el.findall("readme") | |
46 if len(readme_els) > 0: | |
47 readme = readme_els[0].text | |
48 else: | |
49 readme = None | |
50 assert len(install_els) in (0, 1) | |
51 if len(install_els) == 1: | |
52 install_el = install_els[0] | |
53 package = package_factory( | |
54 self, | |
55 package_el, | |
56 install_el, | |
57 readme=readme | |
58 ) | |
59 packages.append(package) | |
60 else: | |
61 repository_el = package_el.find("repository") | |
62 if repository_el is None: | |
63 message = "no repository in package el for %s" % repo | |
64 raise AssertionError(message) | |
65 dependency = Dependency(self, package_el, repository_el) | |
66 dependencies.append(dependency) | |
67 | |
68 self.packages = packages | |
69 self.dependencies = dependencies | |
70 | |
71 def single_package(self): | |
72 return len(self.packages) == 1 | |
73 | |
74 def __repr__(self): | |
75 return "Dependencies[for_repo=%s]" % self.repo | |
76 | |
77 | |
78 class Repo(object): | |
79 | |
80 def __init__(self, **kwds): | |
81 for key, value in iteritems(kwds): | |
82 setattr(self, key, value) | |
83 | |
84 def recipe_base_name(self): | |
85 owner = self.owner.replace("-", "") | |
86 name = self.name | |
87 name = name.replace("_", "").replace("-", "") | |
88 base = "%s_%s" % (owner, name) | |
89 return base | |
90 | |
91 @staticmethod | |
92 def from_xml(elem): | |
93 tool_shed_url = elem.attrib.get("toolshed", None) | |
94 if tool_shed_url and ("testtoolshed" in tool_shed_url): | |
95 prefix = "testtoolshed" | |
96 else: | |
97 prefix = "toolshed" | |
98 prior = elem.attrib.get("prior_installation_required", False) | |
99 return Repo( | |
100 prefix=prefix, | |
101 name=elem.attrib["name"], | |
102 owner=elem.attrib["owner"], | |
103 tool_shed_url=tool_shed_url, | |
104 changeset_revision=elem.attrib.get("changeset_revision", None), | |
105 prior_installation_required=prior, | |
106 ) | |
107 | |
108 @staticmethod | |
109 def from_api(prefix, repo_json): | |
110 return Repo( | |
111 prefix=prefix, | |
112 name=repo_json["name"], | |
113 owner=repo_json["owner"], | |
114 tool_shed_url=TOOLSHED_MAP[prefix], | |
115 ) | |
116 | |
117 def get_file(self, path): | |
118 try: | |
119 url_template = "%s/repos/%s/%s/raw-file/tip/%s" | |
120 url = url_template % ( | |
121 self.tool_shed_url, | |
122 self.owner, | |
123 self.name, | |
124 path | |
125 ) | |
126 path, headers = urlretrieve(url) | |
127 return path | |
128 except Exception as e: | |
129 print(e) | |
130 return None | |
131 | |
132 def __repr__(self): | |
133 return "Repository[name=%s,owner=%s]" % (self.name, self.owner) | |
134 | |
135 | |
136 class Dependency(object): | |
137 | |
138 def __init__(self, dependencies, package_el, repository_el): | |
139 self.dependencies = dependencies | |
140 self.package_el = package_el | |
141 self.repository_el = repository_el | |
142 self.repo = Repo.from_xml(repository_el) | |
143 | |
144 def __repr__(self): | |
145 temp = "Dependency[package_name=%s,version=%s,dependent_package=%s]" | |
146 return temp % ( | |
147 self.package_el.attrib["name"], | |
148 self.package_el.attrib["version"], | |
149 self.repository_el.attrib["name"] | |
150 ) | |
151 | |
152 | |
153 class BasePackage(object): | |
154 | |
155 def __init__(self, dependencies, package_el, install_el, readme): | |
156 self.dependencies = dependencies | |
157 self.package_el = package_el | |
158 self.install_el = install_el | |
159 self.readme = readme | |
160 self.all_actions = self.get_all_actions() | |
161 self.no_arch_option = self.has_no_achitecture_install() | |
162 | |
163 def get_all_actions(self): | |
164 action_or_group = self.install_el[0] | |
165 parsed_actions = [] | |
166 if action_or_group.tag == "actions": | |
167 parsed_actions.append(self.parse_actions(action_or_group)) | |
168 elif action_or_group.tag == "actions_group": | |
169 actions_els = action_or_group.findall("actions") | |
170 assert actions_els is not None | |
171 for actions in actions_els: | |
172 parsed_actions.append(self.parse_actions(actions)) | |
173 action_els = action_or_group.findall("action") | |
174 assert action_els is not None | |
175 for action in action_els: | |
176 for parsed_a in parsed_actions: | |
177 parsed_a.actions.append(self.parse_action(action)) | |
178 return parsed_actions | |
179 | |
180 def has_no_achitecture_install(self): | |
181 all_actions = self.all_actions | |
182 if len(all_actions) < 2: | |
183 return False | |
184 else: | |
185 last_action = all_actions[-1] | |
186 return (not last_action.architecture) and (not last_action.os) | |
187 | |
188 def has_explicit_set_environments(self): | |
189 all_actions = self.all_actions | |
190 for actions in all_actions: | |
191 for action in actions.actions: | |
192 if action.explicit_variables: | |
193 return True | |
194 return False | |
195 | |
196 def has_multiple_set_environments(self): | |
197 all_actions = self.all_actions | |
198 for actions in all_actions: | |
199 count = 0 | |
200 for action in actions.actions: | |
201 if action.explicit_variables: | |
202 count += 1 | |
203 if count > 1: | |
204 return True | |
205 return False | |
206 | |
207 def parse_actions(self, actions): | |
208 os = actions.attrib.get("os", None) | |
209 architecture = actions.get("architecture", None) | |
210 action_els = actions.findall("action") | |
211 assert action_els is not None | |
212 parsed_actions = list(imap(self.parse_action, action_els)) | |
213 action_packages = [] | |
214 for package in actions.findall("package"): | |
215 action_packages.append(self.parse_action_package(package)) | |
216 return Actions(parsed_actions, os, architecture, action_packages) | |
217 | |
218 def parse_action_package(self, elem): | |
219 name = elem.attrib["name"] | |
220 version = elem.attrib["version"] | |
221 repo = Repo.from_xml(elem.find("repository")) | |
222 return ActionPackage(name, version, repo) | |
223 | |
224 def parse_action(self, action): | |
225 return BaseAction.from_elem(action, package=self) | |
226 | |
227 def __repr__(self): | |
228 actions = self.all_actions | |
229 parts = ( | |
230 self.package_el.attrib["name"], | |
231 self.package_el.attrib["version"], | |
232 self.dependencies, | |
233 actions | |
234 ) | |
235 template = "Install[name=%s,version=%s,dependencies=%s,actions=%s]" | |
236 return template % parts | |
237 | |
238 | |
239 class Actions(object): | |
240 | |
241 def __init__( | |
242 self, | |
243 actions, | |
244 os=None, | |
245 architecture=None, | |
246 action_packages=[] | |
247 ): | |
248 self.os = os | |
249 self.architecture = architecture | |
250 self.actions = actions or [] | |
251 self.action_packages = action_packages | |
252 | |
253 def first_download(self): | |
254 for action in self.actions: | |
255 if action.action_type in ["download_by_url", "download_file"]: | |
256 return action | |
257 return None | |
258 | |
259 def downloads(self): | |
260 actions = [] | |
261 for action in self.actions: | |
262 if action.action_type in ["download_by_url", "download_file"]: | |
263 actions.append(action) | |
264 return actions | |
265 | |
266 def __repr__(self): | |
267 platform = "" | |
268 if self.os or self.architecture: | |
269 platform = "os=%s,arch=%s," % (self.os, self.architecture) | |
270 return "Actions[%s%s]" % (platform, map(str, self.actions)) | |
271 | |
272 def _indent_extend(self, target, new_entries, indent=" "): | |
273 for line in new_entries: | |
274 target.append(indent + line) | |
275 | |
276 def to_bash(self): | |
277 # Use self.os.title() to match "Linux" or "Darwin" in bash where case matters: | |
278 if self.os and self.architecture: | |
279 condition = '("%s" == `uname`) && ("%s" == `arch`)' % (self.os.title(), self.architecture) | |
280 elif self.os: | |
281 condition = '"%s" == `uname`' % self.os.title() | |
282 elif self.architecture: | |
283 condition = '"%s" == `arch`' % self.architecture | |
284 else: | |
285 condition = None | |
286 | |
287 install_cmds = [] | |
288 env_cmds = [] | |
289 | |
290 if condition: | |
291 # Conditional actions block | |
292 install_cmds = [ | |
293 '#' + '-' * 60, | |
294 'if [[ $specifc_action_done == 0 && %s ]]' % condition, | |
295 'then', | |
296 ' echo "Platform-specific action for os=%s, arch=%s"' % (self.os, self.architecture)] | |
297 env_cmds = install_cmds[:] | |
298 # TODO - Refactor block indentation? | |
299 for action in self.actions: | |
300 i_cmds, e_cmds = action.to_bash() | |
301 self._indent_extend(install_cmds, i_cmds) | |
302 self._indent_extend(env_cmds, e_cmds) | |
303 # If we run the action, do not want to run any later actions! | |
304 install_cmds.extend([' specifc_action_done=1', 'fi']) | |
305 env_cmds.extend([' specifc_action_done=1', 'fi']) | |
306 else: | |
307 # Non-specific default action... | |
308 install_cmds = [ | |
309 '#' + '-' * 60, | |
310 'if [[ $specifc_action_done == 0 ]]', | |
311 'then', | |
312 ' echo "Non-platform-specific actions"'] | |
313 env_cmds = install_cmds[:] | |
314 for action in self.actions: | |
315 i_cmds, e_cmds = action.to_bash() | |
316 self._indent_extend(install_cmds, i_cmds) | |
317 self._indent_extend(env_cmds, e_cmds) | |
318 install_cmds.append('fi') | |
319 env_cmds.append('fi') | |
320 return install_cmds, env_cmds | |
321 | |
322 | |
323 class ActionPackage(object): | |
324 | |
325 def __init__(self, name, version, repo): | |
326 self.name = name | |
327 self.version = version | |
328 self.repo = repo | |
329 | |
330 | |
331 class BaseAction(object): | |
332 | |
333 def __repr__(self): | |
334 return "Action[type=%s]" % self.action_type | |
335 | |
336 def same_as(self, other): | |
337 if self._keys != other._keys: | |
338 return False | |
339 else: | |
340 for key in self._keys: | |
341 if getattr(self, key) != getattr(other, key): | |
342 return False | |
343 | |
344 return True | |
345 | |
346 def parse_action_repo(self, elem): | |
347 repo_elem = elem.find("repository") | |
348 repo = Repo.from_xml(repo_elem) | |
349 self.repo = repo | |
350 | |
351 def parse_package_elems(self, elem): | |
352 package_els = elem.findall("package") | |
353 packages = [] | |
354 assert package_els is not None | |
355 for package_el in package_els: | |
356 packages.append(package_el.text) | |
357 self.packages = packages | |
358 | |
359 @classmethod | |
360 def from_elem(cls, elem, package): | |
361 type = elem.attrib["type"] | |
362 action_class = actions_by_type[type] | |
363 return action_class(elem) | |
364 | |
365 def to_bash(self): | |
366 """Return lists of bash shell commands to execute this action. | |
367 | |
368 This method is be implemented by each sub-class, and will | |
369 return two list of strings (for ``dep_install.sh`` and | |
370 ``env.sh`` respectively). | |
371 """ | |
372 raise NotImplementedError("No to_bash defined for %r" % self) | |
373 | |
374 | |
375 def _tar_folders(filename): | |
376 with tarfile.open(filename, "r", errorlevel=0) as archive: | |
377 folders = set() | |
378 for i in archive.getmembers(): | |
379 if i.isdir(): | |
380 folders.add(i.name.rstrip("/")) | |
381 else: | |
382 folders.add(os.path.split(i.name)[0]) | |
383 return list(folders) | |
384 | |
385 | |
386 def _zip_folders(filename): | |
387 archive = zipfile.ZipFile(filename, "r") | |
388 return list(set(i.filename.rstrip("/") for i in archive.infolist() if i.filename.endswith("/"))) | |
389 | |
390 | |
391 def _common_prefix(folders): | |
392 common_prefix = "" | |
393 if len(folders) == 1: | |
394 common_prefix = list(folders)[0] | |
395 else: | |
396 common_prefix = os.path.commonprefix(folders) | |
397 assert not os.path.isabs(common_prefix), folders | |
398 return common_prefix | |
399 | |
400 | |
401 def _cache_download(url, filename, sha256sum=None): | |
402 """Returns local path to cached copy of URL using given filename.""" | |
403 cache = os.environ.get("DOWNLOAD_CACHE", "./download_cache/") | |
404 # TODO - expose this as a command line option | |
405 | |
406 if not os.path.isdir(cache): | |
407 os.mkdir(cache) | |
408 | |
409 local = os.path.join(cache, filename) | |
410 | |
411 if not os.path.isfile(local): | |
412 # Must download it... | |
413 try: | |
414 # TODO - log this nicely... | |
415 sys.stderr.write("Downloading %s to %r\n" % (url, local)) | |
416 urlretrieve(url, local) | |
417 except URLError: | |
418 # Most likely server is down, could be bad URL in XML action: | |
419 raise RuntimeError("Unable to download %s" % url) | |
420 except FTPErrors: | |
421 # Most likely server is down, could be bad URL in XML action: | |
422 raise RuntimeError("Unable to download %s" % url) | |
423 | |
424 # Verifying the checksum is slow, only do this on a fresh | |
425 # download. Assume locally cached files are already OK. | |
426 if sha256sum: | |
427 # TODO - log this nicely... | |
428 sys.stderr.write("Verifying checksum for %s\n" % filename) | |
429 filehash = subprocess.check_output(['shasum', '-a', '256', local])[0:64].strip() | |
430 filehash = unicodify(filehash) | |
431 if filehash != sha256sum: | |
432 raise RuntimeError("Checksum failure for %s, got %r but wanted %r" % (local, filehash, sha256sum)) | |
433 | |
434 return local | |
435 | |
436 | |
437 def _determine_compressed_file_folder(url, downloaded_filename, target_filename=None, sha256sum=None): | |
438 """Determine how to decompress the file & its directory structure. | |
439 | |
440 Returns a list of shell commands. Consider this example where the | |
441 folder to change to cannot be guessed from the tar-ball filename: | |
442 | |
443 $ curl -o "ncbi-blast-2.2.30+-ia32-linux.tar.gz" \ | |
444 "ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.2.30/ncbi-blast-2.2.30+-ia32-linux.tar.gz" | |
445 $ tar -zxvf ncbi-blast-2.2.30+-ia32-linux.tar.gz | |
446 $ cd ncbi-blast-2.2.30+ | |
447 | |
448 Here it would return: | |
449 | |
450 ['tar -zxvf ncbi-blast-2.2.30+-ia32-linux.tar.gz', 'cd ncbi-blast-2.2.30+'] | |
451 | |
452 If not cached, this function will download the file to the | |
453 $DOWNLOAD_CACHE folder, and then open it / decompress it in | |
454 order to find common folder prefix used. This will also verify | |
455 how to decompress the file, and the checksum if given. | |
456 """ | |
457 answer = [] | |
458 | |
459 local = _cache_download(url, downloaded_filename, sha256sum) | |
460 | |
461 if not target_filename: | |
462 target_filename = downloaded_filename | |
463 | |
464 if tarfile.is_tarfile(local): | |
465 folders = _tar_folders(local) | |
466 if target_filename.endswith((".tar.gz", ".tgz")): | |
467 answer.append('tar -zxvf %s' % target_filename) | |
468 elif target_filename.endswith(".tar.bz2"): | |
469 answer.append('tar -jxvf %s' % target_filename) | |
470 elif target_filename.endswith(".tar"): | |
471 answer.extend('tar -xvf %s' % target_filename) | |
472 else: | |
473 # Quite possibly this file doesn't need decompressing, | |
474 # but until we've tested lots of real world tool_dependencies.xml | |
475 # files I'd like to check these cases to confirm this. | |
476 raise NotImplementedError("How to decompress tar file %s?" % target_filename) | |
477 elif zipfile.is_zipfile(local): | |
478 if target_filename.endswith(".jar"): | |
479 # Do not decompress! | |
480 return answer | |
481 folders = _zip_folders(local) | |
482 answer.append('unzip %s' % target_filename) | |
483 elif target_filename.endswith(".dmg"): | |
484 # Do not decompress! | |
485 return answer | |
486 else: | |
487 # No compression? Leave as it is? | |
488 raise NotImplementedError("What kind of compression is %s using?" % local) | |
489 | |
490 common_prefix = _common_prefix(folders) | |
491 if common_prefix: | |
492 answer.append('cd "%s"' % common_prefix) | |
493 | |
494 return answer | |
495 | |
496 | |
497 def _commands_and_downloaded_file(url, target_filename=None, sha256sum=None): | |
498 # We preserve the filename from the URL in the cache. | |
499 # i.e. We do NOT use the target_filename in the cache. | |
500 # This because some Galaxy recipes normalise platform specific downloads | |
501 # to use a single target filename, which would therefore break checksums etc | |
502 # e.g. tests/data/repos/package_1/tool_dependencies.xml | |
503 downloaded_filename = os.path.split(url)[-1] | |
504 if "?" in downloaded_filename: | |
505 downloaded_filename = downloaded_filename[:downloaded_filename.index("?")] | |
506 if "#" in downloaded_filename: | |
507 downloaded_filename = downloaded_filename[:downloaded_filename.index("#")] | |
508 | |
509 if not target_filename: | |
510 target_filename = downloaded_filename | |
511 | |
512 # Curl is present on Mac OS X, can we assume it will be on Linux? | |
513 # Cannot assume that wget will be on Mac OS X. | |
514 answer = [ | |
515 'if [[ -f "%s" ]]' % target_filename, | |
516 'then', | |
517 ' echo "Reusing existing %s"' % target_filename, | |
518 'elif [[ -f "$DOWNLOAD_CACHE/%s" ]]' % downloaded_filename, | |
519 'then', | |
520 ' echo "Reusing cached %s"' % downloaded_filename, | |
521 ' cp "$DOWNLOAD_CACHE/%s" "%s"' % (downloaded_filename, target_filename), | |
522 'else', | |
523 ' echo "Downloading %s"' % downloaded_filename, | |
524 ' curl -L -o "$DOWNLOAD_CACHE/%s" "%s"' % (downloaded_filename, url), | |
525 ' cp "$DOWNLOAD_CACHE/%s" "%s"' % (downloaded_filename, target_filename), | |
526 ] | |
527 if sha256sum: | |
528 # This is inserted into the if-else for a fresh download only. | |
529 # Note double space between checksum and filename: | |
530 answer.append(' echo "%s %s" | shasum -a 256 -c -' % (sha256sum, target_filename)) | |
531 answer.append('fi') | |
532 | |
533 return answer, downloaded_filename | |
534 | |
535 | |
536 def _commands_to_download_and_extract(url, target_filename=None, sha256sum=None): | |
537 answer, downloaded_filename = _commands_and_downloaded_file(url, target_filename, sha256sum) | |
538 # Now should we unpack the tar-ball etc? | |
539 answer.extend(_determine_compressed_file_folder(url, downloaded_filename, target_filename, sha256sum)) | |
540 return answer, [] | |
541 | |
542 | |
543 class DownloadByUrlAction(BaseAction): | |
544 action_type = "download_by_url" | |
545 _keys = ["url"] | |
546 | |
547 def __init__(self, elem): | |
548 self.url = elem.text.strip() | |
549 assert self.url | |
550 self.sha256sum = elem.attrib.get("sha256sum", None) | |
551 self.target_filename = elem.attrib.get("target_filename", None) | |
552 | |
553 def to_bash(self): | |
554 # See class DownloadByUrl in Galaxy, | |
555 # lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py | |
556 return _commands_to_download_and_extract(self.url, self.target_filename, self.sha256sum) | |
557 | |
558 | |
559 class DownloadFileAction(BaseAction): | |
560 action_type = "download_file" | |
561 _keys = ["url", "extract"] | |
562 | |
563 def __init__(self, elem): | |
564 self.url = elem.text.strip() | |
565 self.extract = asbool(elem.attrib.get("extract", False)) | |
566 self.sha256sum = elem.attrib.get("sha256sum", None) | |
567 self.target_filename = elem.attrib.get("target_filename", None) | |
568 | |
569 def to_bash(self): | |
570 if self.extract: | |
571 return _commands_to_download_and_extract(self.url, self.target_filename, self.sha256sum) | |
572 else: | |
573 commands, downloaded_file = _commands_and_downloaded_file(self.url, self.target_filename, self.sha256sum) | |
574 return commands, [] | |
575 | |
576 | |
577 class DownloadBinary(BaseAction): | |
578 action_type = "download_binary" | |
579 _keys = ["url_template", "target_directory"] | |
580 | |
581 def __init__(self, elem): | |
582 self.url_template = elem.text | |
583 assert self.url_template | |
584 self.target_directory = elem.get('target_directory', None) | |
585 | |
586 | |
587 class ShellCommandAction(BaseAction): | |
588 action_type = "shell_command" | |
589 _keys = ["command"] | |
590 | |
591 def __init__(self, elem): | |
592 self.command = elem.text | |
593 | |
594 def to_bash(self): | |
595 # Galaxy would run each action from the same temp | |
596 # working directory - possible that tool_dependencies.xml | |
597 # shell_command could change $PWD so reset this: | |
598 return ["pushd . > /dev/null", self.command, "popd > /dev/null"], [] | |
599 | |
600 | |
601 class TemplateShellCommandAction(BaseAction): | |
602 action_type = "template_command" | |
603 _keys = ["language", "command"] | |
604 | |
605 def __init__(self, elem): | |
606 self.command = elem.text | |
607 self.language = elem.get('language', 'cheetah').lower() | |
608 assert self.command | |
609 assert self.language == "cheetah" | |
610 | |
611 | |
612 class MoveFileAction(BaseAction): | |
613 action_type = "move_file" | |
614 _keys = ["move_file"] | |
615 | |
616 def __init__(self, elem): | |
617 self.source = elem.find("source").text | |
618 self.destination = elem.find("destination").text | |
619 | |
620 def to_bash(self): | |
621 return ["mv %s %s" % (self.source, self.destination)], [] | |
622 | |
623 | |
624 class MoveDirectoryFilesAction(BaseAction): | |
625 action_type = "move_directory_files" | |
626 _keys = ["source_directory", "destination_directory"] | |
627 | |
628 def __init__(self, elem): | |
629 source_directory = elem.find("source_directory").text | |
630 destination_directory = elem.find("destination_directory").text | |
631 self.source_directory = source_directory | |
632 self.destination_directory = destination_directory | |
633 | |
634 def to_bash(self): | |
635 return ["mv %s/* %s/" % (self.source_directory, self.destination_directory)], [] | |
636 | |
637 | |
638 class SetEnvironmentAction(BaseAction): | |
639 action_type = "set_environment" | |
640 _keys = ["variables"] | |
641 | |
642 def __init__(self, elem): | |
643 variables = [] | |
644 var_els = elem.findall("environment_variable") | |
645 assert var_els is not None | |
646 for ev_elem in var_els: | |
647 var = SetVariable(ev_elem) | |
648 variables.append(var) | |
649 self.variables = variables | |
650 assert self.variables | |
651 | |
652 def to_bash(self): | |
653 answer = [] | |
654 for var in self.variables: | |
655 # Expand $INSTALL_DIR here? | |
656 if var.action == "set_to": | |
657 answer.append('export %s=%s' % (var.name, var.raw_value)) | |
658 elif var.action == "prepend_to": | |
659 answer.append('export %s=%s:$%s' % (var.name, var.raw_value, var.name)) | |
660 elif var.action == "append_to": | |
661 answer.append('export %s=$%s:%s' % (var.name, var.name, var.raw_value)) | |
662 else: | |
663 raise ValueError("Undefined environment variable action %r" % var.action) | |
664 return answer, answer # Actions needed in env.sh here! | |
665 | |
666 | |
667 class ChmodAction(BaseAction): | |
668 action_type = "chmod" | |
669 _keys = ["mods"] | |
670 | |
671 def __init__(self, elem): | |
672 mods = [] | |
673 file_els = elem.findall("file") | |
674 assert file_els is not None | |
675 for mod_elem in file_els: | |
676 mod = {} | |
677 mod["mode"] = mod_elem.attrib["mode"] | |
678 mod["target"] = mod_elem.text | |
679 mods.append(mod) | |
680 self.mods = mods | |
681 assert self.mods | |
682 | |
683 def to_bash(self): | |
684 return ["chmod %s %s" % (m["mode"], m["target"]) for m in self.mods], [] | |
685 | |
686 | |
687 class MakeInstallAction(BaseAction): | |
688 action_type = "make_install" | |
689 _keys = [] | |
690 | |
691 def __init__(self, elem): | |
692 pass | |
693 | |
694 def to_bash(self): | |
695 return ["make install"], [] | |
696 | |
697 | |
698 class AutoconfAction(BaseAction): | |
699 action_type = "autoconf" | |
700 _keys = ["options"] | |
701 | |
702 def __init__(self, elem): | |
703 self.options = elem.text | |
704 | |
705 def to_bash(self): | |
706 if self.options: | |
707 raise NotImplementedError("Options with action autoconf not implemented yet.") | |
708 return ['./configure', 'make', 'make install'], [] | |
709 | |
710 | |
711 class ChangeDirectoryAction(BaseAction): | |
712 action_type = "change_directory" | |
713 _keys = ["directory"] | |
714 | |
715 def __init__(self, elem): | |
716 self.directory = elem.text | |
717 assert self.directory | |
718 | |
719 def to_bash(self): | |
720 return ["cd %s" % self.directory], [] | |
721 | |
722 | |
723 class MakeDirectoryAction(BaseAction): | |
724 action_type = "make_directory" | |
725 _keys = ["directory"] | |
726 | |
727 def __init__(self, elem): | |
728 self.directory = elem.text | |
729 | |
730 def to_bash(self): | |
731 return ["mkdir -p %s" % self.directory], [] | |
732 | |
733 | |
734 class SetupPerlEnvironmentAction(BaseAction): | |
735 action_type = "setup_perl_environment" | |
736 _keys = ["repo", "packages"] | |
737 | |
738 def __init__(self, elem): | |
739 self.parse_action_repo(elem) | |
740 self.parse_package_elems(elem) | |
741 | |
742 | |
743 class SetupRubyEnvironmentAction(BaseAction): | |
744 action_type = "setup_ruby_environment" | |
745 _keys = ["repo", "packages"] | |
746 | |
747 def __init__(self, elem): | |
748 self.parse_action_repo(elem) | |
749 self.parse_package_elems(elem) | |
750 | |
751 | |
752 class SetupPythonEnvironmentAction(BaseAction): | |
753 action_type = "setup_python_environment" | |
754 _keys = ["repo", "packages"] | |
755 | |
756 def __init__(self, elem): | |
757 self.parse_action_repo(elem) | |
758 self.parse_package_elems(elem) | |
759 | |
760 | |
761 class SetupVirtualenvAction(BaseAction): | |
762 action_type = "setup_virtualenv" | |
763 _keys = ["use_requirements_file", "python", "requirements"] | |
764 | |
765 def __init__(self, elem): | |
766 use_reqs = elem.attrib.get("use_requirements_file", "True") | |
767 self.use_requirements_file = asbool(use_reqs) | |
768 self.python = elem.get('python', 'python') | |
769 self.requirements = elem.text or 'requirements.txt' | |
770 | |
771 | |
772 class SetupREnvironmentAction(BaseAction): | |
773 action_type = "setup_r_environment" | |
774 _keys = ["repo", "packages"] | |
775 | |
776 def __init__(self, elem): | |
777 self.parse_action_repo(elem) | |
778 self.parse_package_elems(elem) | |
779 | |
780 | |
781 class SetEnvironmentForInstallAction(BaseAction): | |
782 action_type = "set_environment_for_install" | |
783 | |
784 def __init__(self, elem): | |
785 pass | |
786 | |
787 def to_bash(self): | |
788 # TODO - How could we resolve/check the dependencies? | |
789 return ['echo "WARNING: Assuming packages already installed!"'], [] | |
790 | |
791 | |
792 class SetVariable(object): | |
793 | |
794 def __init__(self, elem): | |
795 self.action = elem.attrib["action"] | |
796 self.name = elem.attrib["name"] | |
797 self.raw_value = elem.text | |
798 | |
799 | |
800 truthy = frozenset(['true', 'yes', 'on', 'y', 't', '1']) | |
801 falsy = frozenset(['false', 'no', 'off', 'n', 'f', '0']) | |
802 | |
803 | |
804 def asbool(obj): | |
805 if isinstance(obj, string_types): | |
806 obj = obj.strip().lower() | |
807 if obj in truthy: | |
808 return True | |
809 elif obj in falsy: | |
810 return False | |
811 else: | |
812 raise ValueError("String is not true/false: %r" % obj) | |
813 return bool(obj) | |
814 | |
815 | |
816 action_classes = [ | |
817 DownloadByUrlAction, | |
818 DownloadFileAction, | |
819 DownloadBinary, | |
820 ShellCommandAction, | |
821 TemplateShellCommandAction, | |
822 MoveFileAction, | |
823 MoveDirectoryFilesAction, | |
824 SetEnvironmentAction, | |
825 ChmodAction, | |
826 MakeInstallAction, | |
827 AutoconfAction, | |
828 ChangeDirectoryAction, | |
829 MakeDirectoryAction, | |
830 SetupPerlEnvironmentAction, | |
831 SetupRubyEnvironmentAction, | |
832 SetupPythonEnvironmentAction, | |
833 SetupVirtualenvAction, | |
834 SetupREnvironmentAction, | |
835 SetEnvironmentForInstallAction, | |
836 ] | |
837 | |
838 actions_by_type = dict(map(lambda c: (c.action_type, c), action_classes)) |