Mercurial > repos > earlhaminst > export_to_cluster

annotate export_to_cluster.py @ 0:0dbe46f58158 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
author earlhaminst
date Thu, 11 Aug 2016 15:27:30 -0400
parents
children a9d1991c90e3
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
1 #!/usr/bin/env python
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
2 from __future__ import print_function
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
3
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
4 import optparse
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
5 import os
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
6 import os.path
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
7 import re
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
8 import shutil
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
9 import sys
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
10
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
11 parser = optparse.OptionParser()
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
12 parser.add_option('-d', '--export_dir', help='Directory where to export the datasets')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
13 parser.add_option('-p', '--dir_prefix', help='How the export dir should start')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
14 (options, args) = parser.parse_args()
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
15 if not options.export_dir:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
16 parser.error('Export directory cannot be empty')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
17 if not options.dir_prefix:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
18 parser.error('Directory prefix cannot be empty')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
19 if len(args) < 2:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
20 parser.error('Require at least two arguments')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
21 if len(args) % 2 != 0:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
22 parser.error('Require an even number of arguments')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
23
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
24 real_export_dir = os.path.realpath(options.export_dir)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
25 dir_prefix = options.dir_prefix.rstrip(os.sep)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
26 if not real_export_dir.startswith(dir_prefix):
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
27 sys.exit("%s must be a subdirectory of %s" % (options.export_dir, dir_prefix))
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
28 if not os.path.exists(real_export_dir):
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
29 sys.exit("%s does not exist or it is not accessible by the Galaxy user" % options.export_dir)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
30 if not os.path.isdir(real_export_dir):
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
31 sys.exit("%s is not a directory" % options.export_dir)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
32
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
33 dataset_paths = args[::3]
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
34 dataset_names = args[1::3]
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
35 dataset_exts = args[2::3]
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
36 for dp, dn, de in zip(dataset_paths, dataset_names, dataset_exts):
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
37 """
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
38 Copied from get_valid_filename from django
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
39 https://github.com/django/django/blob/master/django/utils/text.py
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
40 """
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
41 dn_de = "%s.%s" % (dn, de)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
42 dn_de_safe = re.sub(r'(?u)[^-\w.]', '', dn_de.strip().replace(' ', '_'))
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
43 dest = os.path.join(real_export_dir, dn_de_safe)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
44 try:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
45 shutil.copy2(dp, dest)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
46 print("Dataset '%s' copied to '%s'" % (dn, dest))
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
47 except Exception as e:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
48 msg = "Error copying dataset '%s' to '%s', %s" % (dn, dest, e)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
49 print(msg)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
50 sys.stderr.write(msg)