Mercurial > repos > earlhaminst > export_to_cluster

annotate export_to_cluster.py @ 5:8b8466d30d50 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/main/tools/export_to_cluster/ commit d96cd7ee9c686c185b6734aa08ab5a891cc44149-dirty
author earlhaminst
date Mon, 03 Mar 2025 17:49:27 +0000
parents ddd9749d3b00
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
1 import optparse
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
2 import os
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
3 import os.path
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
4 import re
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
5 import shutil
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
6 import sys
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
7
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
8 parser = optparse.OptionParser()
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
9 parser.add_option('-d', '--export_dir', help='Directory where to export the datasets')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
10 parser.add_option('-p', '--dir_prefix', help='How the export dir should start')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
11 (options, args) = parser.parse_args()
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
12 if not options.export_dir:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
13 parser.error('Export directory cannot be empty')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
14 if not options.dir_prefix:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
15 parser.error('Directory prefix cannot be empty')
1
a9d1991c90e3 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 7d8a044afcc2cdf4fa7a3c360d9df3990356589c-dirty
earlhaminst
parents: 0
diff changeset
16 if len(args) < 3:
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
17 parser.error('Require at least two arguments')
1
a9d1991c90e3 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 7d8a044afcc2cdf4fa7a3c360d9df3990356589c-dirty
earlhaminst
parents: 0
diff changeset
18 if len(args) % 3 != 0:
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
19 parser.error('Require an even number of arguments')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
20
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
21 real_export_dir = os.path.realpath(options.export_dir)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
22 dir_prefix = options.dir_prefix.rstrip(os.sep)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
23 if not real_export_dir.startswith(dir_prefix):
3
9838eed606ad planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents: 2
diff changeset
24 raise Exception("'%s' must be a subdirectory of '%s'" % (options.export_dir, dir_prefix))
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
25 if not os.path.exists(real_export_dir):
3
9838eed606ad planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents: 2
diff changeset
26 raise Exception("'%s' directory does not exist or it is not accessible by the Galaxy user" % options.export_dir)
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
27 if not os.path.isdir(real_export_dir):
3
9838eed606ad planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents: 2
diff changeset
28 raise Exception("'%s' is not a directory" % options.export_dir)
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
29
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
30 dataset_paths = args[::3]
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
31 dataset_names = args[1::3]
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
32 dataset_exts = args[2::3]
2
b97775e9fe06 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents: 1
diff changeset
33 exit_code = 0
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
34 for dp, dn, de in zip(dataset_paths, dataset_names, dataset_exts):
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
35 """
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
36 Copied from get_valid_filename from django
4
ddd9749d3b00 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/main/tools/export_to_cluster/ commit 457bb20f9c1c007b42d79b08527f5c558529ab8a-dirty
earlhaminst
parents: 3
diff changeset
37 https://github.com/django/django/blob/main/django/utils/text.py
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
38 """
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
39 dn_de = "%s.%s" % (dn, de)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
40 dn_de_safe = re.sub(r'(?u)[^-\w.]', '', dn_de.strip().replace(' ', '_'))
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
41 dest = os.path.join(real_export_dir, dn_de_safe)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
42 try:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
43 shutil.copy2(dp, dest)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
44 print("Dataset '%s' copied to '%s'" % (dn, dest))
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
45 except Exception as e:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
46 msg = "Error copying dataset '%s' to '%s', %s" % (dn, dest, e)
2
b97775e9fe06 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents: 1
diff changeset
47 print(msg, file=sys.stderr)
b97775e9fe06 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents: 1
diff changeset
48 exit_code = 1
b97775e9fe06 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents: 1
diff changeset
49 sys.exit(exit_code)