Mercurial > repos > peterjc > tmhmm_and_signalp

annotate tools/protein_analysis/seq_analysis_utils.py @ 18:eb6ac44d4b8e draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Suite v0.2.8, record Promoter 2 verion + misc internal updates
author peterjc
date Tue, 01 Sep 2015 09:56:36 -0400
parents e6cc27d182a8
children f3ecd80850e2
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
1 """A few useful functions for working with FASTA files and running jobs.
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
2
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
3 This module was originally written to hold common code used in both the TMHMM
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
4 and SignalP wrappers in Galaxy.
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
5
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
6 Given Galaxy currently supports Python 2.4+ this cannot use the Python module
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
7 multiprocessing so the function run_jobs instead is a simple pool approach
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
8 using just the subprocess library.
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
9 """
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
10 import sys
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
11 import os
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
12 import subprocess
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
13 from time import sleep
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
14
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
15 __version__ = "0.0.1"
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
16
18
eb6ac44d4b8e Suite v0.2.8, record Promoter 2 verion + misc internal updates
peterjc
parents: 17
diff changeset
17 def sys_exit(msg, error_level=1):
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
18 """Print error message to stdout and quit with given error level."""
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
19 sys.stderr.write("%s\n" % msg)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
20 sys.exit(error_level)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
21
9
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
22 try:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
23 from multiprocessing import cpu_count
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
24 except ImportError:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
25 #Must be under Python 2.5, this is copied from multiprocessing:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
26 def cpu_count():
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
27 """Returns the number of CPUs in the system."""
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
28 if sys.platform == 'win32':
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
29 try:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
30 num = int(os.environ['NUMBER_OF_PROCESSORS'])
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
31 except (ValueError, KeyError):
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
32 num = 0
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
33 elif 'bsd' in sys.platform or sys.platform == 'darwin':
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
34 comm = '/sbin/sysctl -n hw.ncpu'
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
35 if sys.platform == 'darwin':
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
36 comm = '/usr' + comm
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
37 try:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
38 with os.popen(comm) as p:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
39 num = int(p.read())
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
40 except ValueError:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
41 num = 0
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
42 else:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
43 try:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
44 num = os.sysconf('SC_NPROCESSORS_ONLN')
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
45 except (ValueError, OSError, AttributeError):
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
46 num = 0
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
47
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
48 if num >= 1:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
49 return num
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
50 else:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
51 raise NotImplementedError('cannot determine number of cpus')
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
52
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
53
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
54 def thread_count(command_line_arg, default=1):
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
55 try:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
56 num = int(command_line_arg)
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
57 except:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
58 num = default
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
59 if num < 1:
18
eb6ac44d4b8e Suite v0.2.8, record Promoter 2 verion + misc internal updates
peterjc
parents: 17
diff changeset
60 sys_exit("Threads argument %r is not a positive integer" % command_line_arg)
9
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
61 #Cap this with the pysical limit of the machine,
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
62 try:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
63 num = min(num, cpu_count())
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
64 except NotImplementedError:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
65 pass
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
66 #For debugging,
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
67 #hostname = os.environ.get("HOSTNAME", "this machine")
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
68 #sys.stderr.write("Using %i cores on %s\n" % (num, hostname))
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
69 return num
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
70
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
71
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
72 def fasta_iterator(filename, max_len=None, truncate=None):
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
73 """Simple FASTA parser yielding tuples of (title, sequence) strings."""
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
74 handle = open(filename)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
75 title, seq = "", ""
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
76 for line in handle:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
77 if line.startswith(">"):
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
78 if title:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
79 if truncate:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
80 seq = seq[:truncate]
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
81 if max_len and len(seq) > max_len:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
82 raise ValueError("Sequence %s is length %i, max length %i" \
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
83 % (title.split()[0], len(seq), max_len))
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
84 yield title, seq
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
85 title = line[1:].rstrip()
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
86 seq = ""
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
87 elif title:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
88 seq += line.strip()
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
89 elif not line.strip() or line.startswith("#"):
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
90 #Ignore blank lines, and any comment lines
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
91 #between records (starting with hash).
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
92 pass
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
93 else:
17
e6cc27d182a8 Uploaded v0.2.6, embedded citations and uses $GALAXY_SLOTS
peterjc
parents: 9
diff changeset
94 handle.close()
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
95 raise ValueError("Bad FASTA line %r" % line)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
96 handle.close()
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
97 if title:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
98 if truncate:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
99 seq = seq[:truncate]
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
100 if max_len and len(seq) > max_len:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
101 raise ValueError("Sequence %s is length %i, max length %i" \
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
102 % (title.split()[0], len(seq), max_len))
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
103 yield title, seq
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
104 raise StopIteration
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
105
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
106 def split_fasta(input_filename, output_filename_base, n=500, truncate=None, keep_descr=False, max_len=None):
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
107 """Split FASTA file into sub-files each of at most n sequences.
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
108
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
109 Returns a list of the filenames used (based on the input filename).
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
110 Each sequence can also be truncated (since we only need the start for
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
111 SignalP), and have its description discarded (since we don't usually
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
112 care about it and some tools don't like very long title lines).
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
113
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
114 If a max_len is given and any sequence exceeds it no temp files are
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
115 created and an exception is raised.
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
116 """
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
117 iterator = fasta_iterator(input_filename, max_len, truncate)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
118 files = []
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
119 try:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
120 while True:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
121 records = []
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
122 for i in range(n):
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
123 try:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
124 records.append(iterator.next())
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
125 except StopIteration:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
126 break
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
127 if not records:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
128 break
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
129 new_filename = "%s.%i.tmp" % (output_filename_base, len(files))
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
130 handle = open(new_filename, "w")
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
131 if keep_descr:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
132 for title, seq in records:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
133 handle.write(">%s\n" % title)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
134 for i in range(0, len(seq), 60):
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
135 handle.write(seq[i:i+60] + "\n")
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
136 else:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
137 for title, seq in records:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
138 handle.write(">%s\n" % title.split()[0])
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
139 for i in range(0, len(seq), 60):
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
140 handle.write(seq[i:i+60] + "\n")
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
141 handle.close()
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
142 files.append(new_filename)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
143 #print "%i records in %s" % (len(records), new_filename)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
144 except ValueError, err:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
145 #Max length failure from parser - clean up
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
146 try:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
147 handle.close()
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
148 except:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
149 pass
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
150 for f in files:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
151 if os.path.isfile(f):
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
152 os.remove(f)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
153 raise err
7
9b45a8743100 Uploaded v0.1.0, which adds a wrapper for Promoter 2.0 (DNA tool) and enables use of Galaxy's <parallelism> tag for SignalP, TMHMM X Promoter wrappers.
peterjc
parents: 6
diff changeset
154 for f in files:
9b45a8743100 Uploaded v0.1.0, which adds a wrapper for Promoter 2.0 (DNA tool) and enables use of Galaxy's <parallelism> tag for SignalP, TMHMM X Promoter wrappers.
peterjc
parents: 6
diff changeset
155 assert os.path.isfile(f), "Missing split file %r (!??)" % f
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
156 return files
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
157
6
a290c6d4e658 Migrated tool version 0.0.9 from old tool shed archive to new tool shed repository
peterjc
parents: 3
diff changeset
158 def run_jobs(jobs, threads, pause=10, verbose=False):
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
159 """Takes list of cmd strings, returns dict with error levels."""
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
160 pending = jobs[:]
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
161 running = []
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
162 results = {}
9
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
163 if threads == 1:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
164 #Special case this for speed, don't need the waits
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
165 for cmd in jobs:
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
166 results[cmd] = subprocess.call(cmd, shell=True)
e52220a9ddad Uploaded v0.1.2
peterjc
parents: 7
diff changeset
167 return results
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
168 while pending or running:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
169 #See if any have finished
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
170 for (cmd, process) in running:
3
f3b373a41f81 Migrated tool version 0.0.6 from old tool shed archive to new tool shed repository
peterjc
parents: 0
diff changeset
171 return_code = process.poll() #non-blocking
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
172 if return_code is not None:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
173 results[cmd] = return_code
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
174 running = [(cmd, process) for (cmd, process) in running \
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
175 if cmd not in results]
3
f3b373a41f81 Migrated tool version 0.0.6 from old tool shed archive to new tool shed repository
peterjc
parents: 0
diff changeset
176 if verbose:
f3b373a41f81 Migrated tool version 0.0.6 from old tool shed archive to new tool shed repository
peterjc
parents: 0
diff changeset
177 print "%i jobs pending, %i running, %i completed" \
f3b373a41f81 Migrated tool version 0.0.6 from old tool shed archive to new tool shed repository
peterjc
parents: 0
diff changeset
178 % (len(pending), len(running), len(results))
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
179 #See if we can start any new threads
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
180 while pending and len(running) < threads:
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
181 cmd = pending.pop(0)
3
f3b373a41f81 Migrated tool version 0.0.6 from old tool shed archive to new tool shed repository
peterjc
parents: 0
diff changeset
182 if verbose:
f3b373a41f81 Migrated tool version 0.0.6 from old tool shed archive to new tool shed repository
peterjc
parents: 0
diff changeset
183 print cmd
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
184 process = subprocess.Popen(cmd, shell=True)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
185 running.append((cmd, process))
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
186 #Loop...
6
a290c6d4e658 Migrated tool version 0.0.9 from old tool shed archive to new tool shed repository
peterjc
parents: 3
diff changeset
187 sleep(pause)
3
f3b373a41f81 Migrated tool version 0.0.6 from old tool shed archive to new tool shed repository
peterjc
parents: 0
diff changeset
188 if verbose:
f3b373a41f81 Migrated tool version 0.0.6 from old tool shed archive to new tool shed repository
peterjc
parents: 0
diff changeset
189 print "%i jobs completed" % len(results)
0
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
190 assert set(jobs) == set(results)
bca9bc7fdaef Migrated tool version 0.0.1 from old tool shed archive to new tool shed repository
peterjc
parents:
diff changeset
191 return results