Mercurial > repos > shellac > guppy_basecaller

comparison env/bin/mturk @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 #!/Users/pldms/Development/Projects/2020/david-matthews-galaxy/guppy_basecaller/env/bin/python3
2 # Copyright 2012, 2014 Kodi Arfer
3 #
4 # Permission is hereby granted, free of charge, to any person obtaining a
5 # copy of this software and associated documentation files (the
6 # "Software"), to deal in the Software without restriction, including
7 # without limitation the rights to use, copy, modify, merge, publish, dis-
8 # tribute, sublicense, and/or sell copies of the Software, and to permit
9 # persons to whom the Software is furnished to do so, subject to the fol-
10 # lowing conditions:
11 #
12 # The above copyright notice and this permission notice shall be included
13 # in all copies or substantial portions of the Software.
14 #
15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21
22 import argparse # Hence, Python 2.7 is required.
23 import sys
24 import os.path
25 import string
26 import inspect
27 import datetime, calendar
28 import boto.mturk.connection, boto.mturk.price, boto.mturk.question, boto.mturk.qualification
29 from boto.compat import json
30
31 # --------------------------------------------------
32 # Globals
33 # -------------------------------------------------
34
35 interactive = False
36 con = None
37 mturk_website = None
38
39 default_nicknames_path = os.path.expanduser('~/.boto_mturkcli_hit_nicknames')
40 nicknames = {}
41 nickname_pool = set(string.ascii_lowercase)
42
43 get_assignments_page_size = 100
44
45 time_units = dict(
46 s = 1,
47 min = 60,
48 h = 60 * 60,
49 d = 24 * 60 * 60)
50
51 qual_requirements = dict(
52 Adult = '00000000000000000060',
53 Locale = '00000000000000000071',
54 NumberHITsApproved = '00000000000000000040',
55 PercentAssignmentsSubmitted = '00000000000000000000',
56 PercentAssignmentsAbandoned = '00000000000000000070',
57 PercentAssignmentsReturned = '000000000000000000E0',
58 PercentAssignmentsApproved = '000000000000000000L0',
59 PercentAssignmentsRejected = '000000000000000000S0')
60
61 qual_comparators = {v : k for k, v in dict(
62 LessThan = '<', LessThanOrEqualTo = '<=',
63 GreaterThan = '>', GreaterThanOrEqualTo = '>=',
64 EqualTo = '==', NotEqualTo = '!=',
65 Exists = 'exists').items()}
66
67 example_config_file = '''Example configuration file:
68
69 {
70 "title": "Pick your favorite color",
71 "description": "In this task, you are asked to pick your favorite color.",
72 "reward": 0.50,
73 "assignments": 10,
74 "duration": "20 min",
75 "keywords": ["color", "favorites", "survey"],
76 "lifetime": "7 d",
77 "approval_delay": "14 d",
78 "qualifications": [
79 "PercentAssignmentsApproved > 90",
80 "Locale == US",
81 "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6 exists"
82 ],
83 "question_url": "http://example.com/myhit",
84 "question_frame_height": 450
85 }'''
86
87 # --------------------------------------------------
88 # Subroutines
89 # --------------------------------------------------
90
91 def unjson(path):
92 with open(path) as o:
93 return json.load(o)
94
95 def add_argparse_arguments(parser):
96 parser.add_argument('-P', '--production',
97 dest = 'sandbox', action = 'store_false', default = True,
98 help = 'use the production site (default: use the sandbox)')
99 parser.add_argument('--nicknames',
100 dest = 'nicknames_path', metavar = 'PATH',
101 default = default_nicknames_path,
102 help = 'where to store HIT nicknames (default: {})'.format(
103 default_nicknames_path))
104
105 def init_by_args(args):
106 init(args.sandbox, args.nicknames_path)
107
108 def init(sandbox = False, nicknames_path = default_nicknames_path):
109 global con, mturk_website, nicknames, original_nicknames
110
111 mturk_website = 'workersandbox.mturk.com' if sandbox else 'www.mturk.com'
112 con = boto.mturk.connection.MTurkConnection(
113 host = 'mechanicalturk.sandbox.amazonaws.com' if sandbox else 'mechanicalturk.amazonaws.com')
114
115 try:
116 nicknames = unjson(nicknames_path)
117 except IOError:
118 nicknames = {}
119 original_nicknames = nicknames.copy()
120
121 def save_nicknames(nicknames_path = default_nicknames_path):
122 if nicknames != original_nicknames:
123 with open(nicknames_path, 'w') as o:
124 json.dump(nicknames, o, sort_keys = True, indent = 4)
125 print >>o
126
127 def parse_duration(s):
128 '''Parses durations like "2 d", "48 h", "2880 min",
129 "172800 s", or "172800".'''
130 x = s.split()
131 return int(x[0]) * time_units['s' if len(x) == 1 else x[1]]
132 def display_duration(n):
133 for unit, m in sorted(time_units.items(), key = lambda x: -x[1]):
134 if n % m == 0:
135 return '{} {}'.format(n / m, unit)
136
137 def parse_qualification(inp):
138 '''Parses qualifications like "PercentAssignmentsApproved > 90",
139 "Locale == US", and "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6 exists".'''
140 inp = inp.split()
141 name, comparator, value = inp.pop(0), inp.pop(0), (inp[0] if len(inp) else None)
142 qtid = qual_requirements.get(name)
143 if qtid is None:
144 # Treat "name" as a Qualification Type ID.
145 qtid = name
146 if qtid == qual_requirements['Locale']:
147 return boto.mturk.qualification.LocaleRequirement(
148 qual_comparators[comparator],
149 value,
150 required_to_preview = False)
151 return boto.mturk.qualification.Requirement(
152 qtid,
153 qual_comparators[comparator],
154 value,
155 required_to_preview = qtid == qual_requirements['Adult'])
156 # Thus required_to_preview is true only for the
157 # Worker_Adult requirement.
158
159 def preview_url(hit):
160 return 'https://{}/mturk/preview?groupId={}'.format(
161 mturk_website, hit.HITTypeId)
162
163 def parse_timestamp(s):
164 '''Takes a timestamp like "2012-11-24T16:34:41Z".
165
166 Returns a datetime object in the local time zone.'''
167 return datetime.datetime.fromtimestamp(
168 calendar.timegm(
169 datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ').timetuple()))
170
171 def get_hitid(nickname_or_hitid):
172 return nicknames.get(nickname_or_hitid) or nickname_or_hitid
173
174 def get_nickname(hitid):
175 for k, v in nicknames.items():
176 if v == hitid:
177 return k
178 return None
179
180 def display_datetime(dt):
181 return dt.strftime('%e %b %Y, %l:%M %P')
182
183 def display_hit(hit, verbose = False):
184 et = parse_timestamp(hit.Expiration)
185 return '\n'.join([
186 '{} - {} ({}, {}, {})'.format(
187 get_nickname(hit.HITId),
188 hit.Title,
189 hit.FormattedPrice,
190 display_duration(int(hit.AssignmentDurationInSeconds)),
191 hit.HITStatus),
192 'HIT ID: ' + hit.HITId,
193 'Type ID: ' + hit.HITTypeId,
194 'Group ID: ' + hit.HITGroupId,
195 'Preview: ' + preview_url(hit),
196 'Created {} {}'.format(
197 display_datetime(parse_timestamp(hit.CreationTime)),
198 'Expired' if et <= datetime.datetime.now() else
199 'Expires ' + display_datetime(et)),
200 'Assignments: {} -- {} avail, {} pending, {} reviewable, {} reviewed'.format(
201 hit.MaxAssignments,
202 hit.NumberOfAssignmentsAvailable,
203 hit.NumberOfAssignmentsPending,
204 int(hit.MaxAssignments) - (int(hit.NumberOfAssignmentsAvailable) + int(hit.NumberOfAssignmentsPending) + int(hit.NumberOfAssignmentsCompleted)),
205 hit.NumberOfAssignmentsCompleted)
206 if hasattr(hit, 'NumberOfAssignmentsAvailable')
207 else 'Assignments: {} total'.format(hit.MaxAssignments),
208 # For some reason, SearchHITs includes the
209 # NumberOfAssignmentsFoobar fields but GetHIT doesn't.
210 ] + ([] if not verbose else [
211 '\nDescription: ' + hit.Description,
212 '\nKeywords: ' + hit.Keywords
213 ])) + '\n'
214
215 def digest_assignment(a):
216 return dict(
217 answers = {str(x.qid): str(x.fields[0]) for x in a.answers[0]},
218 **{k: str(getattr(a, k)) for k in (
219 'AcceptTime', 'SubmitTime',
220 'HITId', 'AssignmentId', 'WorkerId',
221 'AssignmentStatus')})
222
223 # --------------------------------------------------
224 # Commands
225 # --------------------------------------------------
226
227 def get_balance():
228 return con.get_account_balance()
229
230 def show_hit(hit):
231 return display_hit(con.get_hit(hit)[0], verbose = True)
232
233 def list_hits():
234 'Lists your 10 most recently created HITs, with the most recent last.'
235 return '\n'.join(reversed(map(display_hit, con.search_hits(
236 sort_by = 'CreationTime',
237 sort_direction = 'Descending',
238 page_size = 10))))
239
240 def make_hit(title, description, keywords, reward, question_url, question_frame_height, duration, assignments, approval_delay, lifetime, qualifications = []):
241 r = con.create_hit(
242 title = title,
243 description = description,
244 keywords = con.get_keywords_as_string(keywords),
245 reward = con.get_price_as_price(reward),
246 question = boto.mturk.question.ExternalQuestion(
247 question_url,
248 question_frame_height),
249 duration = parse_duration(duration),
250 qualifications = boto.mturk.qualification.Qualifications(
251 map(parse_qualification, qualifications)),
252 max_assignments = assignments,
253 approval_delay = parse_duration(approval_delay),
254 lifetime = parse_duration(lifetime))
255 nick = None
256 available_nicks = nickname_pool - set(nicknames.keys())
257 if available_nicks:
258 nick = min(available_nicks)
259 nicknames[nick] = r[0].HITId
260 if interactive:
261 print 'Nickname:', nick
262 print 'HIT ID:', r[0].HITId
263 print 'Preview:', preview_url(r[0])
264 else:
265 return r[0]
266
267 def extend_hit(hit, assignments_increment = None, expiration_increment = None):
268 con.extend_hit(hit, assignments_increment, expiration_increment)
269
270 def expire_hit(hit):
271 con.expire_hit(hit)
272
273 def delete_hit(hit):
274 '''Deletes a HIT using DisableHIT.
275
276 Unreviewed assignments get automatically approved. Unsubmitted
277 assignments get automatically approved upon submission.
278
279 The API docs say DisableHIT doesn't work with Reviewable HITs,
280 but apparently, it does.'''
281 con.disable_hit(hit)
282 global nicknames
283 nicknames = {k: v for k, v in nicknames.items() if v != hit}
284
285 def list_assignments(hit, only_reviewable = False):
286 # Accumulate all relevant assignments, one page of results at
287 # a time.
288 assignments = []
289 page = 1
290 while True:
291 rs = con.get_assignments(
292 hit_id = hit,
293 page_size = get_assignments_page_size,
294 page_number = page,
295 status = 'Submitted' if only_reviewable else None)
296 assignments += map(digest_assignment, rs)
297 if len(assignments) >= int(rs.TotalNumResults):
298 break
299 page += 1
300 if interactive:
301 print json.dumps(assignments, sort_keys = True, indent = 4)
302 print ' '.join([a['AssignmentId'] for a in assignments])
303 print ' '.join([a['WorkerId'] + ',' + a['AssignmentId'] for a in assignments])
304 else:
305 return assignments
306
307 def grant_bonus(message, amount, pairs):
308 for worker, assignment in pairs:
309 con.grant_bonus(worker, assignment, con.get_price_as_price(amount), message)
310 if interactive: print 'Bonused', worker
311
312 def approve_assignments(message, assignments):
313 for a in assignments:
314 con.approve_assignment(a, message)
315 if interactive: print 'Approved', a
316
317 def reject_assignments(message, assignments):
318 for a in assignments:
319 con.reject_assignment(a, message)
320 if interactive: print 'Rejected', a
321
322 def unreject_assignments(message, assignments):
323 for a in assignments:
324 con.approve_rejected_assignment(a, message)
325 if interactive: print 'Unrejected', a
326
327 def notify_workers(subject, text, workers):
328 con.notify_workers(workers, subject, text)
329
330 def give_qualification(qualification, workers, value = 1, notify = True):
331 for w in workers:
332 con.assign_qualification(qualification, w, value, notify)
333 if interactive: print 'Gave to', w
334
335 def revoke_qualification(qualification, workers, message = None):
336 for w in workers:
337 con.revoke_qualification(w, qualification, message)
338 if interactive: print 'Revoked from', w
339
340 # --------------------------------------------------
341 # Mainline code
342 # --------------------------------------------------
343
344 if __name__ == '__main__':
345 interactive = True
346
347 parser = argparse.ArgumentParser()
348 add_argparse_arguments(parser)
349 subs = parser.add_subparsers()
350
351 sub = subs.add_parser('bal',
352 help = 'display your prepaid balance')
353 sub.set_defaults(f = get_balance, a = lambda: [])
354
355 sub = subs.add_parser('hit',
356 help = 'get information about a HIT')
357 sub.add_argument('HIT',
358 help = 'nickname or ID of the HIT to show')
359 sub.set_defaults(f = show_hit, a = lambda:
360 [get_hitid(args.HIT)])
361
362 sub = subs.add_parser('hits',
363 help = 'list all your HITs')
364 sub.set_defaults(f = list_hits, a = lambda: [])
365
366 sub = subs.add_parser('new',
367 help = 'create a new HIT (external questions only)',
368 epilog = example_config_file,
369 formatter_class = argparse.RawDescriptionHelpFormatter)
370 sub.add_argument('JSON_PATH',
371 help = 'path to JSON configuration file for the HIT')
372 sub.add_argument('-u', '--question-url', dest = 'question_url',
373 metavar = 'URL',
374 help = 'URL for the external question')
375 sub.add_argument('-a', '--assignments', dest = 'assignments',
376 type = int, metavar = 'N',
377 help = 'number of assignments')
378 sub.add_argument('-r', '--reward', dest = 'reward',
379 type = float, metavar = 'PRICE',
380 help = 'reward amount, in USD')
381 sub.set_defaults(f = make_hit, a = lambda: dict(
382 unjson(args.JSON_PATH).items() + [(k, getattr(args, k))
383 for k in ('question_url', 'assignments', 'reward')
384 if getattr(args, k) is not None]))
385
386 sub = subs.add_parser('extend',
387 help = 'add assignments or time to a HIT')
388 sub.add_argument('HIT',
389 help = 'nickname or ID of the HIT to extend')
390 sub.add_argument('-a', '--assignments', dest = 'assignments',
391 metavar = 'N', type = int,
392 help = 'number of assignments to add')
393 sub.add_argument('-t', '--time', dest = 'time',
394 metavar = 'T',
395 help = 'amount of time to add to the expiration date')
396 sub.set_defaults(f = extend_hit, a = lambda:
397 [get_hitid(args.HIT), args.assignments,
398 args.time and parse_duration(args.time)])
399
400 sub = subs.add_parser('expire',
401 help = 'force a HIT to expire without deleting it')
402 sub.add_argument('HIT',
403 help = 'nickname or ID of the HIT to expire')
404 sub.set_defaults(f = expire_hit, a = lambda:
405 [get_hitid(args.HIT)])
406
407 sub = subs.add_parser('rm',
408 help = 'delete a HIT')
409 sub.add_argument('HIT',
410 help = 'nickname or ID of the HIT to delete')
411 sub.set_defaults(f = delete_hit, a = lambda:
412 [get_hitid(args.HIT)])
413
414 sub = subs.add_parser('as',
415 help = "list a HIT's submitted assignments")
416 sub.add_argument('HIT',
417 help = 'nickname or ID of the HIT to get assignments for')
418 sub.add_argument('-r', '--reviewable', dest = 'only_reviewable',
419 action = 'store_true',
420 help = 'show only unreviewed assignments')
421 sub.set_defaults(f = list_assignments, a = lambda:
422 [get_hitid(args.HIT), args.only_reviewable])
423
424 for command, fun, helpmsg in [
425 ('approve', approve_assignments, 'approve assignments'),
426 ('reject', reject_assignments, 'reject assignments'),
427 ('unreject', unreject_assignments, 'approve previously rejected assignments')]:
428 sub = subs.add_parser(command, help = helpmsg)
429 sub.add_argument('ASSIGNMENT', nargs = '+',
430 help = 'ID of an assignment')
431 sub.add_argument('-m', '--message', dest = 'message',
432 metavar = 'TEXT',
433 help = 'feedback message shown to workers')
434 sub.set_defaults(f = fun, a = lambda:
435 [args.message, args.ASSIGNMENT])
436
437 sub = subs.add_parser('bonus',
438 help = 'give some workers a bonus')
439 sub.add_argument('AMOUNT', type = float,
440 help = 'bonus amount, in USD')
441 sub.add_argument('MESSAGE',
442 help = 'the reason for the bonus (shown to workers in an email sent by MTurk)')
443 sub.add_argument('WIDAID', nargs = '+',
444 help = 'a WORKER_ID,ASSIGNMENT_ID pair')
445 sub.set_defaults(f = grant_bonus, a = lambda:
446 [args.MESSAGE, args.AMOUNT,
447 [p.split(',') for p in args.WIDAID]])
448
449 sub = subs.add_parser('notify',
450 help = 'send a message to some workers')
451 sub.add_argument('SUBJECT',
452 help = 'subject of the message')
453 sub.add_argument('MESSAGE',
454 help = 'text of the message')
455 sub.add_argument('WORKER', nargs = '+',
456 help = 'ID of a worker')
457 sub.set_defaults(f = notify_workers, a = lambda:
458 [args.SUBJECT, args.MESSAGE, args.WORKER])
459
460 sub = subs.add_parser('give-qual',
461 help = 'give a qualification to some workers')
462 sub.add_argument('QUAL',
463 help = 'ID of the qualification')
464 sub.add_argument('WORKER', nargs = '+',
465 help = 'ID of a worker')
466 sub.add_argument('-v', '--value', dest = 'value',
467 metavar = 'N', type = int, default = 1,
468 help = 'value of the qualification')
469 sub.add_argument('--dontnotify', dest = 'notify',
470 action = 'store_false', default = True,
471 help = "don't notify workers")
472 sub.set_defaults(f = give_qualification, a = lambda:
473 [args.QUAL, args.WORKER, args.value, args.notify])
474
475 sub = subs.add_parser('revoke-qual',
476 help = 'revoke a qualification from some workers')
477 sub.add_argument('QUAL',
478 help = 'ID of the qualification')
479 sub.add_argument('WORKER', nargs = '+',
480 help = 'ID of a worker')
481 sub.add_argument('-m', '--message', dest = 'message',
482 metavar = 'TEXT',
483 help = 'the reason the qualification was revoked (shown to workers in an email sent by MTurk)')
484 sub.set_defaults(f = revoke_qualification, a = lambda:
485 [args.QUAL, args.WORKER, args.message])
486
487 args = parser.parse_args()
488
489 init_by_args(args)
490
491 f = args.f
492 a = args.a()
493 if isinstance(a, dict):
494 # We do some introspective gymnastics so we can produce a
495 # less incomprehensible error message if some arguments
496 # are missing.
497 spec = inspect.getargspec(f)
498 missing = set(spec.args[: len(spec.args) - len(spec.defaults)]) - set(a.keys())
499 if missing:
500 raise ValueError('Missing arguments: ' + ', '.join(missing))
501 doit = lambda: f(**a)
502 else:
503 doit = lambda: f(*a)
504
505 try:
506 x = doit()
507 except boto.mturk.connection.MTurkRequestError as e:
508 print 'MTurk error:', e.error_message
509 sys.exit(1)
510
511 if x is not None:
512 print x
513
514 save_nicknames()