Mercurial > repos > gga > tripal_entity_publish

comparison tripal.py @ 0:fd775ed5c129 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxy-genome-annotation/galaxy-tools/tree/master/tools/tripal commit 869e5fd8535deca8325777efcd31c70a514b582a
author gga
date Mon, 25 Feb 2019 06:31:21 -0500
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:fd775ed5c129
1 import collections
2 import os
3 import time
4 from abc import abstractmethod
5
6 import tripal
7
8
9 #############################################
10 # BEGIN IMPORT OF CACHING LIBRARY #
11 #############################################
12 # This code is licensed under the MIT #
13 # License and is a copy of code publicly #
14 # available in rev. #
15 # e27332bc82f4e327aedaec17c9b656ae719322ed #
16 # of https://github.com/tkem/cachetools/ #
17 #############################################
18 class DefaultMapping(collections.MutableMapping):
19
20 __slots__ = ()
21
22 @abstractmethod
23 def __contains__(self, key): # pragma: nocover
24 return False
25
26 @abstractmethod
27 def __getitem__(self, key): # pragma: nocover
28 if hasattr(self.__class__, '__missing__'):
29 return self.__class__.__missing__(self, key)
30 else:
31 raise KeyError(key)
32
33 def get(self, key, default=None):
34 if key in self:
35 return self[key]
36 else:
37 return default
38
39 __marker = object()
40
41 def pop(self, key, default=__marker):
42 if key in self:
43 value = self[key]
44 del self[key]
45 elif default is self.__marker:
46 raise KeyError(key)
47 else:
48 value = default
49 return value
50
51 def setdefault(self, key, default=None):
52 if key in self:
53 value = self[key]
54 else:
55 self[key] = value = default
56 return value
57
58
59 DefaultMapping.register(dict)
60
61
62 class _DefaultSize(object):
63 def __getitem__(self, _):
64 return 1
65
66 def __setitem__(self, _, value):
67 assert value == 1
68
69 def pop(self, _):
70 return 1
71
72
73 class Cache(DefaultMapping):
74 """Mutable mapping to serve as a simple cache or cache base class."""
75
76 __size = _DefaultSize()
77
78 def __init__(self, maxsize, missing=None, getsizeof=None):
79 if missing:
80 self.__missing = missing
81 if getsizeof:
82 self.__getsizeof = getsizeof
83 self.__size = dict()
84 self.__data = dict()
85 self.__currsize = 0
86 self.__maxsize = maxsize
87
88 def __repr__(self):
89 return '%s(%r, maxsize=%r, currsize=%r)' % (
90 self.__class__.__name__,
91 list(self.__data.items()),
92 self.__maxsize,
93 self.__currsize,
94 )
95
96 def __getitem__(self, key):
97 try:
98 return self.__data[key]
99 except KeyError:
100 return self.__missing__(key)
101
102 def __setitem__(self, key, value):
103 maxsize = self.__maxsize
104 size = self.getsizeof(value)
105 if size > maxsize:
106 raise ValueError('value too large')
107 if key not in self.__data or self.__size[key] < size:
108 while self.__currsize + size > maxsize:
109 self.popitem()
110 if key in self.__data:
111 diffsize = size - self.__size[key]
112 else:
113 diffsize = size
114 self.__data[key] = value
115 self.__size[key] = size
116 self.__currsize += diffsize
117
118 def __delitem__(self, key):
119 size = self.__size.pop(key)
120 del self.__data[key]
121 self.__currsize -= size
122
123 def __contains__(self, key):
124 return key in self.__data
125
126 def __missing__(self, key):
127 value = self.__missing(key)
128 try:
129 self.__setitem__(key, value)
130 except ValueError:
131 pass # value too large
132 return value
133
134 def __iter__(self):
135 return iter(self.__data)
136
137 def __len__(self):
138 return len(self.__data)
139
140 @staticmethod
141 def __getsizeof(value):
142 return 1
143
144 @staticmethod
145 def __missing(key):
146 raise KeyError(key)
147
148 @property
149 def maxsize(self):
150 """The maximum size of the cache."""
151 return self.__maxsize
152
153 @property
154 def currsize(self):
155 """The current size of the cache."""
156 return self.__currsize
157
158 def getsizeof(self, value):
159 """Return the size of a cache element's value."""
160 return self.__getsizeof(value)
161
162
163 class _Link(object):
164
165 __slots__ = ('key', 'expire', 'next', 'prev')
166
167 def __init__(self, key=None, expire=None):
168 self.key = key
169 self.expire = expire
170
171 def __reduce__(self):
172 return _Link, (self.key, self.expire)
173
174 def unlink(self):
175 next = self.next
176 prev = self.prev
177 prev.next = next
178 next.prev = prev
179
180
181 class _Timer(object):
182
183 def __init__(self, timer):
184 self.__timer = timer
185 self.__nesting = 0
186
187 def __call__(self):
188 if self.__nesting == 0:
189 return self.__timer()
190 else:
191 return self.__time
192
193 def __enter__(self):
194 if self.__nesting == 0:
195 self.__time = time = self.__timer()
196 else:
197 time = self.__time
198 self.__nesting += 1
199 return time
200
201 def __exit__(self, *exc):
202 self.__nesting -= 1
203
204 def __reduce__(self):
205 return _Timer, (self.__timer,)
206
207 def __getattr__(self, name):
208 return getattr(self.__timer, name)
209
210
211 class TTLCache(Cache):
212 """LRU Cache implementation with per-item time-to-live (TTL) value."""
213
214 def __init__(self, maxsize, ttl, timer=time.time, missing=None,
215 getsizeof=None):
216 Cache.__init__(self, maxsize, missing, getsizeof)
217 self.__root = root = _Link()
218 root.prev = root.next = root
219 self.__links = collections.OrderedDict()
220 self.__timer = _Timer(timer)
221 self.__ttl = ttl
222
223 def __contains__(self, key):
224 try:
225 link = self.__links[key] # no reordering
226 except KeyError:
227 return False
228 else:
229 return not (link.expire < self.__timer())
230
231 def __getitem__(self, key, cache_getitem=Cache.__getitem__):
232 try:
233 link = self.__getlink(key)
234 except KeyError:
235 expired = False
236 else:
237 expired = link.expire < self.__timer()
238 if expired:
239 return self.__missing__(key)
240 else:
241 return cache_getitem(self, key)
242
243 def __setitem__(self, key, value, cache_setitem=Cache.__setitem__):
244 with self.__timer as time:
245 self.expire(time)
246 cache_setitem(self, key, value)
247 try:
248 link = self.__getlink(key)
249 except KeyError:
250 self.__links[key] = link = _Link(key)
251 else:
252 link.unlink()
253 link.expire = time + self.__ttl
254 link.next = root = self.__root
255 link.prev = prev = root.prev
256 prev.next = root.prev = link
257
258 def __delitem__(self, key, cache_delitem=Cache.__delitem__):
259 cache_delitem(self, key)
260 link = self.__links.pop(key)
261 link.unlink()
262 if link.expire < self.__timer():
263 raise KeyError(key)
264
265 def __iter__(self):
266 root = self.__root
267 curr = root.next
268 while curr is not root:
269 # "freeze" time for iterator access
270 with self.__timer as time:
271 if not (curr.expire < time):
272 yield curr.key
273 curr = curr.next
274
275 def __len__(self):
276 root = self.__root
277 curr = root.next
278 time = self.__timer()
279 count = len(self.__links)
280 while curr is not root and curr.expire < time:
281 count -= 1
282 curr = curr.next
283 return count
284
285 def __setstate__(self, state):
286 self.__dict__.update(state)
287 root = self.__root
288 root.prev = root.next = root
289 for link in sorted(self.__links.values(), key=lambda obj: obj.expire):
290 link.next = root
291 link.prev = prev = root.prev
292 prev.next = root.prev = link
293 self.expire(self.__timer())
294
295 def __repr__(self, cache_repr=Cache.__repr__):
296 with self.__timer as time:
297 self.expire(time)
298 return cache_repr(self)
299
300 @property
301 def currsize(self):
302 with self.__timer as time:
303 self.expire(time)
304 return super(TTLCache, self).currsize
305
306 @property
307 def timer(self):
308 """The timer function used by the cache."""
309 return self.__timer
310
311 @property
312 def ttl(self):
313 """The time-to-live value of the cache's items."""
314 return self.__ttl
315
316 def expire(self, time=None):
317 """Remove expired items from the cache."""
318 if time is None:
319 time = self.__timer()
320 root = self.__root
321 curr = root.next
322 links = self.__links
323 cache_delitem = Cache.__delitem__
324 while curr is not root and curr.expire < time:
325 cache_delitem(self, curr.key)
326 del links[curr.key]
327 next = curr.next
328 curr.unlink()
329 curr = next
330
331 def clear(self):
332 with self.__timer as time:
333 self.expire(time)
334 Cache.clear(self)
335
336 def get(self, *args, **kwargs):
337 with self.__timer:
338 return Cache.get(self, *args, **kwargs)
339
340 def pop(self, *args, **kwargs):
341 with self.__timer:
342 return Cache.pop(self, *args, **kwargs)
343
344 def setdefault(self, *args, **kwargs):
345 with self.__timer:
346 return Cache.setdefault(self, *args, **kwargs)
347
348 def popitem(self):
349 """Remove and return the `(key, value)` pair least recently used that
350 has not already expired.
351
352 """
353 with self.__timer as time:
354 self.expire(time)
355 try:
356 key = next(iter(self.__links))
357 except StopIteration:
358 raise KeyError('%s is empty' % self.__class__.__name__)
359 else:
360 return (key, self.pop(key))
361
362 if hasattr(collections.OrderedDict, 'move_to_end'):
363 def __getlink(self, key):
364 value = self.__links[key]
365 self.__links.move_to_end(key)
366 return value
367 else:
368 def __getlink(self, key):
369 value = self.__links.pop(key)
370 self.__links[key] = value
371 return value
372
373
374 #############################################
375 # END IMPORT OF CACHING LIBRARY #
376 #############################################
377
378 cache = TTLCache(
379 100, # Up to 100 items
380 1 * 60 # 5 minute cache life
381 )
382
383
384 def _get_instance():
385 return tripal.TripalInstance(
386 os.environ['GALAXY_TRIPAL_URL'],
387 os.environ['GALAXY_TRIPAL_USER'],
388 os.environ['GALAXY_TRIPAL_PASSWORD']
389 )
390
391
392 def list_organisms(*args, **kwargs):
393
394 ti = _get_instance()
395
396 # Key for cached data
397 cacheKey = 'orgs'
398 # We don't want to trust "if key in cache" because between asking and fetch
399 # it might through key error.
400 if cacheKey not in cache:
401 # However if it ISN'T there, we know we're safe to fetch + put in
402 # there.
403 data = _list_organisms(ti, *args, **kwargs)
404 cache[cacheKey] = data
405 return data
406 try:
407 # The cache key may or may not be in the cache at this point, it
408 # /likely/ is. However we take no chances that it wasn't evicted between
409 # when we checked above and now, so we reference the object from the
410 # cache in preparation to return.
411 data = cache[cacheKey]
412 return data
413 except KeyError:
414 # If access fails due to eviction, we will fail over and can ensure that
415 # data is inserted.
416 data = _list_organisms(ti, *args, **kwargs)
417 cache[cacheKey] = data
418 return data
419
420
421 def _list_organisms(ti, *args, **kwargs):
422 # Fetch the orgs.
423 orgs_data = []
424 for org in ti.organism.get_organisms():
425 clean_name = '%s %s' % (org['genus'], org['species'])
426 if org['infraspecific_name']:
427 clean_name += ' (%s)' % (org['infraspecific_name'])
428 orgs_data.append((clean_name, org['organism_id'], False))
429 return orgs_data
430
431
432 def list_analyses(*args, **kwargs):
433
434 ti = _get_instance()
435
436 # Key for cached data
437 cacheKey = 'analyses'
438 # We don't want to trust "if key in cache" because between asking and fetch
439 # it might through key error.
440 if cacheKey not in cache:
441 # However if it ISN'T there, we know we're safe to fetch + put in
442 # there.<?xml version="1.0"?>
443
444 data = _list_analyses(ti, *args, **kwargs)
445 cache[cacheKey] = data
446 return data
447 try:
448 # The cache key may or may not be in the cache at this point, it
449 # /likely/ is. However we take no chances that it wasn't evicted between
450 # when we checked above and now, so we reference the object from the
451 # cache in preparation to return.
452 data = cache[cacheKey]
453 return data
454 except KeyError:
455 # If access fails due to eviction, we will fail over and can ensure that
456 # data is inserted.
457 data = _list_analyses(ti, *args, **kwargs)
458 cache[cacheKey] = data
459 return data
460
461
462 def _list_analyses(ti, *args, **kwargs):
463 ans_data = []
464 for an in ti.analysis.get_analyses():
465 ans_data.append((an['name'], an['analysis_id'], False))
466 return ans_data
467
468
469 def list_blastdbs(*args, **kwargs):
470
471 ti = _get_instance()
472
473 # Key for cached data
474 cacheKey = 'blastdbs'
475 # We don't want to trust "if key in cache" because between asking and fetch
476 # it might through key error.
477 if cacheKey not in cache:
478 # However if it ISN'T there, we know we're safe to fetch + put in
479 # there.
480 data = _list_blastdbs(ti, *args, **kwargs)
481 cache[cacheKey] = data
482 return data
483 try:
484 # The cache key may or may not be in the cache at this point, it
485 # /likely/ is. However we take no chances that it wasn't evicted between
486 # when we checked above and now, so we reference the object from the
487 # cache in preparation to return.
488 data = cache[cacheKey]
489 return data
490 except KeyError:
491 # If access fails due to eviction, we will fail over and can ensure that
492 # data is inserted.
493 data = _list_blastdbs(ti, *args, **kwargs)
494 cache[cacheKey] = data
495 return data
496
497
498 def _list_blastdbs(ti, *args, **kwargs):
499 dbs_data = []
500 for db in ti.db.get_dbs():
501 dbs_data.append((db['name'], db['db_id'], False))
502 return dbs_data