Mercurial > repos > gga > chado_load_blast

comparison chado.py @ 0:1daff3b60f39 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxy-genome-annotation/galaxy-tools/tree/master/tools/chado commit 1640878827bdc3870b6f34eded3a3f7571a1849f"
author gga
date Wed, 21 Aug 2019 05:13:05 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:1daff3b60f39
1 import collections
2 import os
3 import time
4 from abc import abstractmethod
5
6 import chado
7
8
9 #############################################
10 # BEGIN IMPORT OF CACHING LIBRARY #
11 #############################################
12 # This code is licensed under the MIT #
13 # License and is a copy of code publicly #
14 # available in rev. #
15 # e27332bc82f4e327aedaec17c9b656ae719322ed #
16 # of https://github.com/tkem/cachetools/ #
17 #############################################
18 class DefaultMapping(collections.MutableMapping):
19
20 __slots__ = ()
21
22 @abstractmethod
23 def __contains__(self, key): # pragma: nocover
24 return False
25
26 @abstractmethod
27 def __getitem__(self, key): # pragma: nocover
28 if hasattr(self.__class__, '__missing__'):
29 return self.__class__.__missing__(self, key)
30 else:
31 raise KeyError(key)
32
33 def get(self, key, default=None):
34 if key in self:
35 return self[key]
36 else:
37 return default
38
39 __marker = object()
40
41 def pop(self, key, default=__marker):
42 if key in self:
43 value = self[key]
44 del self[key]
45 elif default is self.__marker:
46 raise KeyError(key)
47 else:
48 value = default
49 return value
50
51 def setdefault(self, key, default=None):
52 if key in self:
53 value = self[key]
54 else:
55 self[key] = value = default
56 return value
57
58
59 DefaultMapping.register(dict)
60
61
62 class _DefaultSize(object):
63 def __getitem__(self, _):
64 return 1
65
66 def __setitem__(self, _, value):
67 assert value == 1
68
69 def pop(self, _):
70 return 1
71
72
73 class Cache(DefaultMapping):
74 """Mutable mapping to serve as a simple cache or cache base class."""
75
76 __size = _DefaultSize()
77
78 def __init__(self, maxsize, missing=None, getsizeof=None):
79 if missing:
80 self.__missing = missing
81 if getsizeof:
82 self.__getsizeof = getsizeof
83 self.__size = dict()
84 self.__data = dict()
85 self.__currsize = 0
86 self.__maxsize = maxsize
87
88 def __repr__(self):
89 return '%s(%r, maxsize=%r, currsize=%r)' % (
90 self.__class__.__name__,
91 list(self.__data.items()),
92 self.__maxsize,
93 self.__currsize,
94 )
95
96 def __getitem__(self, key):
97 try:
98 return self.__data[key]
99 except KeyError:
100 return self.__missing__(key)
101
102 def __setitem__(self, key, value):
103 maxsize = self.__maxsize
104 size = self.getsizeof(value)
105 if size > maxsize:
106 raise ValueError('value too large')
107 if key not in self.__data or self.__size[key] < size:
108 while self.__currsize + size > maxsize:
109 self.popitem()
110 if key in self.__data:
111 diffsize = size - self.__size[key]
112 else:
113 diffsize = size
114 self.__data[key] = value
115 self.__size[key] = size
116 self.__currsize += diffsize
117
118 def __delitem__(self, key):
119 size = self.__size.pop(key)
120 del self.__data[key]
121 self.__currsize -= size
122
123 def __contains__(self, key):
124 return key in self.__data
125
126 def __missing__(self, key):
127 value = self.__missing(key)
128 try:
129 self.__setitem__(key, value)
130 except ValueError:
131 pass # value too large
132 return value
133
134 def __iter__(self):
135 return iter(self.__data)
136
137 def __len__(self):
138 return len(self.__data)
139
140 @staticmethod
141 def __getsizeof(value):
142 return 1
143
144 @staticmethod
145 def __missing(key):
146 raise KeyError(key)
147
148 @property
149 def maxsize(self):
150 """The maximum size of the cache."""
151 return self.__maxsize
152
153 @property
154 def currsize(self):
155 """The current size of the cache."""
156 return self.__currsize
157
158 def getsizeof(self, value):
159 """Return the size of a cache element's value."""
160 return self.__getsizeof(value)
161
162
163 class _Link(object):
164
165 __slots__ = ('key', 'expire', 'next', 'prev')
166
167 def __init__(self, key=None, expire=None):
168 self.key = key
169 self.expire = expire
170
171 def __reduce__(self):
172 return _Link, (self.key, self.expire)
173
174 def unlink(self):
175 next = self.next
176 prev = self.prev
177 prev.next = next
178 next.prev = prev
179
180
181 class _Timer(object):
182
183 def __init__(self, timer):
184 self.__timer = timer
185 self.__nesting = 0
186
187 def __call__(self):
188 if self.__nesting == 0:
189 return self.__timer()
190 else:
191 return self.__time
192
193 def __enter__(self):
194 if self.__nesting == 0:
195 self.__time = time = self.__timer()
196 else:
197 time = self.__time
198 self.__nesting += 1
199 return time
200
201 def __exit__(self, *exc):
202 self.__nesting -= 1
203
204 def __reduce__(self):
205 return _Timer, (self.__timer,)
206
207 def __getattr__(self, name):
208 return getattr(self.__timer, name)
209
210
211 class TTLCache(Cache):
212 """LRU Cache implementation with per-item time-to-live (TTL) value."""
213
214 def __init__(self, maxsize, ttl, timer=time.time, missing=None,
215 getsizeof=None):
216 Cache.__init__(self, maxsize, missing, getsizeof)
217 self.__root = root = _Link()
218 root.prev = root.next = root
219 self.__links = collections.OrderedDict()
220 self.__timer = _Timer(timer)
221 self.__ttl = ttl
222
223 def __contains__(self, key):
224 try:
225 link = self.__links[key] # no reordering
226 except KeyError:
227 return False
228 else:
229 return not (link.expire < self.__timer())
230
231 def __getitem__(self, key, cache_getitem=Cache.__getitem__):
232 try:
233 link = self.__getlink(key)
234 except KeyError:
235 expired = False
236 else:
237 expired = link.expire < self.__timer()
238 if expired:
239 return self.__missing__(key)
240 else:
241 return cache_getitem(self, key)
242
243 def __setitem__(self, key, value, cache_setitem=Cache.__setitem__):
244 with self.__timer as time:
245 self.expire(time)
246 cache_setitem(self, key, value)
247 try:
248 link = self.__getlink(key)
249 except KeyError:
250 self.__links[key] = link = _Link(key)
251 else:
252 link.unlink()
253 link.expire = time + self.__ttl
254 link.next = root = self.__root
255 link.prev = prev = root.prev
256 prev.next = root.prev = link
257
258 def __delitem__(self, key, cache_delitem=Cache.__delitem__):
259 cache_delitem(self, key)
260 link = self.__links.pop(key)
261 link.unlink()
262 if link.expire < self.__timer():
263 raise KeyError(key)
264
265 def __iter__(self):
266 root = self.__root
267 curr = root.next
268 while curr is not root:
269 # "freeze" time for iterator access
270 with self.__timer as time:
271 if not (curr.expire < time):
272 yield curr.key
273 curr = curr.next
274
275 def __len__(self):
276 root = self.__root
277 curr = root.next
278 time = self.__timer()
279 count = len(self.__links)
280 while curr is not root and curr.expire < time:
281 count -= 1
282 curr = curr.next
283 return count
284
285 def __setstate__(self, state):
286 self.__dict__.update(state)
287 root = self.__root
288 root.prev = root.next = root
289 for link in sorted(self.__links.values(), key=lambda obj: obj.expire):
290 link.next = root
291 link.prev = prev = root.prev
292 prev.next = root.prev = link
293 self.expire(self.__timer())
294
295 def __repr__(self, cache_repr=Cache.__repr__):
296 with self.__timer as time:
297 self.expire(time)
298 return cache_repr(self)
299
300 @property
301 def currsize(self):
302 with self.__timer as time:
303 self.expire(time)
304 return super(TTLCache, self).currsize
305
306 @property
307 def timer(self):
308 """The timer function used by the cache."""
309 return self.__timer
310
311 @property
312 def ttl(self):
313 """The time-to-live value of the cache's items."""
314 return self.__ttl
315
316 def expire(self, time=None):
317 """Remove expired items from the cache."""
318 if time is None:
319 time = self.__timer()
320 root = self.__root
321 curr = root.next
322 links = self.__links
323 cache_delitem = Cache.__delitem__
324 while curr is not root and curr.expire < time:
325 cache_delitem(self, curr.key)
326 del links[curr.key]
327 next = curr.next
328 curr.unlink()
329 curr = next
330
331 def clear(self):
332 with self.__timer as time:
333 self.expire(time)
334 Cache.clear(self)
335
336 def get(self, *args, **kwargs):
337 with self.__timer:
338 return Cache.get(self, *args, **kwargs)
339
340 def pop(self, *args, **kwargs):
341 with self.__timer:
342 return Cache.pop(self, *args, **kwargs)
343
344 def setdefault(self, *args, **kwargs):
345 with self.__timer:
346 return Cache.setdefault(self, *args, **kwargs)
347
348 def popitem(self):
349 """Remove and return the `(key, value)` pair least recently used that
350 has not already expired.
351
352 """
353 with self.__timer as time:
354 self.expire(time)
355 try:
356 key = next(iter(self.__links))
357 except StopIteration:
358 raise KeyError('%s is empty' % self.__class__.__name__)
359 else:
360 return (key, self.pop(key))
361
362 if hasattr(collections.OrderedDict, 'move_to_end'):
363 def __getlink(self, key):
364 value = self.__links[key]
365 self.__links.move_to_end(key)
366 return value
367 else:
368 def __getlink(self, key):
369 value = self.__links.pop(key)
370 self.__links[key] = value
371 return value
372
373
374 #############################################
375 # END IMPORT OF CACHING LIBRARY #
376 #############################################
377
378 cache = TTLCache(
379 100, # Up to 100 items
380 1 * 60 # 5 minute cache life
381 )
382
383
384 def _get_instance():
385 return chado.ChadoInstance(
386 os.environ['GALAXY_CHADO_DBHOST'],
387 os.environ['GALAXY_CHADO_DBNAME'],
388 os.environ['GALAXY_CHADO_DBUSER'],
389 os.environ['GALAXY_CHADO_DBPASS'],
390 os.environ['GALAXY_CHADO_DBSCHEMA'],
391 os.environ['GALAXY_CHADO_DBPORT'],
392 no_reflect=True,
393 pool_connections=False
394 )
395
396
397 def list_organisms(*args, **kwargs):
398
399 ci = _get_instance()
400
401 # Key for cached data
402 cacheKey = 'orgs'
403 # We don't want to trust "if key in cache" because between asking and fetch
404 # it might through key error.
405 if cacheKey not in cache:
406 # However if it ISN'T there, we know we're safe to fetch + put in
407 # there.
408 data = _list_organisms(ci, *args, **kwargs)
409 cache[cacheKey] = data
410 ci.session.close()
411 return data
412 try:
413 # The cache key may or may not be in the cache at this point, it
414 # /likely/ is. However we take no chances that it wasn't evicted between
415 # when we checked above and now, so we reference the object from the
416 # cache in preparation to return.
417 data = cache[cacheKey]
418 ci.session.close()
419 return data
420 except KeyError:
421 # If access fails due to eviction, we will fail over and can ensure that
422 # data is inserted.
423 data = _list_organisms(ci, *args, **kwargs)
424 cache[cacheKey] = data
425 ci.session.close()
426 return data
427
428
429 def _list_organisms(ci, *args, **kwargs):
430 # Fetch the orgs.
431 orgs_data = []
432 for org in ci.organism.get_organisms():
433 clean_name = '%s %s' % (org['genus'], org['species'])
434 if 'infraspecific_name' in org and org['infraspecific_name']:
435 clean_name += ' (%s)' % (org['infraspecific_name'])
436 orgs_data.append((clean_name, str(org['organism_id']), False))
437 return orgs_data
438
439
440 def list_analyses(*args, **kwargs):
441
442 ci = _get_instance()
443
444 # Key for cached data
445 cacheKey = 'analyses'
446 # We don't want to trust "if key in cache" because between asking and fetch
447 # it might through key error.
448 if cacheKey not in cache:
449 # However if it ISN'T there, we know we're safe to fetch + put in
450 # there.<?xml version="1.0"?>
451
452 data = _list_analyses(ci, *args, **kwargs)
453 cache[cacheKey] = data
454 ci.session.close()
455 return data
456 try:
457 # The cache key may or may not be in the cache at this point, it
458 # /likely/ is. However we take no chances that it wasn't evicted between
459 # when we checked above and now, so we reference the object from the
460 # cache in preparation to return.
461 data = cache[cacheKey]
462 ci.session.close()
463 return data
464 except KeyError:
465 # If access fails due to eviction, we will fail over and can ensure that
466 # data is inserted.
467 data = _list_analyses(ci, *args, **kwargs)
468 cache[cacheKey] = data
469 ci.session.close()
470 return data
471
472
473 def _list_analyses(ci, *args, **kwargs):
474 ans_data = []
475 for an in ci.analysis.get_analyses():
476 ans_data.append((an['name'], str(an['analysis_id']), False))
477 return ans_data
478
479
480 def list_dbs(*args, **kwargs):
481
482 ci = _get_instance()
483
484 # Key for cached data
485 cacheKey = 'dbs'
486 # We don't want to trust "if key in cache" because between asking and fetch
487 # it might through key error.
488 if cacheKey not in cache:
489 # However if it ISN'T there, we know we're safe to fetch + put in
490 # there.<?xml version="1.0"?>
491
492 data = _list_dbs(ci, *args, **kwargs)
493 cache[cacheKey] = data
494 ci.session.close()
495 return data
496 try:
497 # The cache key may or may not be in the cache at this point, it
498 # /likely/ is. However we take no chances that it wasn't evicted between
499 # when we checked above and now, so we reference the object from the
500 # cache in preparation to return.
501 data = cache[cacheKey]
502 ci.session.close()
503 return data
504 except KeyError:
505 # If access fails due to eviction, we will fail over and can ensure that
506 # data is inserted.
507 data = _list_dbs(ci, *args, **kwargs)
508 cache[cacheKey] = data
509 ci.session.close()
510 return data
511
512
513 def _list_dbs(ci, *args, **kwargs):
514 dbs_data = []
515 for db in ci.load._get_dbs():
516 dbs_data.append((db['name'], str(db['db_id']), False))
517 return dbs_data