Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/jinja2/bccache.py @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 # -*- coding: utf-8 -*-
2 """The optional bytecode cache system. This is useful if you have very
3 complex template situations and the compilation of all those templates
4 slows down your application too much.
5
6 Situations where this is useful are often forking web applications that
7 are initialized on the first request.
8 """
9 import errno
10 import fnmatch
11 import os
12 import stat
13 import sys
14 import tempfile
15 from hashlib import sha1
16 from os import listdir
17 from os import path
18
19 from ._compat import BytesIO
20 from ._compat import marshal_dump
21 from ._compat import marshal_load
22 from ._compat import pickle
23 from ._compat import text_type
24 from .utils import open_if_exists
25
26 bc_version = 4
27 # Magic bytes to identify Jinja bytecode cache files. Contains the
28 # Python major and minor version to avoid loading incompatible bytecode
29 # if a project upgrades its Python version.
30 bc_magic = (
31 b"j2"
32 + pickle.dumps(bc_version, 2)
33 + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2)
34 )
35
36
37 class Bucket(object):
38 """Buckets are used to store the bytecode for one template. It's created
39 and initialized by the bytecode cache and passed to the loading functions.
40
41 The buckets get an internal checksum from the cache assigned and use this
42 to automatically reject outdated cache material. Individual bytecode
43 cache subclasses don't have to care about cache invalidation.
44 """
45
46 def __init__(self, environment, key, checksum):
47 self.environment = environment
48 self.key = key
49 self.checksum = checksum
50 self.reset()
51
52 def reset(self):
53 """Resets the bucket (unloads the bytecode)."""
54 self.code = None
55
56 def load_bytecode(self, f):
57 """Loads bytecode from a file or file like object."""
58 # make sure the magic header is correct
59 magic = f.read(len(bc_magic))
60 if magic != bc_magic:
61 self.reset()
62 return
63 # the source code of the file changed, we need to reload
64 checksum = pickle.load(f)
65 if self.checksum != checksum:
66 self.reset()
67 return
68 # if marshal_load fails then we need to reload
69 try:
70 self.code = marshal_load(f)
71 except (EOFError, ValueError, TypeError):
72 self.reset()
73 return
74
75 def write_bytecode(self, f):
76 """Dump the bytecode into the file or file like object passed."""
77 if self.code is None:
78 raise TypeError("can't write empty bucket")
79 f.write(bc_magic)
80 pickle.dump(self.checksum, f, 2)
81 marshal_dump(self.code, f)
82
83 def bytecode_from_string(self, string):
84 """Load bytecode from a string."""
85 self.load_bytecode(BytesIO(string))
86
87 def bytecode_to_string(self):
88 """Return the bytecode as string."""
89 out = BytesIO()
90 self.write_bytecode(out)
91 return out.getvalue()
92
93
94 class BytecodeCache(object):
95 """To implement your own bytecode cache you have to subclass this class
96 and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of
97 these methods are passed a :class:`~jinja2.bccache.Bucket`.
98
99 A very basic bytecode cache that saves the bytecode on the file system::
100
101 from os import path
102
103 class MyCache(BytecodeCache):
104
105 def __init__(self, directory):
106 self.directory = directory
107
108 def load_bytecode(self, bucket):
109 filename = path.join(self.directory, bucket.key)
110 if path.exists(filename):
111 with open(filename, 'rb') as f:
112 bucket.load_bytecode(f)
113
114 def dump_bytecode(self, bucket):
115 filename = path.join(self.directory, bucket.key)
116 with open(filename, 'wb') as f:
117 bucket.write_bytecode(f)
118
119 A more advanced version of a filesystem based bytecode cache is part of
120 Jinja.
121 """
122
123 def load_bytecode(self, bucket):
124 """Subclasses have to override this method to load bytecode into a
125 bucket. If they are not able to find code in the cache for the
126 bucket, it must not do anything.
127 """
128 raise NotImplementedError()
129
130 def dump_bytecode(self, bucket):
131 """Subclasses have to override this method to write the bytecode
132 from a bucket back to the cache. If it unable to do so it must not
133 fail silently but raise an exception.
134 """
135 raise NotImplementedError()
136
137 def clear(self):
138 """Clears the cache. This method is not used by Jinja but should be
139 implemented to allow applications to clear the bytecode cache used
140 by a particular environment.
141 """
142
143 def get_cache_key(self, name, filename=None):
144 """Returns the unique hash key for this template name."""
145 hash = sha1(name.encode("utf-8"))
146 if filename is not None:
147 filename = "|" + filename
148 if isinstance(filename, text_type):
149 filename = filename.encode("utf-8")
150 hash.update(filename)
151 return hash.hexdigest()
152
153 def get_source_checksum(self, source):
154 """Returns a checksum for the source."""
155 return sha1(source.encode("utf-8")).hexdigest()
156
157 def get_bucket(self, environment, name, filename, source):
158 """Return a cache bucket for the given template. All arguments are
159 mandatory but filename may be `None`.
160 """
161 key = self.get_cache_key(name, filename)
162 checksum = self.get_source_checksum(source)
163 bucket = Bucket(environment, key, checksum)
164 self.load_bytecode(bucket)
165 return bucket
166
167 def set_bucket(self, bucket):
168 """Put the bucket into the cache."""
169 self.dump_bytecode(bucket)
170
171
172 class FileSystemBytecodeCache(BytecodeCache):
173 """A bytecode cache that stores bytecode on the filesystem. It accepts
174 two arguments: The directory where the cache items are stored and a
175 pattern string that is used to build the filename.
176
177 If no directory is specified a default cache directory is selected. On
178 Windows the user's temp directory is used, on UNIX systems a directory
179 is created for the user in the system temp directory.
180
181 The pattern can be used to have multiple separate caches operate on the
182 same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s``
183 is replaced with the cache key.
184
185 >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
186
187 This bytecode cache supports clearing of the cache using the clear method.
188 """
189
190 def __init__(self, directory=None, pattern="__jinja2_%s.cache"):
191 if directory is None:
192 directory = self._get_default_cache_dir()
193 self.directory = directory
194 self.pattern = pattern
195
196 def _get_default_cache_dir(self):
197 def _unsafe_dir():
198 raise RuntimeError(
199 "Cannot determine safe temp directory. You "
200 "need to explicitly provide one."
201 )
202
203 tmpdir = tempfile.gettempdir()
204
205 # On windows the temporary directory is used specific unless
206 # explicitly forced otherwise. We can just use that.
207 if os.name == "nt":
208 return tmpdir
209 if not hasattr(os, "getuid"):
210 _unsafe_dir()
211
212 dirname = "_jinja2-cache-%d" % os.getuid()
213 actual_dir = os.path.join(tmpdir, dirname)
214
215 try:
216 os.mkdir(actual_dir, stat.S_IRWXU)
217 except OSError as e:
218 if e.errno != errno.EEXIST:
219 raise
220 try:
221 os.chmod(actual_dir, stat.S_IRWXU)
222 actual_dir_stat = os.lstat(actual_dir)
223 if (
224 actual_dir_stat.st_uid != os.getuid()
225 or not stat.S_ISDIR(actual_dir_stat.st_mode)
226 or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU
227 ):
228 _unsafe_dir()
229 except OSError as e:
230 if e.errno != errno.EEXIST:
231 raise
232
233 actual_dir_stat = os.lstat(actual_dir)
234 if (
235 actual_dir_stat.st_uid != os.getuid()
236 or not stat.S_ISDIR(actual_dir_stat.st_mode)
237 or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU
238 ):
239 _unsafe_dir()
240
241 return actual_dir
242
243 def _get_cache_filename(self, bucket):
244 return path.join(self.directory, self.pattern % bucket.key)
245
246 def load_bytecode(self, bucket):
247 f = open_if_exists(self._get_cache_filename(bucket), "rb")
248 if f is not None:
249 try:
250 bucket.load_bytecode(f)
251 finally:
252 f.close()
253
254 def dump_bytecode(self, bucket):
255 f = open(self._get_cache_filename(bucket), "wb")
256 try:
257 bucket.write_bytecode(f)
258 finally:
259 f.close()
260
261 def clear(self):
262 # imported lazily here because google app-engine doesn't support
263 # write access on the file system and the function does not exist
264 # normally.
265 from os import remove
266
267 files = fnmatch.filter(listdir(self.directory), self.pattern % "*")
268 for filename in files:
269 try:
270 remove(path.join(self.directory, filename))
271 except OSError:
272 pass
273
274
275 class MemcachedBytecodeCache(BytecodeCache):
276 """This class implements a bytecode cache that uses a memcache cache for
277 storing the information. It does not enforce a specific memcache library
278 (tummy's memcache or cmemcache) but will accept any class that provides
279 the minimal interface required.
280
281 Libraries compatible with this class:
282
283 - `cachelib <https://github.com/pallets/cachelib>`_
284 - `python-memcached <https://pypi.org/project/python-memcached/>`_
285
286 (Unfortunately the django cache interface is not compatible because it
287 does not support storing binary data, only unicode. You can however pass
288 the underlying cache client to the bytecode cache which is available
289 as `django.core.cache.cache._client`.)
290
291 The minimal interface for the client passed to the constructor is this:
292
293 .. class:: MinimalClientInterface
294
295 .. method:: set(key, value[, timeout])
296
297 Stores the bytecode in the cache. `value` is a string and
298 `timeout` the timeout of the key. If timeout is not provided
299 a default timeout or no timeout should be assumed, if it's
300 provided it's an integer with the number of seconds the cache
301 item should exist.
302
303 .. method:: get(key)
304
305 Returns the value for the cache key. If the item does not
306 exist in the cache the return value must be `None`.
307
308 The other arguments to the constructor are the prefix for all keys that
309 is added before the actual cache key and the timeout for the bytecode in
310 the cache system. We recommend a high (or no) timeout.
311
312 This bytecode cache does not support clearing of used items in the cache.
313 The clear method is a no-operation function.
314
315 .. versionadded:: 2.7
316 Added support for ignoring memcache errors through the
317 `ignore_memcache_errors` parameter.
318 """
319
320 def __init__(
321 self,
322 client,
323 prefix="jinja2/bytecode/",
324 timeout=None,
325 ignore_memcache_errors=True,
326 ):
327 self.client = client
328 self.prefix = prefix
329 self.timeout = timeout
330 self.ignore_memcache_errors = ignore_memcache_errors
331
332 def load_bytecode(self, bucket):
333 try:
334 code = self.client.get(self.prefix + bucket.key)
335 except Exception:
336 if not self.ignore_memcache_errors:
337 raise
338 code = None
339 if code is not None:
340 bucket.bytecode_from_string(code)
341
342 def dump_bytecode(self, bucket):
343 args = (self.prefix + bucket.key, bucket.bytecode_to_string())
344 if self.timeout is not None:
345 args += (self.timeout,)
346 try:
347 self.client.set(*args)
348 except Exception:
349 if not self.ignore_memcache_errors:
350 raise