Mercurial > repos > guerler > springsuite

comparison planemo/lib/python3.7/site-packages/pip/_internal/cache.py @ 1:56ad4e20f292 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author guerler
date Fri, 31 Jul 2020 00:32:28 -0400
parents
children
comparison
equal deleted inserted replaced
0:d30785e31577 1:56ad4e20f292
1 """Cache Management
2 """
3
4 import errno
5 import hashlib
6 import logging
7 import os
8
9 from pip._vendor.packaging.utils import canonicalize_name
10
11 from pip._internal.models.link import Link
12 from pip._internal.utils.compat import expanduser
13 from pip._internal.utils.misc import path_to_url
14 from pip._internal.utils.temp_dir import TempDirectory
15 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
16 from pip._internal.wheel import InvalidWheelFilename, Wheel
17
18 if MYPY_CHECK_RUNNING:
19 from typing import Optional, Set, List, Any
20 from pip._internal.index import FormatControl
21
22 logger = logging.getLogger(__name__)
23
24
25 class Cache(object):
26 """An abstract class - provides cache directories for data from links
27
28
29 :param cache_dir: The root of the cache.
30 :param format_control: An object of FormatControl class to limit
31 binaries being read from the cache.
32 :param allowed_formats: which formats of files the cache should store.
33 ('binary' and 'source' are the only allowed values)
34 """
35
36 def __init__(self, cache_dir, format_control, allowed_formats):
37 # type: (str, FormatControl, Set[str]) -> None
38 super(Cache, self).__init__()
39 self.cache_dir = expanduser(cache_dir) if cache_dir else None
40 self.format_control = format_control
41 self.allowed_formats = allowed_formats
42
43 _valid_formats = {"source", "binary"}
44 assert self.allowed_formats.union(_valid_formats) == _valid_formats
45
46 def _get_cache_path_parts(self, link):
47 # type: (Link) -> List[str]
48 """Get parts of part that must be os.path.joined with cache_dir
49 """
50
51 # We want to generate an url to use as our cache key, we don't want to
52 # just re-use the URL because it might have other items in the fragment
53 # and we don't care about those.
54 key_parts = [link.url_without_fragment]
55 if link.hash_name is not None and link.hash is not None:
56 key_parts.append("=".join([link.hash_name, link.hash]))
57 key_url = "#".join(key_parts)
58
59 # Encode our key url with sha224, we'll use this because it has similar
60 # security properties to sha256, but with a shorter total output (and
61 # thus less secure). However the differences don't make a lot of
62 # difference for our use case here.
63 hashed = hashlib.sha224(key_url.encode()).hexdigest()
64
65 # We want to nest the directories some to prevent having a ton of top
66 # level directories where we might run out of sub directories on some
67 # FS.
68 parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]]
69
70 return parts
71
72 def _get_candidates(self, link, package_name):
73 # type: (Link, Optional[str]) -> List[Any]
74 can_not_cache = (
75 not self.cache_dir or
76 not package_name or
77 not link
78 )
79 if can_not_cache:
80 return []
81
82 canonical_name = canonicalize_name(package_name)
83 formats = self.format_control.get_allowed_formats(
84 canonical_name
85 )
86 if not self.allowed_formats.intersection(formats):
87 return []
88
89 root = self.get_path_for_link(link)
90 try:
91 return os.listdir(root)
92 except OSError as err:
93 if err.errno in {errno.ENOENT, errno.ENOTDIR}:
94 return []
95 raise
96
97 def get_path_for_link(self, link):
98 # type: (Link) -> str
99 """Return a directory to store cached items in for link.
100 """
101 raise NotImplementedError()
102
103 def get(self, link, package_name):
104 # type: (Link, Optional[str]) -> Link
105 """Returns a link to a cached item if it exists, otherwise returns the
106 passed link.
107 """
108 raise NotImplementedError()
109
110 def _link_for_candidate(self, link, candidate):
111 # type: (Link, str) -> Link
112 root = self.get_path_for_link(link)
113 path = os.path.join(root, candidate)
114
115 return Link(path_to_url(path))
116
117 def cleanup(self):
118 # type: () -> None
119 pass
120
121
122 class SimpleWheelCache(Cache):
123 """A cache of wheels for future installs.
124 """
125
126 def __init__(self, cache_dir, format_control):
127 # type: (str, FormatControl) -> None
128 super(SimpleWheelCache, self).__init__(
129 cache_dir, format_control, {"binary"}
130 )
131
132 def get_path_for_link(self, link):
133 # type: (Link) -> str
134 """Return a directory to store cached wheels for link
135
136 Because there are M wheels for any one sdist, we provide a directory
137 to cache them in, and then consult that directory when looking up
138 cache hits.
139
140 We only insert things into the cache if they have plausible version
141 numbers, so that we don't contaminate the cache with things that were
142 not unique. E.g. ./package might have dozens of installs done for it
143 and build a version of 0.0...and if we built and cached a wheel, we'd
144 end up using the same wheel even if the source has been edited.
145
146 :param link: The link of the sdist for which this will cache wheels.
147 """
148 parts = self._get_cache_path_parts(link)
149
150 # Store wheels within the root cache_dir
151 return os.path.join(self.cache_dir, "wheels", *parts)
152
153 def get(self, link, package_name):
154 # type: (Link, Optional[str]) -> Link
155 candidates = []
156
157 for wheel_name in self._get_candidates(link, package_name):
158 try:
159 wheel = Wheel(wheel_name)
160 except InvalidWheelFilename:
161 continue
162 if not wheel.supported():
163 # Built for a different python/arch/etc
164 continue
165 candidates.append((wheel.support_index_min(), wheel_name))
166
167 if not candidates:
168 return link
169
170 return self._link_for_candidate(link, min(candidates)[1])
171
172
173 class EphemWheelCache(SimpleWheelCache):
174 """A SimpleWheelCache that creates it's own temporary cache directory
175 """
176
177 def __init__(self, format_control):
178 # type: (FormatControl) -> None
179 self._temp_dir = TempDirectory(kind="ephem-wheel-cache")
180 self._temp_dir.create()
181
182 super(EphemWheelCache, self).__init__(
183 self._temp_dir.path, format_control
184 )
185
186 def cleanup(self):
187 # type: () -> None
188 self._temp_dir.cleanup()
189
190
191 class WheelCache(Cache):
192 """Wraps EphemWheelCache and SimpleWheelCache into a single Cache
193
194 This Cache allows for gracefully degradation, using the ephem wheel cache
195 when a certain link is not found in the simple wheel cache first.
196 """
197
198 def __init__(self, cache_dir, format_control):
199 # type: (str, FormatControl) -> None
200 super(WheelCache, self).__init__(
201 cache_dir, format_control, {'binary'}
202 )
203 self._wheel_cache = SimpleWheelCache(cache_dir, format_control)
204 self._ephem_cache = EphemWheelCache(format_control)
205
206 def get_path_for_link(self, link):
207 # type: (Link) -> str
208 return self._wheel_cache.get_path_for_link(link)
209
210 def get_ephem_path_for_link(self, link):
211 # type: (Link) -> str
212 return self._ephem_cache.get_path_for_link(link)
213
214 def get(self, link, package_name):
215 # type: (Link, Optional[str]) -> Link
216 retval = self._wheel_cache.get(link, package_name)
217 if retval is link:
218 retval = self._ephem_cache.get(link, package_name)
219 return retval
220
221 def cleanup(self):
222 # type: () -> None
223 self._wheel_cache.cleanup()
224 self._ephem_cache.cleanup()