Mercurial > repos > guerler > springsuite

comparison planemo/lib/python3.7/site-packages/virtualenv/discovery/py_info.py @ 1:56ad4e20f292 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author guerler
date Fri, 31 Jul 2020 00:32:28 -0400
parents
children
comparison
equal deleted inserted replaced
0:d30785e31577 1:56ad4e20f292
1 """
2 The PythonInfo contains information about a concrete instance of a Python interpreter
3
4 Note: this file is also used to query target interpreters, so can only use standard library methods
5 """
6 from __future__ import absolute_import, print_function
7
8 import json
9 import logging
10 import os
11 import platform
12 import re
13 import sys
14 import sysconfig
15 from collections import OrderedDict, namedtuple
16 from distutils import dist
17 from distutils.command.install import SCHEME_KEYS
18 from string import digits
19
20 VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"])
21
22
23 def _get_path_extensions():
24 return list(OrderedDict.fromkeys([""] + os.environ.get("PATHEXT", "").lower().split(os.pathsep)))
25
26
27 EXTENSIONS = _get_path_extensions()
28 _CONF_VAR_RE = re.compile(r"\{\w+\}")
29
30
31 class PythonInfo(object):
32 """Contains information for a Python interpreter"""
33
34 def __init__(self):
35 def u(v):
36 return v.decode("utf-8") if isinstance(v, bytes) else v
37
38 def abs_path(v):
39 return None if v is None else os.path.abspath(v) # unroll relative elements from path (e.g. ..)
40
41 # qualifies the python
42 self.platform = u(sys.platform)
43 self.implementation = u(platform.python_implementation())
44 if self.implementation == "PyPy":
45 self.pypy_version_info = tuple(u(i) for i in sys.pypy_version_info)
46
47 # this is a tuple in earlier, struct later, unify to our own named tuple
48 self.version_info = VersionInfo(*list(u(i) for i in sys.version_info))
49 self.architecture = 64 if sys.maxsize > 2 ** 32 else 32
50
51 self.version = u(sys.version)
52 self.os = u(os.name)
53
54 # information about the prefix - determines python home
55 self.prefix = u(abs_path(getattr(sys, "prefix", None))) # prefix we think
56 self.base_prefix = u(abs_path(getattr(sys, "base_prefix", None))) # venv
57 self.real_prefix = u(abs_path(getattr(sys, "real_prefix", None))) # old virtualenv
58
59 # information about the exec prefix - dynamic stdlib modules
60 self.base_exec_prefix = u(abs_path(getattr(sys, "base_exec_prefix", None)))
61 self.exec_prefix = u(abs_path(getattr(sys, "exec_prefix", None)))
62
63 self.executable = u(abs_path(sys.executable)) # the executable we were invoked via
64 self.original_executable = u(abs_path(self.executable)) # the executable as known by the interpreter
65 self.system_executable = self._fast_get_system_executable() # the executable we are based of (if available)
66
67 try:
68 __import__("venv")
69 has = True
70 except ImportError:
71 has = False
72 self.has_venv = has
73 self.path = [u(i) for i in sys.path]
74 self.file_system_encoding = u(sys.getfilesystemencoding())
75 self.stdout_encoding = u(getattr(sys.stdout, "encoding", None))
76
77 self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()}
78 # https://bugs.python.org/issue22199
79 makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None))
80 self.sysconfig = {
81 u(k): u(v)
82 for k, v in [
83 # a list of content to store from sysconfig
84 ("makefile_filename", makefile()),
85 ]
86 if k is not None
87 }
88
89 config_var_keys = set()
90 for element in self.sysconfig_paths.values():
91 for k in _CONF_VAR_RE.findall(element):
92 config_var_keys.add(u(k[1:-1]))
93 config_var_keys.add("PYTHONFRAMEWORK")
94
95 self.sysconfig_vars = {u(i): u(sysconfig.get_config_var(i) or "") for i in config_var_keys}
96 if self.implementation == "PyPy" and sys.version_info.major == 2:
97 self.sysconfig_vars[u"implementation_lower"] = u"python"
98
99 self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()}
100 confs = {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()}
101 self.system_stdlib = self.sysconfig_path("stdlib", confs)
102 self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs)
103 self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None))
104 self._creators = None
105
106 def _fast_get_system_executable(self):
107 """Try to get the system executable by just looking at properties"""
108 if self.real_prefix or (
109 self.base_prefix is not None and self.base_prefix != self.prefix
110 ): # if this is a virtual environment
111 if self.real_prefix is None:
112 base_executable = getattr(sys, "_base_executable", None) # some platforms may set this to help us
113 if base_executable is not None: # use the saved system executable if present
114 if sys.executable != base_executable: # we know we're in a virtual environment, cannot be us
115 return base_executable
116 return None # in this case we just can't tell easily without poking around FS and calling them, bail
117 # if we're not in a virtual environment, this is already a system python, so return the original executable
118 # note we must choose the original and not the pure executable as shim scripts might throw us off
119 return self.original_executable
120
121 @staticmethod
122 def _distutils_install():
123 # follow https://github.com/pypa/pip/blob/master/src/pip/_internal/locations.py#L95
124 # note here we don't import Distribution directly to allow setuptools to patch it
125 d = dist.Distribution({"script_args": "--no-user-cfg"}) # conf files not parsed so they do not hijack paths
126 if hasattr(sys, "_framework"):
127 sys._framework = None # disable macOS static paths for framework
128 i = d.get_command_obj("install", create=True)
129 i.prefix = os.sep # paths generated are relative to prefix that contains the path sep, this makes it relative
130 i.finalize_options()
131 result = {key: (getattr(i, "install_{}".format(key))[1:]).lstrip(os.sep) for key in SCHEME_KEYS}
132 return result
133
134 @property
135 def version_str(self):
136 return ".".join(str(i) for i in self.version_info[0:3])
137
138 @property
139 def version_release_str(self):
140 return ".".join(str(i) for i in self.version_info[0:2])
141
142 @property
143 def python_name(self):
144 version_info = self.version_info
145 return "python{}.{}".format(version_info.major, version_info.minor)
146
147 @property
148 def is_old_virtualenv(self):
149 return self.real_prefix is not None
150
151 @property
152 def is_venv(self):
153 return self.base_prefix is not None and self.version_info.major == 3
154
155 def sysconfig_path(self, key, config_var=None, sep=os.sep):
156 pattern = self.sysconfig_paths[key]
157 if config_var is None:
158 config_var = self.sysconfig_vars
159 else:
160 base = {k: v for k, v in self.sysconfig_vars.items()}
161 base.update(config_var)
162 config_var = base
163 return pattern.format(**config_var).replace(u"/", sep)
164
165 def creators(self, refresh=False):
166 if self._creators is None or refresh is True:
167 from virtualenv.run.plugin.creators import CreatorSelector
168
169 self._creators = CreatorSelector.for_interpreter(self)
170 return self._creators
171
172 @property
173 def system_include(self):
174 path = self.sysconfig_path(
175 "include",
176 {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()},
177 )
178 if not os.path.exists(path): # some broken packaging don't respect the sysconfig, fallback to distutils path
179 # the pattern include the distribution name too at the end, remove that via the parent call
180 fallback = os.path.join(self.prefix, os.path.dirname(self.distutils_install["headers"]))
181 if os.path.exists(fallback):
182 path = fallback
183 return path
184
185 @property
186 def system_prefix(self):
187 return self.real_prefix or self.base_prefix or self.prefix
188
189 @property
190 def system_exec_prefix(self):
191 return self.real_prefix or self.base_exec_prefix or self.exec_prefix
192
193 def __unicode__(self):
194 content = repr(self)
195 if sys.version_info == 2:
196 content = content.decode("utf-8")
197 return content
198
199 def __repr__(self):
200 return "{}({!r})".format(
201 self.__class__.__name__, {k: v for k, v in self.__dict__.items() if not k.startswith("_")},
202 )
203
204 def __str__(self):
205 content = "{}({})".format(
206 self.__class__.__name__,
207 ", ".join(
208 "{}={}".format(k, v)
209 for k, v in (
210 ("spec", self.spec),
211 (
212 "system"
213 if self.system_executable is not None and self.system_executable != self.executable
214 else None,
215 self.system_executable,
216 ),
217 (
218 "original"
219 if (
220 self.original_executable != self.system_executable
221 and self.original_executable != self.executable
222 )
223 else None,
224 self.original_executable,
225 ),
226 ("exe", self.executable),
227 ("platform", self.platform),
228 ("version", repr(self.version)),
229 ("encoding_fs_io", "{}-{}".format(self.file_system_encoding, self.stdout_encoding)),
230 )
231 if k is not None
232 ),
233 )
234 return content
235
236 @property
237 def spec(self):
238 return "{}{}-{}".format(self.implementation, ".".join(str(i) for i in self.version_info), self.architecture)
239
240 @classmethod
241 def clear_cache(cls, app_data):
242 # this method is not used by itself, so here and called functions can import stuff locally
243 from virtualenv.discovery.cached_py_info import clear
244
245 clear(app_data)
246 cls._cache_exe_discovery.clear()
247
248 def satisfies(self, spec, impl_must_match):
249 """check if a given specification can be satisfied by the this python interpreter instance"""
250 if spec.path:
251 if self.executable == os.path.abspath(spec.path):
252 return True # if the path is a our own executable path we're done
253 if not spec.is_abs:
254 # if path set, and is not our original executable name, this does not match
255 basename = os.path.basename(self.original_executable)
256 spec_path = spec.path
257 if sys.platform == "win32":
258 basename, suffix = os.path.splitext(basename)
259 if spec_path.endswith(suffix):
260 spec_path = spec_path[: -len(suffix)]
261 if basename != spec_path:
262 return False
263
264 if impl_must_match:
265 if spec.implementation is not None and spec.implementation.lower() != self.implementation.lower():
266 return False
267
268 if spec.architecture is not None and spec.architecture != self.architecture:
269 return False
270
271 for our, req in zip(self.version_info[0:3], (spec.major, spec.minor, spec.micro)):
272 if req is not None and our is not None and our != req:
273 return False
274 return True
275
276 _current_system = None
277 _current = None
278
279 @classmethod
280 def current(cls, app_data=None):
281 """
282 This locates the current host interpreter information. This might be different than what we run into in case
283 the host python has been upgraded from underneath us.
284 """
285 if cls._current is None:
286 cls._current = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=False)
287 return cls._current
288
289 @classmethod
290 def current_system(cls, app_data=None):
291 """
292 This locates the current host interpreter information. This might be different than what we run into in case
293 the host python has been upgraded from underneath us.
294 """
295 if cls._current_system is None:
296 cls._current_system = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=True)
297 return cls._current_system
298
299 def _to_json(self):
300 # don't save calculated paths, as these are non primitive types
301 return json.dumps(self._to_dict(), indent=2)
302
303 def _to_dict(self):
304 data = {var: (getattr(self, var) if var not in ("_creators",) else None) for var in vars(self)}
305 # noinspection PyProtectedMember
306 data["version_info"] = data["version_info"]._asdict() # namedtuple to dictionary
307 return data
308
309 @classmethod
310 def from_exe(cls, exe, app_data=None, raise_on_error=True, ignore_cache=False, resolve_to_host=True):
311 """Given a path to an executable get the python information"""
312 # this method is not used by itself, so here and called functions can import stuff locally
313 from virtualenv.discovery.cached_py_info import from_exe
314
315 proposed = from_exe(cls, app_data, exe, raise_on_error=raise_on_error, ignore_cache=ignore_cache)
316 # noinspection PyProtectedMember
317 if isinstance(proposed, PythonInfo) and resolve_to_host:
318 try:
319 proposed = proposed._resolve_to_system(app_data, proposed)
320 except Exception as exception:
321 if raise_on_error:
322 raise exception
323 logging.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception)
324 proposed = None
325 return proposed
326
327 @classmethod
328 def _from_json(cls, payload):
329 # the dictionary unroll here is to protect against pypy bug of interpreter crashing
330 raw = json.loads(payload)
331 return cls._from_dict({k: v for k, v in raw.items()})
332
333 @classmethod
334 def _from_dict(cls, data):
335 data["version_info"] = VersionInfo(**data["version_info"]) # restore this to a named tuple structure
336 result = cls()
337 result.__dict__ = {k: v for k, v in data.items()}
338 return result
339
340 @classmethod
341 def _resolve_to_system(cls, app_data, target):
342 start_executable = target.executable
343 prefixes = OrderedDict()
344 while target.system_executable is None:
345 prefix = target.real_prefix or target.base_prefix or target.prefix
346 if prefix in prefixes:
347 if len(prefixes) == 1:
348 # if we're linking back to ourselves accept ourselves with a WARNING
349 logging.info("%r links back to itself via prefixes", target)
350 target.system_executable = target.executable
351 break
352 for at, (p, t) in enumerate(prefixes.items(), start=1):
353 logging.error("%d: prefix=%s, info=%r", at, p, t)
354 logging.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target)
355 raise RuntimeError("prefixes are causing a circle {}".format("|".join(prefixes.keys())))
356 prefixes[prefix] = target
357 target = target.discover_exe(app_data, prefix=prefix, exact=False)
358 if target.executable != target.system_executable:
359 target = cls.from_exe(target.system_executable, app_data)
360 target.executable = start_executable
361 return target
362
363 _cache_exe_discovery = {}
364
365 def discover_exe(self, app_data, prefix, exact=True):
366 key = prefix, exact
367 if key in self._cache_exe_discovery and prefix:
368 logging.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key])
369 return self._cache_exe_discovery[key]
370 logging.debug("discover exe for %s in %s", self, prefix)
371 # we don't know explicitly here, do some guess work - our executable name should tell
372 possible_names = self._find_possible_exe_names()
373 possible_folders = self._find_possible_folders(prefix)
374 discovered = []
375 for folder in possible_folders:
376 for name in possible_names:
377 info = self._check_exe(app_data, folder, name, exact, discovered)
378 if info is not None:
379 self._cache_exe_discovery[key] = info
380 return info
381 if exact is False and discovered:
382 info = self._select_most_likely(discovered, self)
383 folders = os.pathsep.join(possible_folders)
384 self._cache_exe_discovery[key] = info
385 logging.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders)
386 return info
387 msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders))
388 raise RuntimeError(msg)
389
390 def _check_exe(self, app_data, folder, name, exact, discovered):
391 exe_path = os.path.join(folder, name)
392 if not os.path.exists(exe_path):
393 return None
394 info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False)
395 if info is None: # ignore if for some reason we can't query
396 return None
397 for item in ["implementation", "architecture", "version_info"]:
398 found = getattr(info, item)
399 searched = getattr(self, item)
400 if found != searched:
401 if item == "version_info":
402 found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched)
403 executable = info.executable
404 logging.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched)
405 if exact is False:
406 discovered.append(info)
407 break
408 else:
409 return info
410 return None
411
412 @staticmethod
413 def _select_most_likely(discovered, target):
414 # no exact match found, start relaxing our requirements then to facilitate system package upgrades that
415 # could cause this (when using copy strategy of the host python)
416 def sort_by(info):
417 # we need to setup some priority of traits, this is as follows:
418 # implementation, major, minor, micro, architecture, tag, serial
419 matches = [
420 info.implementation == target.implementation,
421 info.version_info.major == target.version_info.major,
422 info.version_info.minor == target.version_info.minor,
423 info.architecture == target.architecture,
424 info.version_info.micro == target.version_info.micro,
425 info.version_info.releaselevel == target.version_info.releaselevel,
426 info.version_info.serial == target.version_info.serial,
427 ]
428 priority = sum((1 << pos if match else 0) for pos, match in enumerate(reversed(matches)))
429 return priority
430
431 sorted_discovered = sorted(discovered, key=sort_by, reverse=True) # sort by priority in decreasing order
432 most_likely = sorted_discovered[0]
433 return most_likely
434
435 def _find_possible_folders(self, inside_folder):
436 candidate_folder = OrderedDict()
437 executables = OrderedDict()
438 executables[os.path.realpath(self.executable)] = None
439 executables[self.executable] = None
440 executables[os.path.realpath(self.original_executable)] = None
441 executables[self.original_executable] = None
442 for exe in executables.keys():
443 base = os.path.dirname(exe)
444 # following path pattern of the current
445 if base.startswith(self.prefix):
446 relative = base[len(self.prefix) :]
447 candidate_folder["{}{}".format(inside_folder, relative)] = None
448
449 # or at root level
450 candidate_folder[inside_folder] = None
451 return list(i for i in candidate_folder.keys() if os.path.exists(i))
452
453 def _find_possible_exe_names(self):
454 name_candidate = OrderedDict()
455 for name in self._possible_base():
456 for at in (3, 2, 1, 0):
457 version = ".".join(str(i) for i in self.version_info[:at])
458 for arch in ["-{}".format(self.architecture), ""]:
459 for ext in EXTENSIONS:
460 candidate = "{}{}{}{}".format(name, version, arch, ext)
461 name_candidate[candidate] = None
462 return list(name_candidate.keys())
463
464 def _possible_base(self):
465 possible_base = OrderedDict()
466 basename = os.path.splitext(os.path.basename(self.executable))[0].rstrip(digits)
467 possible_base[basename] = None
468 possible_base[self.implementation] = None
469 # python is always the final option as in practice is used by multiple implementation as exe name
470 if "python" in possible_base:
471 del possible_base["python"]
472 possible_base["python"] = None
473 for base in possible_base:
474 lower = base.lower()
475 yield lower
476 from virtualenv.info import fs_is_case_sensitive
477
478 if fs_is_case_sensitive():
479 if base != lower:
480 yield base
481 upper = base.upper()
482 if upper != base:
483 yield upper
484
485
486 if __name__ == "__main__":
487 # dump a JSON representation of the current python
488 # noinspection PyProtectedMember
489 print(PythonInfo()._to_json())