Mercurial > repos > guerler > springsuite
diff planemo/lib/python3.7/site-packages/virtualenv/seed/wheels/periodic_update.py @ 1:56ad4e20f292 draft
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author | guerler |
---|---|
date | Fri, 31 Jul 2020 00:32:28 -0400 |
parents | |
children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/planemo/lib/python3.7/site-packages/virtualenv/seed/wheels/periodic_update.py Fri Jul 31 00:32:28 2020 -0400 @@ -0,0 +1,364 @@ +""" +Periodically update bundled versions. +""" + +from __future__ import absolute_import, unicode_literals + +import json +import logging +import os +import ssl +import subprocess +import sys +from datetime import datetime, timedelta +from itertools import groupby +from shutil import copy2 +from textwrap import dedent +from threading import Thread + +from six.moves.urllib.error import URLError +from six.moves.urllib.request import urlopen + +from virtualenv.app_data import AppDataDiskFolder +from virtualenv.info import PY2 +from virtualenv.util.path import Path +from virtualenv.util.subprocess import DETACHED_PROCESS, Popen + +from ..wheels.embed import BUNDLE_SUPPORT +from ..wheels.util import Wheel + +if PY2: + # on Python 2 datetime.strptime throws the error below if the import did not trigger on main thread + # Failed to import _strptime because the import lock is held by + try: + import _strptime # noqa + except ImportError: # pragma: no cov + pass # pragma: no cov + + +def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update): + if do_periodic_update: + handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data) + + now = datetime.now() + + u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) + u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False + for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]): + version = next(group) # use only latest patch version per minor, earlier assumed to be buggy + if wheel is not None and Path(version.filename).name == wheel.name: + break + if u_log.periodic is False or (u_log_older_than_hour and version.use(now)): + updated_wheel = Wheel(app_data.house / version.filename) + logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) + wheel = updated_wheel + break + + return wheel + + +def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data): + embed_update_log = app_data.embed_update_log(distribution, for_py_version) + u_log = UpdateLog.from_dict(embed_update_log.read()) + if u_log.needs_update: + u_log.periodic = True + u_log.started = datetime.now() + embed_update_log.write(u_log.to_dict()) + trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True) + + +DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +def dump_datetime(value): + return None if value is None else value.strftime(DATETIME_FMT) + + +def load_datetime(value): + return None if value is None else datetime.strptime(value, DATETIME_FMT) + + +class NewVersion(object): + def __init__(self, filename, found_date, release_date): + self.filename = filename + self.found_date = found_date + self.release_date = release_date + + @classmethod + def from_dict(cls, dictionary): + return cls( + filename=dictionary["filename"], + found_date=load_datetime(dictionary["found_date"]), + release_date=load_datetime(dictionary["release_date"]), + ) + + def to_dict(self): + return { + "filename": self.filename, + "release_date": dump_datetime(self.release_date), + "found_date": dump_datetime(self.found_date), + } + + def use(self, now): + compare_from = self.release_date or self.found_date + return now - compare_from >= timedelta(days=28) + + def __repr__(self): + return "{}(filename={}), found_date={}, release_date={})".format( + self.__class__.__name__, self.filename, self.found_date, self.release_date, + ) + + def __eq__(self, other): + return type(self) == type(other) and all( + getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"] + ) + + def __ne__(self, other): + return not (self == other) + + @property + def wheel(self): + return Wheel(Path(self.filename)) + + +class UpdateLog(object): + def __init__(self, started, completed, versions, periodic): + self.started = started + self.completed = completed + self.versions = versions + self.periodic = periodic + + @classmethod + def from_dict(cls, dictionary): + if dictionary is None: + dictionary = {} + return cls( + load_datetime(dictionary.get("started")), + load_datetime(dictionary.get("completed")), + [NewVersion.from_dict(v) for v in dictionary.get("versions", [])], + dictionary.get("periodic"), + ) + + @classmethod + def from_app_data(cls, app_data, distribution, for_py_version): + raw_json = app_data.embed_update_log(distribution, for_py_version).read() + return cls.from_dict(raw_json) + + def to_dict(self): + return { + "started": dump_datetime(self.started), + "completed": dump_datetime(self.completed), + "periodic": self.periodic, + "versions": [r.to_dict() for r in self.versions], + } + + @property + def needs_update(self): + now = datetime.now() + if self.completed is None: # never completed + return self._check_start(now) + else: + if now - self.completed <= timedelta(days=14): + return False + return self._check_start(now) + + def _check_start(self, now): + return self.started is None or now - self.started > timedelta(hours=1) + + +def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic): + wheel_path = None if wheel is None else str(wheel.path) + cmd = [ + sys.executable, + "-c", + dedent( + """ + from virtualenv.report import setup_report, MAX_LEVEL + from virtualenv.seed.wheels.periodic_update import do_update + setup_report(MAX_LEVEL, show_pid=True) + do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r}) + """, + ) + .strip() + .format(distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic), + ] + debug = os.environ.get(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1") + pipe = None if debug else subprocess.PIPE + kwargs = {"stdout": pipe, "stderr": pipe} + if not debug and sys.platform == "win32": + kwargs["creationflags"] = DETACHED_PROCESS + process = Popen(cmd, **kwargs) + logging.info( + "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", + distribution, + "" if wheel is None else "=={}".format(wheel.version), + for_py_version, + process.pid, + ) + if debug: + process.communicate() # on purpose not called to make it a background process + + +def do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic): + versions = None + try: + versions = _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs) + finally: + logging.debug("done %s %s with %s", distribution, for_py_version, versions) + return versions + + +def _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs): + from virtualenv.seed.wheels import acquire + + wheel_filename = None if embed_filename is None else Path(embed_filename) + embed_version = None if wheel_filename is None else Wheel(wheel_filename).version_tuple + app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data + search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs] + wheelhouse = app_data.house + embed_update_log = app_data.embed_update_log(distribution, for_py_version) + u_log = UpdateLog.from_dict(embed_update_log.read()) + now = datetime.now() + if wheel_filename is not None: + dest = wheelhouse / wheel_filename.name + if not dest.exists(): + copy2(str(wheel_filename), str(wheelhouse)) + last, last_version, versions = None, None, [] + while last is None or not last.use(now): + download_time = datetime.now() + dest = acquire.download_wheel( + distribution=distribution, + version_spec=None if last_version is None else "<{}".format(last_version), + for_py_version=for_py_version, + search_dirs=search_dirs, + app_data=app_data, + to_folder=wheelhouse, + ) + if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name): + break + release_date = release_date_for_wheel_path(dest.path) + last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time) + logging.info("detected %s in %s", last, datetime.now() - download_time) + versions.append(last) + last_wheel = Wheel(Path(last.filename)) + last_version = last_wheel.version + if embed_version is not None: + if embed_version >= last_wheel.version_tuple: # stop download if we reach the embed version + break + u_log.periodic = periodic + if not u_log.periodic: + u_log.started = now + u_log.versions = versions + u_log.versions + u_log.completed = datetime.now() + embed_update_log.write(u_log.to_dict()) + return versions + + +def release_date_for_wheel_path(dest): + wheel = Wheel(dest) + # the most accurate is to ask PyPi - e.g. https://pypi.org/pypi/pip/json, + # see https://warehouse.pypa.io/api-reference/json/ for more details + content = _pypi_get_distribution_info_cached(wheel.distribution) + if content is not None: + try: + upload_time = content["releases"][wheel.version][0]["upload_time"] + return datetime.strptime(upload_time, "%Y-%m-%dT%H:%M:%S") + except Exception as exception: + logging.error("could not load release date %s because %r", content, exception) + return None + + +def _request_context(): + yield None + # fallback to non verified HTTPS (the information we request is not sensitive, so fallback) + yield ssl._create_unverified_context() # noqa + + +_PYPI_CACHE = {} + + +def _pypi_get_distribution_info_cached(distribution): + if distribution not in _PYPI_CACHE: + _PYPI_CACHE[distribution] = _pypi_get_distribution_info(distribution) + return _PYPI_CACHE[distribution] + + +def _pypi_get_distribution_info(distribution): + content, url = None, "https://pypi.org/pypi/{}/json".format(distribution) + try: + for context in _request_context(): + try: + with urlopen(url, context=context) as file_handler: + content = json.load(file_handler) + break + except URLError as exception: + logging.error("failed to access %s because %r", url, exception) + except Exception as exception: + logging.error("failed to access %s because %r", url, exception) + return content + + +def manual_upgrade(app_data): + threads = [] + + for for_py_version, distribution_to_package in BUNDLE_SUPPORT.items(): + # load extra search dir for the given for_py + for distribution in distribution_to_package.keys(): + thread = Thread(target=_run_manual_upgrade, args=(app_data, distribution, for_py_version)) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + +def _run_manual_upgrade(app_data, distribution, for_py_version): + start = datetime.now() + from .bundle import from_bundle + + current = from_bundle( + distribution=distribution, + version=None, + for_py_version=for_py_version, + search_dirs=[], + app_data=app_data, + do_periodic_update=False, + ) + logging.warning( + "upgrade %s for python %s with current %s", + distribution, + for_py_version, + "" if current is None else current.name, + ) + versions = do_update( + distribution=distribution, + for_py_version=for_py_version, + embed_filename=current.path, + app_data=app_data, + search_dirs=[], + periodic=False, + ) + msg = "upgraded %s for python %s in %s {}".format( + "new entries found:\n%s" if versions else "no new versions found", + ) + args = [ + distribution, + for_py_version, + datetime.now() - start, + ] + if versions: + args.append("\n".join("\t{}".format(v) for v in versions)) + logging.warning(msg, *args) + + +__all__ = ( + "periodic_update", + "do_update", + "manual_upgrade", + "NewVersion", + "UpdateLog", + "load_datetime", + "dump_datetime", + "trigger_update", + "release_date_for_wheel_path", +)