Mercurial > repos > guerler > springsuite

comparison planemo/lib/python3.7/site-packages/psutil/tests/__init__.py @ 1:56ad4e20f292 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
author guerler
date Fri, 31 Jul 2020 00:32:28 -0400
parents
children
comparison
equal deleted inserted replaced
0:d30785e31577 1:56ad4e20f292
1 # -*- coding: utf-8 -*-
2
3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
6
7 """
8 Test utilities.
9 """
10
11 from __future__ import print_function
12 import atexit
13 import contextlib
14 import ctypes
15 import errno
16 import functools
17 import gc
18 import inspect
19 import os
20 import random
21 import re
22 import select
23 import shutil
24 import signal
25 import socket
26 import stat
27 import subprocess
28 import sys
29 import tempfile
30 import textwrap
31 import threading
32 import time
33 import warnings
34 from socket import AF_INET
35 from socket import AF_INET6
36 from socket import SOCK_STREAM
37
38 import psutil
39 from psutil import AIX
40 from psutil import LINUX
41 from psutil import MACOS
42 from psutil import POSIX
43 from psutil import SUNOS
44 from psutil import WINDOWS
45 from psutil._common import bytes2human
46 from psutil._common import print_color
47 from psutil._common import supports_ipv6
48 from psutil._compat import FileExistsError
49 from psutil._compat import FileNotFoundError
50 from psutil._compat import PY3
51 from psutil._compat import range
52 from psutil._compat import super
53 from psutil._compat import u
54 from psutil._compat import unicode
55 from psutil._compat import which
56
57 if PY3:
58 import unittest
59 else:
60 import unittest2 as unittest # requires "pip install unittest2"
61
62 try:
63 from unittest import mock # py3
64 except ImportError:
65 with warnings.catch_warnings():
66 warnings.simplefilter("ignore")
67 import mock # NOQA - requires "pip install mock"
68
69 if sys.version_info >= (3, 4):
70 import enum
71 else:
72 enum = None
73
74
75 __all__ = [
76 # constants
77 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES',
78 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFN_PREFIX',
79 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', 'TRAVIS', 'CIRRUS',
80 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', 'IS_64BIT',
81 "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS",
82 "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT",
83 "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS",
84 "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO",
85 # subprocesses
86 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie',
87 'spawn_children_pair',
88 # threads
89 'ThreadTask'
90 # test utils
91 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented',
92 'retry_on_failure', 'TestMemoryLeak', 'PsutilTestCase',
93 'process_namespace', 'system_namespace', 'print_sysinfo',
94 # install utils
95 'install_pip', 'install_test_deps',
96 # fs utils
97 'chdir', 'safe_rmpath', 'create_exe', 'decode_path', 'encode_path',
98 'get_testfn',
99 # os
100 'get_winver', 'get_kernel_version',
101 # sync primitives
102 'call_until', 'wait_for_pid', 'wait_for_file',
103 # network
104 'check_net_address',
105 'get_free_port', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair',
106 'unix_socketpair', 'create_sockets',
107 # compat
108 'reload_module', 'import_module_by_path',
109 # others
110 'warn', 'copyload_shared_lib', 'is_namedtuple',
111 ]
112
113
114 # ===================================================================
115 # --- constants
116 # ===================================================================
117
118 # --- platforms
119
120 PYPY = '__pypy__' in sys.builtin_module_names
121 # whether we're running this test suite on a Continuous Integration service
122 TRAVIS = 'TRAVIS' in os.environ
123 APPVEYOR = 'APPVEYOR' in os.environ
124 CIRRUS = 'CIRRUS' in os.environ
125 GITHUB_WHEELS = 'CIBUILDWHEEL' in os.environ
126 CI_TESTING = TRAVIS or APPVEYOR or CIRRUS or GITHUB_WHEELS
127 # are we a 64 bit process?
128 IS_64BIT = sys.maxsize > 2 ** 32
129
130
131 # --- configurable defaults
132
133 # how many times retry_on_failure() decorator will retry
134 NO_RETRIES = 10
135 # bytes tolerance for system-wide related tests
136 TOLERANCE_SYS_MEM = 5 * 1024 * 1024 # 5MB
137 TOLERANCE_DISK_USAGE = 10 * 1024 * 1024 # 10MB
138 # the timeout used in functions which have to wait
139 GLOBAL_TIMEOUT = 5
140 # be more tolerant if we're on travis / appveyor in order to avoid
141 # false positives
142 if CI_TESTING:
143 NO_RETRIES *= 3
144 GLOBAL_TIMEOUT *= 3
145 TOLERANCE_SYS_MEM *= 3
146 TOLERANCE_DISK_USAGE *= 3
147
148 # --- file names
149
150 # Disambiguate TESTFN for parallel testing.
151 if os.name == 'java':
152 # Jython disallows @ in module names
153 TESTFN_PREFIX = '$psutil-%s-' % os.getpid()
154 else:
155 TESTFN_PREFIX = '@psutil-%s-' % os.getpid()
156 UNICODE_SUFFIX = u("-ƒőő")
157 # An invalid unicode string.
158 if PY3:
159 INVALID_UNICODE_SUFFIX = b"f\xc0\x80".decode('utf8', 'surrogateescape')
160 else:
161 INVALID_UNICODE_SUFFIX = "f\xc0\x80"
162 ASCII_FS = sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii')
163
164 # --- paths
165
166 ROOT_DIR = os.path.realpath(
167 os.path.join(os.path.dirname(__file__), '..', '..'))
168 SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts')
169 HERE = os.path.realpath(os.path.dirname(__file__))
170
171 # --- support
172
173 HAS_CONNECTIONS_UNIX = POSIX and not SUNOS
174 HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity")
175 HAS_CPU_FREQ = hasattr(psutil, "cpu_freq")
176 HAS_GETLOADAVG = hasattr(psutil, "getloadavg")
177 HAS_ENVIRON = hasattr(psutil.Process, "environ")
178 HAS_IONICE = hasattr(psutil.Process, "ionice")
179 HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps")
180 HAS_NET_IO_COUNTERS = hasattr(psutil, "net_io_counters")
181 HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num")
182 HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters")
183 HAS_RLIMIT = hasattr(psutil.Process, "rlimit")
184 HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery")
185 try:
186 HAS_BATTERY = HAS_SENSORS_BATTERY and bool(psutil.sensors_battery())
187 except Exception:
188 HAS_BATTERY = True
189 HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans")
190 HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures")
191 HAS_THREADS = hasattr(psutil.Process, "threads")
192 SKIP_SYSCONS = (MACOS or AIX) and os.getuid() != 0
193
194 # --- misc
195
196
197 def _get_py_exe():
198 def attempt(exe):
199 try:
200 subprocess.check_call(
201 [exe, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
202 except Exception:
203 return None
204 else:
205 return exe
206
207 if GITHUB_WHEELS:
208 if PYPY:
209 return which("pypy3") if PY3 else which("pypy")
210 else:
211 return which('python')
212 elif MACOS:
213 exe = \
214 attempt(sys.executable) or \
215 attempt(os.path.realpath(sys.executable)) or \
216 attempt(which("python%s.%s" % sys.version_info[:2])) or \
217 attempt(psutil.Process().exe())
218 if not exe:
219 raise ValueError("can't find python exe real abspath")
220 return exe
221 else:
222 exe = os.path.realpath(sys.executable)
223 assert os.path.exists(exe), exe
224 return exe
225
226
227 PYTHON_EXE = _get_py_exe()
228 DEVNULL = open(os.devnull, 'r+')
229 atexit.register(DEVNULL.close)
230
231 VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil)
232 if x.startswith('STATUS_')]
233 AF_UNIX = getattr(socket, "AF_UNIX", object())
234
235 _subprocesses_started = set()
236 _pids_started = set()
237
238
239 # ===================================================================
240 # --- threads
241 # ===================================================================
242
243
244 class ThreadTask(threading.Thread):
245 """A thread task which does nothing expect staying alive."""
246
247 def __init__(self):
248 super().__init__()
249 self._running = False
250 self._interval = 0.001
251 self._flag = threading.Event()
252
253 def __repr__(self):
254 name = self.__class__.__name__
255 return '<%s running=%s at %#x>' % (name, self._running, id(self))
256
257 def __enter__(self):
258 self.start()
259 return self
260
261 def __exit__(self, *args, **kwargs):
262 self.stop()
263
264 def start(self):
265 """Start thread and keep it running until an explicit
266 stop() request. Polls for shutdown every 'timeout' seconds.
267 """
268 if self._running:
269 raise ValueError("already started")
270 threading.Thread.start(self)
271 self._flag.wait()
272
273 def run(self):
274 self._running = True
275 self._flag.set()
276 while self._running:
277 time.sleep(self._interval)
278
279 def stop(self):
280 """Stop thread execution and and waits until it is stopped."""
281 if not self._running:
282 raise ValueError("already stopped")
283 self._running = False
284 self.join()
285
286
287 # ===================================================================
288 # --- subprocesses
289 # ===================================================================
290
291
292 def _reap_children_on_err(fun):
293 @functools.wraps(fun)
294 def wrapper(*args, **kwargs):
295 try:
296 return fun(*args, **kwargs)
297 except Exception:
298 reap_children()
299 raise
300 return wrapper
301
302
303 @_reap_children_on_err
304 def spawn_testproc(cmd=None, **kwds):
305 """Creates a python subprocess which does nothing for 60 secs and
306 return it as a subprocess.Popen instance.
307 If "cmd" is specified that is used instead of python.
308 By default stdin and stdout are redirected to /dev/null.
309 It also attemps to make sure the process is in a reasonably
310 initialized state.
311 The process is registered for cleanup on reap_children().
312 """
313 kwds.setdefault("stdin", DEVNULL)
314 kwds.setdefault("stdout", DEVNULL)
315 kwds.setdefault("cwd", os.getcwd())
316 kwds.setdefault("env", os.environ)
317 if WINDOWS:
318 # Prevents the subprocess to open error dialogs. This will also
319 # cause stderr to be suppressed, which is suboptimal in order
320 # to debug broken tests.
321 CREATE_NO_WINDOW = 0x8000000
322 kwds.setdefault("creationflags", CREATE_NO_WINDOW)
323 if cmd is None:
324 testfn = get_testfn()
325 try:
326 safe_rmpath(testfn)
327 pyline = "from time import sleep;" \
328 "open(r'%s', 'w').close();" \
329 "sleep(60);" % testfn
330 cmd = [PYTHON_EXE, "-c", pyline]
331 sproc = subprocess.Popen(cmd, **kwds)
332 _subprocesses_started.add(sproc)
333 wait_for_file(testfn, delete=True, empty=True)
334 finally:
335 safe_rmpath(testfn)
336 else:
337 sproc = subprocess.Popen(cmd, **kwds)
338 _subprocesses_started.add(sproc)
339 wait_for_pid(sproc.pid)
340 return sproc
341
342
343 @_reap_children_on_err
344 def spawn_children_pair():
345 """Create a subprocess which creates another one as in:
346 A (us) -> B (child) -> C (grandchild).
347 Return a (child, grandchild) tuple.
348 The 2 processes are fully initialized and will live for 60 secs
349 and are registered for cleanup on reap_children().
350 """
351 tfile = None
352 testfn = get_testfn(dir=os.getcwd())
353 try:
354 s = textwrap.dedent("""\
355 import subprocess, os, sys, time
356 s = "import os, time;"
357 s += "f = open('%s', 'w');"
358 s += "f.write(str(os.getpid()));"
359 s += "f.close();"
360 s += "time.sleep(60);"
361 p = subprocess.Popen([r'%s', '-c', s])
362 p.wait()
363 """ % (os.path.basename(testfn), PYTHON_EXE))
364 # On Windows if we create a subprocess with CREATE_NO_WINDOW flag
365 # set (which is the default) a "conhost.exe" extra process will be
366 # spawned as a child. We don't want that.
367 if WINDOWS:
368 subp, tfile = pyrun(s, creationflags=0)
369 else:
370 subp, tfile = pyrun(s)
371 child = psutil.Process(subp.pid)
372 grandchild_pid = int(wait_for_file(testfn, delete=True, empty=False))
373 _pids_started.add(grandchild_pid)
374 grandchild = psutil.Process(grandchild_pid)
375 return (child, grandchild)
376 finally:
377 safe_rmpath(testfn)
378 if tfile is not None:
379 safe_rmpath(tfile)
380
381
382 def spawn_zombie():
383 """Create a zombie process and return a (parent, zombie) process tuple.
384 In order to kill the zombie parent must be terminate()d first, then
385 zombie must be wait()ed on.
386 """
387 assert psutil.POSIX
388 unix_file = get_testfn()
389 src = textwrap.dedent("""\
390 import os, sys, time, socket, contextlib
391 child_pid = os.fork()
392 if child_pid > 0:
393 time.sleep(3000)
394 else:
395 # this is the zombie process
396 s = socket.socket(socket.AF_UNIX)
397 with contextlib.closing(s):
398 s.connect('%s')
399 if sys.version_info < (3, ):
400 pid = str(os.getpid())
401 else:
402 pid = bytes(str(os.getpid()), 'ascii')
403 s.sendall(pid)
404 """ % unix_file)
405 tfile = None
406 sock = bind_unix_socket(unix_file)
407 try:
408 sock.settimeout(GLOBAL_TIMEOUT)
409 parent, tfile = pyrun(src)
410 conn, _ = sock.accept()
411 try:
412 select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT)
413 zpid = int(conn.recv(1024))
414 _pids_started.add(zpid)
415 zombie = psutil.Process(zpid)
416 call_until(lambda: zombie.status(), "ret == psutil.STATUS_ZOMBIE")
417 return (parent, zombie)
418 finally:
419 conn.close()
420 finally:
421 sock.close()
422 safe_rmpath(unix_file)
423 if tfile is not None:
424 safe_rmpath(tfile)
425
426
427 @_reap_children_on_err
428 def pyrun(src, **kwds):
429 """Run python 'src' code string in a separate interpreter.
430 Returns a subprocess.Popen instance and the test file where the source
431 code was written.
432 """
433 kwds.setdefault("stdout", None)
434 kwds.setdefault("stderr", None)
435 srcfile = get_testfn()
436 try:
437 with open(srcfile, 'wt') as f:
438 f.write(src)
439 subp = spawn_testproc([PYTHON_EXE, f.name], **kwds)
440 wait_for_pid(subp.pid)
441 return (subp, srcfile)
442 except Exception:
443 safe_rmpath(srcfile)
444 raise
445
446
447 @_reap_children_on_err
448 def sh(cmd, **kwds):
449 """run cmd in a subprocess and return its output.
450 raises RuntimeError on error.
451 """
452 shell = True if isinstance(cmd, (str, unicode)) else False
453 # Prevents subprocess to open error dialogs in case of error.
454 flags = 0x8000000 if WINDOWS and shell else 0
455 kwds.setdefault("shell", shell)
456 kwds.setdefault("stdout", subprocess.PIPE)
457 kwds.setdefault("stderr", subprocess.PIPE)
458 kwds.setdefault("universal_newlines", True)
459 kwds.setdefault("creationflags", flags)
460 p = subprocess.Popen(cmd, **kwds)
461 _subprocesses_started.add(p)
462 if PY3:
463 stdout, stderr = p.communicate(timeout=GLOBAL_TIMEOUT)
464 else:
465 stdout, stderr = p.communicate()
466 if p.returncode != 0:
467 raise RuntimeError(stderr)
468 if stderr:
469 warn(stderr)
470 if stdout.endswith('\n'):
471 stdout = stdout[:-1]
472 return stdout
473
474
475 def terminate(proc_or_pid, sig=signal.SIGTERM, wait_timeout=GLOBAL_TIMEOUT):
476 """Terminate a process and wait() for it.
477 Process can be a PID or an instance of psutil.Process(),
478 subprocess.Popen() or psutil.Popen().
479 If it's a subprocess.Popen() or psutil.Popen() instance also closes
480 its stdin / stdout / stderr fds.
481 PID is wait()ed even if the process is already gone (kills zombies).
482 Does nothing if the process does not exist.
483 Return process exit status.
484 """
485 if POSIX:
486 from psutil._psposix import wait_pid
487
488 def wait(proc, timeout):
489 if isinstance(proc, subprocess.Popen) and not PY3:
490 proc.wait()
491 else:
492 proc.wait(timeout)
493 if WINDOWS and isinstance(proc, subprocess.Popen):
494 # Otherwise PID may still hang around.
495 try:
496 return psutil.Process(proc.pid).wait(timeout)
497 except psutil.NoSuchProcess:
498 pass
499
500 def sendsig(proc, sig):
501 # XXX: otherwise the build hangs for some reason.
502 if MACOS and GITHUB_WHEELS:
503 sig = signal.SIGKILL
504 # If the process received SIGSTOP, SIGCONT is necessary first,
505 # otherwise SIGTERM won't work.
506 if POSIX and sig != signal.SIGKILL:
507 proc.send_signal(signal.SIGCONT)
508 proc.send_signal(sig)
509
510 def term_subproc(proc, timeout):
511 try:
512 sendsig(proc, sig)
513 except OSError as err:
514 if WINDOWS and err.winerror == 6: # "invalid handle"
515 pass
516 elif err.errno != errno.ESRCH:
517 raise
518 return wait(proc, timeout)
519
520 def term_psproc(proc, timeout):
521 try:
522 sendsig(proc, sig)
523 except psutil.NoSuchProcess:
524 pass
525 return wait(proc, timeout)
526
527 def term_pid(pid, timeout):
528 try:
529 proc = psutil.Process(pid)
530 except psutil.NoSuchProcess:
531 # Needed to kill zombies.
532 if POSIX:
533 return wait_pid(pid, timeout)
534 else:
535 return term_psproc(proc, timeout)
536
537 def flush_popen(proc):
538 if proc.stdout:
539 proc.stdout.close()
540 if proc.stderr:
541 proc.stderr.close()
542 # Flushing a BufferedWriter may raise an error.
543 if proc.stdin:
544 proc.stdin.close()
545
546 p = proc_or_pid
547 try:
548 if isinstance(p, int):
549 return term_pid(p, wait_timeout)
550 elif isinstance(p, (psutil.Process, psutil.Popen)):
551 return term_psproc(p, wait_timeout)
552 elif isinstance(p, subprocess.Popen):
553 return term_subproc(p, wait_timeout)
554 else:
555 raise TypeError("wrong type %r" % p)
556 finally:
557 if isinstance(p, (subprocess.Popen, psutil.Popen)):
558 flush_popen(p)
559 pid = p if isinstance(p, int) else p.pid
560 assert not psutil.pid_exists(pid), pid
561
562
563 def reap_children(recursive=False):
564 """Terminate and wait() any subprocess started by this test suite
565 and any children currently running, ensuring that no processes stick
566 around to hog resources.
567 If resursive is True it also tries to terminate and wait()
568 all grandchildren started by this process.
569 """
570 # Get the children here before terminating them, as in case of
571 # recursive=True we don't want to lose the intermediate reference
572 # pointing to the grandchildren.
573 children = psutil.Process().children(recursive=recursive)
574
575 # Terminate subprocess.Popen.
576 while _subprocesses_started:
577 subp = _subprocesses_started.pop()
578 terminate(subp)
579
580 # Collect started pids.
581 while _pids_started:
582 pid = _pids_started.pop()
583 terminate(pid)
584
585 # Terminate children.
586 if children:
587 for p in children:
588 terminate(p, wait_timeout=None)
589 gone, alive = psutil.wait_procs(children, timeout=GLOBAL_TIMEOUT)
590 for p in alive:
591 warn("couldn't terminate process %r; attempting kill()" % p)
592 terminate(p, sig=signal.SIGKILL)
593
594
595 # ===================================================================
596 # --- OS
597 # ===================================================================
598
599
600 def get_kernel_version():
601 """Return a tuple such as (2, 6, 36)."""
602 if not POSIX:
603 raise NotImplementedError("not POSIX")
604 s = ""
605 uname = os.uname()[2]
606 for c in uname:
607 if c.isdigit() or c == '.':
608 s += c
609 else:
610 break
611 if not s:
612 raise ValueError("can't parse %r" % uname)
613 minor = 0
614 micro = 0
615 nums = s.split('.')
616 major = int(nums[0])
617 if len(nums) >= 2:
618 minor = int(nums[1])
619 if len(nums) >= 3:
620 micro = int(nums[2])
621 return (major, minor, micro)
622
623
624 def get_winver():
625 if not WINDOWS:
626 raise NotImplementedError("not WINDOWS")
627 wv = sys.getwindowsversion()
628 if hasattr(wv, 'service_pack_major'): # python >= 2.7
629 sp = wv.service_pack_major or 0
630 else:
631 r = re.search(r"\s\d$", wv[4])
632 if r:
633 sp = int(r.group(0))
634 else:
635 sp = 0
636 return (wv[0], wv[1], sp)
637
638
639 # ===================================================================
640 # --- sync primitives
641 # ===================================================================
642
643
644 class retry(object):
645 """A retry decorator."""
646
647 def __init__(self,
648 exception=Exception,
649 timeout=None,
650 retries=None,
651 interval=0.001,
652 logfun=None,
653 ):
654 if timeout and retries:
655 raise ValueError("timeout and retries args are mutually exclusive")
656 self.exception = exception
657 self.timeout = timeout
658 self.retries = retries
659 self.interval = interval
660 self.logfun = logfun
661
662 def __iter__(self):
663 if self.timeout:
664 stop_at = time.time() + self.timeout
665 while time.time() < stop_at:
666 yield
667 elif self.retries:
668 for _ in range(self.retries):
669 yield
670 else:
671 while True:
672 yield
673
674 def sleep(self):
675 if self.interval is not None:
676 time.sleep(self.interval)
677
678 def __call__(self, fun):
679 @functools.wraps(fun)
680 def wrapper(*args, **kwargs):
681 exc = None
682 for _ in self:
683 try:
684 return fun(*args, **kwargs)
685 except self.exception as _: # NOQA
686 exc = _
687 if self.logfun is not None:
688 self.logfun(exc)
689 self.sleep()
690 continue
691 if PY3:
692 raise exc
693 else:
694 raise
695
696 # This way the user of the decorated function can change config
697 # parameters.
698 wrapper.decorator = self
699 return wrapper
700
701
702 @retry(exception=psutil.NoSuchProcess, logfun=None, timeout=GLOBAL_TIMEOUT,
703 interval=0.001)
704 def wait_for_pid(pid):
705 """Wait for pid to show up in the process list then return.
706 Used in the test suite to give time the sub process to initialize.
707 """
708 psutil.Process(pid)
709 if WINDOWS:
710 # give it some more time to allow better initialization
711 time.sleep(0.01)
712
713
714 @retry(exception=(FileNotFoundError, AssertionError), logfun=None,
715 timeout=GLOBAL_TIMEOUT, interval=0.001)
716 def wait_for_file(fname, delete=True, empty=False):
717 """Wait for a file to be written on disk with some content."""
718 with open(fname, "rb") as f:
719 data = f.read()
720 if not empty:
721 assert data
722 if delete:
723 safe_rmpath(fname)
724 return data
725
726
727 @retry(exception=AssertionError, logfun=None, timeout=GLOBAL_TIMEOUT,
728 interval=0.001)
729 def call_until(fun, expr):
730 """Keep calling function for timeout secs and exit if eval()
731 expression is True.
732 """
733 ret = fun()
734 assert eval(expr)
735 return ret
736
737
738 # ===================================================================
739 # --- fs
740 # ===================================================================
741
742
743 def safe_rmpath(path):
744 "Convenience function for removing temporary test files or dirs"
745 def retry_fun(fun):
746 # On Windows it could happen that the file or directory has
747 # open handles or references preventing the delete operation
748 # to succeed immediately, so we retry for a while. See:
749 # https://bugs.python.org/issue33240
750 stop_at = time.time() + GLOBAL_TIMEOUT
751 while time.time() < stop_at:
752 try:
753 return fun()
754 except FileNotFoundError:
755 pass
756 except WindowsError as _:
757 err = _
758 warn("ignoring %s" % (str(err)))
759 time.sleep(0.01)
760 raise err
761
762 try:
763 st = os.stat(path)
764 if stat.S_ISDIR(st.st_mode):
765 fun = functools.partial(shutil.rmtree, path)
766 else:
767 fun = functools.partial(os.remove, path)
768 if POSIX:
769 fun()
770 else:
771 retry_fun(fun)
772 except FileNotFoundError:
773 pass
774
775
776 def safe_mkdir(dir):
777 "Convenience function for creating a directory"
778 try:
779 os.mkdir(dir)
780 except FileExistsError:
781 pass
782
783
784 @contextlib.contextmanager
785 def chdir(dirname):
786 "Context manager which temporarily changes the current directory."
787 curdir = os.getcwd()
788 try:
789 os.chdir(dirname)
790 yield
791 finally:
792 os.chdir(curdir)
793
794
795 def create_exe(outpath, c_code=None):
796 """Creates an executable file in the given location."""
797 assert not os.path.exists(outpath), outpath
798 if c_code:
799 if not which("gcc"):
800 raise ValueError("gcc is not installed")
801 if isinstance(c_code, bool): # c_code is True
802 c_code = textwrap.dedent(
803 """
804 #include <unistd.h>
805 int main() {
806 pause();
807 return 1;
808 }
809 """)
810 assert isinstance(c_code, str), c_code
811 with open(get_testfn(suffix='.c'), 'wt') as f:
812 f.write(c_code)
813 try:
814 subprocess.check_call(["gcc", f.name, "-o", outpath])
815 finally:
816 safe_rmpath(f.name)
817 else:
818 # copy python executable
819 shutil.copyfile(PYTHON_EXE, outpath)
820 if POSIX:
821 st = os.stat(outpath)
822 os.chmod(outpath, st.st_mode | stat.S_IEXEC)
823
824
825 def get_testfn(suffix="", dir=None):
826 """Return an absolute pathname of a file or dir that did not
827 exist at the time this call is made. Also schedule it for safe
828 deletion at interpreter exit. It's technically racy but probably
829 not really due to the time variant.
830 """
831 while True:
832 name = tempfile.mktemp(prefix=TESTFN_PREFIX, suffix=suffix, dir=dir)
833 if not os.path.exists(name): # also include dirs
834 return os.path.realpath(name) # needed for OSX
835
836
837 # ===================================================================
838 # --- testing
839 # ===================================================================
840
841
842 class TestCase(unittest.TestCase):
843
844 # Print a full path representation of the single unit tests
845 # being run.
846 def __str__(self):
847 fqmod = self.__class__.__module__
848 if not fqmod.startswith('psutil.'):
849 fqmod = 'psutil.tests.' + fqmod
850 return "%s.%s.%s" % (
851 fqmod, self.__class__.__name__, self._testMethodName)
852
853 # assertRaisesRegexp renamed to assertRaisesRegex in 3.3;
854 # add support for the new name.
855 if not hasattr(unittest.TestCase, 'assertRaisesRegex'):
856 assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
857
858 # ...otherwise multiprocessing.Pool complains
859 if not PY3:
860 def runTest(self):
861 pass
862
863
864 # monkey patch default unittest.TestCase
865 unittest.TestCase = TestCase
866
867
868 class PsutilTestCase(TestCase):
869 """Test class providing auto-cleanup wrappers on top of process
870 test utilities.
871 """
872
873 def get_testfn(self, suffix="", dir=None):
874 fname = get_testfn(suffix=suffix, dir=dir)
875 self.addCleanup(safe_rmpath, fname)
876 return fname
877
878 def spawn_testproc(self, *args, **kwds):
879 sproc = spawn_testproc(*args, **kwds)
880 self.addCleanup(terminate, sproc)
881 return sproc
882
883 def spawn_children_pair(self):
884 child1, child2 = spawn_children_pair()
885 self.addCleanup(terminate, child2)
886 self.addCleanup(terminate, child1) # executed first
887 return (child1, child2)
888
889 def spawn_zombie(self):
890 parent, zombie = spawn_zombie()
891 self.addCleanup(terminate, zombie)
892 self.addCleanup(terminate, parent) # executed first
893 return (parent, zombie)
894
895 def pyrun(self, *args, **kwds):
896 sproc, srcfile = pyrun(*args, **kwds)
897 self.addCleanup(safe_rmpath, srcfile)
898 self.addCleanup(terminate, sproc) # executed first
899 return sproc
900
901 def assertProcessGone(self, proc):
902 self.assertRaises(psutil.NoSuchProcess, psutil.Process, proc.pid)
903 if isinstance(proc, (psutil.Process, psutil.Popen)):
904 assert not proc.is_running()
905 self.assertRaises(psutil.NoSuchProcess, proc.status)
906 proc.wait(timeout=0) # assert not raise TimeoutExpired
907 assert not psutil.pid_exists(proc.pid), proc.pid
908 self.assertNotIn(proc.pid, psutil.pids())
909
910
911 @unittest.skipIf(PYPY, "unreliable on PYPY")
912 class TestMemoryLeak(PsutilTestCase):
913 """Test framework class for detecting function memory leaks,
914 typically functions implemented in C which forgot to free() memory
915 from the heap. It does so by checking whether the process memory
916 usage increased before and after calling the function many times.
917
918 Note that this is hard (probably impossible) to do reliably, due
919 to how the OS handles memory, the GC and so on (memory can even
920 decrease!). In order to avoid false positives, in case of failure
921 (mem > 0) we retry the test for up to 5 times, increasing call
922 repetitions each time. If the memory keeps increasing then it's a
923 failure.
924
925 If available (Linux, OSX, Windows), USS memory is used for comparison,
926 since it's supposed to be more precise, see:
927 https://gmpy.dev/blog/2016/real-process-memory-and-environ-in-python
928 If not, RSS memory is used. mallinfo() on Linux and _heapwalk() on
929 Windows may give even more precision, but at the moment are not
930 implemented.
931
932 PyPy appears to be completely unstable for this framework, probably
933 because of its JIT, so tests on PYPY are skipped.
934
935 Usage:
936
937 class TestLeaks(psutil.tests.TestMemoryLeak):
938
939 def test_fun(self):
940 self.execute(some_function)
941 """
942 # Configurable class attrs.
943 times = 200
944 warmup_times = 10
945 tolerance = 0 # memory
946 retries = 10 if CI_TESTING else 5
947 verbose = True
948 _thisproc = psutil.Process()
949
950 def _get_mem(self):
951 # USS is the closest thing we have to "real" memory usage and it
952 # should be less likely to produce false positives.
953 mem = self._thisproc.memory_full_info()
954 return getattr(mem, "uss", mem.rss)
955
956 def _get_num_fds(self):
957 if POSIX:
958 return self._thisproc.num_fds()
959 else:
960 return self._thisproc.num_handles()
961
962 def _log(self, msg):
963 if self.verbose:
964 print_color(msg, color="yellow", file=sys.stderr)
965
966 def _check_fds(self, fun):
967 """Makes sure num_fds() (POSIX) or num_handles() (Windows) does
968 not increase after calling a function. Used to discover forgotten
969 close(2) and CloseHandle syscalls.
970 """
971 before = self._get_num_fds()
972 self.call(fun)
973 after = self._get_num_fds()
974 diff = after - before
975 if diff < 0:
976 raise self.fail("negative diff %r (gc probably collected a "
977 "resource from a previous test)" % diff)
978 if diff > 0:
979 type_ = "fd" if POSIX else "handle"
980 if diff > 1:
981 type_ += "s"
982 msg = "%s unclosed %s after calling %r" % (diff, type_, fun)
983 raise self.fail(msg)
984
985 def _call_ntimes(self, fun, times):
986 """Get 2 distinct memory samples, before and after having
987 called fun repeadetly, and return the memory difference.
988 """
989 gc.collect(generation=1)
990 mem1 = self._get_mem()
991 for x in range(times):
992 ret = self.call(fun)
993 del x, ret
994 gc.collect(generation=1)
995 mem2 = self._get_mem()
996 self.assertEqual(gc.garbage, [])
997 diff = mem2 - mem1 # can also be negative
998 return diff
999
1000 def _check_mem(self, fun, times, warmup_times, retries, tolerance):
1001 messages = []
1002 prev_mem = 0
1003 increase = times
1004 for idx in range(1, retries + 1):
1005 mem = self._call_ntimes(fun, times)
1006 msg = "Run #%s: extra-mem=%s, per-call=%s, calls=%s" % (
1007 idx, bytes2human(mem), bytes2human(mem / times), times)
1008 messages.append(msg)
1009 success = mem <= tolerance or mem <= prev_mem
1010 if success:
1011 if idx > 1:
1012 self._log(msg)
1013 return
1014 else:
1015 if idx == 1:
1016 print() # NOQA
1017 self._log(msg)
1018 times += increase
1019 prev_mem = mem
1020 raise self.fail(". ".join(messages))
1021
1022 # ---
1023
1024 def call(self, fun):
1025 return fun()
1026
1027 def execute(self, fun, times=None, warmup_times=None, retries=None,
1028 tolerance=None):
1029 """Test a callable."""
1030 times = times if times is not None else self.times
1031 warmup_times = warmup_times if warmup_times is not None \
1032 else self.warmup_times
1033 retries = retries if retries is not None else self.retries
1034 tolerance = tolerance if tolerance is not None else self.tolerance
1035 try:
1036 assert times >= 1, "times must be >= 1"
1037 assert warmup_times >= 0, "warmup_times must be >= 0"
1038 assert retries >= 0, "retries must be >= 0"
1039 assert tolerance >= 0, "tolerance must be >= 0"
1040 except AssertionError as err:
1041 raise ValueError(str(err))
1042
1043 self._call_ntimes(fun, warmup_times) # warm up
1044 self._check_fds(fun)
1045 self._check_mem(fun, times=times, warmup_times=warmup_times,
1046 retries=retries, tolerance=tolerance)
1047
1048 def execute_w_exc(self, exc, fun, **kwargs):
1049 """Convenience method to test a callable while making sure it
1050 raises an exception on every call.
1051 """
1052 def call():
1053 self.assertRaises(exc, fun)
1054
1055 self.execute(call, **kwargs)
1056
1057
1058 def print_sysinfo():
1059 import collections
1060 import datetime
1061 import getpass
1062 import platform
1063
1064 info = collections.OrderedDict()
1065 info['OS'] = platform.system()
1066 if psutil.OSX:
1067 info['version'] = str(platform.mac_ver())
1068 elif psutil.WINDOWS:
1069 info['version'] = ' '.join(map(str, platform.win32_ver()))
1070 if hasattr(platform, 'win32_edition'):
1071 info['edition'] = platform.win32_edition()
1072 else:
1073 info['version'] = platform.version()
1074 if psutil.POSIX:
1075 info['kernel'] = '.'.join(map(str, get_kernel_version()))
1076 info['arch'] = ', '.join(
1077 list(platform.architecture()) + [platform.machine()])
1078 info['hostname'] = platform.node()
1079 info['python'] = ', '.join([
1080 platform.python_implementation(),
1081 platform.python_version(),
1082 platform.python_compiler()])
1083 if psutil.POSIX:
1084 s = platform.libc_ver()[1]
1085 if s:
1086 info['glibc'] = s
1087 info['fs-encoding'] = sys.getfilesystemencoding()
1088 info['time'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
1089 info['user'] = getpass.getuser()
1090 info['pid'] = os.getpid()
1091 print("=" * 70) # NOQA
1092 for k, v in info.items():
1093 print("%-14s %s" % (k + ':', v)) # NOQA
1094 print("=" * 70) # NOQA
1095
1096
1097 def _get_eligible_cpu():
1098 p = psutil.Process()
1099 if hasattr(p, "cpu_num"):
1100 return p.cpu_num()
1101 elif hasattr(p, "cpu_affinity"):
1102 return p.cpu_affinity()[0]
1103 return 0
1104
1105
1106 class process_namespace:
1107 """A container that lists all Process class method names + some
1108 reasonable parameters to be called with. Utility methods (parent(),
1109 children(), ...) are excluded.
1110
1111 >>> ns = process_namespace(psutil.Process())
1112 >>> for fun, name in ns.iter(ns.getters):
1113 ... fun()
1114 """
1115 utils = [
1116 ('cpu_percent', (), {}),
1117 ('memory_percent', (), {}),
1118 ]
1119
1120 ignored = [
1121 ('as_dict', (), {}),
1122 ('children', (), {'recursive': True}),
1123 ('is_running', (), {}),
1124 ('memory_info_ex', (), {}),
1125 ('oneshot', (), {}),
1126 ('parent', (), {}),
1127 ('parents', (), {}),
1128 ('pid', (), {}),
1129 ('wait', (0, ), {}),
1130 ]
1131
1132 getters = [
1133 ('cmdline', (), {}),
1134 ('connections', (), {'kind': 'all'}),
1135 ('cpu_times', (), {}),
1136 ('create_time', (), {}),
1137 ('cwd', (), {}),
1138 ('exe', (), {}),
1139 ('memory_full_info', (), {}),
1140 ('memory_info', (), {}),
1141 ('name', (), {}),
1142 ('nice', (), {}),
1143 ('num_ctx_switches', (), {}),
1144 ('num_threads', (), {}),
1145 ('open_files', (), {}),
1146 ('ppid', (), {}),
1147 ('status', (), {}),
1148 ('threads', (), {}),
1149 ('username', (), {}),
1150 ]
1151 if POSIX:
1152 getters += [('uids', (), {})]
1153 getters += [('gids', (), {})]
1154 getters += [('terminal', (), {})]
1155 getters += [('num_fds', (), {})]
1156 if HAS_PROC_IO_COUNTERS:
1157 getters += [('io_counters', (), {})]
1158 if HAS_IONICE:
1159 getters += [('ionice', (), {})]
1160 if HAS_RLIMIT:
1161 getters += [('rlimit', (psutil.RLIMIT_NOFILE, ), {})]
1162 if HAS_CPU_AFFINITY:
1163 getters += [('cpu_affinity', (), {})]
1164 if HAS_PROC_CPU_NUM:
1165 getters += [('cpu_num', (), {})]
1166 if HAS_ENVIRON:
1167 getters += [('environ', (), {})]
1168 if WINDOWS:
1169 getters += [('num_handles', (), {})]
1170 if HAS_MEMORY_MAPS:
1171 getters += [('memory_maps', (), {'grouped': False})]
1172
1173 setters = []
1174 if POSIX:
1175 setters += [('nice', (0, ), {})]
1176 else:
1177 setters += [('nice', (psutil.NORMAL_PRIORITY_CLASS, ), {})]
1178 if HAS_RLIMIT:
1179 setters += [('rlimit', (psutil.RLIMIT_NOFILE, (1024, 4096)), {})]
1180 if HAS_IONICE:
1181 if LINUX:
1182 setters += [('ionice', (psutil.IOPRIO_CLASS_NONE, 0), {})]
1183 else:
1184 setters += [('ionice', (psutil.IOPRIO_NORMAL, ), {})]
1185 if HAS_CPU_AFFINITY:
1186 setters += [('cpu_affinity', ([_get_eligible_cpu()], ), {})]
1187
1188 killers = [
1189 ('send_signal', (signal.SIGTERM, ), {}),
1190 ('suspend', (), {}),
1191 ('resume', (), {}),
1192 ('terminate', (), {}),
1193 ('kill', (), {}),
1194 ]
1195 if WINDOWS:
1196 killers += [('send_signal', (signal.CTRL_C_EVENT, ), {})]
1197 killers += [('send_signal', (signal.CTRL_BREAK_EVENT, ), {})]
1198
1199 all = utils + getters + setters + killers
1200
1201 def __init__(self, proc):
1202 self._proc = proc
1203
1204 def iter(self, ls, clear_cache=True):
1205 """Given a list of tuples yields a set of (fun, fun_name) tuples
1206 in random order.
1207 """
1208 ls = list(ls)
1209 random.shuffle(ls)
1210 for fun_name, args, kwds in ls:
1211 if clear_cache:
1212 self.clear_cache()
1213 fun = getattr(self._proc, fun_name)
1214 fun = functools.partial(fun, *args, **kwds)
1215 yield (fun, fun_name)
1216
1217 def clear_cache(self):
1218 """Clear the cache of a Process instance."""
1219 self._proc._init(self._proc.pid, _ignore_nsp=True)
1220
1221 @classmethod
1222 def test_class_coverage(cls, test_class, ls):
1223 """Given a TestCase instance and a list of tuples checks that
1224 the class defines the required test method names.
1225 """
1226 for fun_name, _, _ in ls:
1227 meth_name = 'test_' + fun_name
1228 if not hasattr(test_class, meth_name):
1229 msg = "%r class should define a '%s' method" % (
1230 test_class.__class__.__name__, meth_name)
1231 raise AttributeError(msg)
1232
1233 @classmethod
1234 def test(cls):
1235 this = set([x[0] for x in cls.all])
1236 ignored = set([x[0] for x in cls.ignored])
1237 klass = set([x for x in dir(psutil.Process) if x[0] != '_'])
1238 leftout = (this | ignored) ^ klass
1239 if leftout:
1240 raise ValueError("uncovered Process class names: %r" % leftout)
1241
1242
1243 class system_namespace:
1244 """A container that lists all the module-level, system-related APIs.
1245 Utilities such as cpu_percent() are excluded. Usage:
1246
1247 >>> ns = system_namespace
1248 >>> for fun, name in ns.iter(ns.getters):
1249 ... fun()
1250 """
1251 getters = [
1252 ('boot_time', (), {}),
1253 ('cpu_count', (), {'logical': False}),
1254 ('cpu_count', (), {'logical': True}),
1255 ('cpu_stats', (), {}),
1256 ('cpu_times', (), {'percpu': False}),
1257 ('cpu_times', (), {'percpu': True}),
1258 ('disk_io_counters', (), {'perdisk': True}),
1259 ('disk_partitions', (), {'all': True}),
1260 ('disk_usage', (os.getcwd(), ), {}),
1261 ('net_connections', (), {'kind': 'all'}),
1262 ('net_if_addrs', (), {}),
1263 ('net_if_stats', (), {}),
1264 ('net_io_counters', (), {'pernic': True}),
1265 ('pid_exists', (os.getpid(), ), {}),
1266 ('pids', (), {}),
1267 ('swap_memory', (), {}),
1268 ('users', (), {}),
1269 ('virtual_memory', (), {}),
1270 ]
1271 if HAS_CPU_FREQ:
1272 getters += [('cpu_freq', (), {'percpu': True})]
1273 if HAS_GETLOADAVG:
1274 getters += [('getloadavg', (), {})]
1275 if HAS_SENSORS_TEMPERATURES:
1276 getters += [('sensors_temperatures', (), {})]
1277 if HAS_SENSORS_FANS:
1278 getters += [('sensors_fans', (), {})]
1279 if HAS_SENSORS_BATTERY:
1280 getters += [('sensors_battery', (), {})]
1281 if WINDOWS:
1282 getters += [('win_service_iter', (), {})]
1283 getters += [('win_service_get', ('alg', ), {})]
1284
1285 ignored = [
1286 ('process_iter', (), {}),
1287 ('wait_procs', ([psutil.Process()], ), {}),
1288 ('cpu_percent', (), {}),
1289 ('cpu_times_percent', (), {}),
1290 ]
1291
1292 all = getters
1293
1294 @staticmethod
1295 def iter(ls):
1296 """Given a list of tuples yields a set of (fun, fun_name) tuples
1297 in random order.
1298 """
1299 ls = list(ls)
1300 random.shuffle(ls)
1301 for fun_name, args, kwds in ls:
1302 fun = getattr(psutil, fun_name)
1303 fun = functools.partial(fun, *args, **kwds)
1304 yield (fun, fun_name)
1305
1306 test_class_coverage = process_namespace.test_class_coverage
1307
1308
1309 def serialrun(klass):
1310 """A decorator to mark a TestCase class. When running parallel tests,
1311 class' unit tests will be run serially (1 process).
1312 """
1313 # assert issubclass(klass, unittest.TestCase), klass
1314 assert inspect.isclass(klass), klass
1315 klass._serialrun = True
1316 return klass
1317
1318
1319 def retry_on_failure(retries=NO_RETRIES):
1320 """Decorator which runs a test function and retries N times before
1321 actually failing.
1322 """
1323 def logfun(exc):
1324 print("%r, retrying" % exc, file=sys.stderr) # NOQA
1325
1326 return retry(exception=AssertionError, timeout=None, retries=retries,
1327 logfun=logfun)
1328
1329
1330 def skip_on_access_denied(only_if=None):
1331 """Decorator to Ignore AccessDenied exceptions."""
1332 def decorator(fun):
1333 @functools.wraps(fun)
1334 def wrapper(*args, **kwargs):
1335 try:
1336 return fun(*args, **kwargs)
1337 except psutil.AccessDenied:
1338 if only_if is not None:
1339 if not only_if:
1340 raise
1341 raise unittest.SkipTest("raises AccessDenied")
1342 return wrapper
1343 return decorator
1344
1345
1346 def skip_on_not_implemented(only_if=None):
1347 """Decorator to Ignore NotImplementedError exceptions."""
1348 def decorator(fun):
1349 @functools.wraps(fun)
1350 def wrapper(*args, **kwargs):
1351 try:
1352 return fun(*args, **kwargs)
1353 except NotImplementedError:
1354 if only_if is not None:
1355 if not only_if:
1356 raise
1357 msg = "%r was skipped because it raised NotImplementedError" \
1358 % fun.__name__
1359 raise unittest.SkipTest(msg)
1360 return wrapper
1361 return decorator
1362
1363
1364 # ===================================================================
1365 # --- network
1366 # ===================================================================
1367
1368
1369 def get_free_port(host='127.0.0.1'):
1370 """Return an unused TCP port."""
1371 with contextlib.closing(socket.socket()) as sock:
1372 sock.bind((host, 0))
1373 return sock.getsockname()[1]
1374
1375
1376 def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None):
1377 """Binds a generic socket."""
1378 if addr is None and family in (AF_INET, AF_INET6):
1379 addr = ("", 0)
1380 sock = socket.socket(family, type)
1381 try:
1382 if os.name not in ('nt', 'cygwin'):
1383 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
1384 sock.bind(addr)
1385 if type == socket.SOCK_STREAM:
1386 sock.listen(5)
1387 return sock
1388 except Exception:
1389 sock.close()
1390 raise
1391
1392
1393 def bind_unix_socket(name, type=socket.SOCK_STREAM):
1394 """Bind a UNIX socket."""
1395 assert psutil.POSIX
1396 assert not os.path.exists(name), name
1397 sock = socket.socket(socket.AF_UNIX, type)
1398 try:
1399 sock.bind(name)
1400 if type == socket.SOCK_STREAM:
1401 sock.listen(5)
1402 except Exception:
1403 sock.close()
1404 raise
1405 return sock
1406
1407
1408 def tcp_socketpair(family, addr=("", 0)):
1409 """Build a pair of TCP sockets connected to each other.
1410 Return a (server, client) tuple.
1411 """
1412 with contextlib.closing(socket.socket(family, SOCK_STREAM)) as ll:
1413 ll.bind(addr)
1414 ll.listen(5)
1415 addr = ll.getsockname()
1416 c = socket.socket(family, SOCK_STREAM)
1417 try:
1418 c.connect(addr)
1419 caddr = c.getsockname()
1420 while True:
1421 a, addr = ll.accept()
1422 # check that we've got the correct client
1423 if addr == caddr:
1424 return (a, c)
1425 a.close()
1426 except OSError:
1427 c.close()
1428 raise
1429
1430
1431 def unix_socketpair(name):
1432 """Build a pair of UNIX sockets connected to each other through
1433 the same UNIX file name.
1434 Return a (server, client) tuple.
1435 """
1436 assert psutil.POSIX
1437 server = client = None
1438 try:
1439 server = bind_unix_socket(name, type=socket.SOCK_STREAM)
1440 server.setblocking(0)
1441 client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
1442 client.setblocking(0)
1443 client.connect(name)
1444 # new = server.accept()
1445 except Exception:
1446 if server is not None:
1447 server.close()
1448 if client is not None:
1449 client.close()
1450 raise
1451 return (server, client)
1452
1453
1454 @contextlib.contextmanager
1455 def create_sockets():
1456 """Open as many socket families / types as possible."""
1457 socks = []
1458 fname1 = fname2 = None
1459 try:
1460 socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM))
1461 socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM))
1462 if supports_ipv6():
1463 socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM))
1464 socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM))
1465 if POSIX and HAS_CONNECTIONS_UNIX:
1466 fname1 = get_testfn()
1467 fname2 = get_testfn()
1468 s1, s2 = unix_socketpair(fname1)
1469 s3 = bind_unix_socket(fname2, type=socket.SOCK_DGRAM)
1470 for s in (s1, s2, s3):
1471 socks.append(s)
1472 yield socks
1473 finally:
1474 for s in socks:
1475 s.close()
1476 for fname in (fname1, fname2):
1477 if fname is not None:
1478 safe_rmpath(fname)
1479
1480
1481 def check_net_address(addr, family):
1482 """Check a net address validity. Supported families are IPv4,
1483 IPv6 and MAC addresses.
1484 """
1485 import ipaddress # python >= 3.3 / requires "pip install ipaddress"
1486 if enum and PY3 and not PYPY:
1487 assert isinstance(family, enum.IntEnum), family
1488 if family == socket.AF_INET:
1489 octs = [int(x) for x in addr.split('.')]
1490 assert len(octs) == 4, addr
1491 for num in octs:
1492 assert 0 <= num <= 255, addr
1493 if not PY3:
1494 addr = unicode(addr)
1495 ipaddress.IPv4Address(addr)
1496 elif family == socket.AF_INET6:
1497 assert isinstance(addr, str), addr
1498 if not PY3:
1499 addr = unicode(addr)
1500 ipaddress.IPv6Address(addr)
1501 elif family == psutil.AF_LINK:
1502 assert re.match(r'([a-fA-F0-9]{2}[:|\-]?){6}', addr) is not None, addr
1503 else:
1504 raise ValueError("unknown family %r", family)
1505
1506
1507 # ===================================================================
1508 # --- compatibility
1509 # ===================================================================
1510
1511
1512 def reload_module(module):
1513 """Backport of importlib.reload of Python 3.3+."""
1514 try:
1515 import importlib
1516 if not hasattr(importlib, 'reload'): # python <=3.3
1517 raise ImportError
1518 except ImportError:
1519 import imp
1520 return imp.reload(module)
1521 else:
1522 return importlib.reload(module)
1523
1524
1525 def import_module_by_path(path):
1526 name = os.path.splitext(os.path.basename(path))[0]
1527 if sys.version_info[0] == 2:
1528 import imp
1529 return imp.load_source(name, path)
1530 elif sys.version_info[:2] <= (3, 4):
1531 from importlib.machinery import SourceFileLoader
1532 return SourceFileLoader(name, path).load_module()
1533 else:
1534 import importlib.util
1535 spec = importlib.util.spec_from_file_location(name, path)
1536 mod = importlib.util.module_from_spec(spec)
1537 spec.loader.exec_module(mod)
1538 return mod
1539
1540
1541 # ===================================================================
1542 # --- others
1543 # ===================================================================
1544
1545
1546 def warn(msg):
1547 """Raise a warning msg."""
1548 warnings.warn(msg, UserWarning)
1549
1550
1551 def is_namedtuple(x):
1552 """Check if object is an instance of namedtuple."""
1553 t = type(x)
1554 b = t.__bases__
1555 if len(b) != 1 or b[0] != tuple:
1556 return False
1557 f = getattr(t, '_fields', None)
1558 if not isinstance(f, tuple):
1559 return False
1560 return all(type(n) == str for n in f)
1561
1562
1563 if POSIX:
1564 @contextlib.contextmanager
1565 def copyload_shared_lib(suffix=""):
1566 """Ctx manager which picks up a random shared CO lib used
1567 by this process, copies it in another location and loads it
1568 in memory via ctypes. Return the new absolutized path.
1569 """
1570 exe = 'pypy' if PYPY else 'python'
1571 ext = ".so"
1572 dst = get_testfn(suffix=suffix + ext)
1573 libs = [x.path for x in psutil.Process().memory_maps() if
1574 os.path.splitext(x.path)[1] == ext and
1575 exe in x.path.lower()]
1576 src = random.choice(libs)
1577 shutil.copyfile(src, dst)
1578 try:
1579 ctypes.CDLL(dst)
1580 yield dst
1581 finally:
1582 safe_rmpath(dst)
1583 else:
1584 @contextlib.contextmanager
1585 def copyload_shared_lib(suffix=""):
1586 """Ctx manager which picks up a random shared DLL lib used
1587 by this process, copies it in another location and loads it
1588 in memory via ctypes.
1589 Return the new absolutized, normcased path.
1590 """
1591 from ctypes import wintypes
1592 from ctypes import WinError
1593 ext = ".dll"
1594 dst = get_testfn(suffix=suffix + ext)
1595 libs = [x.path for x in psutil.Process().memory_maps() if
1596 x.path.lower().endswith(ext) and
1597 'python' in os.path.basename(x.path).lower() and
1598 'wow64' not in x.path.lower()]
1599 if PYPY and not libs:
1600 libs = [x.path for x in psutil.Process().memory_maps() if
1601 'pypy' in os.path.basename(x.path).lower()]
1602 src = random.choice(libs)
1603 shutil.copyfile(src, dst)
1604 cfile = None
1605 try:
1606 cfile = ctypes.WinDLL(dst)
1607 yield dst
1608 finally:
1609 # Work around OverflowError:
1610 # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/
1611 # job/o53330pbnri9bcw7
1612 # - http://bugs.python.org/issue30286
1613 # - http://stackoverflow.com/questions/23522055
1614 if cfile is not None:
1615 FreeLibrary = ctypes.windll.kernel32.FreeLibrary
1616 FreeLibrary.argtypes = [wintypes.HMODULE]
1617 ret = FreeLibrary(cfile._handle)
1618 if ret == 0:
1619 WinError()
1620 safe_rmpath(dst)
1621
1622
1623 # ===================================================================
1624 # --- Exit funs (first is executed last)
1625 # ===================================================================
1626
1627
1628 # this is executed first
1629 @atexit.register
1630 def cleanup_test_procs():
1631 reap_children(recursive=True)
1632
1633
1634 # atexit module does not execute exit functions in case of SIGTERM, which
1635 # gets sent to test subprocesses, which is a problem if they import this
1636 # module. With this it will. See:
1637 # https://gmpy.dev/blog/2016/how-to-always-execute-exit-functions-in-python
1638 if POSIX:
1639 signal.signal(signal.SIGTERM, lambda sig, frame: sys.exit(sig))