blob: f1d829a6f1724ef0eafd8240f2e57bca9992261c [file] [log] [blame]
Olivier Deprezf4ef2d02021-04-20 13:36:24 +02001# subprocess - Subprocesses with accessible I/O streams
2#
3# For more information about this module, see PEP 324.
4#
5# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
6#
7# Licensed to PSF under a Contributor Agreement.
8# See http://www.python.org/2.4/license for licensing details.
9
10r"""Subprocesses with accessible I/O streams
11
12This module allows you to spawn processes, connect to their
13input/output/error pipes, and obtain their return codes.
14
15For a complete description of this module see the Python documentation.
16
17Main API
18========
19run(...): Runs a command, waits for it to complete, then returns a
20 CompletedProcess instance.
21Popen(...): A class for flexibly executing a command in a new process
22
23Constants
24---------
25DEVNULL: Special value that indicates that os.devnull should be used
26PIPE: Special value that indicates a pipe should be created
27STDOUT: Special value that indicates that stderr should go to stdout
28
29
30Older API
31=========
32call(...): Runs a command, waits for it to complete, then returns
33 the return code.
34check_call(...): Same as call() but raises CalledProcessError()
35 if return code is not 0
36check_output(...): Same as check_call() but returns the contents of
37 stdout instead of a return code
38getoutput(...): Runs a command in the shell, waits for it to complete,
39 then returns the output
40getstatusoutput(...): Runs a command in the shell, waits for it to complete,
41 then returns a (exitcode, output) tuple
42"""
43
44import builtins
45import errno
46import io
47import os
48import time
49import signal
50import sys
51import threading
52import warnings
53import contextlib
54from time import monotonic as _time
55import types
56
57try:
58 import pwd
59except ImportError:
60 pwd = None
61try:
62 import grp
63except ImportError:
64 grp = None
65
66__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
67 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
68 "SubprocessError", "TimeoutExpired", "CompletedProcess"]
69 # NOTE: We intentionally exclude list2cmdline as it is
70 # considered an internal implementation detail. issue10838.
71
72try:
73 import msvcrt
74 import _winapi
75 _mswindows = True
76except ModuleNotFoundError:
77 _mswindows = False
78 import _posixsubprocess
79 import select
80 import selectors
81else:
82 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
83 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
84 STD_ERROR_HANDLE, SW_HIDE,
85 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW,
86 ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS,
87 HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS,
88 NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS,
89 CREATE_NO_WINDOW, DETACHED_PROCESS,
90 CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB)
91
92 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
93 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
94 "STD_ERROR_HANDLE", "SW_HIDE",
95 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW",
96 "STARTUPINFO",
97 "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS",
98 "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS",
99 "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS",
100 "CREATE_NO_WINDOW", "DETACHED_PROCESS",
101 "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"])
102
103
104# Exception classes used by this module.
105class SubprocessError(Exception): pass
106
107
108class CalledProcessError(SubprocessError):
109 """Raised when run() is called with check=True and the process
110 returns a non-zero exit status.
111
112 Attributes:
113 cmd, returncode, stdout, stderr, output
114 """
115 def __init__(self, returncode, cmd, output=None, stderr=None):
116 self.returncode = returncode
117 self.cmd = cmd
118 self.output = output
119 self.stderr = stderr
120
121 def __str__(self):
122 if self.returncode and self.returncode < 0:
123 try:
124 return "Command '%s' died with %r." % (
125 self.cmd, signal.Signals(-self.returncode))
126 except ValueError:
127 return "Command '%s' died with unknown signal %d." % (
128 self.cmd, -self.returncode)
129 else:
130 return "Command '%s' returned non-zero exit status %d." % (
131 self.cmd, self.returncode)
132
133 @property
134 def stdout(self):
135 """Alias for output attribute, to match stderr"""
136 return self.output
137
138 @stdout.setter
139 def stdout(self, value):
140 # There's no obvious reason to set this, but allow it anyway so
141 # .stdout is a transparent alias for .output
142 self.output = value
143
144
145class TimeoutExpired(SubprocessError):
146 """This exception is raised when the timeout expires while waiting for a
147 child process.
148
149 Attributes:
150 cmd, output, stdout, stderr, timeout
151 """
152 def __init__(self, cmd, timeout, output=None, stderr=None):
153 self.cmd = cmd
154 self.timeout = timeout
155 self.output = output
156 self.stderr = stderr
157
158 def __str__(self):
159 return ("Command '%s' timed out after %s seconds" %
160 (self.cmd, self.timeout))
161
162 @property
163 def stdout(self):
164 return self.output
165
166 @stdout.setter
167 def stdout(self, value):
168 # There's no obvious reason to set this, but allow it anyway so
169 # .stdout is a transparent alias for .output
170 self.output = value
171
172
173if _mswindows:
174 class STARTUPINFO:
175 def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None,
176 hStdError=None, wShowWindow=0, lpAttributeList=None):
177 self.dwFlags = dwFlags
178 self.hStdInput = hStdInput
179 self.hStdOutput = hStdOutput
180 self.hStdError = hStdError
181 self.wShowWindow = wShowWindow
182 self.lpAttributeList = lpAttributeList or {"handle_list": []}
183
184 def copy(self):
185 attr_list = self.lpAttributeList.copy()
186 if 'handle_list' in attr_list:
187 attr_list['handle_list'] = list(attr_list['handle_list'])
188
189 return STARTUPINFO(dwFlags=self.dwFlags,
190 hStdInput=self.hStdInput,
191 hStdOutput=self.hStdOutput,
192 hStdError=self.hStdError,
193 wShowWindow=self.wShowWindow,
194 lpAttributeList=attr_list)
195
196
197 class Handle(int):
198 closed = False
199
200 def Close(self, CloseHandle=_winapi.CloseHandle):
201 if not self.closed:
202 self.closed = True
203 CloseHandle(self)
204
205 def Detach(self):
206 if not self.closed:
207 self.closed = True
208 return int(self)
209 raise ValueError("already closed")
210
211 def __repr__(self):
212 return "%s(%d)" % (self.__class__.__name__, int(self))
213
214 __del__ = Close
215else:
216 # When select or poll has indicated that the file is writable,
217 # we can write up to _PIPE_BUF bytes without risk of blocking.
218 # POSIX defines PIPE_BUF as >= 512.
219 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
220
221 # poll/select have the advantage of not requiring any extra file
222 # descriptor, contrarily to epoll/kqueue (also, they require a single
223 # syscall).
224 if hasattr(selectors, 'PollSelector'):
225 _PopenSelector = selectors.PollSelector
226 else:
227 _PopenSelector = selectors.SelectSelector
228
229
230if _mswindows:
231 # On Windows we just need to close `Popen._handle` when we no longer need
232 # it, so that the kernel can free it. `Popen._handle` gets closed
233 # implicitly when the `Popen` instance is finalized (see `Handle.__del__`,
234 # which is calling `CloseHandle` as requested in [1]), so there is nothing
235 # for `_cleanup` to do.
236 #
237 # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/
238 # creating-processes
239 _active = None
240
241 def _cleanup():
242 pass
243else:
244 # This lists holds Popen instances for which the underlying process had not
245 # exited at the time its __del__ method got called: those processes are
246 # wait()ed for synchronously from _cleanup() when a new Popen object is
247 # created, to avoid zombie processes.
248 _active = []
249
250 def _cleanup():
251 if _active is None:
252 return
253 for inst in _active[:]:
254 res = inst._internal_poll(_deadstate=sys.maxsize)
255 if res is not None:
256 try:
257 _active.remove(inst)
258 except ValueError:
259 # This can happen if two threads create a new Popen instance.
260 # It's harmless that it was already removed, so ignore.
261 pass
262
263PIPE = -1
264STDOUT = -2
265DEVNULL = -3
266
267
268# XXX This function is only used by multiprocessing and the test suite,
269# but it's here so that it can be imported when Python is compiled without
270# threads.
271
272def _optim_args_from_interpreter_flags():
273 """Return a list of command-line arguments reproducing the current
274 optimization settings in sys.flags."""
275 args = []
276 value = sys.flags.optimize
277 if value > 0:
278 args.append('-' + 'O' * value)
279 return args
280
281
282def _args_from_interpreter_flags():
283 """Return a list of command-line arguments reproducing the current
284 settings in sys.flags, sys.warnoptions and sys._xoptions."""
285 flag_opt_map = {
286 'debug': 'd',
287 # 'inspect': 'i',
288 # 'interactive': 'i',
289 'dont_write_bytecode': 'B',
290 'no_site': 'S',
291 'verbose': 'v',
292 'bytes_warning': 'b',
293 'quiet': 'q',
294 # -O is handled in _optim_args_from_interpreter_flags()
295 }
296 args = _optim_args_from_interpreter_flags()
297 for flag, opt in flag_opt_map.items():
298 v = getattr(sys.flags, flag)
299 if v > 0:
300 args.append('-' + opt * v)
301
302 if sys.flags.isolated:
303 args.append('-I')
304 else:
305 if sys.flags.ignore_environment:
306 args.append('-E')
307 if sys.flags.no_user_site:
308 args.append('-s')
309
310 # -W options
311 warnopts = sys.warnoptions[:]
312 bytes_warning = sys.flags.bytes_warning
313 xoptions = getattr(sys, '_xoptions', {})
314 dev_mode = ('dev' in xoptions)
315
316 if bytes_warning > 1:
317 warnopts.remove("error::BytesWarning")
318 elif bytes_warning:
319 warnopts.remove("default::BytesWarning")
320 if dev_mode:
321 warnopts.remove('default')
322 for opt in warnopts:
323 args.append('-W' + opt)
324
325 # -X options
326 if dev_mode:
327 args.extend(('-X', 'dev'))
328 for opt in ('faulthandler', 'tracemalloc', 'importtime',
329 'showrefcount', 'utf8', 'oldparser'):
330 if opt in xoptions:
331 value = xoptions[opt]
332 if value is True:
333 arg = opt
334 else:
335 arg = '%s=%s' % (opt, value)
336 args.extend(('-X', arg))
337
338 return args
339
340
341def call(*popenargs, timeout=None, **kwargs):
342 """Run command with arguments. Wait for command to complete or
343 timeout, then return the returncode attribute.
344
345 The arguments are the same as for the Popen constructor. Example:
346
347 retcode = call(["ls", "-l"])
348 """
349 with Popen(*popenargs, **kwargs) as p:
350 try:
351 return p.wait(timeout=timeout)
352 except: # Including KeyboardInterrupt, wait handled that.
353 p.kill()
354 # We don't call p.wait() again as p.__exit__ does that for us.
355 raise
356
357
358def check_call(*popenargs, **kwargs):
359 """Run command with arguments. Wait for command to complete. If
360 the exit code was zero then return, otherwise raise
361 CalledProcessError. The CalledProcessError object will have the
362 return code in the returncode attribute.
363
364 The arguments are the same as for the call function. Example:
365
366 check_call(["ls", "-l"])
367 """
368 retcode = call(*popenargs, **kwargs)
369 if retcode:
370 cmd = kwargs.get("args")
371 if cmd is None:
372 cmd = popenargs[0]
373 raise CalledProcessError(retcode, cmd)
374 return 0
375
376
377def check_output(*popenargs, timeout=None, **kwargs):
378 r"""Run command with arguments and return its output.
379
380 If the exit code was non-zero it raises a CalledProcessError. The
381 CalledProcessError object will have the return code in the returncode
382 attribute and output in the output attribute.
383
384 The arguments are the same as for the Popen constructor. Example:
385
386 >>> check_output(["ls", "-l", "/dev/null"])
387 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
388
389 The stdout argument is not allowed as it is used internally.
390 To capture standard error in the result, use stderr=STDOUT.
391
392 >>> check_output(["/bin/sh", "-c",
393 ... "ls -l non_existent_file ; exit 0"],
394 ... stderr=STDOUT)
395 b'ls: non_existent_file: No such file or directory\n'
396
397 There is an additional optional argument, "input", allowing you to
398 pass a string to the subprocess's stdin. If you use this argument
399 you may not also use the Popen constructor's "stdin" argument, as
400 it too will be used internally. Example:
401
402 >>> check_output(["sed", "-e", "s/foo/bar/"],
403 ... input=b"when in the course of fooman events\n")
404 b'when in the course of barman events\n'
405
406 By default, all communication is in bytes, and therefore any "input"
407 should be bytes, and the return value will be bytes. If in text mode,
408 any "input" should be a string, and the return value will be a string
409 decoded according to locale encoding, or by "encoding" if set. Text mode
410 is triggered by setting any of text, encoding, errors or universal_newlines.
411 """
412 if 'stdout' in kwargs:
413 raise ValueError('stdout argument not allowed, it will be overridden.')
414
415 if 'input' in kwargs and kwargs['input'] is None:
416 # Explicitly passing input=None was previously equivalent to passing an
417 # empty string. That is maintained here for backwards compatibility.
418 kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
419
420 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
421 **kwargs).stdout
422
423
424class CompletedProcess(object):
425 """A process that has finished running.
426
427 This is returned by run().
428
429 Attributes:
430 args: The list or str args passed to run().
431 returncode: The exit code of the process, negative for signals.
432 stdout: The standard output (None if not captured).
433 stderr: The standard error (None if not captured).
434 """
435 def __init__(self, args, returncode, stdout=None, stderr=None):
436 self.args = args
437 self.returncode = returncode
438 self.stdout = stdout
439 self.stderr = stderr
440
441 def __repr__(self):
442 args = ['args={!r}'.format(self.args),
443 'returncode={!r}'.format(self.returncode)]
444 if self.stdout is not None:
445 args.append('stdout={!r}'.format(self.stdout))
446 if self.stderr is not None:
447 args.append('stderr={!r}'.format(self.stderr))
448 return "{}({})".format(type(self).__name__, ', '.join(args))
449
450 __class_getitem__ = classmethod(types.GenericAlias)
451
452
453 def check_returncode(self):
454 """Raise CalledProcessError if the exit code is non-zero."""
455 if self.returncode:
456 raise CalledProcessError(self.returncode, self.args, self.stdout,
457 self.stderr)
458
459
460def run(*popenargs,
461 input=None, capture_output=False, timeout=None, check=False, **kwargs):
462 """Run command with arguments and return a CompletedProcess instance.
463
464 The returned instance will have attributes args, returncode, stdout and
465 stderr. By default, stdout and stderr are not captured, and those attributes
466 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
467
468 If check is True and the exit code was non-zero, it raises a
469 CalledProcessError. The CalledProcessError object will have the return code
470 in the returncode attribute, and output & stderr attributes if those streams
471 were captured.
472
473 If timeout is given, and the process takes too long, a TimeoutExpired
474 exception will be raised.
475
476 There is an optional argument "input", allowing you to
477 pass bytes or a string to the subprocess's stdin. If you use this argument
478 you may not also use the Popen constructor's "stdin" argument, as
479 it will be used internally.
480
481 By default, all communication is in bytes, and therefore any "input" should
482 be bytes, and the stdout and stderr will be bytes. If in text mode, any
483 "input" should be a string, and stdout and stderr will be strings decoded
484 according to locale encoding, or by "encoding" if set. Text mode is
485 triggered by setting any of text, encoding, errors or universal_newlines.
486
487 The other arguments are the same as for the Popen constructor.
488 """
489 if input is not None:
490 if kwargs.get('stdin') is not None:
491 raise ValueError('stdin and input arguments may not both be used.')
492 kwargs['stdin'] = PIPE
493
494 if capture_output:
495 if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None:
496 raise ValueError('stdout and stderr arguments may not be used '
497 'with capture_output.')
498 kwargs['stdout'] = PIPE
499 kwargs['stderr'] = PIPE
500
501 with Popen(*popenargs, **kwargs) as process:
502 try:
503 stdout, stderr = process.communicate(input, timeout=timeout)
504 except TimeoutExpired as exc:
505 process.kill()
506 if _mswindows:
507 # Windows accumulates the output in a single blocking
508 # read() call run on child threads, with the timeout
509 # being done in a join() on those threads. communicate()
510 # _after_ kill() is required to collect that and add it
511 # to the exception.
512 exc.stdout, exc.stderr = process.communicate()
513 else:
514 # POSIX _communicate already populated the output so
515 # far into the TimeoutExpired exception.
516 process.wait()
517 raise
518 except: # Including KeyboardInterrupt, communicate handled that.
519 process.kill()
520 # We don't call process.wait() as .__exit__ does that for us.
521 raise
522 retcode = process.poll()
523 if check and retcode:
524 raise CalledProcessError(retcode, process.args,
525 output=stdout, stderr=stderr)
526 return CompletedProcess(process.args, retcode, stdout, stderr)
527
528
529def list2cmdline(seq):
530 """
531 Translate a sequence of arguments into a command line
532 string, using the same rules as the MS C runtime:
533
534 1) Arguments are delimited by white space, which is either a
535 space or a tab.
536
537 2) A string surrounded by double quotation marks is
538 interpreted as a single argument, regardless of white space
539 contained within. A quoted string can be embedded in an
540 argument.
541
542 3) A double quotation mark preceded by a backslash is
543 interpreted as a literal double quotation mark.
544
545 4) Backslashes are interpreted literally, unless they
546 immediately precede a double quotation mark.
547
548 5) If backslashes immediately precede a double quotation mark,
549 every pair of backslashes is interpreted as a literal
550 backslash. If the number of backslashes is odd, the last
551 backslash escapes the next double quotation mark as
552 described in rule 3.
553 """
554
555 # See
556 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
557 # or search http://msdn.microsoft.com for
558 # "Parsing C++ Command-Line Arguments"
559 result = []
560 needquote = False
561 for arg in map(os.fsdecode, seq):
562 bs_buf = []
563
564 # Add a space to separate this argument from the others
565 if result:
566 result.append(' ')
567
568 needquote = (" " in arg) or ("\t" in arg) or not arg
569 if needquote:
570 result.append('"')
571
572 for c in arg:
573 if c == '\\':
574 # Don't know if we need to double yet.
575 bs_buf.append(c)
576 elif c == '"':
577 # Double backslashes.
578 result.append('\\' * len(bs_buf)*2)
579 bs_buf = []
580 result.append('\\"')
581 else:
582 # Normal char
583 if bs_buf:
584 result.extend(bs_buf)
585 bs_buf = []
586 result.append(c)
587
588 # Add remaining backslashes, if any.
589 if bs_buf:
590 result.extend(bs_buf)
591
592 if needquote:
593 result.extend(bs_buf)
594 result.append('"')
595
596 return ''.join(result)
597
598
599# Various tools for executing commands and looking at their output and status.
600#
601
602def getstatusoutput(cmd):
603 """Return (exitcode, output) of executing cmd in a shell.
604
605 Execute the string 'cmd' in a shell with 'check_output' and
606 return a 2-tuple (status, output). The locale encoding is used
607 to decode the output and process newlines.
608
609 A trailing newline is stripped from the output.
610 The exit status for the command can be interpreted
611 according to the rules for the function 'wait'. Example:
612
613 >>> import subprocess
614 >>> subprocess.getstatusoutput('ls /bin/ls')
615 (0, '/bin/ls')
616 >>> subprocess.getstatusoutput('cat /bin/junk')
617 (1, 'cat: /bin/junk: No such file or directory')
618 >>> subprocess.getstatusoutput('/bin/junk')
619 (127, 'sh: /bin/junk: not found')
620 >>> subprocess.getstatusoutput('/bin/kill $$')
621 (-15, '')
622 """
623 try:
624 data = check_output(cmd, shell=True, text=True, stderr=STDOUT)
625 exitcode = 0
626 except CalledProcessError as ex:
627 data = ex.output
628 exitcode = ex.returncode
629 if data[-1:] == '\n':
630 data = data[:-1]
631 return exitcode, data
632
633def getoutput(cmd):
634 """Return output (stdout or stderr) of executing cmd in a shell.
635
636 Like getstatusoutput(), except the exit status is ignored and the return
637 value is a string containing the command's output. Example:
638
639 >>> import subprocess
640 >>> subprocess.getoutput('ls /bin/ls')
641 '/bin/ls'
642 """
643 return getstatusoutput(cmd)[1]
644
645
646def _use_posix_spawn():
647 """Check if posix_spawn() can be used for subprocess.
648
649 subprocess requires a posix_spawn() implementation that properly reports
650 errors to the parent process, & sets errno on the following failures:
651
652 * Process attribute actions failed.
653 * File actions failed.
654 * exec() failed.
655
656 Prefer an implementation which can use vfork() in some cases for best
657 performance.
658 """
659 if _mswindows or not hasattr(os, 'posix_spawn'):
660 # os.posix_spawn() is not available
661 return False
662
663 if sys.platform == 'darwin':
664 # posix_spawn() is a syscall on macOS and properly reports errors
665 return True
666
667 # Check libc name and runtime libc version
668 try:
669 ver = os.confstr('CS_GNU_LIBC_VERSION')
670 # parse 'glibc 2.28' as ('glibc', (2, 28))
671 parts = ver.split(maxsplit=1)
672 if len(parts) != 2:
673 # reject unknown format
674 raise ValueError
675 libc = parts[0]
676 version = tuple(map(int, parts[1].split('.')))
677
678 if sys.platform == 'linux' and libc == 'glibc' and version >= (2, 24):
679 # glibc 2.24 has a new Linux posix_spawn implementation using vfork
680 # which properly reports errors to the parent process.
681 return True
682 # Note: Don't use the implementation in earlier glibc because it doesn't
683 # use vfork (even if glibc 2.26 added a pipe to properly report errors
684 # to the parent process).
685 except (AttributeError, ValueError, OSError):
686 # os.confstr() or CS_GNU_LIBC_VERSION value not available
687 pass
688
689 # By default, assume that posix_spawn() does not properly report errors.
690 return False
691
692
693_USE_POSIX_SPAWN = _use_posix_spawn()
694
695
696class Popen(object):
697 """ Execute a child program in a new process.
698
699 For a complete description of the arguments see the Python documentation.
700
701 Arguments:
702 args: A string, or a sequence of program arguments.
703
704 bufsize: supplied as the buffering argument to the open() function when
705 creating the stdin/stdout/stderr pipe file objects
706
707 executable: A replacement program to execute.
708
709 stdin, stdout and stderr: These specify the executed programs' standard
710 input, standard output and standard error file handles, respectively.
711
712 preexec_fn: (POSIX only) An object to be called in the child process
713 just before the child is executed.
714
715 close_fds: Controls closing or inheriting of file descriptors.
716
717 shell: If true, the command will be executed through the shell.
718
719 cwd: Sets the current directory before the child is executed.
720
721 env: Defines the environment variables for the new process.
722
723 text: If true, decode stdin, stdout and stderr using the given encoding
724 (if set) or the system default otherwise.
725
726 universal_newlines: Alias of text, provided for backwards compatibility.
727
728 startupinfo and creationflags (Windows only)
729
730 restore_signals (POSIX only)
731
732 start_new_session (POSIX only)
733
734 group (POSIX only)
735
736 extra_groups (POSIX only)
737
738 user (POSIX only)
739
740 umask (POSIX only)
741
742 pass_fds (POSIX only)
743
744 encoding and errors: Text mode encoding and error handling to use for
745 file objects stdin, stdout and stderr.
746
747 Attributes:
748 stdin, stdout, stderr, pid, returncode
749 """
750 _child_created = False # Set here since __del__ checks it
751
752 def __init__(self, args, bufsize=-1, executable=None,
753 stdin=None, stdout=None, stderr=None,
754 preexec_fn=None, close_fds=True,
755 shell=False, cwd=None, env=None, universal_newlines=None,
756 startupinfo=None, creationflags=0,
757 restore_signals=True, start_new_session=False,
758 pass_fds=(), *, user=None, group=None, extra_groups=None,
759 encoding=None, errors=None, text=None, umask=-1):
760 """Create new Popen instance."""
761 _cleanup()
762 # Held while anything is calling waitpid before returncode has been
763 # updated to prevent clobbering returncode if wait() or poll() are
764 # called from multiple threads at once. After acquiring the lock,
765 # code must re-check self.returncode to see if another thread just
766 # finished a waitpid() call.
767 self._waitpid_lock = threading.Lock()
768
769 self._input = None
770 self._communication_started = False
771 if bufsize is None:
772 bufsize = -1 # Restore default
773 if not isinstance(bufsize, int):
774 raise TypeError("bufsize must be an integer")
775
776 if _mswindows:
777 if preexec_fn is not None:
778 raise ValueError("preexec_fn is not supported on Windows "
779 "platforms")
780 else:
781 # POSIX
782 if pass_fds and not close_fds:
783 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
784 close_fds = True
785 if startupinfo is not None:
786 raise ValueError("startupinfo is only supported on Windows "
787 "platforms")
788 if creationflags != 0:
789 raise ValueError("creationflags is only supported on Windows "
790 "platforms")
791
792 self.args = args
793 self.stdin = None
794 self.stdout = None
795 self.stderr = None
796 self.pid = None
797 self.returncode = None
798 self.encoding = encoding
799 self.errors = errors
800
801 # Validate the combinations of text and universal_newlines
802 if (text is not None and universal_newlines is not None
803 and bool(universal_newlines) != bool(text)):
804 raise SubprocessError('Cannot disambiguate when both text '
805 'and universal_newlines are supplied but '
806 'different. Pass one or the other.')
807
808 # Input and output objects. The general principle is like
809 # this:
810 #
811 # Parent Child
812 # ------ -----
813 # p2cwrite ---stdin---> p2cread
814 # c2pread <--stdout--- c2pwrite
815 # errread <--stderr--- errwrite
816 #
817 # On POSIX, the child objects are file descriptors. On
818 # Windows, these are Windows file handles. The parent objects
819 # are file descriptors on both platforms. The parent objects
820 # are -1 when not using PIPEs. The child objects are -1
821 # when not redirecting.
822
823 (p2cread, p2cwrite,
824 c2pread, c2pwrite,
825 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
826
827 # We wrap OS handles *before* launching the child, otherwise a
828 # quickly terminating child could make our fds unwrappable
829 # (see #8458).
830
831 if _mswindows:
832 if p2cwrite != -1:
833 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
834 if c2pread != -1:
835 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
836 if errread != -1:
837 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
838
839 self.text_mode = encoding or errors or text or universal_newlines
840
841 # How long to resume waiting on a child after the first ^C.
842 # There is no right value for this. The purpose is to be polite
843 # yet remain good for interactive users trying to exit a tool.
844 self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber()
845
846 self._closed_child_pipe_fds = False
847
848 if self.text_mode:
849 if bufsize == 1:
850 line_buffering = True
851 # Use the default buffer size for the underlying binary streams
852 # since they don't support line buffering.
853 bufsize = -1
854 else:
855 line_buffering = False
856
857 gid = None
858 if group is not None:
859 if not hasattr(os, 'setregid'):
860 raise ValueError("The 'group' parameter is not supported on the "
861 "current platform")
862
863 elif isinstance(group, str):
864 if grp is None:
865 raise ValueError("The group parameter cannot be a string "
866 "on systems without the grp module")
867
868 gid = grp.getgrnam(group).gr_gid
869 elif isinstance(group, int):
870 gid = group
871 else:
872 raise TypeError("Group must be a string or an integer, not {}"
873 .format(type(group)))
874
875 if gid < 0:
876 raise ValueError(f"Group ID cannot be negative, got {gid}")
877
878 gids = None
879 if extra_groups is not None:
880 if not hasattr(os, 'setgroups'):
881 raise ValueError("The 'extra_groups' parameter is not "
882 "supported on the current platform")
883
884 elif isinstance(extra_groups, str):
885 raise ValueError("Groups must be a list, not a string")
886
887 gids = []
888 for extra_group in extra_groups:
889 if isinstance(extra_group, str):
890 if grp is None:
891 raise ValueError("Items in extra_groups cannot be "
892 "strings on systems without the "
893 "grp module")
894
895 gids.append(grp.getgrnam(extra_group).gr_gid)
896 elif isinstance(extra_group, int):
897 gids.append(extra_group)
898 else:
899 raise TypeError("Items in extra_groups must be a string "
900 "or integer, not {}"
901 .format(type(extra_group)))
902
903 # make sure that the gids are all positive here so we can do less
904 # checking in the C code
905 for gid_check in gids:
906 if gid_check < 0:
907 raise ValueError(f"Group ID cannot be negative, got {gid_check}")
908
909 uid = None
910 if user is not None:
911 if not hasattr(os, 'setreuid'):
912 raise ValueError("The 'user' parameter is not supported on "
913 "the current platform")
914
915 elif isinstance(user, str):
916 if pwd is None:
917 raise ValueError("The user parameter cannot be a string "
918 "on systems without the pwd module")
919
920 uid = pwd.getpwnam(user).pw_uid
921 elif isinstance(user, int):
922 uid = user
923 else:
924 raise TypeError("User must be a string or an integer")
925
926 if uid < 0:
927 raise ValueError(f"User ID cannot be negative, got {uid}")
928
929 try:
930 if p2cwrite != -1:
931 self.stdin = io.open(p2cwrite, 'wb', bufsize)
932 if self.text_mode:
933 self.stdin = io.TextIOWrapper(self.stdin, write_through=True,
934 line_buffering=line_buffering,
935 encoding=encoding, errors=errors)
936 if c2pread != -1:
937 self.stdout = io.open(c2pread, 'rb', bufsize)
938 if self.text_mode:
939 self.stdout = io.TextIOWrapper(self.stdout,
940 encoding=encoding, errors=errors)
941 if errread != -1:
942 self.stderr = io.open(errread, 'rb', bufsize)
943 if self.text_mode:
944 self.stderr = io.TextIOWrapper(self.stderr,
945 encoding=encoding, errors=errors)
946
947 self._execute_child(args, executable, preexec_fn, close_fds,
948 pass_fds, cwd, env,
949 startupinfo, creationflags, shell,
950 p2cread, p2cwrite,
951 c2pread, c2pwrite,
952 errread, errwrite,
953 restore_signals,
954 gid, gids, uid, umask,
955 start_new_session)
956 except:
957 # Cleanup if the child failed starting.
958 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
959 try:
960 f.close()
961 except OSError:
962 pass # Ignore EBADF or other errors.
963
964 if not self._closed_child_pipe_fds:
965 to_close = []
966 if stdin == PIPE:
967 to_close.append(p2cread)
968 if stdout == PIPE:
969 to_close.append(c2pwrite)
970 if stderr == PIPE:
971 to_close.append(errwrite)
972 if hasattr(self, '_devnull'):
973 to_close.append(self._devnull)
974 for fd in to_close:
975 try:
976 if _mswindows and isinstance(fd, Handle):
977 fd.Close()
978 else:
979 os.close(fd)
980 except OSError:
981 pass
982
983 raise
984
985 def __repr__(self):
986 obj_repr = (
987 f"<{self.__class__.__name__}: "
988 f"returncode: {self.returncode} args: {list(self.args)!r}>"
989 )
990 if len(obj_repr) > 80:
991 obj_repr = obj_repr[:76] + "...>"
992 return obj_repr
993
994 __class_getitem__ = classmethod(types.GenericAlias)
995
996 @property
997 def universal_newlines(self):
998 # universal_newlines as retained as an alias of text_mode for API
999 # compatibility. bpo-31756
1000 return self.text_mode
1001
1002 @universal_newlines.setter
1003 def universal_newlines(self, universal_newlines):
1004 self.text_mode = bool(universal_newlines)
1005
1006 def _translate_newlines(self, data, encoding, errors):
1007 data = data.decode(encoding, errors)
1008 return data.replace("\r\n", "\n").replace("\r", "\n")
1009
1010 def __enter__(self):
1011 return self
1012
1013 def __exit__(self, exc_type, value, traceback):
1014 if self.stdout:
1015 self.stdout.close()
1016 if self.stderr:
1017 self.stderr.close()
1018 try: # Flushing a BufferedWriter may raise an error
1019 if self.stdin:
1020 self.stdin.close()
1021 finally:
1022 if exc_type == KeyboardInterrupt:
1023 # https://bugs.python.org/issue25942
1024 # In the case of a KeyboardInterrupt we assume the SIGINT
1025 # was also already sent to our child processes. We can't
1026 # block indefinitely as that is not user friendly.
1027 # If we have not already waited a brief amount of time in
1028 # an interrupted .wait() or .communicate() call, do so here
1029 # for consistency.
1030 if self._sigint_wait_secs > 0:
1031 try:
1032 self._wait(timeout=self._sigint_wait_secs)
1033 except TimeoutExpired:
1034 pass
1035 self._sigint_wait_secs = 0 # Note that this has been done.
1036 return # resume the KeyboardInterrupt
1037
1038 # Wait for the process to terminate, to avoid zombies.
1039 self.wait()
1040
1041 def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn):
1042 if not self._child_created:
1043 # We didn't get to successfully create a child process.
1044 return
1045 if self.returncode is None:
1046 # Not reading subprocess exit status creates a zombie process which
1047 # is only destroyed at the parent python process exit
1048 _warn("subprocess %s is still running" % self.pid,
1049 ResourceWarning, source=self)
1050 # In case the child hasn't been waited on, check if it's done.
1051 self._internal_poll(_deadstate=_maxsize)
1052 if self.returncode is None and _active is not None:
1053 # Child is still running, keep us alive until we can wait on it.
1054 _active.append(self)
1055
1056 def _get_devnull(self):
1057 if not hasattr(self, '_devnull'):
1058 self._devnull = os.open(os.devnull, os.O_RDWR)
1059 return self._devnull
1060
1061 def _stdin_write(self, input):
1062 if input:
1063 try:
1064 self.stdin.write(input)
1065 except BrokenPipeError:
1066 pass # communicate() must ignore broken pipe errors.
1067 except OSError as exc:
1068 if exc.errno == errno.EINVAL:
1069 # bpo-19612, bpo-30418: On Windows, stdin.write() fails
1070 # with EINVAL if the child process exited or if the child
1071 # process is still running but closed the pipe.
1072 pass
1073 else:
1074 raise
1075
1076 try:
1077 self.stdin.close()
1078 except BrokenPipeError:
1079 pass # communicate() must ignore broken pipe errors.
1080 except OSError as exc:
1081 if exc.errno == errno.EINVAL:
1082 pass
1083 else:
1084 raise
1085
1086 def communicate(self, input=None, timeout=None):
1087 """Interact with process: Send data to stdin and close it.
1088 Read data from stdout and stderr, until end-of-file is
1089 reached. Wait for process to terminate.
1090
1091 The optional "input" argument should be data to be sent to the
1092 child process, or None, if no data should be sent to the child.
1093 communicate() returns a tuple (stdout, stderr).
1094
1095 By default, all communication is in bytes, and therefore any
1096 "input" should be bytes, and the (stdout, stderr) will be bytes.
1097 If in text mode (indicated by self.text_mode), any "input" should
1098 be a string, and (stdout, stderr) will be strings decoded
1099 according to locale encoding, or by "encoding" if set. Text mode
1100 is triggered by setting any of text, encoding, errors or
1101 universal_newlines.
1102 """
1103
1104 if self._communication_started and input:
1105 raise ValueError("Cannot send input after starting communication")
1106
1107 # Optimization: If we are not worried about timeouts, we haven't
1108 # started communicating, and we have one or zero pipes, using select()
1109 # or threads is unnecessary.
1110 if (timeout is None and not self._communication_started and
1111 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
1112 stdout = None
1113 stderr = None
1114 if self.stdin:
1115 self._stdin_write(input)
1116 elif self.stdout:
1117 stdout = self.stdout.read()
1118 self.stdout.close()
1119 elif self.stderr:
1120 stderr = self.stderr.read()
1121 self.stderr.close()
1122 self.wait()
1123 else:
1124 if timeout is not None:
1125 endtime = _time() + timeout
1126 else:
1127 endtime = None
1128
1129 try:
1130 stdout, stderr = self._communicate(input, endtime, timeout)
1131 except KeyboardInterrupt:
1132 # https://bugs.python.org/issue25942
1133 # See the detailed comment in .wait().
1134 if timeout is not None:
1135 sigint_timeout = min(self._sigint_wait_secs,
1136 self._remaining_time(endtime))
1137 else:
1138 sigint_timeout = self._sigint_wait_secs
1139 self._sigint_wait_secs = 0 # nothing else should wait.
1140 try:
1141 self._wait(timeout=sigint_timeout)
1142 except TimeoutExpired:
1143 pass
1144 raise # resume the KeyboardInterrupt
1145
1146 finally:
1147 self._communication_started = True
1148
1149 sts = self.wait(timeout=self._remaining_time(endtime))
1150
1151 return (stdout, stderr)
1152
1153
1154 def poll(self):
1155 """Check if child process has terminated. Set and return returncode
1156 attribute."""
1157 return self._internal_poll()
1158
1159
1160 def _remaining_time(self, endtime):
1161 """Convenience for _communicate when computing timeouts."""
1162 if endtime is None:
1163 return None
1164 else:
1165 return endtime - _time()
1166
1167
1168 def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
1169 skip_check_and_raise=False):
1170 """Convenience for checking if a timeout has expired."""
1171 if endtime is None:
1172 return
1173 if skip_check_and_raise or _time() > endtime:
1174 raise TimeoutExpired(
1175 self.args, orig_timeout,
1176 output=b''.join(stdout_seq) if stdout_seq else None,
1177 stderr=b''.join(stderr_seq) if stderr_seq else None)
1178
1179
1180 def wait(self, timeout=None):
1181 """Wait for child process to terminate; returns self.returncode."""
1182 if timeout is not None:
1183 endtime = _time() + timeout
1184 try:
1185 return self._wait(timeout=timeout)
1186 except KeyboardInterrupt:
1187 # https://bugs.python.org/issue25942
1188 # The first keyboard interrupt waits briefly for the child to
1189 # exit under the common assumption that it also received the ^C
1190 # generated SIGINT and will exit rapidly.
1191 if timeout is not None:
1192 sigint_timeout = min(self._sigint_wait_secs,
1193 self._remaining_time(endtime))
1194 else:
1195 sigint_timeout = self._sigint_wait_secs
1196 self._sigint_wait_secs = 0 # nothing else should wait.
1197 try:
1198 self._wait(timeout=sigint_timeout)
1199 except TimeoutExpired:
1200 pass
1201 raise # resume the KeyboardInterrupt
1202
1203 def _close_pipe_fds(self,
1204 p2cread, p2cwrite,
1205 c2pread, c2pwrite,
1206 errread, errwrite):
1207 # self._devnull is not always defined.
1208 devnull_fd = getattr(self, '_devnull', None)
1209
1210 with contextlib.ExitStack() as stack:
1211 if _mswindows:
1212 if p2cread != -1:
1213 stack.callback(p2cread.Close)
1214 if c2pwrite != -1:
1215 stack.callback(c2pwrite.Close)
1216 if errwrite != -1:
1217 stack.callback(errwrite.Close)
1218 else:
1219 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
1220 stack.callback(os.close, p2cread)
1221 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
1222 stack.callback(os.close, c2pwrite)
1223 if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
1224 stack.callback(os.close, errwrite)
1225
1226 if devnull_fd is not None:
1227 stack.callback(os.close, devnull_fd)
1228
1229 # Prevent a double close of these handles/fds from __init__ on error.
1230 self._closed_child_pipe_fds = True
1231
1232 if _mswindows:
1233 #
1234 # Windows methods
1235 #
1236 def _get_handles(self, stdin, stdout, stderr):
1237 """Construct and return tuple with IO objects:
1238 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1239 """
1240 if stdin is None and stdout is None and stderr is None:
1241 return (-1, -1, -1, -1, -1, -1)
1242
1243 p2cread, p2cwrite = -1, -1
1244 c2pread, c2pwrite = -1, -1
1245 errread, errwrite = -1, -1
1246
1247 if stdin is None:
1248 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
1249 if p2cread is None:
1250 p2cread, _ = _winapi.CreatePipe(None, 0)
1251 p2cread = Handle(p2cread)
1252 _winapi.CloseHandle(_)
1253 elif stdin == PIPE:
1254 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
1255 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
1256 elif stdin == DEVNULL:
1257 p2cread = msvcrt.get_osfhandle(self._get_devnull())
1258 elif isinstance(stdin, int):
1259 p2cread = msvcrt.get_osfhandle(stdin)
1260 else:
1261 # Assuming file-like object
1262 p2cread = msvcrt.get_osfhandle(stdin.fileno())
1263 p2cread = self._make_inheritable(p2cread)
1264
1265 if stdout is None:
1266 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
1267 if c2pwrite is None:
1268 _, c2pwrite = _winapi.CreatePipe(None, 0)
1269 c2pwrite = Handle(c2pwrite)
1270 _winapi.CloseHandle(_)
1271 elif stdout == PIPE:
1272 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
1273 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
1274 elif stdout == DEVNULL:
1275 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
1276 elif isinstance(stdout, int):
1277 c2pwrite = msvcrt.get_osfhandle(stdout)
1278 else:
1279 # Assuming file-like object
1280 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
1281 c2pwrite = self._make_inheritable(c2pwrite)
1282
1283 if stderr is None:
1284 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
1285 if errwrite is None:
1286 _, errwrite = _winapi.CreatePipe(None, 0)
1287 errwrite = Handle(errwrite)
1288 _winapi.CloseHandle(_)
1289 elif stderr == PIPE:
1290 errread, errwrite = _winapi.CreatePipe(None, 0)
1291 errread, errwrite = Handle(errread), Handle(errwrite)
1292 elif stderr == STDOUT:
1293 errwrite = c2pwrite
1294 elif stderr == DEVNULL:
1295 errwrite = msvcrt.get_osfhandle(self._get_devnull())
1296 elif isinstance(stderr, int):
1297 errwrite = msvcrt.get_osfhandle(stderr)
1298 else:
1299 # Assuming file-like object
1300 errwrite = msvcrt.get_osfhandle(stderr.fileno())
1301 errwrite = self._make_inheritable(errwrite)
1302
1303 return (p2cread, p2cwrite,
1304 c2pread, c2pwrite,
1305 errread, errwrite)
1306
1307
1308 def _make_inheritable(self, handle):
1309 """Return a duplicate of handle, which is inheritable"""
1310 h = _winapi.DuplicateHandle(
1311 _winapi.GetCurrentProcess(), handle,
1312 _winapi.GetCurrentProcess(), 0, 1,
1313 _winapi.DUPLICATE_SAME_ACCESS)
1314 return Handle(h)
1315
1316
1317 def _filter_handle_list(self, handle_list):
1318 """Filter out console handles that can't be used
1319 in lpAttributeList["handle_list"] and make sure the list
1320 isn't empty. This also removes duplicate handles."""
1321 # An handle with it's lowest two bits set might be a special console
1322 # handle that if passed in lpAttributeList["handle_list"], will
1323 # cause it to fail.
1324 return list({handle for handle in handle_list
1325 if handle & 0x3 != 0x3
1326 or _winapi.GetFileType(handle) !=
1327 _winapi.FILE_TYPE_CHAR})
1328
1329
1330 def _execute_child(self, args, executable, preexec_fn, close_fds,
1331 pass_fds, cwd, env,
1332 startupinfo, creationflags, shell,
1333 p2cread, p2cwrite,
1334 c2pread, c2pwrite,
1335 errread, errwrite,
1336 unused_restore_signals,
1337 unused_gid, unused_gids, unused_uid,
1338 unused_umask,
1339 unused_start_new_session):
1340 """Execute program (MS Windows version)"""
1341
1342 assert not pass_fds, "pass_fds not supported on Windows."
1343
1344 if isinstance(args, str):
1345 pass
1346 elif isinstance(args, bytes):
1347 if shell:
1348 raise TypeError('bytes args is not allowed on Windows')
1349 args = list2cmdline([args])
1350 elif isinstance(args, os.PathLike):
1351 if shell:
1352 raise TypeError('path-like args is not allowed when '
1353 'shell is true')
1354 args = list2cmdline([args])
1355 else:
1356 args = list2cmdline(args)
1357
1358 if executable is not None:
1359 executable = os.fsdecode(executable)
1360
1361 # Process startup details
1362 if startupinfo is None:
1363 startupinfo = STARTUPINFO()
1364 else:
1365 # bpo-34044: Copy STARTUPINFO since it is modified above,
1366 # so the caller can reuse it multiple times.
1367 startupinfo = startupinfo.copy()
1368
1369 use_std_handles = -1 not in (p2cread, c2pwrite, errwrite)
1370 if use_std_handles:
1371 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
1372 startupinfo.hStdInput = p2cread
1373 startupinfo.hStdOutput = c2pwrite
1374 startupinfo.hStdError = errwrite
1375
1376 attribute_list = startupinfo.lpAttributeList
1377 have_handle_list = bool(attribute_list and
1378 "handle_list" in attribute_list and
1379 attribute_list["handle_list"])
1380
1381 # If we were given an handle_list or need to create one
1382 if have_handle_list or (use_std_handles and close_fds):
1383 if attribute_list is None:
1384 attribute_list = startupinfo.lpAttributeList = {}
1385 handle_list = attribute_list["handle_list"] = \
1386 list(attribute_list.get("handle_list", []))
1387
1388 if use_std_handles:
1389 handle_list += [int(p2cread), int(c2pwrite), int(errwrite)]
1390
1391 handle_list[:] = self._filter_handle_list(handle_list)
1392
1393 if handle_list:
1394 if not close_fds:
1395 warnings.warn("startupinfo.lpAttributeList['handle_list'] "
1396 "overriding close_fds", RuntimeWarning)
1397
1398 # When using the handle_list we always request to inherit
1399 # handles but the only handles that will be inherited are
1400 # the ones in the handle_list
1401 close_fds = False
1402
1403 if shell:
1404 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
1405 startupinfo.wShowWindow = _winapi.SW_HIDE
1406 comspec = os.environ.get("COMSPEC", "cmd.exe")
1407 args = '{} /c "{}"'.format (comspec, args)
1408
1409 if cwd is not None:
1410 cwd = os.fsdecode(cwd)
1411
1412 sys.audit("subprocess.Popen", executable, args, cwd, env)
1413
1414 # Start the process
1415 try:
1416 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
1417 # no special security
1418 None, None,
1419 int(not close_fds),
1420 creationflags,
1421 env,
1422 cwd,
1423 startupinfo)
1424 finally:
1425 # Child is launched. Close the parent's copy of those pipe
1426 # handles that only the child should have open. You need
1427 # to make sure that no handles to the write end of the
1428 # output pipe are maintained in this process or else the
1429 # pipe will not close when the child process exits and the
1430 # ReadFile will hang.
1431 self._close_pipe_fds(p2cread, p2cwrite,
1432 c2pread, c2pwrite,
1433 errread, errwrite)
1434
1435 # Retain the process handle, but close the thread handle
1436 self._child_created = True
1437 self._handle = Handle(hp)
1438 self.pid = pid
1439 _winapi.CloseHandle(ht)
1440
1441 def _internal_poll(self, _deadstate=None,
1442 _WaitForSingleObject=_winapi.WaitForSingleObject,
1443 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1444 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
1445 """Check if child process has terminated. Returns returncode
1446 attribute.
1447
1448 This method is called by __del__, so it can only refer to objects
1449 in its local scope.
1450
1451 """
1452 if self.returncode is None:
1453 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1454 self.returncode = _GetExitCodeProcess(self._handle)
1455 return self.returncode
1456
1457
1458 def _wait(self, timeout):
1459 """Internal implementation of wait() on Windows."""
1460 if timeout is None:
1461 timeout_millis = _winapi.INFINITE
1462 else:
1463 timeout_millis = int(timeout * 1000)
1464 if self.returncode is None:
1465 # API note: Returns immediately if timeout_millis == 0.
1466 result = _winapi.WaitForSingleObject(self._handle,
1467 timeout_millis)
1468 if result == _winapi.WAIT_TIMEOUT:
1469 raise TimeoutExpired(self.args, timeout)
1470 self.returncode = _winapi.GetExitCodeProcess(self._handle)
1471 return self.returncode
1472
1473
1474 def _readerthread(self, fh, buffer):
1475 buffer.append(fh.read())
1476 fh.close()
1477
1478
1479 def _communicate(self, input, endtime, orig_timeout):
1480 # Start reader threads feeding into a list hanging off of this
1481 # object, unless they've already been started.
1482 if self.stdout and not hasattr(self, "_stdout_buff"):
1483 self._stdout_buff = []
1484 self.stdout_thread = \
1485 threading.Thread(target=self._readerthread,
1486 args=(self.stdout, self._stdout_buff))
1487 self.stdout_thread.daemon = True
1488 self.stdout_thread.start()
1489 if self.stderr and not hasattr(self, "_stderr_buff"):
1490 self._stderr_buff = []
1491 self.stderr_thread = \
1492 threading.Thread(target=self._readerthread,
1493 args=(self.stderr, self._stderr_buff))
1494 self.stderr_thread.daemon = True
1495 self.stderr_thread.start()
1496
1497 if self.stdin:
1498 self._stdin_write(input)
1499
1500 # Wait for the reader threads, or time out. If we time out, the
1501 # threads remain reading and the fds left open in case the user
1502 # calls communicate again.
1503 if self.stdout is not None:
1504 self.stdout_thread.join(self._remaining_time(endtime))
1505 if self.stdout_thread.is_alive():
1506 raise TimeoutExpired(self.args, orig_timeout)
1507 if self.stderr is not None:
1508 self.stderr_thread.join(self._remaining_time(endtime))
1509 if self.stderr_thread.is_alive():
1510 raise TimeoutExpired(self.args, orig_timeout)
1511
1512 # Collect the output from and close both pipes, now that we know
1513 # both have been read successfully.
1514 stdout = None
1515 stderr = None
1516 if self.stdout:
1517 stdout = self._stdout_buff
1518 self.stdout.close()
1519 if self.stderr:
1520 stderr = self._stderr_buff
1521 self.stderr.close()
1522
1523 # All data exchanged. Translate lists into strings.
1524 if stdout is not None:
1525 stdout = stdout[0]
1526 if stderr is not None:
1527 stderr = stderr[0]
1528
1529 return (stdout, stderr)
1530
1531 def send_signal(self, sig):
1532 """Send a signal to the process."""
1533 # Don't signal a process that we know has already died.
1534 if self.returncode is not None:
1535 return
1536 if sig == signal.SIGTERM:
1537 self.terminate()
1538 elif sig == signal.CTRL_C_EVENT:
1539 os.kill(self.pid, signal.CTRL_C_EVENT)
1540 elif sig == signal.CTRL_BREAK_EVENT:
1541 os.kill(self.pid, signal.CTRL_BREAK_EVENT)
1542 else:
1543 raise ValueError("Unsupported signal: {}".format(sig))
1544
1545 def terminate(self):
1546 """Terminates the process."""
1547 # Don't terminate a process that we know has already died.
1548 if self.returncode is not None:
1549 return
1550 try:
1551 _winapi.TerminateProcess(self._handle, 1)
1552 except PermissionError:
1553 # ERROR_ACCESS_DENIED (winerror 5) is received when the
1554 # process already died.
1555 rc = _winapi.GetExitCodeProcess(self._handle)
1556 if rc == _winapi.STILL_ACTIVE:
1557 raise
1558 self.returncode = rc
1559
1560 kill = terminate
1561
1562 else:
1563 #
1564 # POSIX methods
1565 #
1566 def _get_handles(self, stdin, stdout, stderr):
1567 """Construct and return tuple with IO objects:
1568 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1569 """
1570 p2cread, p2cwrite = -1, -1
1571 c2pread, c2pwrite = -1, -1
1572 errread, errwrite = -1, -1
1573
1574 if stdin is None:
1575 pass
1576 elif stdin == PIPE:
1577 p2cread, p2cwrite = os.pipe()
1578 elif stdin == DEVNULL:
1579 p2cread = self._get_devnull()
1580 elif isinstance(stdin, int):
1581 p2cread = stdin
1582 else:
1583 # Assuming file-like object
1584 p2cread = stdin.fileno()
1585
1586 if stdout is None:
1587 pass
1588 elif stdout == PIPE:
1589 c2pread, c2pwrite = os.pipe()
1590 elif stdout == DEVNULL:
1591 c2pwrite = self._get_devnull()
1592 elif isinstance(stdout, int):
1593 c2pwrite = stdout
1594 else:
1595 # Assuming file-like object
1596 c2pwrite = stdout.fileno()
1597
1598 if stderr is None:
1599 pass
1600 elif stderr == PIPE:
1601 errread, errwrite = os.pipe()
1602 elif stderr == STDOUT:
1603 if c2pwrite != -1:
1604 errwrite = c2pwrite
1605 else: # child's stdout is not set, use parent's stdout
1606 errwrite = sys.__stdout__.fileno()
1607 elif stderr == DEVNULL:
1608 errwrite = self._get_devnull()
1609 elif isinstance(stderr, int):
1610 errwrite = stderr
1611 else:
1612 # Assuming file-like object
1613 errwrite = stderr.fileno()
1614
1615 return (p2cread, p2cwrite,
1616 c2pread, c2pwrite,
1617 errread, errwrite)
1618
1619
1620 def _posix_spawn(self, args, executable, env, restore_signals,
1621 p2cread, p2cwrite,
1622 c2pread, c2pwrite,
1623 errread, errwrite):
1624 """Execute program using os.posix_spawn()."""
1625 if env is None:
1626 env = os.environ
1627
1628 kwargs = {}
1629 if restore_signals:
1630 # See _Py_RestoreSignals() in Python/pylifecycle.c
1631 sigset = []
1632 for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'):
1633 signum = getattr(signal, signame, None)
1634 if signum is not None:
1635 sigset.append(signum)
1636 kwargs['setsigdef'] = sigset
1637
1638 file_actions = []
1639 for fd in (p2cwrite, c2pread, errread):
1640 if fd != -1:
1641 file_actions.append((os.POSIX_SPAWN_CLOSE, fd))
1642 for fd, fd2 in (
1643 (p2cread, 0),
1644 (c2pwrite, 1),
1645 (errwrite, 2),
1646 ):
1647 if fd != -1:
1648 file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2))
1649 if file_actions:
1650 kwargs['file_actions'] = file_actions
1651
1652 self.pid = os.posix_spawn(executable, args, env, **kwargs)
1653 self._child_created = True
1654
1655 self._close_pipe_fds(p2cread, p2cwrite,
1656 c2pread, c2pwrite,
1657 errread, errwrite)
1658
1659 def _execute_child(self, args, executable, preexec_fn, close_fds,
1660 pass_fds, cwd, env,
1661 startupinfo, creationflags, shell,
1662 p2cread, p2cwrite,
1663 c2pread, c2pwrite,
1664 errread, errwrite,
1665 restore_signals,
1666 gid, gids, uid, umask,
1667 start_new_session):
1668 """Execute program (POSIX version)"""
1669
1670 if isinstance(args, (str, bytes)):
1671 args = [args]
1672 elif isinstance(args, os.PathLike):
1673 if shell:
1674 raise TypeError('path-like args is not allowed when '
1675 'shell is true')
1676 args = [args]
1677 else:
1678 args = list(args)
1679
1680 if shell:
1681 # On Android the default shell is at '/system/bin/sh'.
1682 unix_shell = ('/system/bin/sh' if
1683 hasattr(sys, 'getandroidapilevel') else '/bin/sh')
1684 args = [unix_shell, "-c"] + args
1685 if executable:
1686 args[0] = executable
1687
1688 if executable is None:
1689 executable = args[0]
1690
1691 sys.audit("subprocess.Popen", executable, args, cwd, env)
1692
1693 if (_USE_POSIX_SPAWN
1694 and os.path.dirname(executable)
1695 and preexec_fn is None
1696 and not close_fds
1697 and not pass_fds
1698 and cwd is None
1699 and (p2cread == -1 or p2cread > 2)
1700 and (c2pwrite == -1 or c2pwrite > 2)
1701 and (errwrite == -1 or errwrite > 2)
1702 and not start_new_session
1703 and gid is None
1704 and gids is None
1705 and uid is None
1706 and umask < 0):
1707 self._posix_spawn(args, executable, env, restore_signals,
1708 p2cread, p2cwrite,
1709 c2pread, c2pwrite,
1710 errread, errwrite)
1711 return
1712
1713 orig_executable = executable
1714
1715 # For transferring possible exec failure from child to parent.
1716 # Data format: "exception name:hex errno:description"
1717 # Pickle is not used; it is complex and involves memory allocation.
1718 errpipe_read, errpipe_write = os.pipe()
1719 # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
1720 low_fds_to_close = []
1721 while errpipe_write < 3:
1722 low_fds_to_close.append(errpipe_write)
1723 errpipe_write = os.dup(errpipe_write)
1724 for low_fd in low_fds_to_close:
1725 os.close(low_fd)
1726 try:
1727 try:
1728 # We must avoid complex work that could involve
1729 # malloc or free in the child process to avoid
1730 # potential deadlocks, thus we do all this here.
1731 # and pass it to fork_exec()
1732
1733 if env is not None:
1734 env_list = []
1735 for k, v in env.items():
1736 k = os.fsencode(k)
1737 if b'=' in k:
1738 raise ValueError("illegal environment variable name")
1739 env_list.append(k + b'=' + os.fsencode(v))
1740 else:
1741 env_list = None # Use execv instead of execve.
1742 executable = os.fsencode(executable)
1743 if os.path.dirname(executable):
1744 executable_list = (executable,)
1745 else:
1746 # This matches the behavior of os._execvpe().
1747 executable_list = tuple(
1748 os.path.join(os.fsencode(dir), executable)
1749 for dir in os.get_exec_path(env))
1750 fds_to_keep = set(pass_fds)
1751 fds_to_keep.add(errpipe_write)
1752 self.pid = _posixsubprocess.fork_exec(
1753 args, executable_list,
1754 close_fds, tuple(sorted(map(int, fds_to_keep))),
1755 cwd, env_list,
1756 p2cread, p2cwrite, c2pread, c2pwrite,
1757 errread, errwrite,
1758 errpipe_read, errpipe_write,
1759 restore_signals, start_new_session,
1760 gid, gids, uid, umask,
1761 preexec_fn)
1762 self._child_created = True
1763 finally:
1764 # be sure the FD is closed no matter what
1765 os.close(errpipe_write)
1766
1767 self._close_pipe_fds(p2cread, p2cwrite,
1768 c2pread, c2pwrite,
1769 errread, errwrite)
1770
1771 # Wait for exec to fail or succeed; possibly raising an
1772 # exception (limited in size)
1773 errpipe_data = bytearray()
1774 while True:
1775 part = os.read(errpipe_read, 50000)
1776 errpipe_data += part
1777 if not part or len(errpipe_data) > 50000:
1778 break
1779 finally:
1780 # be sure the FD is closed no matter what
1781 os.close(errpipe_read)
1782
1783 if errpipe_data:
1784 try:
1785 pid, sts = os.waitpid(self.pid, 0)
1786 if pid == self.pid:
1787 self._handle_exitstatus(sts)
1788 else:
1789 self.returncode = sys.maxsize
1790 except ChildProcessError:
1791 pass
1792
1793 try:
1794 exception_name, hex_errno, err_msg = (
1795 errpipe_data.split(b':', 2))
1796 # The encoding here should match the encoding
1797 # written in by the subprocess implementations
1798 # like _posixsubprocess
1799 err_msg = err_msg.decode()
1800 except ValueError:
1801 exception_name = b'SubprocessError'
1802 hex_errno = b'0'
1803 err_msg = 'Bad exception data from child: {!r}'.format(
1804 bytes(errpipe_data))
1805 child_exception_type = getattr(
1806 builtins, exception_name.decode('ascii'),
1807 SubprocessError)
1808 if issubclass(child_exception_type, OSError) and hex_errno:
1809 errno_num = int(hex_errno, 16)
1810 child_exec_never_called = (err_msg == "noexec")
1811 if child_exec_never_called:
1812 err_msg = ""
1813 # The error must be from chdir(cwd).
1814 err_filename = cwd
1815 else:
1816 err_filename = orig_executable
1817 if errno_num != 0:
1818 err_msg = os.strerror(errno_num)
1819 raise child_exception_type(errno_num, err_msg, err_filename)
1820 raise child_exception_type(err_msg)
1821
1822
1823 def _handle_exitstatus(self, sts,
1824 waitstatus_to_exitcode=os.waitstatus_to_exitcode,
1825 _WIFSTOPPED=os.WIFSTOPPED,
1826 _WSTOPSIG=os.WSTOPSIG):
1827 """All callers to this function MUST hold self._waitpid_lock."""
1828 # This method is called (indirectly) by __del__, so it cannot
1829 # refer to anything outside of its local scope.
1830 if _WIFSTOPPED(sts):
1831 self.returncode = -_WSTOPSIG(sts)
1832 else:
1833 self.returncode = waitstatus_to_exitcode(sts)
1834
1835 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
1836 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
1837 """Check if child process has terminated. Returns returncode
1838 attribute.
1839
1840 This method is called by __del__, so it cannot reference anything
1841 outside of the local scope (nor can any methods it calls).
1842
1843 """
1844 if self.returncode is None:
1845 if not self._waitpid_lock.acquire(False):
1846 # Something else is busy calling waitpid. Don't allow two
1847 # at once. We know nothing yet.
1848 return None
1849 try:
1850 if self.returncode is not None:
1851 return self.returncode # Another thread waited.
1852 pid, sts = _waitpid(self.pid, _WNOHANG)
1853 if pid == self.pid:
1854 self._handle_exitstatus(sts)
1855 except OSError as e:
1856 if _deadstate is not None:
1857 self.returncode = _deadstate
1858 elif e.errno == _ECHILD:
1859 # This happens if SIGCLD is set to be ignored or
1860 # waiting for child processes has otherwise been
1861 # disabled for our process. This child is dead, we
1862 # can't get the status.
1863 # http://bugs.python.org/issue15756
1864 self.returncode = 0
1865 finally:
1866 self._waitpid_lock.release()
1867 return self.returncode
1868
1869
1870 def _try_wait(self, wait_flags):
1871 """All callers to this function MUST hold self._waitpid_lock."""
1872 try:
1873 (pid, sts) = os.waitpid(self.pid, wait_flags)
1874 except ChildProcessError:
1875 # This happens if SIGCLD is set to be ignored or waiting
1876 # for child processes has otherwise been disabled for our
1877 # process. This child is dead, we can't get the status.
1878 pid = self.pid
1879 sts = 0
1880 return (pid, sts)
1881
1882
1883 def _wait(self, timeout):
1884 """Internal implementation of wait() on POSIX."""
1885 if self.returncode is not None:
1886 return self.returncode
1887
1888 if timeout is not None:
1889 endtime = _time() + timeout
1890 # Enter a busy loop if we have a timeout. This busy loop was
1891 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1892 delay = 0.0005 # 500 us -> initial delay of 1 ms
1893 while True:
1894 if self._waitpid_lock.acquire(False):
1895 try:
1896 if self.returncode is not None:
1897 break # Another thread waited.
1898 (pid, sts) = self._try_wait(os.WNOHANG)
1899 assert pid == self.pid or pid == 0
1900 if pid == self.pid:
1901 self._handle_exitstatus(sts)
1902 break
1903 finally:
1904 self._waitpid_lock.release()
1905 remaining = self._remaining_time(endtime)
1906 if remaining <= 0:
1907 raise TimeoutExpired(self.args, timeout)
1908 delay = min(delay * 2, remaining, .05)
1909 time.sleep(delay)
1910 else:
1911 while self.returncode is None:
1912 with self._waitpid_lock:
1913 if self.returncode is not None:
1914 break # Another thread waited.
1915 (pid, sts) = self._try_wait(0)
1916 # Check the pid and loop as waitpid has been known to
1917 # return 0 even without WNOHANG in odd situations.
1918 # http://bugs.python.org/issue14396.
1919 if pid == self.pid:
1920 self._handle_exitstatus(sts)
1921 return self.returncode
1922
1923
1924 def _communicate(self, input, endtime, orig_timeout):
1925 if self.stdin and not self._communication_started:
1926 # Flush stdio buffer. This might block, if the user has
1927 # been writing to .stdin in an uncontrolled fashion.
1928 try:
1929 self.stdin.flush()
1930 except BrokenPipeError:
1931 pass # communicate() must ignore BrokenPipeError.
1932 if not input:
1933 try:
1934 self.stdin.close()
1935 except BrokenPipeError:
1936 pass # communicate() must ignore BrokenPipeError.
1937
1938 stdout = None
1939 stderr = None
1940
1941 # Only create this mapping if we haven't already.
1942 if not self._communication_started:
1943 self._fileobj2output = {}
1944 if self.stdout:
1945 self._fileobj2output[self.stdout] = []
1946 if self.stderr:
1947 self._fileobj2output[self.stderr] = []
1948
1949 if self.stdout:
1950 stdout = self._fileobj2output[self.stdout]
1951 if self.stderr:
1952 stderr = self._fileobj2output[self.stderr]
1953
1954 self._save_input(input)
1955
1956 if self._input:
1957 input_view = memoryview(self._input)
1958
1959 with _PopenSelector() as selector:
1960 if self.stdin and input:
1961 selector.register(self.stdin, selectors.EVENT_WRITE)
1962 if self.stdout and not self.stdout.closed:
1963 selector.register(self.stdout, selectors.EVENT_READ)
1964 if self.stderr and not self.stderr.closed:
1965 selector.register(self.stderr, selectors.EVENT_READ)
1966
1967 while selector.get_map():
1968 timeout = self._remaining_time(endtime)
1969 if timeout is not None and timeout < 0:
1970 self._check_timeout(endtime, orig_timeout,
1971 stdout, stderr,
1972 skip_check_and_raise=True)
1973 raise RuntimeError( # Impossible :)
1974 '_check_timeout(..., skip_check_and_raise=True) '
1975 'failed to raise TimeoutExpired.')
1976
1977 ready = selector.select(timeout)
1978 self._check_timeout(endtime, orig_timeout, stdout, stderr)
1979
1980 # XXX Rewrite these to use non-blocking I/O on the file
1981 # objects; they are no longer using C stdio!
1982
1983 for key, events in ready:
1984 if key.fileobj is self.stdin:
1985 chunk = input_view[self._input_offset :
1986 self._input_offset + _PIPE_BUF]
1987 try:
1988 self._input_offset += os.write(key.fd, chunk)
1989 except BrokenPipeError:
1990 selector.unregister(key.fileobj)
1991 key.fileobj.close()
1992 else:
1993 if self._input_offset >= len(self._input):
1994 selector.unregister(key.fileobj)
1995 key.fileobj.close()
1996 elif key.fileobj in (self.stdout, self.stderr):
1997 data = os.read(key.fd, 32768)
1998 if not data:
1999 selector.unregister(key.fileobj)
2000 key.fileobj.close()
2001 self._fileobj2output[key.fileobj].append(data)
2002
2003 self.wait(timeout=self._remaining_time(endtime))
2004
2005 # All data exchanged. Translate lists into strings.
2006 if stdout is not None:
2007 stdout = b''.join(stdout)
2008 if stderr is not None:
2009 stderr = b''.join(stderr)
2010
2011 # Translate newlines, if requested.
2012 # This also turns bytes into strings.
2013 if self.text_mode:
2014 if stdout is not None:
2015 stdout = self._translate_newlines(stdout,
2016 self.stdout.encoding,
2017 self.stdout.errors)
2018 if stderr is not None:
2019 stderr = self._translate_newlines(stderr,
2020 self.stderr.encoding,
2021 self.stderr.errors)
2022
2023 return (stdout, stderr)
2024
2025
2026 def _save_input(self, input):
2027 # This method is called from the _communicate_with_*() methods
2028 # so that if we time out while communicating, we can continue
2029 # sending input if we retry.
2030 if self.stdin and self._input is None:
2031 self._input_offset = 0
2032 self._input = input
2033 if input is not None and self.text_mode:
2034 self._input = self._input.encode(self.stdin.encoding,
2035 self.stdin.errors)
2036
2037
2038 def send_signal(self, sig):
2039 """Send a signal to the process."""
2040 # bpo-38630: Polling reduces the risk of sending a signal to the
2041 # wrong process if the process completed, the Popen.returncode
2042 # attribute is still None, and the pid has been reassigned
2043 # (recycled) to a new different process. This race condition can
2044 # happens in two cases.
2045 #
2046 # Case 1. Thread A calls Popen.poll(), thread B calls
2047 # Popen.send_signal(). In thread A, waitpid() succeed and returns
2048 # the exit status. Thread B calls kill() because poll() in thread A
2049 # did not set returncode yet. Calling poll() in thread B prevents
2050 # the race condition thanks to Popen._waitpid_lock.
2051 #
2052 # Case 2. waitpid(pid, 0) has been called directly, without
2053 # using Popen methods: returncode is still None is this case.
2054 # Calling Popen.poll() will set returncode to a default value,
2055 # since waitpid() fails with ProcessLookupError.
2056 self.poll()
2057 if self.returncode is not None:
2058 # Skip signalling a process that we know has already died.
2059 return
2060
2061 # The race condition can still happen if the race condition
2062 # described above happens between the returncode test
2063 # and the kill() call.
2064 try:
2065 os.kill(self.pid, sig)
2066 except ProcessLookupError:
2067 # Supress the race condition error; bpo-40550.
2068 pass
2069
2070 def terminate(self):
2071 """Terminate the process with SIGTERM
2072 """
2073 self.send_signal(signal.SIGTERM)
2074
2075 def kill(self):
2076 """Kill the process with SIGKILL
2077 """
2078 self.send_signal(signal.SIGKILL)