import os
import sys
from pickle import load
from multiprocessing import process, util
from . import spawn
from . import reduction
from .context import get_spawning_popen, set_spawning_popen
if sys.platform == "win32":
# Avoid import error by code introspection tools such as test runners
# trying to import this module while running on non-Windows systems.
import msvcrt
from .compat_win32 import _winapi
from .compat_win32 import Popen as _Popen
from .reduction import duplicate
else:
_Popen = object
if sys.version_info[:2] < (3, 3):
from os import fdopen as open
__all__ = ['Popen']
#
#
#
TERMINATE = 0x10000
WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
def _path_eq(p1, p2):
return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
WINENV = (hasattr(sys, "_base_executable")
and not _path_eq(sys.executable, sys._base_executable))
#
# We define a Popen class similar to the one from subprocess, but
# whose constructor takes a process object as its argument.
#
class Popen(_Popen):
'''
Start a subprocess to run the code of a process object
'''
method = 'loky'
def __init__(self, process_obj):
prep_data = spawn.get_preparation_data(
process_obj._name, getattr(process_obj, "init_main_module", True))
# read end of pipe will be "stolen" by the child process
# -- see spawn_main() in spawn.py.
rfd, wfd = os.pipe()
rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
os.close(rfd)
cmd = get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle)
cmd = ' '.join('"%s"' % x for x in cmd)
python_exe = spawn.get_executable()
# copy the environment variables to set in the child process
child_env = os.environ.copy()
child_env.update(process_obj.env)
# bpo-35797: When running in a venv, we bypass the redirect
# executor and launch our base Python.
if WINENV and _path_eq(python_exe, sys.executable):
python_exe = sys._base_executable
child_env["__PYVENV_LAUNCHER__"] = sys.executable
try:
with open(wfd, 'wb') as to_child:
# start process
try:
# This flag allows to pass inheritable handles from the
# parent to the child process in a python2-3 compatible way
# (see
# https://github.com/tomMoral/loky/pull/204#discussion_r290719629
# for more detail). When support for Python 2 is dropped,
# the cleaner multiprocessing.reduction.steal_handle should
# be used instead.
inherit = True
hp, ht, pid, tid = _winapi.CreateProcess(
python_exe, cmd,
None, None, inherit, 0,
child_env, None, None)
_winapi.CloseHandle(ht)
except BaseException:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
# send information to child
set_spawning_popen(self)
if sys.version_info[:2] < (3, 4):
Popen._tls.process_handle = int(hp)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
set_spawning_popen(None)
if sys.version_info[:2] < (3, 4):
del Popen._tls.process_handle
except IOError as exc:
# IOError 22 happens when the launched subprocess terminated before
# wfd.close is called. Thus we can safely ignore it.
if exc.errno != 22:
raise
util.debug("While starting {}, ignored a IOError 22"
.format(process_obj._name))
def duplicate_for_child(self, handle):
assert self is get_spawning_popen()
return duplicate(handle, self.sentinel)
def get_command_line(pipe_handle, **kwds):
'''
Returns prefix of command line used for spawning a child process
'''
if getattr(sys, 'frozen', False):
return ([sys.executable, '--multiprocessing-fork', pipe_handle])
else:
prog = 'from joblib.externals.loky.backend.popen_loky_win32 import main; main()'
opts = util._args_from_interpreter_flags()
return [spawn.get_executable()] + opts + [
'-c', prog, '--multiprocessing-fork', pipe_handle]
def is_forking(argv):
'''
Return whether commandline indicates we are forking
'''
if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
assert len(argv) == 3
return True
else:
return False
def main():
'''
Run code specified by data received over pipe
'''
assert is_forking(sys.argv)
handle = int(sys.argv[-1])
fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
from_parent = os.fdopen(fd, 'rb')
process.current_process()._inheriting = True
preparation_data = load(from_parent)
spawn.prepare(preparation_data)
self = load(from_parent)
process.current_process()._inheriting = False
from_parent.close()
exitcode = self._bootstrap()
exit(exitcode)