From fe6aaf7b813904f271964ed3b3b5b2db092df5e9 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 30 Sep 2021 23:18:13 +0100 Subject: [PATCH 1/5] bpo-45337: Use the realpath of the new executable when creating a venv on Windows. Also warn if the realpath does not match the user-specified location. --- Lib/test/test_venv.py | 2 +- Lib/venv/__init__.py | 21 ++++++++++++------- .../2021-09-30-23-17-27.bpo-45337.qg7U_h.rst | 4 ++++ 3 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 098ba17af59758e..42a3680fc1fab64 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -150,7 +150,7 @@ def test_prompt(self): def test_upgrade_dependencies(self): builder = venv.EnvBuilder() bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' - python_exe = 'python.exe' if sys.platform == 'win32' else 'python' + python_exe = os.path.split(sys.executable)[1] with tempfile.TemporaryDirectory() as fake_env_dir: def pip_cmd_checker(cmd): diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index b007e25f7ea2cce..326ff9eced010b5 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -140,8 +140,19 @@ def create_if_needed(d): os.symlink('lib', link_path) context.bin_path = binpath = os.path.join(env_dir, binname) context.bin_name = binname - context.env_exe = os.path.join(binpath, exename) + context.env_exe = context.env_cmd = os.path.join(binpath, exename) create_if_needed(binpath) + if sys.platform == 'win32': + # Fixup the env_cmd to account for redirections when we launch + # the newly created environment. + real_env_exe = os.path.realpath(context.env_exe) + if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): + logger.warning('Actual environment location may have moved due to ' + 'redirects, links or junctions.\n' + ' Requested location: "%s"\n' + ' Actual location: "%s"', + context.env_exe, real_env_exe) + context.env_cmd = real_env_exe return context def create_configuration(self, context): @@ -294,7 +305,7 @@ def _setup_pip(self, context): # We run ensurepip in isolated mode to avoid side effects from # environment vars, the current directory and anything else # intended for the global Python environment - cmd = [context.env_exe, '-Im', 'ensurepip', '--upgrade', + cmd = [context.env_cmd, '-Im', 'ensurepip', '--upgrade', '--default-pip'] subprocess.check_output(cmd, stderr=subprocess.STDOUT) @@ -395,11 +406,7 @@ def upgrade_dependencies(self, context): logger.debug( f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' ) - if sys.platform == 'win32': - python_exe = os.path.join(context.bin_path, 'python.exe') - else: - python_exe = os.path.join(context.bin_path, 'python') - cmd = [python_exe, '-m', 'pip', 'install', '--upgrade'] + cmd = [context.env_cmd, '-m', 'pip', 'install', '--upgrade'] cmd.extend(CORE_VENV_DEPS) subprocess.check_call(cmd) diff --git a/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst b/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst new file mode 100644 index 000000000000000..007ee87195d6e31 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst @@ -0,0 +1,4 @@ +venv now warns when the created environment may need to be accessed at a +different path, due to redirections, links or junctions. It also now +correctly installs or upgrades components when the alternate path is +required. From 49fa6642d9ab5217e72cff81ed9a7677c8c7ca57 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 1 Oct 2021 14:38:34 +0100 Subject: [PATCH 2/5] Check against realpath on Windows --- Lib/test/test_venv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 42a3680fc1fab64..9a887f34b3138d4 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -152,12 +152,15 @@ def test_upgrade_dependencies(self): bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' python_exe = os.path.split(sys.executable)[1] with tempfile.TemporaryDirectory() as fake_env_dir: + expect_exe = os.path.join(fake_env_dir, bin_path, python_exe) + if sys.platform == 'win32': + expect_exe = os.path.realpath(expect_exe) def pip_cmd_checker(cmd): self.assertEqual( cmd, [ - os.path.join(fake_env_dir, bin_path, python_exe), + expect_exe, '-m', 'pip', 'install', From 4f2ba184fc61882cd4059fe8cfd3ade4a25e8e1d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 1 Oct 2021 18:43:26 +0100 Subject: [PATCH 3/5] Normalise path case before comparing strings --- Lib/test/test_venv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 9a887f34b3138d4..58ce305aafd98fb 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -157,10 +157,12 @@ def test_upgrade_dependencies(self): expect_exe = os.path.realpath(expect_exe) def pip_cmd_checker(cmd): + cmd[0] = os.path.normcase(cmd[0]) + exe = os.path.normcase(expect_exe) self.assertEqual( cmd, [ - expect_exe, + exe, '-m', 'pip', 'install', From eadb3cd4be06b6f88b74401f5b848a717bb3a241 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 1 Oct 2021 18:44:29 +0100 Subject: [PATCH 4/5] Better way to normalise case --- Lib/test/test_venv.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 58ce305aafd98fb..94d626598bac38e 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -152,17 +152,18 @@ def test_upgrade_dependencies(self): bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' python_exe = os.path.split(sys.executable)[1] with tempfile.TemporaryDirectory() as fake_env_dir: - expect_exe = os.path.join(fake_env_dir, bin_path, python_exe) + expect_exe = os.path.normcase( + os.path.join(fake_env_dir, bin_path, python_exe) + ) if sys.platform == 'win32': - expect_exe = os.path.realpath(expect_exe) + expect_exe = os.path.normcase(os.path.realpath(expect_exe)) def pip_cmd_checker(cmd): cmd[0] = os.path.normcase(cmd[0]) - exe = os.path.normcase(expect_exe) self.assertEqual( cmd, [ - exe, + expect_exe, '-m', 'pip', 'install', From 642720d5144678d39544da6191ca25583b110228 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 6 Oct 2021 20:22:28 +0100 Subject: [PATCH 5/5] Improve obviousness of new member --- Lib/venv/__init__.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 326ff9eced010b5..6f1af294ae63e36 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -140,11 +140,14 @@ def create_if_needed(d): os.symlink('lib', link_path) context.bin_path = binpath = os.path.join(env_dir, binname) context.bin_name = binname - context.env_exe = context.env_cmd = os.path.join(binpath, exename) + context.env_exe = os.path.join(binpath, exename) create_if_needed(binpath) + # Assign and update the command to use when launching the newly created + # environment, in case it isn't simply the executable script (e.g. bpo-45337) + context.env_exec_cmd = context.env_exe if sys.platform == 'win32': - # Fixup the env_cmd to account for redirections when we launch - # the newly created environment. + # bpo-45337: Fix up env_exec_cmd to account for file system redirections. + # Some redirects only apply to CreateFile and not CreateProcess real_env_exe = os.path.realpath(context.env_exe) if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): logger.warning('Actual environment location may have moved due to ' @@ -152,7 +155,7 @@ def create_if_needed(d): ' Requested location: "%s"\n' ' Actual location: "%s"', context.env_exe, real_env_exe) - context.env_cmd = real_env_exe + context.env_exec_cmd = real_env_exe return context def create_configuration(self, context): @@ -305,8 +308,8 @@ def _setup_pip(self, context): # We run ensurepip in isolated mode to avoid side effects from # environment vars, the current directory and anything else # intended for the global Python environment - cmd = [context.env_cmd, '-Im', 'ensurepip', '--upgrade', - '--default-pip'] + cmd = [context.env_exec_cmd, '-Im', 'ensurepip', '--upgrade', + '--default-pip'] subprocess.check_output(cmd, stderr=subprocess.STDOUT) def setup_scripts(self, context): @@ -406,7 +409,7 @@ def upgrade_dependencies(self, context): logger.debug( f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' ) - cmd = [context.env_cmd, '-m', 'pip', 'install', '--upgrade'] + cmd = [context.env_exec_cmd, '-m', 'pip', 'install', '--upgrade'] cmd.extend(CORE_VENV_DEPS) subprocess.check_call(cmd)