bpo-34977: Add Windows App Store package

Author: zooba

.azure-pipelines/windows-appx-test.yml

@@ -0,0 +1,65 @@
+jobs:
+- job: Prebuild
+  displayName: Pre-build checks
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  steps:
+  - template: ./prebuild-checks.yml
+
+
+- job: Windows_Appx_Tests
+  displayName: Windows Appx Tests
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+  pool:
+    vmImage: vs2017-win2016
+
+  strategy:
+    matrix:
+      win64:
+        arch: amd64
+        buildOpt: '-p x64'
+        testRunTitle: '$(Build.SourceBranchName)-win64-appx'
+        testRunPlatform: win64
+    maxParallel: 2
+
+  steps:
+  - checkout: self
+    clean: true
+    fetchDepth: 5
+
+  - powershell: |
+      # Relocate build outputs outside of source directory to make cleaning faster
+      Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
+      # UNDONE: Do not build to a different directory because of broken tests
+      Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
+      Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals'
+    displayName: Update build locations
+
+  - script: PCbuild\build.bat -e $(buildOpt)
+    displayName: 'Build CPython'
+
+  - script: python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-$(arch)" --copy "$(Py_IntDir)\layout-$(arch)" --precompile --preset-appx --include-tests
+    displayName: 'Create APPX layout'
+
+  - script: .\python.exe -m test.pythoninfo
+    workingDirectory: $(Py_IntDir)\layout-$(arch)
+    displayName: 'Display build info'
+
+  - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir "$(Py_IntDir)\tmp-$(arch)"
+    workingDirectory: $(Py_IntDir)\layout-$(arch)
+    displayName: 'Tests'
+    env:
+      PREFIX: $(Py_IntDir)\layout-$(arch)
+
+  - task: PublishTestResults@2
+    displayName: 'Publish Test Results'
+    inputs:
+      testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
+      mergeTestResults: true
+      testRunTitle: $(testRunTitle)
+      platform: $(testRunPlatform)
+    condition: succeededOrFailed()

Doc/make.bat

@@ -117,10 +117,12 @@ if not exist "%BUILDDIR%" mkdir "%BUILDDIR%"
 
 if exist ..\Misc\NEWS (
     echo.Copying Misc\NEWS to build\NEWS
+    if not exist build mkdir build
     copy ..\Misc\NEWS build\NEWS > nul
 ) else if exist ..\Misc\NEWS.D (
     if defined BLURB (
         echo.Merging Misc/NEWS with %BLURB%
+        if not exist build mkdir build
         %BLURB% merge -f build\NEWS
     ) else (
         echo.No Misc/NEWS file and Blurb is not available.

Lib/test/test_venv.py

@@ -243,6 +243,7 @@ def test_isolation(self):
             self.assertIn('include-system-site-packages = %s\n' % s, data)
 
     @unittest.skipUnless(can_symlink(), 'Needs symlinks')
+    @unittest.skipIf(os.name == 'nt', 'Symlinks are never used on Windows')
     def test_symlinking(self):
         """
         Test symlinking works as expected

Lib/venv/__init__.py

@@ -64,10 +64,11 @@ def create(self, env_dir):
         self.system_site_packages = False
         self.create_configuration(context)
         self.setup_python(context)
+        if not self.upgrade:
+            self.setup_scripts(context)
         if self.with_pip:
             self._setup_pip(context)
         if not self.upgrade:
-            self.setup_scripts(context)
             self.post_setup(context)
         if true_system_site_packages:
             # We had set it to False before, now
@@ -158,14 +159,6 @@ def create_configuration(self, context):
             f.write('include-system-site-packages = %s\n' % incl)
             f.write('version = %d.%d.%d\n' % sys.version_info[:3])
 
-    if os.name == 'nt':
-        def include_binary(self, f):
-            if f.endswith(('.pyd', '.dll')):
-                result = True
-            else:
-                result = f.startswith('python') and f.endswith('.exe')
-            return result
-
     def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
         """
         Try symlinking a file, and if that fails, fall back to copying.
@@ -195,9 +188,9 @@ def setup_python(self, context):
         binpath = context.bin_path
         path = context.env_exe
         copier = self.symlink_or_copy
-        copier(context.executable, path)
         dirname = context.python_dir
         if os.name != 'nt':
+            copier(context.executable, path)
             if not os.path.islink(path):
                 os.chmod(path, 0o755)
             for suffix in ('python', 'python3'):
@@ -209,26 +202,22 @@ def setup_python(self, context):
                     if not os.path.islink(path):
                         os.chmod(path, 0o755)
         else:
-            # See bpo-34011. When using a proper install, we should only need to
-            # copy the top-level of DLLs.
-            include = self.include_binary
-            files = [f for f in os.listdir(dirname) if include(f)]
-            for f in files:
-                src = os.path.join(dirname, f)
-                dst = os.path.join(binpath, f)
-                if dst != context.env_exe:  # already done, above
-                    copier(src, dst)
-
-            # When creating from a build directory, we continue to copy all files.
+            # For normal cases, the venvlauncher will be copied from
+            # our scripts folder. For builds, we need to copy it
+            # manually.
             if sysconfig.is_python_build(True):
-                subdir = 'DLLs'
-                dirname = os.path.join(dirname, subdir)
-                if os.path.isdir(dirname):
-                    files = [f for f in os.listdir(dirname) if include(f)]
-                    for f in files:
-                        src = os.path.join(dirname, f)
-                        dst = os.path.join(binpath, f)
-                        copier(src, dst)
+                suffix = '.exe'
+                if context.python_exe.lower().endswith('_d.exe'):
+                    suffix = '_d.exe'
+
+                src = os.path.join(dirname, "venvlauncher" + suffix)
+                dst = os.path.join(binpath, context.python_exe)
+                copier(src, dst)
+
+                src = os.path.join(dirname, "venvwlauncher" + suffix)
+                dst = os.path.join(binpath, "pythonw" + suffix)
+                copier(src, dst)
+
                 # copy init.tcl over
                 for root, dirs, files in os.walk(context.python_dir):
                     if 'init.tcl' in files:

Misc/NEWS.d/next/Windows/2018-10-30-13-39-17.bpo-34977.0l7_QV.rst

@@ -0,0 +1 @@
+Adds support for building a Windows App Store package

Modules/_testcapimodule.c

@@ -4960,7 +4960,7 @@ static PyMethodDef TestMethods[] = {
     {"get_mapping_items", get_mapping_items, METH_O},
     {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS},
     {"hamt", new_hamt, METH_NOARGS},
-    {"bad_get", bad_get, METH_FASTCALL},
+    {"bad_get", (PyCFunction)bad_get, METH_FASTCALL},
     {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
     {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
     {"get_global_config", get_global_config, METH_NOARGS},

PC/getpathp.c

@@ -536,10 +536,16 @@ static _PyInitError
 get_program_full_path(const _PyCoreConfig *core_config,
                       PyCalculatePath *calculate, _PyPathConfig *config)
 {
+    const wchar_t *pyvenv_launcher;
     wchar_t program_full_path[MAXPATHLEN+1];
     memset(program_full_path, 0, sizeof(program_full_path));
 
-    if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
+    /* The launcher may need to force the executable path to a
+     * different environment, so override it here. */
+    pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__");
+    if (pyvenv_launcher && pyvenv_launcher[0]) {
+        wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher);
+    } else if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
         /* GetModuleFileName should never fail when passed NULL */
         return _Py_INIT_ERR("Cannot determine program path");
     }