bpo-45211: Remember the stdlib dir during startup.

Include/cpython/initconfig.h

@@ -184,6 +184,7 @@ typedef struct PyConfig{
  /* --- Path configuration outputs ----------- */
  int module_search_paths_set;
  PyWideStringList module_search_paths;
+ wchar_t *stdlib_dir;
  wchar_t *executable;
  wchar_t *base_executable;
  wchar_t *prefix;

Include/internal/pycore_pathconfig.h

@@ -13,6 +13,7 @@ typedef struct _PyPathConfig{
  wchar_t *program_full_path;
  wchar_t *prefix;
  wchar_t *exec_prefix;
+ wchar_t *stdlib_dir;
  /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */
  wchar_t *module_search_path;
  /* Python program name */

Include/internal/pycore_pylifecycle.h

@@ -122,6 +122,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig(
  const PyConfig *config,
  const struct _PyArgv *args);
 
+PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void);
 
 PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p);
 

Lib/test/test_embed.py

@@ -406,6 +406,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
  'module_search_paths': GET_DEFAULT_CONFIG,
  'module_search_paths_set': 1,
  'platlibdir': sys.platlibdir,
+ 'stdlib_dir': GET_DEFAULT_CONFIG,
 
  'site_import': 1,
  'bytes_warning': 0,
@@ -515,6 +516,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
  'exec_prefix',
  'program_name',
  'home',
+ 'stdlib_dir',
  # program_full_path and module_search_path are copied indirectly from
  # the core configuration in check_path_config().
  ]
@@ -1142,6 +1144,9 @@ def test_init_setpath(self):
  'base_prefix': '',
  'exec_prefix': '',
  'base_exec_prefix': '',
+ # The current getpath.c doesn't determine the stdlib dir
+ # in this case.
+ 'stdlib_dir': '',
  }
  self.default_program_name(config)
  env ={'TESTPATH': os.path.pathsep.join(paths)}
@@ -1162,6 +1167,9 @@ def test_init_setpath_config(self):
  'base_prefix': '',
  'exec_prefix': '',
  'base_exec_prefix': '',
+ # The current getpath.c doesn't determine the stdlib dir
+ # in this case.
+ 'stdlib_dir': '',
  # overriden by PyConfig
  'program_name': 'conf_program_name',
  'base_executable': 'conf_executable',
@@ -1251,6 +1259,7 @@ def test_init_setpythonhome(self):
  'exec_prefix': exec_prefix,
  'base_exec_prefix': exec_prefix,
  'pythonpath_env': paths_str,
+ 'stdlib_dir': home,
  }
  self.default_program_name(config)
  env ={'TESTHOME': home, 'PYTHONPATH': paths_str}
@@ -1288,6 +1297,9 @@ def test_init_pybuilddir(self):
  'base_executable': executable,
  'executable': executable,
  'module_search_paths': module_search_paths,
+ # The current getpath.c doesn't determine the stdlib dir
+ # in this case.
+ 'stdlib_dir': None,
  }
  env = self.copy_paths_by_env(config)
  self.check_all_configs("test_init_compat_config", config,
@@ -1345,6 +1357,7 @@ def test_init_pyvenv_cfg(self):
  if MS_WINDOWS:
  config['base_prefix'] = pyvenv_home
  config['prefix'] = pyvenv_home
+ config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib')
 
  ver = sys.version_info
  dll = f'python{ver.major}'
@@ -1353,6 +1366,10 @@ def test_init_pyvenv_cfg(self):
  dll += '.DLL'
  dll = os.path.join(os.path.dirname(executable), dll)
  path_config['python3_dll'] = dll
+ else:
+ # The current getpath.c doesn't determine the stdlib dir
+ # in this case.
+ config['stdlib_dir'] = None
 
  env = self.copy_paths_by_env(config)
  self.check_all_configs("test_init_compat_config", config,

Lib/test/test_sys.py

@@ -13,6 +13,7 @@
 from test.support import os_helper
 from test.support.script_helper import assert_python_ok, assert_python_failure
 from test.support import threading_helper
+from test.support import import_helper
 import textwrap
 import unittest
 import warnings
@@ -994,6 +995,15 @@ def test_module_names(self):
  for name in sys.stdlib_module_names:
  self.assertIsInstance(name, str)
 
+ def test_stdlib_dir(self):
+ os = import_helper.import_fresh_module('os')
+ marker = getattr(os, '__file__', None)
+ if marker and not os.path.exists(marker):
+ marker = None
+ expected = os.path.dirname(marker) if marker else None
+ actual = sys._stdlib_dir
+ self.assertEqual(actual, expected)
+
 
 @test.support.cpython_only
 class UnraisableHookTest(unittest.TestCase):

Modules/getpath.c

@@ -1492,6 +1492,16 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
  }
  }
 
+ if (pathconfig->stdlib_dir == NULL){
+ if (calculate->prefix_found){
+ /* This must be done *before* calculate_set_prefix() is called. */
+ pathconfig->stdlib_dir = _PyMem_RawWcsdup(calculate->prefix);
+ if (pathconfig->stdlib_dir == NULL){
+ return _PyStatus_NO_MEMORY();
+ }
+ }
+ }
+
  if (pathconfig->prefix == NULL){
  status = calculate_set_prefix(calculate, pathconfig);
  if (_PyStatus_EXCEPTION(status)){

PC/getpathp.c

@@ -116,6 +116,8 @@
  * with a semicolon separated path prior to calling Py_Initialize.
  */
 
+#define STDLIB_SUBDIR L"lib"
+
 #define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow")
 
 
@@ -293,12 +295,12 @@ search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path)
  wcscpy_s(stdlibdir, Py_ARRAY_LENGTH(stdlibdir), prefix);
  /* We initialize with the longest possible path, in case it doesn't fit.
  This also gives us an initial SEP at stdlibdir[wcslen(prefix)]. */
- join(stdlibdir, L"lib");
+ join(stdlibdir, STDLIB_SUBDIR);
  do{
  assert(stdlibdir[wcslen(prefix)] == SEP);
  /* Due to reduce() and our initial value, this result
  is guaranteed to fit. */
- wcscpy(&stdlibdir[wcslen(prefix) + 1], L"lib");
+ wcscpy(&stdlibdir[wcslen(prefix) + 1], STDLIB_SUBDIR);
  if (is_stdlibdir(stdlibdir)){
  return 1;
  }
@@ -1013,6 +1015,12 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
  }
 
 done:
+ if (pathconfig->stdlib_dir == NULL){
+ pathconfig->stdlib_dir = _Py_join_relfile(prefix, STDLIB_SUBDIR);
+ if (pathconfig->stdlib_dir == NULL){
+ return _PyStatus_NO_MEMORY();
+ }
+ }
  if (pathconfig->prefix == NULL){
  pathconfig->prefix = _PyMem_RawWcsdup(prefix);
  if (pathconfig->prefix == NULL){

Python/initconfig.c

@@ -669,6 +669,7 @@ PyConfig_Clear(PyConfig *config)
  _PyWideStringList_Clear(&config->xoptions);
  _PyWideStringList_Clear(&config->module_search_paths);
  config->module_search_paths_set = 0;
+ CLEAR(config->stdlib_dir);
 
  CLEAR(config->executable);
  CLEAR(config->base_executable);
@@ -909,6 +910,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
  COPY_WSTRLIST(xoptions);
  COPY_WSTRLIST(module_search_paths);
  COPY_ATTR(module_search_paths_set);
+ COPY_WSTR_ATTR(stdlib_dir);
 
  COPY_WSTR_ATTR(executable);
  COPY_WSTR_ATTR(base_executable);
@@ -1015,6 +1017,7 @@ _PyConfig_AsDict(const PyConfig *config)
  SET_ITEM_WSTR(home);
  SET_ITEM_INT(module_search_paths_set);
  SET_ITEM_WSTRLIST(module_search_paths);
+ SET_ITEM_WSTR(stdlib_dir);
  SET_ITEM_WSTR(executable);
  SET_ITEM_WSTR(base_executable);
  SET_ITEM_WSTR(prefix);
@@ -1318,6 +1321,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
  // Path configuration output
  GET_UINT(module_search_paths_set);
  GET_WSTRLIST(module_search_paths);
+ GET_WSTR_OPT(stdlib_dir);
  GET_WSTR_OPT(executable);
  GET_WSTR_OPT(base_executable);
  GET_WSTR_OPT(prefix);
@@ -3094,6 +3098,7 @@ _Py_DumpPathConfig(PyThreadState *tstate)
  PySys_WriteStderr(" environment = %i\n", config->use_environment);
  PySys_WriteStderr(" user site = %i\n", config->user_site_directory);
  PySys_WriteStderr(" import site = %i\n", config->site_import);
+ DUMP_CONFIG("stdlib dir", stdlib_dir);
 #undef DUMP_CONFIG
 
 #define DUMP_SYS(NAME) \

Python/pathconfig.c

@@ -54,6 +54,7 @@ pathconfig_clear(_PyPathConfig *config)
  CLEAR(config->program_full_path);
  CLEAR(config->prefix);
  CLEAR(config->exec_prefix);
+ CLEAR(config->stdlib_dir);
  CLEAR(config->module_search_path);
  CLEAR(config->program_name);
  CLEAR(config->home);
@@ -83,6 +84,7 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
  COPY_ATTR(prefix);
  COPY_ATTR(exec_prefix);
  COPY_ATTR(module_search_path);
+ COPY_ATTR(stdlib_dir);
  COPY_ATTR(program_name);
  COPY_ATTR(home);
 #ifdef MS_WINDOWS
@@ -167,6 +169,7 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
  COPY_CONFIG(program_full_path, executable);
  COPY_CONFIG(prefix, prefix);
  COPY_CONFIG(exec_prefix, exec_prefix);
+ COPY_CONFIG(stdlib_dir, stdlib_dir);
  COPY_CONFIG(program_name, program_name);
  COPY_CONFIG(home, home);
 #ifdef MS_WINDOWS
@@ -218,6 +221,7 @@ _PyPathConfig_AsDict(void)
  SET_ITEM_STR(prefix);
  SET_ITEM_STR(exec_prefix);
  SET_ITEM_STR(module_search_path);
+ SET_ITEM_STR(stdlib_dir);
  SET_ITEM_STR(program_name);
  SET_ITEM_STR(home);
 #ifdef MS_WINDOWS
@@ -311,6 +315,7 @@ config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
 
  - exec_prefix
  - module_search_path
+ - stdlib_dir
  - prefix
  - program_full_path
 
@@ -401,6 +406,7 @@ config_init_pathconfig(PyConfig *config, int compute_path_config)
  COPY_ATTR(program_full_path, executable);
  COPY_ATTR(prefix, prefix);
  COPY_ATTR(exec_prefix, exec_prefix);
+ COPY_ATTR(stdlib_dir, stdlib_dir);
 
 #undef COPY_ATTR
 
@@ -486,16 +492,25 @@ Py_SetPath(const wchar_t *path)
 
  PyMem_RawFree(_Py_path_config.prefix);
  PyMem_RawFree(_Py_path_config.exec_prefix);
+ PyMem_RawFree(_Py_path_config.stdlib_dir);
  PyMem_RawFree(_Py_path_config.module_search_path);
 
  _Py_path_config.prefix = _PyMem_RawWcsdup(L"");
  _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
+ // XXX Copy this from the new module_search_path?
+ if (_Py_path_config.home != NULL){
+ _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(_Py_path_config.home);
+ }
+ else{
+ _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L"");
+ }
  _Py_path_config.module_search_path = _PyMem_RawWcsdup(path);
 
  PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
  if (_Py_path_config.prefix == NULL
  || _Py_path_config.exec_prefix == NULL
+ || _Py_path_config.stdlib_dir == NULL
  || _Py_path_config.module_search_path == NULL)
 {
  path_out_of_memory(__func__);
@@ -515,10 +530,13 @@ Py_SetPythonHome(const wchar_t *home)
 
  PyMem_RawFree(_Py_path_config.home);
  _Py_path_config.home = _PyMem_RawWcsdup(home);
+ if (_Py_path_config.home != NULL){
+ _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(home);
+ }
 
  PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
- if (_Py_path_config.home == NULL){
+ if (_Py_path_config.home == NULL || _Py_path_config.stdlib_dir == NULL){
  path_out_of_memory(__func__);
  }
 }
@@ -572,6 +590,17 @@ Py_GetPath(void)
 }
 
 
+wchar_t *
+_Py_GetStdlibDir(void)
+{
+ wchar_t *stdlib_dir = _Py_path_config.stdlib_dir;
+ if (stdlib_dir != NULL && stdlib_dir[0] != L'\0'){
+ return stdlib_dir;
+ }
+ return NULL;
+}
+
+
 wchar_t *
 Py_GetPrefix(void)
 {

Python/sysmodule.c

@@ -2974,6 +2974,14 @@ _PySys_UpdateConfig(PyThreadState *tstate)
 
  SET_SYS("_xoptions", sys_create_xoptions_dict(config));
 
+ const wchar_t *stdlibdir = _Py_GetStdlibDir();
+ if (stdlibdir != NULL){
+ SET_SYS_FROM_WSTR("_stdlib_dir", stdlibdir);
+ }
+ else{
+ PyDict_SetItemString(sysdict, "_stdlib_dir", Py_None);
+ }
+
 #undef SET_SYS_FROM_WSTR
 #undef COPY_LIST
 #undef COPY_WSTR