Skip to content

bpo-41100: Support macOS 11 and Apple Silicon#22855

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ronaldoussoren merged 49 commits into from
Nov 8, 2020
Merged

bpo-41100: Support macOS 11 and Apple Silicon #22855

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
919efcc
Add support for ARM64 to the pythonw tool
ronaldoussoren Jul 20, 2020
69c39f3
Add support for "universal2" as a fat binary target on macOS
ronaldoussoren Jul 20, 2020
3940c86
Tweak ctypes.macholib.dyld to work with the shared library cache.
ronaldoussoren Jul 20, 2020
deda5f0
Silence compile time warning
ronaldoussoren Jul 20, 2020
ea3c200
macOS/arm64 support, based on GH-21249
ronaldoussoren Jul 20, 2020
552bca8
Fix test for macOS before 10.10
ronaldoussoren Jul 20, 2020
e0c23a1
ARM64 is a valid platform for macOS
ronaldoussoren Jul 20, 2020
e637a77
Unconditionally use uint32_t here.
ronaldoussoren Jul 20, 2020
02fb660
First stab at deploying to an earlier version of macOS than the build…
ronaldoussoren Jul 21, 2020
ba2f5a3
Update build-installer.py for universal2 builds
ronaldoussoren Jul 26, 2020
8e3b454
Merge ctypes changes from Apple (PR 21241)
ronaldoussoren Jul 26, 2020
87c942b
The archive-deps.py script isn't really useful
ronaldoussoren Jul 26, 2020
515fbe6
Merge branch 'macos-deploy-to-earlier' into macos11-deploy-earlier-br…
ronaldoussoren Oct 20, 2020
eee5437
Cleanup the code to check at runtime for a couple of APIs.
ronaldoussoren Oct 20, 2020
3a1d4f2
Dynamicly fill posix._have_functions as well
ronaldoussoren Oct 20, 2020
2f019f4
Changes after testing with Xcode 6.2 on macOS 10.9
ronaldoussoren Oct 21, 2020
cec3da7
Code formatting
ronaldoussoren Oct 21, 2020
86b5cf3
Fix a number of errors in weaklinking support
ronaldoussoren Oct 21, 2020
d604cef
Merge branch 'macos11-deploy-earlier-branch' of github.com:ronaldouss…
ronaldoussoren Oct 21, 2020
dde0ba4
Implement tests for weak linked symbols.
ronaldoussoren Oct 21, 2020
7ac26c4
Finish the test for os.link behaviour
ronaldoussoren Oct 21, 2020
191a2d7
posixmodule.c uses multi-phase initialization
ronaldoussoren Oct 21, 2020
e6d195b
Make sure some form of prep_cif is called, even if the variadic versi…
ronaldoussoren Oct 21, 2020
cfb02ba
Move the removal of unavailable functions to the module exec
ronaldoussoren Oct 21, 2020
003dae8
Improved error handling
ronaldoussoren Oct 21, 2020
004ba4e
Remove '-arch arm64' from the compiler arguments when necessary
ronaldoussoren Oct 21, 2020
6019346
check that weak linking works for the time module
ronaldoussoren Oct 22, 2020
54576ab
Drop the DEPS_ONLY changes
ronaldoussoren Oct 22, 2020
6af77ab
Undo change to default PATH
ronaldoussoren Oct 22, 2020
b653df9
Use configure to detect the presence of _dyld_shared_cache_contains_path
ronaldoussoren Oct 22, 2020
98af7b3
Revert unnecessary change
ronaldoussoren Oct 22, 2020
817d9bf
Updates to timemodule.c
ronaldoussoren Oct 22, 2020
8684d9d
Stricter checking of __builtin_available
ronaldoussoren Oct 22, 2020
0b44610
Clean up setup.py
ronaldoussoren Oct 22, 2020
36deb92
Fix build failures on Linux
ronaldoussoren Oct 22, 2020
33d5710
Fix typo
ronaldoussoren Oct 22, 2020
c3113eb
Remove reference to non-existing probe function
ronaldoussoren Oct 22, 2020
587e53e
One more attempt at building on Linux...
ronaldoussoren Oct 22, 2020
24ef276
A bit of clean up of the setup.py logic.
ronaldoussoren Oct 22, 2020
e0614bc
Fix patchcheck issues
ronaldoussoren Oct 22, 2020
e6478fa
add prefix to f-strings
ronaldoussoren Oct 22, 2020
3f72949
Add missing preprocessor guards
ronaldoussoren Oct 22, 2020
5daff99
Use the correct cut-off for the availability of clock_gettime and
ronaldoussoren Oct 22, 2020
6681261
Fix error in preprocessor logic for os.chmod implementation
ronaldoussoren Oct 22, 2020
b17a3d5
Merge branch 'master' into macos11-deploy-earlier-branch
ronaldoussoren Oct 22, 2020
7e64e95
the sid attribute for posix_spawn is only available on recent version…
ronaldoussoren Oct 31, 2020
fef2e93
Changes due to review by Victor Stinner
ronaldoussoren Oct 31, 2020
24eb0d0
Merge branch 'macos11-deploy-earlier-branch' of github.com:ronaldouss…
ronaldoussoren Oct 31, 2020
296666b
inor changes
ronaldoussoren Nov 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 38 additions & 6 deletions Lib/_osx_support.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -110,6 +110,26 @@ def _get_system_version():

return _SYSTEM_VERSION

_SYSTEM_VERSION_TUPLE = None
def _get_system_version_tuple():
"""
Return the macOS system version as a tuple

The return value is safe to use to compare
two version numbers.
"""
global _SYSTEM_VERSION_TUPLE
if _SYSTEM_VERSION_TUPLE is None:
osx_version = _get_system_version()
if osx_version:
try:
_SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.'))
except ValueError:
_SYSTEM_VERSION_TUPLE = ()

return _SYSTEM_VERSION_TUPLE


def _remove_original_values(_config_vars):
"""Remove original unmodified values for testing"""
# This is needed for higher-level cross-platform tests of get_platform.
Expand All@@ -132,14 +152,18 @@ def _supports_universal_builds():
# builds, in particular -isysroot and -arch arguments to the compiler. This
# is in support of allowing 10.4 universal builds to run on 10.3.x systems.

osx_version = _get_system_version()
if osx_version:
try:
osx_version = tuple(int(i) for i in osx_version.split('.'))
except ValueError:
osx_version = ''
osx_version = _get_system_version_tuple()
return bool(osx_version >= (10, 4)) if osx_version else False

def _supports_arm64_builds():
"""Returns True if arm64 builds are supported on this system"""
# There are two sets of systems supporting macOS/arm64 builds:
# 1. macOS 11 and later, unconditionally
# 2. macOS 10.15 with Xcode 12.2 or later
# For now the second category is ignored.
osx_version = _get_system_version_tuple()
return osx_version >= (11, 0) if osx_version else False


def _find_appropriate_compiler(_config_vars):
"""Find appropriate C compiler for extension module builds"""
Expand DownExpand Up@@ -331,6 +355,12 @@ def compiler_fixup(compiler_so, cc_args):
except ValueError:
break

elif not _supports_arm64_builds():
# Look for "-arch arm64" and drop that
for idx in range(len(compiler_so)):
Copy link

@ddl-jamesleeddl-jamesleeDec 1, 2021•
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When idx is len(compiler_so)-1, it will throw an IndexError exception.
So, it should be for idx in range(len(compiler_so)-1):

Copy link

@smoofrasmoofraDec 1, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dang, looks like you're right

Copy link
Member

@ned-deilyned-deilyDec 1, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This PR was merged over a year ago and has long since been released to the field. Please open a new issue on the bug tracker and, if possible, provide a PR. Otherwise, it will likely be forgotten about it. Thanks!

if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64":
del compiler_so[idx:idx+2]

if 'ARCHFLAGS' in os.environ and not stripArch:
# User specified different -arch flags in the environ,
# see also distutils.sysconfig
Expand DownExpand Up@@ -481,6 +511,8 @@ def get_platform_osx(_config_vars, osname, release, machine):

if len(archs) == 1:
machine = archs[0]
elif archs == ('arm64', 'x86_64'):
machine = 'universal2'
elif archs == ('i386', 'ppc'):
machine = 'fat'
elif archs == ('i386', 'x86_64'):
Expand Down
12 changes: 12 additions & 0 deletions Lib/ctypes/macholib/dyld.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,6 +6,11 @@
from ctypes.macholib.framework import framework_info
from ctypes.macholib.dylib import dylib_info
from itertools import *
try:
from _ctypes import _dyld_shared_cache_contains_path
except ImportError:
def _dyld_shared_cache_contains_path(*args):
raise NotImplementedError

__all__ = [
'dyld_find', 'framework_find',
Expand DownExpand Up@@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None):
dyld_executable_path_search(name, executable_path),
dyld_default_search(name, env),
), env):

if os.path.isfile(path):
return path
try:
if _dyld_shared_cache_contains_path(path):
return path
except NotImplementedError:
pass

raise ValueError("dylib %s could not be found" % (name,))

def framework_find(fn, executable_path=None, env=None):
Expand Down
15 changes: 9 additions & 6 deletions Lib/ctypes/test/test_macholib.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,19 +45,22 @@ def find_lib(name):
class MachOTest(unittest.TestCase):
@unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
def test_find(self):

self.assertEqual(find_lib('pthread'),
'/usr/lib/libSystem.B.dylib')
# On Mac OS 11, system dylibs are only present in the shared cache,
# so symlinks like libpthread.dylib -> libSystem.B.dylib will not
# be resolved by dyld_find
self.assertIn(find_lib('pthread'),
('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib'))

result = find_lib('z')
# Issue #21093: dyld default search path includes $HOME/lib and
# /usr/local/lib before /usr/lib, which caused test failures if
# a local copy of libz exists in one of them. Now ignore the head
# of the path.
self.assertRegex(result, r".*/lib/libz\..*.*\.dylib")
self.assertRegex(result, r".*/lib/libz.*\.dylib")

self.assertEqual(find_lib('IOKit'),
'/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit')
self.assertIn(find_lib('IOKit'),
('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit',
'/System/Library/Frameworks/IOKit.framework/IOKit'))

if __name__ == "__main__":
unittest.main()
2 changes: 1 addition & 1 deletion Lib/distutils/tests/test_build_ext.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -493,7 +493,7 @@ def _try_compile_deployment_target(self, operator, target):
# format the target value as defined in the Apple
# Availability Macros. We can't use the macro names since
# at least one value we test with will not exist yet.
if target[1] < 10:
if target[:2] < (10, 10):
# for 10.1 through 10.9.x -> "10n0"
target = '%02d%01d0' % target
else:
Expand Down
1 change: 1 addition & 0 deletions Lib/test/test_bytes.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1036,6 +1036,7 @@ def test_from_format(self):
c_char_p)

PyBytes_FromFormat = pythonapi.PyBytes_FromFormat
PyBytes_FromFormat.argtypes = (c_char_p,)
PyBytes_FromFormat.restype = py_object

# basic tests
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_platform.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -246,7 +246,7 @@ def test_mac_ver(self):
self.assertEqual(res[1], ('', '', ''))

if sys.byteorder == 'little':
self.assertIn(res[2], ('i386', 'x86_64'))
self.assertIn(res[2], ('i386', 'x86_64', 'arm64'))
else:
self.assertEqual(res[2], 'PowerPC')

Expand Down
228 changes: 228 additions & 0 deletions Lib/test/test_posix.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1925,13 +1925,241 @@ def test_posix_spawnp(self):
assert_python_ok(*args, PATH=path)


@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS")
class TestPosixWeaklinking(unittest.TestCase):
# These test cases verify that weak linking support on macOS works
# as expected. These cases only test new behaviour introduced by weak linking,
# regular behaviour is tested by the normal test cases.
#
# See the section on Weak Linking in Mac/README.txt for more information.
def setUp(self):
import sysconfig
import platform

config_vars = sysconfig.get_config_vars()
self.available ={nm for nm in config_vars if nm.startswith("HAVE_") and config_vars[nm] }
self.mac_ver = tuple(int(part) for part in platform.mac_ver()[0].split("."))

def _verify_available(self, name):
if name not in self.available:
raise unittest.SkipTest(f"{name} not weak-linked")

def test_pwritev(self):
self._verify_available("HAVE_PWRITEV")
if self.mac_ver >= (10, 16):
self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available")
self.assertTrue(hasattr(os, "preadv"), "os.readv is not available")

else:
self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available")
self.assertFalse(hasattr(os, "preadv"), "os.readv is available")

def test_stat(self):
self._verify_available("HAVE_FSTATAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_FSTATAT", posix._have_functions)

else:
self.assertNotIn("HAVE_FSTATAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.stat("file", dir_fd=0)

def test_access(self):
self._verify_available("HAVE_FACCESSAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_FACCESSAT", posix._have_functions)

else:
self.assertNotIn("HAVE_FACCESSAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.access("file", os.R_OK, dir_fd=0)

with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"):
os.access("file", os.R_OK, follow_symlinks=False)

with self.assertRaisesRegex(NotImplementedError, "effective_ids unavailable"):
os.access("file", os.R_OK, effective_ids=True)

def test_chmod(self):
self._verify_available("HAVE_FCHMODAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_FCHMODAT", posix._have_functions)

else:
self.assertNotIn("HAVE_FCHMODAT", posix._have_functions)
self.assertIn("HAVE_LCHMOD", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.chmod("file", 0o644, dir_fd=0)

def test_chown(self):
self._verify_available("HAVE_FCHOWNAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_FCHOWNAT", posix._have_functions)

else:
self.assertNotIn("HAVE_FCHOWNAT", posix._have_functions)
self.assertIn("HAVE_LCHOWN", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.chown("file", 0, 0, dir_fd=0)

def test_link(self):
self._verify_available("HAVE_LINKAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_LINKAT", posix._have_functions)

else:
self.assertNotIn("HAVE_LINKAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"):
os.link("source", "target", src_dir_fd=0)

with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"):
os.link("source", "target", dst_dir_fd=0)

with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"):
os.link("source", "target", src_dir_fd=0, dst_dir_fd=0)

# issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag
with os_helper.temp_dir() as base_path:
link_path = os.path.join(base_path, "link")
target_path = os.path.join(base_path, "target")
source_path = os.path.join(base_path, "source")

with open(source_path, "w") as fp:
fp.write("data")

os.symlink("target", link_path)

# Calling os.link should fail in the link(2) call, and
# should not reject *follow_symlinks* (to match the
# behaviour you'd get when building on a platform without
# linkat)
with self.assertRaises(FileExistsError):
os.link(source_path, link_path, follow_symlinks=True)

with self.assertRaises(FileExistsError):
os.link(source_path, link_path, follow_symlinks=False)


def test_listdir_scandir(self):
self._verify_available("HAVE_FDOPENDIR")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_FDOPENDIR", posix._have_functions)

else:
self.assertNotIn("HAVE_FDOPENDIR", posix._have_functions)

with self.assertRaisesRegex(TypeError, "listdir: path should be string, bytes, os.PathLike or None, not int"):
os.listdir(0)

with self.assertRaisesRegex(TypeError, "scandir: path should be string, bytes, os.PathLike or None, not int"):
os.scandir(0)

def test_mkdir(self):
self._verify_available("HAVE_MKDIRAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_MKDIRAT", posix._have_functions)

else:
self.assertNotIn("HAVE_MKDIRAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.mkdir("dir", dir_fd=0)

def test_rename_replace(self):
self._verify_available("HAVE_RENAMEAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_RENAMEAT", posix._have_functions)

else:
self.assertNotIn("HAVE_RENAMEAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
os.rename("a", "b", src_dir_fd=0)

with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
os.rename("a", "b", dst_dir_fd=0)

with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
os.replace("a", "b", src_dir_fd=0)

with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
os.replace("a", "b", dst_dir_fd=0)

def test_unlink_rmdir(self):
self._verify_available("HAVE_UNLINKAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_UNLINKAT", posix._have_functions)

else:
self.assertNotIn("HAVE_UNLINKAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.unlink("path", dir_fd=0)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.rmdir("path", dir_fd=0)

def test_open(self):
self._verify_available("HAVE_OPENAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_OPENAT", posix._have_functions)

else:
self.assertNotIn("HAVE_OPENAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.open("path", os.O_RDONLY, dir_fd=0)

def test_readlink(self):
self._verify_available("HAVE_READLINKAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_READLINKAT", posix._have_functions)

else:
self.assertNotIn("HAVE_READLINKAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.readlink("path", dir_fd=0)

def test_symlink(self):
self._verify_available("HAVE_SYMLINKAT")
if self.mac_ver >= (10, 10):
self.assertIn("HAVE_SYMLINKAT", posix._have_functions)

else:
self.assertNotIn("HAVE_SYMLINKAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.symlink("a", "b", dir_fd=0)

def test_utime(self):
self._verify_available("HAVE_FUTIMENS")
self._verify_available("HAVE_UTIMENSAT")
if self.mac_ver >= (10, 13):
self.assertIn("HAVE_FUTIMENS", posix._have_functions)
self.assertIn("HAVE_UTIMENSAT", posix._have_functions)

else:
self.assertNotIn("HAVE_FUTIMENS", posix._have_functions)
self.assertNotIn("HAVE_UTIMENSAT", posix._have_functions)

with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
os.utime("path", dir_fd=0)


def test_main():
try:
support.run_unittest(
PosixTester,
PosixGroupsTester,
TestPosixSpawn,
TestPosixSpawnP,
TestPosixWeaklinking
)
finally:
support.reap_children()
Expand Down
Loading