_manylinux.py 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278
  1. from __future__ import annotations
  2. import collections
  3. import contextlib
  4. import functools
  5. import os
  6. import re
  7. import sys
  8. import warnings
  9. from typing import TYPE_CHECKING, NamedTuple
  10. from ._elffile import EIClass, EIData, ELFFile, EMachine
  11. if TYPE_CHECKING:
  12. import types
  13. from collections.abc import Generator, Iterator, Sequence
  14. EF_ARM_ABIMASK = 0xFF000000
  15. EF_ARM_ABI_VER5 = 0x05000000
  16. EF_ARM_ABI_FLOAT_HARD = 0x00000400
  17. _ALLOWED_ARCHS = {
  18. "x86_64",
  19. "aarch64",
  20. "ppc64",
  21. "ppc64le",
  22. "s390x",
  23. "loongarch64",
  24. "riscv64",
  25. }
  26. @contextlib.contextmanager
  27. def _parse_elf(path: str) -> Generator[ELFFile | None, None, None]:
  28. try:
  29. with open(path, "rb") as f:
  30. yield ELFFile(f)
  31. except (OSError, TypeError, ValueError):
  32. yield None
  33. def _is_linux_armhf(executable: str) -> bool:
  34. # hard-float ABI can be detected from the ELF header of the running
  35. # process
  36. # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf
  37. with _parse_elf(executable) as f:
  38. return (
  39. f is not None
  40. and f.capacity == EIClass.C32
  41. and f.encoding == EIData.Lsb
  42. and f.machine == EMachine.Arm
  43. and f.flags & EF_ARM_ABIMASK == EF_ARM_ABI_VER5
  44. and f.flags & EF_ARM_ABI_FLOAT_HARD == EF_ARM_ABI_FLOAT_HARD
  45. )
  46. def _is_linux_i686(executable: str) -> bool:
  47. with _parse_elf(executable) as f:
  48. return (
  49. f is not None
  50. and f.capacity == EIClass.C32
  51. and f.encoding == EIData.Lsb
  52. and f.machine == EMachine.I386
  53. )
  54. def _have_compatible_abi(executable: str, archs: Sequence[str]) -> bool:
  55. if "armv7l" in archs:
  56. return _is_linux_armhf(executable)
  57. if "i686" in archs:
  58. return _is_linux_i686(executable)
  59. return any(arch in _ALLOWED_ARCHS for arch in archs)
  60. # If glibc ever changes its major version, we need to know what the last
  61. # minor version was, so we can build the complete list of all versions.
  62. # For now, guess what the highest minor version might be, assume it will
  63. # be 50 for testing. Once this actually happens, update the dictionary
  64. # with the actual value.
  65. _LAST_GLIBC_MINOR: dict[int, int] = collections.defaultdict(lambda: 50)
  66. class _GLibCVersion(NamedTuple):
  67. major: int
  68. minor: int
  69. def _glibc_version_string_confstr() -> str | None:
  70. """
  71. Primary implementation of glibc_version_string using os.confstr.
  72. """
  73. # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely
  74. # to be broken or missing. This strategy is used in the standard library
  75. # platform module.
  76. # https://github.com/python/cpython/blob/fcf1d003bf4f0100c/Lib/platform.py#L175-L183
  77. try:
  78. # Should be a string like "glibc 2.17".
  79. version_string: str | None = os.confstr("CS_GNU_LIBC_VERSION")
  80. assert version_string is not None
  81. _, version = version_string.rsplit()
  82. except (AssertionError, AttributeError, OSError, ValueError):
  83. # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)...
  84. return None
  85. return version
  86. def _glibc_version_string_ctypes() -> str | None:
  87. """
  88. Fallback implementation of glibc_version_string using ctypes.
  89. """
  90. try:
  91. import ctypes # noqa: PLC0415
  92. except ImportError:
  93. return None
  94. # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen
  95. # manpage says, "If filename is NULL, then the returned handle is for the
  96. # main program". This way we can let the linker do the work to figure out
  97. # which libc our process is actually using.
  98. #
  99. # We must also handle the special case where the executable is not a
  100. # dynamically linked executable. This can occur when using musl libc,
  101. # for example. In this situation, dlopen() will error, leading to an
  102. # OSError. Interestingly, at least in the case of musl, there is no
  103. # errno set on the OSError. The single string argument used to construct
  104. # OSError comes from libc itself and is therefore not portable to
  105. # hard code here. In any case, failure to call dlopen() means we
  106. # can proceed, so we bail on our attempt.
  107. try:
  108. process_namespace = ctypes.CDLL(None)
  109. except OSError:
  110. return None
  111. try:
  112. gnu_get_libc_version = process_namespace.gnu_get_libc_version
  113. except AttributeError:
  114. # Symbol doesn't exist -> therefore, we are not linked to
  115. # glibc.
  116. return None
  117. # Call gnu_get_libc_version, which returns a string like "2.5".
  118. gnu_get_libc_version.restype = ctypes.c_char_p
  119. # A c_char_p restype comes back as bytes, so decode to text.
  120. version_str: str | bytes = gnu_get_libc_version()
  121. if isinstance(version_str, bytes):
  122. version_str = version_str.decode("ascii")
  123. return version_str
  124. def _glibc_version_string() -> str | None:
  125. """Returns glibc version string, or None if not using glibc."""
  126. return _glibc_version_string_confstr() or _glibc_version_string_ctypes()
  127. def _parse_glibc_version(version_str: str) -> _GLibCVersion:
  128. """Parse glibc version.
  129. We use a regexp instead of str.split because we want to discard any
  130. random junk that might come after the minor version -- this might happen
  131. in patched/forked versions of glibc (e.g. Linaro's version of glibc
  132. uses version strings like "2.20-2014.11"). See gh-3588.
  133. """
  134. m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str)
  135. if not m:
  136. warnings.warn(
  137. f"Expected glibc version with 2 components major.minor, got: {version_str}",
  138. RuntimeWarning,
  139. stacklevel=2,
  140. )
  141. return _GLibCVersion(-1, -1)
  142. return _GLibCVersion(int(m.group("major")), int(m.group("minor")))
  143. @functools.lru_cache
  144. def _get_glibc_version() -> _GLibCVersion:
  145. version_str = _glibc_version_string()
  146. if version_str is None:
  147. return _GLibCVersion(-1, -1)
  148. return _parse_glibc_version(version_str)
  149. # From PEP 513, PEP 600
  150. @functools.lru_cache(maxsize=1)
  151. def _get_manylinux_module() -> types.ModuleType | None:
  152. """Return the ``_manylinux`` C extension module, or None if unavailable.
  153. The result is cached for the lifetime of the process, since the presence
  154. of the module does not change while running.
  155. """
  156. try:
  157. return __import__("_manylinux")
  158. except ImportError:
  159. return None
  160. def _is_compatible(arch: str, version: _GLibCVersion) -> bool:
  161. sys_glibc = _get_glibc_version()
  162. if sys_glibc < version:
  163. return False
  164. # Check for presence of _manylinux module.
  165. manylinux_mod = _get_manylinux_module()
  166. if manylinux_mod is None:
  167. return True
  168. if hasattr(manylinux_mod, "manylinux_compatible"):
  169. result = manylinux_mod.manylinux_compatible(version[0], version[1], arch)
  170. if result is not None:
  171. return bool(result)
  172. return True
  173. if version == _GLibCVersion(2, 5) and hasattr(
  174. manylinux_mod, "manylinux1_compatible"
  175. ):
  176. return bool(manylinux_mod.manylinux1_compatible)
  177. if version == _GLibCVersion(2, 12) and hasattr(
  178. manylinux_mod, "manylinux2010_compatible"
  179. ):
  180. return bool(manylinux_mod.manylinux2010_compatible)
  181. if version == _GLibCVersion(2, 17) and hasattr(
  182. manylinux_mod, "manylinux2014_compatible"
  183. ):
  184. return bool(manylinux_mod.manylinux2014_compatible)
  185. return True
  186. _LEGACY_MANYLINUX_MAP: dict[_GLibCVersion, str] = {
  187. # CentOS 7 w/ glibc 2.17 (PEP 599)
  188. _GLibCVersion(2, 17): "manylinux2014",
  189. # CentOS 6 w/ glibc 2.12 (PEP 571)
  190. _GLibCVersion(2, 12): "manylinux2010",
  191. # CentOS 5 w/ glibc 2.5 (PEP 513)
  192. _GLibCVersion(2, 5): "manylinux1",
  193. }
  194. def platform_tags(archs: Sequence[str]) -> Iterator[str]:
  195. """Generate manylinux tags compatible to the current platform.
  196. :param archs: Sequence of compatible architectures.
  197. The first one shall be the closest to the actual architecture and be the part of
  198. platform tag after the ``linux_`` prefix, e.g. ``x86_64``.
  199. The ``linux_`` prefix is assumed as a prerequisite for the current platform to
  200. be manylinux-compatible.
  201. :returns: An iterator of compatible manylinux tags.
  202. """
  203. if not _have_compatible_abi(sys.executable, archs):
  204. return
  205. # Oldest glibc to be supported regardless of architecture is (2, 17).
  206. too_old_glibc2 = _GLibCVersion(2, 16)
  207. if set(archs) & {"x86_64", "i686"}:
  208. # On x86/i686 also oldest glibc to be supported is (2, 5).
  209. too_old_glibc2 = _GLibCVersion(2, 4)
  210. current_glibc = _GLibCVersion(*_get_glibc_version())
  211. glibc_max_list = [current_glibc]
  212. # We can assume compatibility across glibc major versions.
  213. # https://sourceware.org/bugzilla/show_bug.cgi?id=24636
  214. #
  215. # Build a list of maximum glibc versions so that we can
  216. # output the canonical list of all glibc from current_glibc
  217. # down to too_old_glibc2, including all intermediary versions.
  218. for glibc_major in range(current_glibc.major - 1, 1, -1):
  219. glibc_minor = _LAST_GLIBC_MINOR[glibc_major]
  220. glibc_max_list.append(_GLibCVersion(glibc_major, glibc_minor))
  221. for arch in archs:
  222. for glibc_max in glibc_max_list:
  223. if glibc_max.major == too_old_glibc2.major:
  224. min_minor = too_old_glibc2.minor
  225. else:
  226. # For other glibc major versions oldest supported is (x, 0).
  227. min_minor = -1
  228. for glibc_minor in range(glibc_max.minor, min_minor, -1):
  229. glibc_version = _GLibCVersion(glibc_max.major, glibc_minor)
  230. if _is_compatible(arch, glibc_version):
  231. yield "manylinux_{}_{}_{}".format(*glibc_version, arch)
  232. # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags.
  233. if legacy_tag := _LEGACY_MANYLINUX_MAP.get(glibc_version):
  234. yield f"{legacy_tag}_{arch}"