Skip to content

Commit 2114dd5

Browse files
freakboy3742freakboy3742
committed
Remove redundant copies of the standard library.
1 parent 2606b8f commit 2114dd5

File tree

5 files changed

+201
-125
lines changed

5 files changed

+201
-125
lines changed

‎Apple/__main__.py‎

Lines changed: 67 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ def build_python_path() -> Path:
225225

226226
@contextmanager
227227
def group(text: str):
228-
"""A context manager that output a log marker around a section of a build.
228+
"""A context manager that outputs a log marker around a section of a build.
229229
230230
If running in a GitHub Actions environment, the GitHub syntax for
231231
collapsible log sections is used.
@@ -460,6 +460,43 @@ def package_version(prefix_path: Path) -> str:
460460
sys.exit("Unable to determine Python version being packaged.")
461461

462462

463+
def lib_platform_files(dirname, names):
464+
"""A file filter that ignores platform-specific files in the lib directory.
465+
"""
466+
path = Path(dirname)
467+
if (
468+
path.parts[-3] == "lib"
469+
and path.parts[-2].startswith("python")
470+
and path.parts[-1] == "lib-dynload"
471+
):
472+
return names
473+
elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"):
474+
ignored_names = set(
475+
name
476+
for name in names
477+
if (
478+
name.startswith("_sysconfigdata_")
479+
or name.startswith("_sysconfig_vars_")
480+
or name == "build-details.json"
481+
)
482+
)
483+
else:
484+
ignored_names = set()
485+
486+
return ignored_names
487+
488+
489+
def lib_non_platform_files(dirname, names):
490+
"""A file filter that ignores anything *except* platform-specific files
491+
in the lib directory.
492+
"""
493+
path = Path(dirname)
494+
if path.parts[-2] == "lib" and path.parts[-1].startswith("python"):
495+
return set(names) - lib_platform_files(dirname, names) - {"lib-dynload"}
496+
else:
497+
return set()
498+
499+
463500
def create_xcframework(platform: str) -> str:
464501
"""Build an XCframework from the component parts for the platform.
465502
@@ -547,6 +584,7 @@ def create_xcframework(platform: str) -> str:
547584
# to be copied in separately.
548585
print()
549586
print("Copy additional resources...")
587+
has_common_stdlib = False
550588
for slice_name, slice_parts in HOSTS[platform].items():
551589
# Some parts are the same across all slices, so we can any of the
552590
# host frameworks as the source for the merged version.
@@ -575,71 +613,26 @@ def create_xcframework(platform: str) -> str:
575613
slice_framework / "Headers/pyconfig.h",
576614
)
577615

578-
# Copy the lib folder. If there's only one slice, we can copy the .so
579-
# binary modules as is. Otherwise, we ignore .so files, and merge them
580-
# into fat binaries in the next step.
581-
print(f" - {slice_name} standard library")
582-
shutil.copytree(
583-
first_path / "lib",
584-
slice_path / "lib",
585-
ignore=(
586-
None
587-
if len(slice_parts) == 1
588-
else shutil.ignore_patterns("*.so")
589-
),
590-
)
591-
592-
# If there's more than one slice, merge binary .so modules.
593-
if len(slice_parts) > 1:
594-
print(f" - {slice_name} merging binary modules")
595-
lib_dirs = [
596-
CROSS_BUILD_DIR
597-
/ f"{host_triple}/Apple/iOS/Frameworks"
598-
/ f"{multiarch}/lib"
599-
for host_triple, multiarch in slice_parts.items()
600-
]
601-
602-
# The list of .so binary modules should be the same in each slice.
603-
# Find all .so files in each slice; then sort and zip those lists.
604-
# Zipping with strict=True means any length discrepancy will raise
605-
# an error.
606-
for lib_set in zip(
607-
*(sorted(lib_dir.glob("**/*.so")) for lib_dir in lib_dirs),
608-
strict=True,
609-
):
610-
# An additional safety check - not only must the two lists of
611-
# libraries be the same length, but they must have the same
612-
# module names. Raise an error if there's any discrepancy.
613-
relative_libs = set(
614-
lib.relative_to(lib_dir.parent)
615-
for lib_dir, lib in zip(lib_dirs, lib_set)
616-
)
617-
if len(relative_libs) != 1:
618-
raise RuntimeError(
619-
f"Cannot merge non-matching libraries: {relative_libs}"
620-
) from None
621-
622-
# Merge the per-arch .so files into a single "fat" binary.
623-
relative_lib = next(iter(relative_libs))
624-
run(
625-
[
626-
"lipo",
627-
"-create",
628-
"-output",
629-
slice_path / relative_lib,
630-
]
631-
+ [
632-
(
633-
CROSS_BUILD_DIR
634-
/ f"{host_triple}/Apple/iOS/Frameworks/{multiarch}"
635-
/ relative_lib
636-
)
637-
for host_triple, multiarch in slice_parts.items()
638-
]
639-
)
640-
641616
print(f" - {slice_name} architecture-specific files")
642617
for host_triple, multiarch in slice_parts.items():
618+
print(f" - {multiarch} standard library")
619+
arch, _ = multiarch.split("-", 1)
620+
621+
if not has_common_stdlib:
622+
print(" - using this architecture as the common stdlib")
623+
shutil.copytree(
624+
framework_path(host_triple, multiarch) / "lib",
625+
package_path / "Python.xcframework/lib",
626+
ignore=lib_platform_files,
627+
)
628+
has_common_stdlib = True
629+
630+
shutil.copytree(
631+
framework_path(host_triple, multiarch) / "lib",
632+
slice_path / f"lib-{arch}",
633+
ignore=lib_non_platform_files,
634+
)
635+
643636
# Copy the host's pyconfig.h to an architecture-specific name.
644637
arch = multiarch.split("-")[0]
645638
host_path = (
@@ -654,12 +647,11 @@ def create_xcframework(platform: str) -> str:
654647
slice_framework / f"Headers/pyconfig-{arch}.h",
655648
)
656649

657-
# Copy any files (such as sysconfig) that are multiarch-specific.
658-
for path in host_path.glob(f"lib/**/*_{multiarch}.*"):
659-
shutil.copy(
660-
path,
661-
slice_path / (path.relative_to(host_path)),
662-
)
650+
print(" - build tools")
651+
shutil.copytree(
652+
PYTHON_DIR / "Apple/iOS/Resources/build",
653+
package_path / "Python.xcframework/build",
654+
)
663655

664656
return version
665657

@@ -925,6 +917,11 @@ def parse_args() -> argparse.Namespace:
925917
"an ARM64 iPhone 16 Pro simulator running iOS 26.0."
926918
),
927919
)
920+
cmd.add_argument(
921+
"--slow",
922+
action="store_true",
923+
help="Run tests with --slow-ci options.",
924+
)
928925

929926
for subcommand in [configure_build, configure_host, build, ci]:
930927
subcommand.add_argument(

Apple/iOS/Resources/dylib-Info-template.plist renamed to Apple/iOS/Resources/build/dylib-Info-template.plist

File renamed without changes.

‎Apple/iOS/Resources/build/utils.sh‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
# Utility methods for use in an Xcode project.
2+
#
3+
# An iOS XCframework cannot include any content other than the library binary
4+
# and relevant metadata. However, Python requires a standard library at runtime.
5+
# Therefore, it is necessary to add a build step to an Xcode app target that
6+
# processes the standard library and puts the content into the final app.
7+
#
8+
# In general, these tools will be invoked after bundle resources have been
9+
# copied into the app, but before framework embedding (and signing).
10+
#
11+
# The following is an example script, assuming that:
12+
# * Python.xcframework is in the root of the project
13+
# * There is an `app` folder that contains the app code
14+
# * There is an `app_packages` folder that contains installed Python packages.
15+
# -----
16+
# set -e
17+
# source $PROJECT_DIR/Python.xcframework/build/build_utils.sh
18+
# install_python Python.xcframework app app_packages
19+
# -----
20+
21+
# Copy the standard library from the XCframework into the app bundle.
22+
#
23+
# Accepts one argument:
24+
# 1. The path, relative to the root of the Xcode project, where the Python
25+
# XCframework can be found.
26+
install_stdlib() {
27+
PYTHON_XCFRAMEWORK_PATH=$1
28+
29+
mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib"
30+
if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then
31+
echo "Installing Python modules for iOS Simulator"
32+
if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/ios-arm64-simulator" ]; then
33+
SLICE_FOLDER="ios-arm64-simulator/lib-$ARCHS"
34+
else
35+
SLICE_FOLDER="ios-arm64_x86_64-simulator/lib-$ARCHS"
36+
fi
37+
else
38+
echo "Installing Python modules for iOS Device"
39+
SLICE_FOLDER="ios-arm64/lib-$ARCHS/"
40+
fi
41+
42+
rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/"
43+
rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/" "$CODESIGNING_FOLDER_PATH/python/lib/"
44+
}
45+
46+
# Convert a single .so library into a framework that iOS can load.
47+
#
48+
# Accepts three arguments:
49+
# 1. The path, relative to the root of the Xcode project, where the Python
50+
# XCframework can be found.
51+
# 2. The base path, relative to the installed location in the app bundle, that
52+
# needs to be processed. Any .so file found in this path (or a subdirectory
53+
# of it) will be processed.
54+
# 2. The full path to a single .so file to process. This path should include
55+
# the base path.
56+
install_dylib () {
57+
PYTHON_XCFRAMEWORK_PATH=$1
58+
INSTALL_BASE=$2
59+
FULL_EXT=$3
60+
61+
# The name of the extension file
62+
EXT=$(basename "$FULL_EXT")
63+
# The location of the extension file, relative to the bundle
64+
RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/}
65+
# The path to the extension file, relative to the install base
66+
PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}
67+
# The full dotted name of the extension module, constructed from the file path.
68+
FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" ".");
69+
# A bundle identifier; not actually used, but required by Xcode framework packaging
70+
FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-")
71+
# The name of the framework folder.
72+
FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework"
73+
74+
# If the framework folder doesn't exist, create it.
75+
if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then
76+
echo "Creating framework for $RELATIVE_EXT"
77+
mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER"
78+
cp "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/build/dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist"
79+
plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist"
80+
plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist"
81+
fi
82+
83+
echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME"
84+
mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME"
85+
# Create a placeholder .fwork file where the .so was
86+
echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork
87+
# Create a back reference to the .so file location in the framework
88+
echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin"
89+
90+
echo "Signing framework as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..."
91+
/usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER"
92+
}
93+
94+
# Process all the dynamic libraries in a path into Framework format.
95+
#
96+
# Accepts two arguments:
97+
# 1. The path, relative to the root of the Xcode project, where the Python
98+
# XCframework can be found.
99+
# 2. The base path, relative to the installed location in the app bundle, that
100+
# needs to be processed. Any .so file found in this path (or a subdirectory
101+
# of it) will be processed.
102+
process_dylibs () {
103+
PYTHON_XCFRAMEWORK_PATH=$1
104+
LIB_PATH=$2
105+
find "$CODESIGNING_FOLDER_PATH/$LIB_PATH" -name "*.so" | while read FULL_EXT; do
106+
install_dylib $PYTHON_XCFRAMEWORK_PATH "$LIB_PATH/" "$FULL_EXT"
107+
done
108+
}
109+
110+
# The entry point for post-processing a Python XCframework.
111+
#
112+
# Accepts 1 or more arguments:
113+
# 1. The path, relative to the root of the Xcode project, where the Python
114+
# XCframework can be found. If the XCframework is in the root of the project,
115+
# 2+. The path of a package, relative to the root of the packaged app, that contains
116+
# library content that should be processed for binary libraries.
117+
install_python() {
118+
PYTHON_XCFRAMEWORK_PATH=$1
119+
shift
120+
121+
install_stdlib $PYTHON_XCFRAMEWORK_PATH
122+
PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib")
123+
echo "Install Python $PYTHON_VER standard library extension modules..."
124+
process_dylibs $PYTHON_XCFRAMEWORK_PATH python/lib/$PYTHON_VER/lib-dynload
125+
126+
for package_path in $@; do
127+
echo "Installing $package_path extension modules ..."
128+
process_dylibs $PYTHON_XCFRAMEWORK_PATH $package_path
129+
done
130+
}

0 commit comments

Comments
 (0)