From c036cc74b12e20d846747035573e8b94b3917a51 Mon Sep 17 00:00:00 2001 From: David Revay Date: Fri, 5 Dec 2025 15:50:54 +1100 Subject: [PATCH] chore: improve release --- docs/releases.md | 31 +++- platform_cli/groups/release.py | 328 ++++++++++++++++++++++----------- 2 files changed, 247 insertions(+), 112 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index 9d62a19..c8c948d 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -56,8 +56,27 @@ If you try and release a package that has already been released (and does not ha Releases are done using [semantic-release](https://github.com/semantic-release/semantic-release). There are 2 tricky things here... 1. We want to build and version each package separately which is why we use [multi-semantic-release](https://github.com/qiwi/multi-semantic-release). -2. We want to build `.deb`s for both **AMD64** and **ARM64** and possibly multiple versions of ROS which is why we use docker to build the `.deb` files. We then need to mount these back to the host machine so they can be linked into a single release. +2. We want to build `.deb`s for **AMD64** and/or **ARM64** architectures and possibly multiple versions of ROS, with automatic optimization for native builds. We use docker to build the `.deb` files and mount them back to the host machine so they can be linked into a single release. +#### Architecture Build Strategies: + +The build process automatically optimizes based on target architectures: + +**Native Architecture Builds:** +- Target architecture matches the current system (e.g., amd64 → amd64) +- Skips QEMU emulation setup for better performance +- Skips local registry setup +- Uses buildx with local loading +- Faster build times and simpler setup + +**Cross-Platform Builds:** +- Multiple architectures OR non-native single architecture +- Sets up QEMU for emulation +- Creates local docker registry for multi-platform images +- Uses buildx with registry output +- Supports building arm64 on amd64 machines and vice versa + +> **Performance Note**: When building for only the current system architecture, the build process automatically optimizes by skipping cross-platform emulation setup, resulting in faster build times. #### Setup: `platform release setup` is run from the root of a platform module repo. This will: @@ -78,9 +97,13 @@ Releases are done using [semantic-release](https://github.com/semantic-release/s 2. Generating releases notes for each package based on the commits 3. Generating a changelog for each package based in the commits 4. Running [`platform release deb-prepare`](../platform_cli/groups/release.py#:~:text=deb_prepare) which builds the `.deb` in a docker container. - 1. Sets up `tonistiigi/binfmt` which allows docker to run `arm64` containers on `amd64` machines - 2. Create a local docker registry on [localhost:5000](http://localhost:5000) to store the built images - 3. Uses `buildx` to build for both `amd64` and `arm64` and push to the local registry + 1. **Architecture Detection**: Determines if cross-platform emulation is needed + 2. **Conditional Setup**: + - For cross-platform builds: Sets up QEMU (`tonistiigi/binfmt`) and local registry on [localhost:5000](http://localhost:5000) + - For native builds: Skips QEMU and registry for better performance + 3. **Buildx Build**: Uses `buildx` for all builds (enables secrets support) + - Cross-platform: Builds for multiple architectures and pushes to local registry + - Native: Builds for current architecture only and loads locally 4. Executes [`platform pkg build`](../platform_cli/groups/packaging.py#:~:text=build) inside each docker container to build the `.deb` with a docker volume to mount the resultant `.deb` back to the host machine. 5. Running [`platform release deb-publish`](../platform_cli/groups/release.py#:~:text=deb_publish) to publish the `.deb` to the apt repo 6. Uploading the `.deb` to the github release diff --git a/platform_cli/groups/release.py b/platform_cli/groups/release.py index ac7551b..5d5434a 100644 --- a/platform_cli/groups/release.py +++ b/platform_cli/groups/release.py @@ -1,5 +1,6 @@ import click import shutil +import platform from glob import glob import os @@ -62,6 +63,37 @@ def check_parents_for_file(filename: str, path: Optional[Path] = None) -> Path: raise Exception(f"Could not find {filename} in any parent directory") +def get_current_system_architecture() -> Architecture: + """ + Returns the current system architecture mapped to our Architecture enum. + """ + machine = platform.machine() + if machine in ("x86_64", "AMD64"): + return Architecture.amd64 + elif machine in ("aarch64", "arm64"): + return Architecture.arm64 + else: + # Default to amd64 for unknown architectures + return Architecture.amd64 + + +def should_build_with_qemu(target_architectures: List[Architecture]) -> bool: + """ + Determines if we need to use QEMU for cross-platform emulation. + Returns True if we need QEMU (cross-platform or multiple architectures), + False if we can build natively (single native architecture). + Note: We still use buildx in both cases for secrets support. + """ + if len(target_architectures) == 0: + return False + + if len(target_architectures) > 1: + return True + + current_arch = get_current_system_architecture() + return target_architectures[0] != current_arch + + def get_module_info(path: Optional[Path] = None) -> Optional[ModuleInfo]: """ Returns the module info for the directory (or CWD). @@ -274,10 +306,14 @@ def _get_release_mode(self) -> ReleaseMode: return ReleaseMode.SINGLE return ReleaseMode.MULTI - def _get_docker_image_name(self, platform_module_name: str) -> str: + def _get_docker_image_name(self, platform_module_name: str, use_registry: bool = False) -> str: """Returns the docker image name for a package""" # Note, uppercase is not allowed in docker image names - return f"{DOCKER_REGISTRY}/{platform_module_name.lower()}:latest" + image_name = f"{platform_module_name.lower()}:latest" + if use_registry: + return f"{DOCKER_REGISTRY}/{image_name}" + else: + return image_name def _write_root_yarn_lock(self, src: Path): dest = Path.cwd() / "yarn.lock" @@ -301,11 +337,11 @@ def _write_package_jsons_for_each_package(self, packages: Iterable[PackageInfo]) self._write_package_json(package_json_path, package_info.package_name) def _get_docker_image_name_with_digest( - self, image_name: str, image_manifests: Manifest, architecture: Architecture + self, image_name: str, image_manifests: Optional[Manifest], architecture: Architecture ) -> str: """Returns the digest for a docker image given an architecture""" - if image_manifests.manifests is None: - # If there are no manifests, then there is only one image so we don't need the digest + if image_manifests is None or image_manifests.manifests is None: + # If there are no manifests (native build), then there is only one image so we don't need the digest return image_name # Find the image for the platform and archicture @@ -323,7 +359,7 @@ def _build_deb_in_docker( package_info: PackageInfo, docker_image_name: str, architecture: Architecture, - image_manifests: Manifest, + image_manifests: Optional[Manifest], ): """ Runs the build command in a docker container @@ -374,6 +410,157 @@ def _build_deb_in_docker( tty=True, ) + def _setup_qemu(self): + """Install qemu binfmt support for other architectures""" + echo("Setting up QEMU...") + try: + docker.run( + "multiarch/qemu-user-static", + ["--reset", "-p", "yes", "--credential", "yes"], + privileged=True, + remove=True, + ) + except Exception as e: + # docker on ZFS causes this to error and there is no known fix + echo(f"QEMU already running: {e}", "yellow") + pass + + def _setup_local_registry(self): + """Start a local docker registry on port 5000""" + echo("Setting up local docker registry...") + try: + docker.run( + "registry:2", + publish=[(5000, 5000)], + detach=True, + name="registry", + remove=True, + ) + except Exception as e: + echo(f"Local registry already running: {e}", "yellow") + + def _setup_buildx_environment(self): + """Configure docker to use the platform buildx builder""" + try: + # Network host is required for the local registry to work + docker.buildx.create( + name="platform", + driver="docker-container", + use=True, + driver_options={"network": "host"}, + ) + except Exception: + echo("docker buildx environment already exists", "yellow") + echo( + "Consider running `docker buildx rm platform` if you want to reset the build environment", + "yellow", + ) + + docker.buildx.use("platform") + + def _parse_secrets_for_buildx(self, secrets: str) -> List[str]: + """Parse secrets JSON and prepare buildx secrets format""" + buildx_secrets = [] + try: + secrets_dict = json.loads(secrets) + for secret_id, secret_path in secrets_dict.items(): + buildx_secrets.append(f"id={secret_id},src={secret_path}") + except json.JSONDecodeError: + if secrets != "{}": + echo(f"Warning: Invalid secrets JSON format: {secrets}", "yellow") + + return buildx_secrets + + def _build_docker_image_with_buildx( + self, + package_info: PackageInfo, + docker_image_name: str, + buildx_secrets: List[str], + package_dir: str, + ros_distro: str, + package: str, + docker_platforms: Optional[List[str]] = None, + use_registry: bool = False, + ) -> Optional[Manifest]: + """Build docker image using buildx with unified logic for both native and multiplatform builds""" + if not package_info.module_info: + raise Exception("Module info is required to build docker images") + + # Dynamic echo message based on build type + if docker_platforms: + echo("Building docker container with buildx...", group_start=True) + else: + echo( + f"Building docker container for native architecture ({get_current_system_architecture().value}) with buildx...", + group_start=True, + ) + + # Prepare build arguments + build_kwargs = { + "context": package_info.module_info.platform_module_path, + "tags": [docker_image_name], + "secrets": buildx_secrets, + "build_args": { + "API_TOKEN_GITHUB": os.environ["API_TOKEN_GITHUB"], + "GPU": os.environ["GPU"], + "PLATFORM_MODULE": package_info.module_info.platform_module_name, + "PACKAGE_DIR": package_dir, + "ROS_DISTRO": ros_distro, + "PACKAGE_NAME": package, + }, + } + + # Add platform-specific options + if docker_platforms: + build_kwargs["platforms"] = docker_platforms + + if use_registry: + build_kwargs["output"] = {"type": "registry"} + else: + build_kwargs["load"] = True + + # Build the image + docker.buildx.build(**build_kwargs) + echo(group_end=True) + + # Return manifest for registry builds, None for local builds + if use_registry: + return docker.buildx.imagetools.inspect(docker_image_name) + else: + return None + + def _build_all_architecture_debs( + self, + arch: List[Architecture], + version: str, + package_info: PackageInfo, + docker_image_name: str, + image_manifests: Optional[Manifest], + ): + """Build .deb files for all target architectures""" + for architecture in arch: + echo( + f"Building .deb for package {package_info.package_name} for {architecture.value}", + "blue", + group_start=True, + ) + try: + self._build_deb_in_docker( + version=version if version else package_info.package_version, + package_info=package_info, + docker_image_name=docker_image_name, + architecture=architecture, + image_manifests=image_manifests, + ) + except Exception as e: + echo( + f"Failed to build .deb for {architecture}", + "red", + level=LogLevels.ERROR, + ) + raise e + echo(group_end=True) + def create(self, cli: click.Group): @cli.group(help="CLI handlers associated releasing a platform module") def release(): @@ -605,7 +792,7 @@ def deb_prepare(version: str, arch: List[Architecture], package: str, package_di if "API_TOKEN_GITHUB" not in os.environ: raise Exception("API_TOKEN_GITHUB must be set") - # resolve path of package + # Resolve path of package if package: packages = find_packages() package_info = packages[package] @@ -615,115 +802,40 @@ def deb_prepare(version: str, arch: List[Architecture], package: str, package_di if not package_info.module_info: raise Exception("Module info is required to build debs") + # Determine if we need to use QEMU for cross-platform emulation + needs_qemu = should_build_with_qemu(arch) + docker_image_name = self._get_docker_image_name( - package_info.module_info.platform_module_name + package_info.module_info.platform_module_name, use_registry=needs_qemu ) - # Install qemu binfmt support for other architectures - echo("Setting up QEMU...") - try: - docker.run( - "multiarch/qemu-user-static", - ["--reset", "-p", "yes", "--credential", "yes"], - privileged=True, - remove=True, - ) - except Exception as e: - # docker on ZFS causes this to error and there is no known fix - echo(f"QEMU already running: {e}", "yellow") - pass - - # Start a local registry on port 5000 - echo("Setting up local docker registry...") - try: - docker.run( - "registry:2", - publish=[(5000, 5000)], - detach=True, - name="registry", - remove=True, - ) - except Exception as e: - echo(f"Local registry already running: {e}", "yellow") + # Setup cross-platform emulation if needed + if needs_qemu: + self._setup_qemu() + self._setup_local_registry() echo(group_end=True) - echo("Building docker container with buildx...", group_start=True) - try: - # Configure docker to use the platform buildx builder - # Network host is required for the local registry to work - docker.buildx.create( - name="platform", - driver="docker-container", - use=True, - driver_options={"network": "host"}, - ) - except Exception: - echo("docker buildx environment already exists", "yellow") - echo( - "Consider running `docker buildx rm platform` if you want to reset the build environment", - "yellow", - ) - - docker.buildx.use("platform") - - # Parse secrets JSON and prepare buildx secrets - buildx_secrets = [] - try: - secrets_dict = json.loads(secrets) - for secret_id, secret_path in secrets_dict.items(): - buildx_secrets.append(f"id={secret_id},src={secret_path}") - except json.JSONDecodeError: - if secrets != "{}": - echo(f"Warning: Invalid secrets JSON format: {secrets}", "yellow") - - # Build the images for arm and amd using buildx - docker.buildx.build( - package_info.module_info.platform_module_path, - platforms=docker_platforms, - tags=[docker_image_name], - secrets=buildx_secrets, - build_args={ - "API_TOKEN_GITHUB": os.environ["API_TOKEN_GITHUB"], - "GPU": os.environ["GPU"], - "PLATFORM_MODULE": package_info.module_info.platform_module_name, - "PACKAGE_DIR": package_dir, - "ROS_DISTRO": ros_distro, - # Note we pass the PACKAGE_NAME here so we can rosdep install and build only the package we want - # If we don't pass this, it will try to build all packages in the workspace - # When in MULTI mode, we want to build all packages without installing deps / building each package - # So we DON'T pass this. It will just be an empty string - "PACKAGE_NAME": package, - }, - output={"type": "registry"}, + # Always use buildx (for secrets support), but configure differently + self._setup_buildx_environment() + buildx_secrets = self._parse_secrets_for_buildx(secrets) + + # Build docker image with appropriate strategy + image_manifests = self._build_docker_image_with_buildx( + package_info, + docker_image_name, + buildx_secrets, + package_dir, + ros_distro, + package, + docker_platforms=docker_platforms if needs_qemu else None, + use_registry=needs_qemu, ) - echo(group_end=True) - # Inspect the image to get the manifest - image_manifests = docker.buildx.imagetools.inspect(docker_image_name) - - for architecture in arch: - echo( - f"Building .deb for package {package_info.package_name} for {architecture.value}", - "blue", - group_start=True, - ) - try: - self._build_deb_in_docker( - version=version if version else package_info.package_version, - package_info=package_info, - docker_image_name=docker_image_name, - architecture=architecture, - image_manifests=image_manifests, - ) - except Exception as e: - echo( - f"Failed to build .deb for {architecture}", - "red", - level=LogLevels.ERROR, - ) - raise e - echo(group_end=True) + # Build .deb files for all architectures + self._build_all_architecture_debs( + arch, version, package_info, docker_image_name, image_manifests + ) @release.command(name="deb-publish") @click.option(