test: move bootc-image-builder integration tests here

.github/workflows/bibtests.yaml

@@ -0,0 +1,118 @@
+---
+name: Bib tests
+
+on:  # yamllint disable-line rule:truthy
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - "main"
+  # for merge queue
+  merge_group:
+
+env:
+  GO_VERSION: 1.23
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  collect_tests:
+    runs-on: ubuntu-latest
+    outputs:
+      test_files: ${{ steps.collect.outputs.test_files }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Collect test files
+        id: collect
+        run: |
+          TEST_FILES=$(ls test/bib/test_*.py | sort)
+          JSON_FILES=$(echo "${TEST_FILES}" | jq -R | jq -cs )
+          echo "test_files=${JSON_FILES}" >> $GITHUB_OUTPUT
+
+  integration:
+    name: "Integration"
+    runs-on: ubuntu-24.04
+    needs: collect_tests
+    strategy:
+      matrix:
+        test_file: ${{ fromJson(needs.collect_tests.outputs.test_files) }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Setup up python
+        uses: actions/setup-python@v6
+      - name: Apt update
+        run: sudo apt update
+      - name: Install test dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y python3-pytest python3-boto3 flake8 pylint libosinfo-bin squashfs-tools sshpass
+      - name: Diskspace (before)
+        run: |
+          df -h
+          sudo du -sh * /var/tmp /tmp /var/lib/containers | sort -sh
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+        with:
+          tool-cache: true
+          # The following line runs apt remove which is slow
+          large-packages: false
+      - name: Workaround podman issues in GH actions
+        run: |
+          # see https://github.com/osbuild/bootc-image-builder/issues/446
+          sudo rm -rf /var/lib/containers/storage
+          sudo mkdir -p /etc/containers
+          echo -e "[storage]\ndriver = \"overlay\"\nrunroot = \"/run/containers/storage\"\ngraphroot = \"/var/lib/containers/storage\"" | sudo tee /etc/containers/storage.conf
+      - name: Updating qemu-user
+        run: |
+          # get qemu-9 with openat2 patches via qemu-user-static, that
+          # has no dependencies so just install.
+          # XXX: remove once ubuntu ships qemu-9.1
+          sudo apt install -y software-properties-common
+          sudo apt-add-repository -y ppa:mvo/qemu
+          sudo apt install --no-install-recommends -y qemu-user-static
+          # Now remove ppa again, the metadata confuses apt. Then install
+          # qemu-system-* from the regular repo again.
+          sudo apt-add-repository --remove -y ppa:mvo/qemu
+          sudo apt install -y qemu-system-arm qemu-system-x86 qemu-efi-aarch64
+      - name: Install python test deps
+        run: |
+          # make sure test deps are available for root
+          sudo -E pip install --user -r test/bib/requirements.txt
+      - name: Workarounds for GH runner diskspace
+        run: |
+          # use custom basetemp here because /var/tmp is on a smaller disk
+          # than /mnt
+          sudo mkdir -p /mnt/var/tmp/bib-tests
+          # on GH runners /mnt has 70G free space, use that for our container
+          # storage
+          sudo mkdir -p /mnt/var/lib/containers
+          sudo mount -o bind /mnt/var/lib/containers /var/lib/containers
+      - run: |
+          mkdir -p /var/tmp/osbuild-test-store
+      - name: Cache osbuild env
+        uses: actions/cache@v4
+        with:
+          path: /var/tmp/osbuild-test-store
+          key: no-key-needed-here
+      - name: Run tests
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          # podman needs (parts of) the environment but will break when
+          # XDG_RUNTIME_DIR is set.
+          # TODO: figure out what exactly podman needs
+          sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest-3 -v --basetemp=/mnt/var/tmp/bib-tests ${{ matrix.test_file }}
+      - name: Diskspace (after)
+        if: ${{ always() }}
+        run: |
+          df -h
+          sudo du -sh * /var/tmp /tmp /var/lib/containers | sort -sh

.pre-commit-config.yaml

@@ -31,15 +31,15 @@ repos:
     rev: v3.2.6
     hooks:
       - id: pylint
-        additional_dependencies: ["PyYAML", "types-PyYAML", "pytest"]
+        additional_dependencies: ["PyYAML", "types-PyYAML", "pytest", "vmtest @ git+https://github.com/osbuild/images.git"]
         files: ^(?!test/|tools/)
       - id: pylint
         name: pylint test/ directory
-        additional_dependencies: ["PyYAML", "types-PyYAML", "pytest"]
+        additional_dependencies: ["PyYAML", "types-PyYAML", "pytest", "vmtest @ git+https://github.com/osbuild/images.git"]
         args: ["--rcfile=test/.pylintrc"]
         files: ^test/
       - id: pylint
         name: pylint tools/ directory
-        additional_dependencies: ["PyYAML", "types-PyYAML", "pytest"]
+        additional_dependencies: ["PyYAML", "types-PyYAML", "pytest", "vmtest @ git+https://github.com/osbuild/images.git"]
         args: ["--rcfile=tools/.pylintrc"]
         files: ^tools/

.pylintrc

@@ -1,2 +1,3 @@
+# Note that there are additional pylintrc files in ./tools, ./test
 [FORMAT]
 max-line-length=120

Containerfile

@@ -4,12 +4,12 @@ ARG GOPROXY=https://proxy.golang.org,direct
 RUN go env -w GOPROXY=$GOPROXY
 COPY . /build
 WORKDIR /build
-# keep in sync with:
-# https://github.com/containers/podman/blob/2981262215f563461d449b9841741339f4d9a894/Makefile#L51
 # disable cgo as
 # a) gcc crashes on fedora41/arm64 regularly
 # b) we don't really need it
-RUN CGO_ENABLED=0 go build -tags "containers_image_openpgp exclude_graphdriver_btrfs exclude_graphdriver_devicemapper" ./cmd/image-builder
+# Note that this cannot move into the makefile as for bib we want to build *with* cgo to have
+# the chmod/chown logic work as those need cgo/libnss (in the general case)
+RUN CGO_ENABLED=0 make build
 
 FROM registry.fedoraproject.org/fedora:43
 
@@ -21,7 +21,7 @@ RUN dnf install -y dnf-plugins-core \
     && dnf install -y libxcrypt-compat wget osbuild osbuild-ostree osbuild-depsolve-dnf osbuild-lvm2 openssl subscription-manager \
     && dnf clean all
 
-COPY --from=builder /build/image-builder /usr/bin/
+COPY --from=builder /build/bin/image-builder /usr/bin/
 
 ENTRYPOINT ["/usr/bin/image-builder"]
 VOLUME /output

Containerfile.bib

@@ -0,0 +1,35 @@
+FROM registry.fedoraproject.org/fedora:43 AS builder
+RUN dnf install -y git-core golang gpgme-devel libassuan-devel libvirt-devel && mkdir -p /build/bib
+COPY go.mod go.sum /build/bib/
+ARG GOPROXY=https://proxy.golang.org,direct
+RUN go env -w GOPROXY=$GOPROXY
+RUN cd /build/bib && go mod download
+# Copy the entire dir to avoid having to conditionally include ".git" as that
+# will not be available when tests are run under tmt
+COPY . /build
+WORKDIR /build
+# keep in sync with main Containerfile, this was part of bib:build.sh
+RUN make build
+
+FROM registry.fedoraproject.org/fedora:43
+# Fast-track osbuild so we don't depend on the "slow" Fedora release process to implement new features in bib
+RUN dnf install -y dnf-plugins-core \
+    && dnf copr enable -y @osbuild/osbuild \
+    && dnf install -y libxcrypt-compat wget osbuild osbuild-ostree osbuild-depsolve-dnf osbuild-lvm2 openssl subscription-manager libvirt-libs \
+    && dnf clean all
+
+# copy as bootc-image-builder
+COPY --from=builder /build/bin/image-builder /usr/bin/bootc-image-builder
+
+ENTRYPOINT ["/usr/bin/bootc-image-builder"]
+VOLUME /output
+WORKDIR /output
+VOLUME /store
+VOLUME /rpmmd
+VOLUME /var/lib/containers/storage
+
+LABEL description="This tools allows to build and deploy disk-images from bootc container inputs."
+LABEL io.k8s.description="This tools allows to build and deploy disk-images from bootc container inputs."
+LABEL io.k8s.display-name="Bootc Image Builder"
+LABEL io.openshift.tags="base fedora42"
+LABEL summary="A container to create disk-images from bootc container inputs"

Makefile

@@ -96,9 +96,18 @@ $(BUILDDIR)/%/:
 # They are not supported nor is their use recommended in scripts.
 #
 
+# keep in sync with:
+# https://github.com/containers/podman/blob/2981262215f563461d449b9841741339f4d9a894/Makefile#L51
+TAGS := containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper
+
 .PHONY: build
 build: $(BUILDDIR)/bin/  ## build the binary from source
-	go build -ldflags="-X main.version=${VERSION}" -o $<image-builder ./cmd/image-builder/
+	go build -tags="$(TAGS)" -ldflags="-X main.version=${VERSION}" -o $<image-builder ./cmd/image-builder/
+	# Note that this is only needed for the bib container to detect if qemu-user is available
+	for arch in amd64 arm64; do \
+	    [ "$$arch" = "$$(go env GOARCH)" ] && continue; \
+	    GOARCH="$$arch" go build -ldflags="-s -w" -o ./bin/bib-canary-"$$arch" ./cmd/cross-arch/; \
+	done
 
 .PHONY: clean
 clean:  ## Remove all built binaries

cmd/cross-arch/canary.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+	println("ok")
+}

test/.pylintrc

@@ -1,6 +1,10 @@
 [MESSAGES CONTROL]
 disable=
+    missing-class-docstring,
     missing-module-docstring,
     missing-function-docstring,
     duplicate-code,
     fixme
+
+[FORMAT]
+max-line-length=120

test/bib/conftest.py

@@ -0,0 +1,29 @@
+import pytest
+
+# pylint: disable=wrong-import-order
+from testcases import TestCase
+from vmtest.util import get_free_port
+
+
+def pytest_addoption(parser):
+    parser.addoption("--force-aws-upload", action="store_true", default=False,
+                     help=("Force AWS upload when building AMI, failing if credentials are not set. "
+                           "If not set, the upload will be performed only when credentials are available."))
+
+
+@pytest.fixture(name="force_aws_upload", scope="session")
+def force_aws_upload_fixture(request):
+    return request.config.getoption("--force-aws-upload")
+
+
+# see https://hackebrot.github.io/pytest-tricks/param_id_func/ and
+# https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_make_parametrize_id
+def pytest_make_parametrize_id(config, val):  # pylint: disable=W0613
+    if isinstance(val, TestCase):
+        return f"{val}"
+    return None
+
+
+@pytest.fixture(name="free_port")
+def free_port_fixture():
+    return get_free_port()