C api / Python bindings

.github/workflows/ci.yml

@@ -13,6 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-22.04, windows-latest, macos-14]
+      fail-fast: false
 
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
@@ -44,6 +45,14 @@ jobs:
           version: 1.4.309.0
           cache: true
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.9.15"
+
+      - name: Set up Python
+        run: uv sync --dev --python 3.12
+
       - name: Configure (Linux)
         if: matrix.os == 'ubuntu-22.04'
         run: >

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.28)
 
-project(vision.cpp VERSION 0.2.0 LANGUAGES CXX)
+project(vision.cpp VERSION 0.3.0 LANGUAGES CXX)
 
 option(BUILD_SHARED_LIBS "Build shared libraries instead of static libraries" ON)
 option(VISP_VULKAN "Enable Vulkan support" OFF)
@@ -145,6 +145,8 @@ if(VISP_CI OR VISP_DEV)
   set_target_properties(vision-cli PROPERTIES INSTALL_RPATH "\$ORIGIN/../${VISP_LIB_INSTALL_DIR}")
 endif()
 
+install(DIRECTORY bindings/python DESTINATION . PATTERN "__pycache__" EXCLUDE)
+
 include(CMakePackageConfigHelpers)
 
 configure_package_config_file(

README.md

@@ -174,7 +174,7 @@ ctest -C Release
 Some tests require a Python environment. It can be set up with [uv](https://docs.astral.sh/uv/):
 ```sh
 # Setup venv and install dependencies (once only)
-uv sync
+uv sync --dev
 
 # Run python tests
 uv run pytest

bindings/python/visioncpp/__init__.py

@@ -0,0 +1,2 @@
+from ._lib import Error  # noqa
+from .vision import *  # noqa

bindings/python/visioncpp/_lib.py

@@ -0,0 +1,191 @@
+import ctypes
+import platform
+from pathlib import Path
+from ctypes import c_byte, c_char_p, c_void_p, c_int32, POINTER
+from PIL import Image
+
+
+class Error(Exception):
+    pass
+
+
+def _image_format_to_string(format: int):
+    match format:
+        case 0:
+            return "RGBA"
+        case 3:
+            return "RGB"
+        case 4:
+            return "L"
+        case _:
+            raise ValueError(f"Unsupported image format: {format}")
+
+
+def _image_mode_from_string(mode: str):
+    match mode:
+        case "RGBA":
+            return 0, 4  # visp::image_format, bytes per pixel
+        case "RGB":
+            return 3, 3
+        case "L":
+            return 4, 1
+        case _:
+            raise ValueError(f"Unsupported image mode: {mode}")
+
+
+class ImageView(ctypes.Structure):
+    _fields_ = [
+        ("width", c_int32),
+        ("height", c_int32),
+        ("stride", c_int32),
+        ("format", c_int32),
+        ("data", c_void_p),
+    ]
+
+    @staticmethod
+    def from_bytes(width: int, height: int, stride: int, format: int, data: bytes):
+        ptr = (c_byte * len(data)).from_buffer_copy(data)
+        return ImageView(width, height, stride, format, ctypes.cast(ptr, ctypes.c_void_p))
+
+    @staticmethod
+    def from_pil_image(image):
+        assert isinstance(image, Image.Image), "Expected a PIL Image"
+        data = image.tobytes()
+        w, h = image.size
+        format, bpp = _image_mode_from_string(image.mode)
+        return ImageView.from_bytes(w, h, w * bpp, format, data)
+
+    def to_pil_image(self):
+        mode = _image_format_to_string(self.format)
+        size = self.height * self.stride
+        data = memoryview((c_byte * size).from_address(self.data))
+        return Image.frombytes(mode, (self.width, self.height), data, "raw", mode, self.stride)
+
+
+class _ImageData(ctypes.Structure):
+    pass
+
+
+class _Device(ctypes.Structure):
+    pass
+
+
+class _Model(ctypes.Structure):
+    pass
+
+
+ImageData = POINTER(_ImageData)
+Device = POINTER(_Device)
+Model = POINTER(_Model)
+
+Handle = ctypes._Pointer
+
+
+def _load():
+    cur_dir = Path(__file__).parent
+    system = platform.system().lower()
+    if system == "windows":
+        prefix = ""
+        suffix = ".dll"
+        libdir = "bin"
+    elif system == "darwin":
+        prefix = "lib"
+        suffix = ".dylib"
+        libdir = "lib"
+    else:  # assume Linux / Unix
+        prefix = "lib"
+        suffix = ".so"
+        libdir = "lib"
+    libname = f"{prefix}visioncpp{suffix}"
+    paths = [
+        cur_dir / libname,
+        cur_dir.parent.parent / libdir / libname,
+        cur_dir.parent.parent.parent / "build" / libdir / libname,
+        cur_dir.parent.parent.parent / "build" / libdir / "Release" / libname,
+    ]
+    error = f"Library {libname} not found in any of the following paths: {paths}"
+    for path in paths:
+        if path.exists():
+            try:
+                lib = ctypes.CDLL(str(path))
+                return lib, path
+            except OSError as e:
+                error = e
+                continue
+    raise OSError(f"Could not load vision.cpp library: {error}")
+
+
+def init():
+    lib, path = _load()
+
+    lib.visp_get_last_error.restype = c_char_p
+
+    lib.visp_backend_load_all.argtypes = [c_char_p]
+    lib.visp_backend_load_all.restype = c_int32
+
+    lib.visp_image_destroy.argtypes = [ImageData]
+    lib.visp_image_destroy.restype = None
+
+    lib.visp_device_init.argtypes = [c_int32, POINTER(Device)]
+    lib.visp_device_init.restype = c_int32
+
+    lib.visp_device_destroy.argtypes = [Device]
+    lib.visp_device_destroy.restype = None
+
+    lib.visp_device_type.argtypes = [Device]
+    lib.visp_device_type.restype = c_int32
+
+    lib.visp_device_name.argtypes = [Device]
+    lib.visp_device_name.restype = c_char_p
+
+    lib.visp_device_description.argtypes = [Device]
+    lib.visp_device_description.restype = c_char_p
+
+    lib.visp_model_detect_family.argtypes = [c_char_p, POINTER(c_int32)]
+    lib.visp_model_detect_family.restype = c_int32
+
+    lib.visp_model_load.argtypes = [c_char_p, Device, c_int32, POINTER(Model)]
+    lib.visp_model_load.restype = c_int32
+
+    lib.visp_model_destroy.argtypes = [Model, c_int32]
+    lib.visp_model_destroy.restype = None
+
+    lib.visp_model_compute.argtypes = [
+        Model,
+        c_int32,
+        POINTER(ImageView),
+        c_int32,
+        POINTER(c_int32),
+        c_int32,
+        POINTER(ImageView),
+        POINTER(ImageData),
+    ]
+    lib.visp_model_compute.restype = c_int32
+
+    # On Linux, libvisioncpp might be in lib/ and ggml backends in bin/
+    if path.parent.name == "lib":
+        bin_dir = path.parent.parent / "bin"
+        if bin_dir.exists():
+            lib.visp_backend_load_all(str(bin_dir).encode())
+
+    return lib
+
+
+_lib: ctypes.CDLL | None = None
+
+
+def get_lib() -> ctypes.CDLL:
+    global _lib
+    if _lib is None:
+        _lib = init()
+    return _lib
+
+
+def check(return_value: int):
+    if return_value == 0:
+        assert _lib is not None, "Library not initialized"
+        raise Error(_lib.visp_get_last_error().decode())
+
+
+def path_to_char_p(p: str | Path):
+    return str(p).encode()

bindings/python/visioncpp/vision.py

@@ -0,0 +1,145 @@
+from ctypes import CDLL, byref, c_int32
+from enum import Enum
+from pathlib import Path
+from typing import NamedTuple, Sequence
+import PIL.Image
+
+from . import _lib as lib
+from ._lib import get_lib, check
+
+
+class ImageFormat(Enum):
+    rgba_u8 = 0
+    bgra_u8 = 1
+    argb_u8 = 2
+    rgb_u8 = 3
+    alpha_u8 = 4
+
+    rgba_f32 = 5
+    rgb_f32 = 6
+    alpha_f32 = 7
+
+
+class ImageRef(NamedTuple):
+    width: int
+    height: int
+    stride: int
+    format: ImageFormat
+    data: bytes
+
+
+Image = ImageRef | PIL.Image.Image
+
+
+class Backend(Enum):
+    auto = 0
+    cpu = 1
+    gpu = 2
+
+    vulkan = gpu | 1 << 8
+
+    @property
+    def is_cpu(self):
+        return self.value & 0xFF00 == Backend.cpu.value
+
+    @property
+    def is_gpu(self):
+        return self.value & 0xFF00 == Backend.gpu.value
+
+
+class Device:
+    @staticmethod
+    def init(backend: Backend = Backend.auto):
+        api = get_lib()
+        handle = lib.Device()
+        check(api.visp_device_init(backend.value, byref(handle)))
+        return Device(api, handle)
+
+    @property
+    def type(self) -> Backend:
+        return Backend(self._api.visp_device_type(self._handle))
+
+    @property
+    def name(self) -> str:
+        return self._api.visp_device_name(self._handle).decode()
+
+    @property
+    def description(self) -> str:
+        return self._api.visp_device_description(self._handle).decode()
+
+    def __init__(self, api: CDLL, handle: lib.Handle):
+        self._api = api
+        self._handle = handle
+
+    def __del__(self):
+        self._api.visp_device_destroy(self._handle)
+
+
+class Arch(Enum):
+    sam = 0
+    birefnet = 1
+    depth_anything = 2
+    migan = 3
+    esrgan = 4
+    unknown = 5
+
+
+class Model:
+    @classmethod
+    def load(cls, path: str | Path, device: Device, arch=Arch.unknown):
+        api = get_lib()
+        handle = lib.Model()
+        path_str = lib.path_to_char_p(path)
+        if arch is Arch.unknown:
+            arch_v = c_int32()
+            check(api.visp_model_detect_family(path_str, byref(arch_v)))
+            arch = Arch(arch_v.value)
+        else:
+            arch_v = arch.value
+
+        check(api.visp_model_load(path_str, device._handle, arch_v, byref(handle)))
+        return cls(api, handle, arch)
+
+    def compute(self, *images: Image, args: Sequence[int] | None = None):
+        if args is None:
+            args = []
+
+        in_views = [_img_view(i) for i in images]
+        in_views_array = (lib.ImageView * len(in_views))(*in_views)
+        args_array = (lib.c_int32 * len(args))(*args)
+        out_view = lib.ImageView()
+        out_data = lib.ImageData()
+        check(
+            self._api.visp_model_compute(
+                self._handle,
+                self.arch.value,
+                in_views_array,
+                len(in_views_array),
+                args_array,
+                len(args_array),
+                byref(out_view),
+                byref(out_data),
+            )
+        )
+        try:
+            result = lib.ImageView.to_pil_image(out_view)
+        finally:
+            self._api.visp_image_destroy(out_data)
+        return result
+
+    def __init__(self, api: CDLL, handle: lib.Handle, arch: Arch):
+        self.arch = arch
+        self._api = api
+        self._handle = handle
+
+    def __del__(self):
+        self._api.visp_model_destroy(self._handle, self.arch.value)
+
+
+def _img_view(i: Image) -> lib.ImageView:
+    if isinstance(i, PIL.Image.Image):
+        return lib.ImageView.from_pil_image(i)
+    elif isinstance(i, ImageRef):
+        return lib.ImageView.from_bytes(i.width, i.height, i.stride, i.format.value, i.data)
+    else:
+        raise TypeError("Expected a PIL Image or ImageRef")