docs(genai): Add Bounding Box Sample for Gen AI SDK

Author: holtskinner

genai/bounding_box/boundingbox_with_txt_img.py

@@ -0,0 +1,112 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def generate_content() -> str:
+    # [START googlegenaisdk_boundingbox_with_txt_img]
+    import requests
+
+    from google import genai
+    from google.genai.types import (
+        GenerateContentConfig,
+        HttpOptions,
+        Part,
+        SafetySetting,
+    )
+
+    from PIL import Image, ImageColor, ImageDraw
+
+    from pydantic import BaseModel
+
+    class BoundingBox(BaseModel):
+        box_2d: list[int]
+        label: str
+
+    def plot_bounding_boxes(image_uri: str, bounding_boxes: list[BoundingBox]) -> None:
+        """
+        Plots bounding boxes on an image with markers for each a name, using PIL, normalized coordinates, and different colors.
+
+        Args:
+            img_path: The path to the image file.
+            bounding_boxes: A list of bounding boxes containing the name of the object
+            and their positions in normalized [y1 x1 y2 x2] format.
+        """
+
+        with Image.open(requests.get(image_uri, stream=True, timeout=10).raw) as im:
+            width, height = im.size
+            draw = ImageDraw.Draw(im)
+
+            colors = list(ImageColor.colormap.keys())
+
+            for i, bbox in enumerate(bounding_boxes):
+                y1, x1, y2, x2 = bbox.box_2d
+                abs_y1 = int(y1 / 1000 * height)
+                abs_x1 = int(x1 / 1000 * width)
+                abs_y2 = int(y2 / 1000 * height)
+                abs_x2 = int(x2 / 1000 * width)
+
+                # Use a single color selection:
+                color = colors[i % len(colors)]
+
+                draw.rectangle(
+                    ((abs_x1, abs_y1), (abs_x2, abs_y2)), outline=color, width=4
+                )
+                if bbox.label:
+                    draw.text((abs_x1 + 8, abs_y1 + 6), bbox.label, fill=color)
+
+            im.show()
+
+    client = genai.Client(http_options=HttpOptions(api_version="v1"))
+
+    config = GenerateContentConfig(
+        system_instruction="""Return bounding boxes as an array with labels. Never return masks. Limit to 25 objects.
+        If an object is present multiple times, give each object a unique label according to its distinct characteristics (colors, size, position, etc..).""",
+        temperature=0.5,
+        safety_settings=[
+            SafetySetting(
+                category="HARM_CATEGORY_DANGEROUS_CONTENT",
+                threshold="BLOCK_ONLY_HIGH",
+            ),
+        ],
+        response_mime_type="application/json",
+        response_schema=list[BoundingBox],
+    )
+
+    image_uri = "https://storage.googleapis.com/generativeai-downloads/images/socks.jpg"
+
+    response = client.models.generate_content(
+        model="gemini-2.0-flash-001",
+        contents=[
+            Part.from_uri(
+                file_uri=image_uri,
+                mime_type="image/jpeg",
+            ),
+            "Output the positions of the socks with a face. Label according to position in the image.",
+        ],
+        config=config,
+    )
+    print(response.text)
+    plot_bounding_boxes(image_uri, response.parsed)
+
+    # Example response:
+    # [
+    #     {"box_2d": [36, 246, 380, 492], "label": "top left sock with face"},
+    #     {"box_2d": [260, 663, 640, 917], "label": "top right sock with face"},
+    # ]
+    # [END googlegenaisdk_boundingbox_with_txt_img]
+    return response.text
+
+
+if __name__ == "__main__":
+    generate_content()

genai/bounding_box/noxfile_config.py

@@ -0,0 +1,42 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default TEST_CONFIG_OVERRIDE for python repos.
+
+# You can copy this file into your directory, then it will be imported from
+# the noxfile.py.
+
+# The source of truth:
+# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py
+
+TEST_CONFIG_OVERRIDE = {
+    # You can opt out from the test for specific Python versions.
+    "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"],
+    # Old samples are opted out of enforcing Python type hints
+    # All new samples should feature them
+    "enforce_type_hints": True,
+    # An envvar key for determining the project id to use. Change it
+    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
+    # build specific Cloud project. You can also use your own string
+    # to use your own Cloud project.
+    "gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
+    # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
+    # If you need to use a specific version of pip,
+    # change pip_version_override to the string representation
+    # of the version number, for example, "20.2.4"
+    "pip_version_override": None,
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    "envs": {},
+}

genai/bounding_box/requirements-test.txt

@@ -0,0 +1,2 @@
+google-api-core==2.24.0
+pytest==8.2.0

genai/bounding_box/requirements.txt

@@ -0,0 +1 @@
+google-genai==1.2.0

genai/bounding_box/test_bounding_box_examples.py

@@ -0,0 +1,31 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Using Google Cloud Vertex AI to test the code samples.
+#
+
+import os
+
+import boundingbox_with_txt_img
+
+os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True"
+os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1"
+# The project name is included in the CICD pipeline
+# os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name"
+
+
+def test_boundingbox_with_txt_img() -> None:
+    response = boundingbox_with_txt_img.generate_content()
+    assert response