feat(genai): Add local tokenizer samples for Count and Compute

Author: msampathkumar

genai/count_tokens/counttoken_with_txt.py

@@ -25,7 +25,7 @@ def count_tokens() -> int:
     )
     print(response)
     # Example output:
-    # total_tokens=10
+    # total_tokens=9
     # cached_content_token_count=None
     # [END googlegenaisdk_counttoken_with_txt]
     return response.total_tokens

genai/count_tokens/requirements.txt

@@ -1 +1,2 @@
 google-genai==1.42.0
+sentencepiece==0.2.1

genai/count_tokens/test_count_tokens_examples.py

@@ -19,6 +19,8 @@
 import os
 
 import counttoken_compute_with_txt
+import counttoken_localtokenizer_compute_with_txt
+import counttoken_localtokenizer_with_txt
 import counttoken_resp_with_txt
 import counttoken_with_txt
 import counttoken_with_txt_vid
@@ -43,3 +45,11 @@ def test_counttoken_with_txt() -> None:
 
 def test_counttoken_with_txt_vid() -> None:
     assert counttoken_with_txt_vid.count_tokens()
+
+
+def test_counttoken_localtokenizer_with_txt() -> None:
+    assert counttoken_localtokenizer_with_txt.counttoken_localtokenizer_with_txt()
+
+
+def test_counttoken_localtokenizer_compute_with_txt() -> None:
+    assert counttoken_localtokenizer_compute_with_txt.counttoken_localtokenizer_compute_with_txt()