Skip to content

Commit 90aecee

Browse files
sararobcopybara-github
authored andcommitted
feat: Add gemini-3-pro-preview support for local tokenizer
PiperOrigin-RevId: 838759612
1 parent c0b175a commit 90aecee

File tree

2 files changed

+3
-2
lines changed

2 files changed

+3
-2
lines changed

google/genai/_local_tokenizer_loader.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
"gemini-2.5-flash-lite-preview-06-17": "gemma3",
4545
"gemini-2.0-flash-001": "gemma3",
4646
"gemini-2.0-flash-lite-001": "gemma3",
47+
"gemini-3-pro-preview": "gemma3",
4748
}
4849

4950

@@ -62,7 +63,7 @@ class _TokenizerConfig:
6263
),
6364
),
6465
"gemma3": _TokenizerConfig(
65-
model_url="https://raw.githubusercontent.com/google/gemma_pytorch/014acb7ac4563a5f77c76d7ff98f31b568c16508/tokenizer/gemma3_cleaned_262144_v2.spiece.model",
66+
model_url="https://raw.githubusercontent.com/google/gemma_pytorch/cb7c0152a369e43908e769eb09e1ce6043afe084/tokenizer/gemma3_cleaned_262144_v2.spiece.model",
6667
model_hash=(
6768
"1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c"
6869
),

google/genai/tests/local_tokenizer/test_local_tokenizer.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def setUp(self):
3737
self.mock_tokenizer = MagicMock()
3838
self.mock_get_sentencepiece.return_value = self.mock_tokenizer
3939

40-
self.tokenizer = local_tokenizer.LocalTokenizer(model_name='gemini-2.5-flash')
40+
self.tokenizer = local_tokenizer.LocalTokenizer(model_name='gemini-3-pro-preview')
4141

4242
def tearDown(self):
4343
patch.stopall()

0 commit comments

Comments
 (0)