Skip to content

Commit fb1c8f2

Browse files
ericyangpanclaude
ericyangpan
and
claude
committed
build: add model comparison script
Add utility script for comparing and analyzing model manifests to aid in data validation and quality assurance. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent f3f7831 commit fb1c8f2

File tree

2 files changed

+245
-70
lines changed

2 files changed

+245
-70
lines changed

‎manifests/models/llama-4-maverick.json‎

Lines changed: 0 additions & 70 deletions
This file was deleted.

‎scripts/fetch/compare-models.mjs‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
#!/usr/bin/env node
2+
3+
import { readdir, readFile } from 'node:fs/promises'
4+
import { dirname, join } from 'node:path'
5+
import { fileURLToPath } from 'node:url'
6+
7+
const __dirname = dirname(fileURLToPath(import.meta.url))
8+
const manifestsDir = join(__dirname, '../../manifests/models')
9+
const apiDataFile = join(__dirname, '../../tmp/models-dev-api.json')
10+
const mappingFile = join(__dirname, '../../manifests/mapping.json')
11+
12+
// Helper to compare values and return match status
13+
function compare(manifestValue, apiValue, _manifestKey, _apiKey) {
14+
if (manifestValue === null && apiValue === undefined) return { match: true, skip: true }
15+
if (manifestValue === null && apiValue === null) return { match: true, skip: false }
16+
if (manifestValue === null && apiValue !== undefined)
17+
return { match: false, manifest: null, api: apiValue }
18+
if (manifestValue !== null && apiValue === undefined)
19+
return { match: false, manifest: manifestValue, api: null }
20+
if (manifestValue === apiValue) return { match: true, skip: false }
21+
return { match: false, manifest: manifestValue, api: apiValue }
22+
}
23+
24+
// Convert API model data to manifest-compatible format for comparison
25+
function normalizeApiModel(apiModel) {
26+
return {
27+
name: apiModel.name,
28+
releaseDate: apiModel.release_date || null,
29+
contextWindow: apiModel.limit?.context || null,
30+
maxOutput: apiModel.limit?.output || null,
31+
inputModalities: apiModel.modalities?.input || [],
32+
tokenPricing: {
33+
input: apiModel.cost?.input || null,
34+
output: apiModel.cost?.output || null,
35+
cache: apiModel.cost?.cache_read || null,
36+
},
37+
capabilities: [
38+
...(apiModel.tool_call ? ['function-calling', 'tool-choice', 'structured-outputs'] : []),
39+
...(apiModel.reasoning ? ['reasoning'] : []),
40+
].sort(),
41+
}
42+
}
43+
44+
async function main() {
45+
// Read API reference data and mapping
46+
const apiData = JSON.parse(await readFile(apiDataFile, 'utf-8'))
47+
const { vendors: vendorMapping, models: modelMapping } = JSON.parse(
48+
await readFile(mappingFile, 'utf-8')
49+
)
50+
51+
// Read all model manifests
52+
const files = await readdir(manifestsDir)
53+
const manifestFiles = files.filter(f => f.endsWith('.json'))
54+
55+
const results = []
56+
57+
for (const file of manifestFiles) {
58+
const manifest = JSON.parse(await readFile(join(manifestsDir, file), 'utf-8'))
59+
60+
const modelId = manifest.id
61+
const vendor = manifest.vendor
62+
// Use mapping.json if available, otherwise lowercase the vendor name
63+
const vendorKey = vendorMapping[vendor] ?? vendor.toLowerCase()
64+
// Use mapping.json for model IDs if available
65+
const apiModelId = modelMapping[modelId] ?? modelId
66+
67+
// Check if vendor exists in API data
68+
const vendorData = apiData[vendorKey]
69+
const vendorExists = !!vendorData
70+
71+
// Check if model exists under that vendor
72+
const apiModel = vendorData?.models?.[apiModelId]
73+
const modelExists = !!apiModel
74+
75+
const comparisons = []
76+
if (modelExists) {
77+
const normalizedApi = normalizeApiModel(apiModel)
78+
79+
// Compare releaseDate
80+
comparisons.push({
81+
field: 'releaseDate',
82+
manifestKey: 'releaseDate',
83+
apiKey: 'release_date',
84+
...compare(manifest.releaseDate, normalizedApi.releaseDate),
85+
})
86+
87+
// Compare contextWindow
88+
comparisons.push({
89+
field: 'contextWindow',
90+
manifestKey: 'contextWindow',
91+
apiKey: 'limit.context',
92+
...compare(manifest.contextWindow, normalizedApi.contextWindow),
93+
})
94+
95+
// Compare maxOutput
96+
comparisons.push({
97+
field: 'maxOutput',
98+
manifestKey: 'maxOutput',
99+
apiKey: 'limit.output',
100+
...compare(manifest.maxOutput, normalizedApi.maxOutput),
101+
})
102+
103+
// Compare inputModalities
104+
const modalitiesMatch =
105+
JSON.stringify(manifest.inputModalities.sort()) ===
106+
JSON.stringify(normalizedApi.inputModalities.sort())
107+
comparisons.push({
108+
field: 'inputModalities',
109+
manifestKey: 'inputModalities',
110+
apiKey: 'modalities.input',
111+
match: !!modalitiesMatch,
112+
skip: false,
113+
...(!modalitiesMatch
114+
? { manifest: manifest.inputModalities, api: normalizedApi.inputModalities }
115+
: {}),
116+
})
117+
118+
// Compare tokenPricing
119+
const inputPriceMatch = compare(
120+
manifest.tokenPricing.input,
121+
normalizedApi.tokenPricing.input,
122+
'input',
123+
'input'
124+
)
125+
comparisons.push({
126+
field: 'tokenPricing.input',
127+
manifestKey: 'tokenPricing.input',
128+
apiKey: 'cost.input',
129+
...inputPriceMatch,
130+
})
131+
132+
const outputPriceMatch = compare(
133+
manifest.tokenPricing.output,
134+
normalizedApi.tokenPricing.output,
135+
'output',
136+
'output'
137+
)
138+
comparisons.push({
139+
field: 'tokenPricing.output',
140+
manifestKey: 'tokenPricing.output',
141+
apiKey: 'cost.output',
142+
...outputPriceMatch,
143+
})
144+
145+
const cachePriceMatch = compare(
146+
manifest.tokenPricing.cache,
147+
normalizedApi.tokenPricing.cache,
148+
'cache',
149+
'cache_read'
150+
)
151+
comparisons.push({
152+
field: 'tokenPricing.cache',
153+
manifestKey: 'tokenPricing.cache',
154+
apiKey: 'cost.cache_read',
155+
...cachePriceMatch,
156+
})
157+
158+
// Compare capabilities
159+
const capabilitiesMatch =
160+
JSON.stringify(manifest.capabilities.sort()) ===
161+
JSON.stringify(normalizedApi.capabilities.sort())
162+
comparisons.push({
163+
field: 'capabilities',
164+
manifestKey: 'capabilities',
165+
apiKey: 'tool_call/reasoning',
166+
match: !!capabilitiesMatch,
167+
skip: false,
168+
...(!capabilitiesMatch
169+
? { manifest: manifest.capabilities, api: normalizedApi.capabilities }
170+
: {}),
171+
})
172+
}
173+
174+
results.push({
175+
modelId,
176+
apiModelId,
177+
vendor,
178+
vendorKey,
179+
vendorExists,
180+
modelExists,
181+
comparisons,
182+
})
183+
}
184+
185+
// Group by match status and field mismatches
186+
const matched = results.filter(r => r.modelExists)
187+
const unmatched = results.filter(r => !r.modelExists)
188+
const withMismatches = matched.filter(r => r.comparisons.some(c => !c.match && !c.skip))
189+
190+
// Output matched models with all fields matching
191+
console.log('Matched Models (no field mismatches):')
192+
console.log('=======================================\n')
193+
const perfectlyMatched = matched.filter(r => !r.comparisons.some(c => !c.match && !c.skip))
194+
for (const r of perfectlyMatched) {
195+
const modelDisplay = r.apiModelId !== r.modelId ? `${r.modelId} → ${r.apiModelId}` : r.modelId
196+
console.log(` ${modelDisplay} | ${r.vendor} (${r.vendorKey})`)
197+
}
198+
199+
// Output matched models with field mismatches
200+
console.log(`\nMatched Models (with field mismatches):`)
201+
console.log('========================================\n')
202+
for (const r of withMismatches) {
203+
const modelDisplay = r.apiModelId !== r.modelId ? `${r.modelId} → ${r.apiModelId}` : r.modelId
204+
console.log(` ${modelDisplay} | ${r.vendor} (${r.vendorKey})`)
205+
for (const comp of r.comparisons.filter(c => !c.match && !c.skip)) {
206+
const mValue = JSON.stringify(comp.manifest)
207+
const aValue = JSON.stringify(comp.api)
208+
console.log(` ✗ ${comp.field} (manifest: ${mValue}, api: ${aValue})`)
209+
}
210+
}
211+
212+
// Output matched models with skipped fields (null in manifest)
213+
console.log(`\nMatched Models (with skipped fields - null in manifest):`)
214+
console.log('========================================================\n')
215+
const withSkipped = matched.filter(r => r.comparisons.some(c => c.skip))
216+
for (const r of withSkipped) {
217+
const modelDisplay = r.apiModelId !== r.modelId ? `${r.modelId} → ${r.apiModelId}` : r.modelId
218+
console.log(` ${modelDisplay} | ${r.vendor} (${r.vendorKey})`)
219+
for (const comp of r.comparisons.filter(c => c.skip)) {
220+
console.log(` → ${comp.field}: null (api: ${JSON.stringify(comp.api ?? 'undefined')})`)
221+
}
222+
}
223+
224+
// Output unmatched models
225+
console.log(`\nUnmatched Models:`)
226+
console.log('=================\n')
227+
for (const r of unmatched) {
228+
const vendorStatus = r.vendorExists ? '✓' : '✗'
229+
const modelDisplay = r.apiModelId !== r.modelId ? `${r.modelId} → ${r.apiModelId}` : r.modelId
230+
console.log(` ${modelDisplay} | ${r.vendor} (${r.vendorKey}) | Vendor: ${vendorStatus}`)
231+
}
232+
233+
// Summary
234+
const total = results.length
235+
console.log(`\nSummary:`)
236+
console.log(` Total models: ${total}`)
237+
console.log(` Perfectly matched: ${perfectlyMatched.length}`)
238+
console.log(` Matched with mismatches: ${withMismatches.length}`)
239+
console.log(
240+
` Matched with skipped fields: ${withSkipped.filter(r => !r.comparisons.some(c => !c.match && !c.skip)).length}`
241+
)
242+
console.log(` Not found in API: ${unmatched.length}`)
243+
}
244+
245+
main().catch(console.error)

0 commit comments

Comments
 (0)