feat: add o3-mini, missing o1 model

Author: dqbd

js/src/core.ts

@@ -281,11 +281,14 @@ export function getEncodingNameForModel(model: TiktokenModel) {
     case "gpt-4o-2024-08-06":
     case "gpt-4o-mini-2024-07-18":
     case "gpt-4o-mini":
+    case "o1":
     case "o1-2024-12-17":
     case "o1-mini":
     case "o1-preview":
     case "o1-preview-2024-09-12":
     case "o1-mini-2024-09-12":
+    case "o3-mini":
+    case "o3-mini-2025-01-31":
     case "chatgpt-4o-latest":
     case "gpt-4o-realtime":
     case "gpt-4o-realtime-preview-2024-10-01":

tiktoken/model_to_encoding.json

@@ -60,11 +60,14 @@
     "gpt-4o-2024-08-06":"o200k_base",
     "gpt-4o-mini-2024-07-18": "o200k_base",
     "gpt-4o-mini": "o200k_base",
+    "o1": "o200k_base",
     "o1-2024-12-17": "o200k_base",
     "o1-mini": "o200k_base",
     "o1-preview": "o200k_base",
     "o1-preview-2024-09-12": "o200k_base",
     "o1-mini-2024-09-12": "o200k_base",
+    "o3-mini": "o200k_base",
+    "o3-mini-2025-01-31": "o200k_base",
     "chatgpt-4o-latest": "o200k_base",
     "gpt-4o-realtime": "o200k_base",
     "gpt-4o-realtime-preview-2024-10-01": "o200k_base"

wasm/src/lib.rs

@@ -423,11 +423,14 @@ export type TiktokenModel =
     | "gpt-4o-2024-08-06"
     | "gpt-4o-mini-2024-07-18"
     | "gpt-4o-mini"
+    | "o1"
     | "o1-2024-12-17"
     | "o1-mini"
     | "o1-preview"
     | "o1-preview-2024-09-12"
     | "o1-mini-2024-09-12"
+    | "o3-mini"
+    | "o3-mini-2025-01-31"
     | "chatgpt-4o-latest"
     | "gpt-4o-realtime"
     | "gpt-4o-realtime-preview-2024-10-01"