Merge pull request #128 from dqbd/dqbd/o3-mini

feat: add missing o3-mini, o1 models

Author: dqbd

.changeset/silver-moons-boil.md

@@ -0,0 +1,7 @@
+---
+"tiktoken": patch
+"js-tiktoken": patch
+"@dqbd/tiktoken": patch
+---
+
+feat: add o3-mini, missing o1 model

js/src/core.ts

@@ -281,11 +281,14 @@ export function getEncodingNameForModel(model: TiktokenModel) {
     case "gpt-4o-2024-08-06":
     case "gpt-4o-mini-2024-07-18":
     case "gpt-4o-mini":
+    case "o1":
     case "o1-2024-12-17":
     case "o1-mini":
     case "o1-preview":
     case "o1-preview-2024-09-12":
     case "o1-mini-2024-09-12":
+    case "o3-mini":
+    case "o3-mini-2025-01-31":
     case "chatgpt-4o-latest":
     case "gpt-4o-realtime":
     case "gpt-4o-realtime-preview-2024-10-01":

tiktoken/model_to_encoding.json

@@ -60,11 +60,14 @@
     "gpt-4o-2024-08-06":"o200k_base",
     "gpt-4o-mini-2024-07-18": "o200k_base",
     "gpt-4o-mini": "o200k_base",
+    "o1": "o200k_base",
     "o1-2024-12-17": "o200k_base",
     "o1-mini": "o200k_base",
     "o1-preview": "o200k_base",
     "o1-preview-2024-09-12": "o200k_base",
     "o1-mini-2024-09-12": "o200k_base",
+    "o3-mini": "o200k_base",
+    "o3-mini-2025-01-31": "o200k_base",
     "chatgpt-4o-latest": "o200k_base",
     "gpt-4o-realtime": "o200k_base",
     "gpt-4o-realtime-preview-2024-10-01": "o200k_base"

wasm/src/lib.rs

@@ -385,6 +385,8 @@ export type TiktokenModel =
     | "text-davinci-edit-001"
     | "code-davinci-edit-001"
     | "text-embedding-ada-002"
+    | "text-embedding-3-small"
+    | "text-embedding-3-large"
     | "text-similarity-davinci-001"
     | "text-similarity-curie-001"
     | "text-similarity-babbage-001"
@@ -423,11 +425,14 @@ export type TiktokenModel =
     | "gpt-4o-2024-08-06"
     | "gpt-4o-mini-2024-07-18"
     | "gpt-4o-mini"
+    | "o1"
     | "o1-2024-12-17"
     | "o1-mini"
     | "o1-preview"
     | "o1-preview-2024-09-12"
     | "o1-mini-2024-09-12"
+    | "o3-mini"
+    | "o3-mini-2025-01-31"
     | "chatgpt-4o-latest"
     | "gpt-4o-realtime"
     | "gpt-4o-realtime-preview-2024-10-01"