feat(api): new models for TTS, STT, + new audio features for Realtime (#1407)

Author: stainless-app[bot]

.stats.yml

@@ -1,2 +1,2 @@
-configured_endpoints: 81
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-b26121d5df6eb5d3032a45a267473798b15fcfec76dd44a3256cf1238be05fa4.yml
+configured_endpoints: 82
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-c22f59c66aec7914b6ee653d3098d1c1c8c16c180d2a158e819c8ddbf476f74b.yml

api.md

@@ -141,7 +141,11 @@ Types:
 Types:
 
 - <code><a href="./src/resources/audio/transcriptions.ts">Transcription</a></code>
+- <code><a href="./src/resources/audio/transcriptions.ts">TranscriptionInclude</a></code>
 - <code><a href="./src/resources/audio/transcriptions.ts">TranscriptionSegment</a></code>
+- <code><a href="./src/resources/audio/transcriptions.ts">TranscriptionStreamEvent</a></code>
+- <code><a href="./src/resources/audio/transcriptions.ts">TranscriptionTextDeltaEvent</a></code>
+- <code><a href="./src/resources/audio/transcriptions.ts">TranscriptionTextDoneEvent</a></code>
 - <code><a href="./src/resources/audio/transcriptions.ts">TranscriptionVerbose</a></code>
 - <code><a href="./src/resources/audio/transcriptions.ts">TranscriptionWord</a></code>
 - <code><a href="./src/resources/audio/transcriptions.ts">TranscriptionCreateResponse</a></code>
@@ -298,7 +302,9 @@ Types:
 - <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemDeleteEvent</a></code>
 - <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemDeletedEvent</a></code>
 - <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemInputAudioTranscriptionCompletedEvent</a></code>
+- <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemInputAudioTranscriptionDeltaEvent</a></code>
 - <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemInputAudioTranscriptionFailedEvent</a></code>
+- <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemRetrieveEvent</a></code>
 - <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemTruncateEvent</a></code>
 - <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemTruncatedEvent</a></code>
 - <code><a href="./src/resources/beta/realtime/realtime.ts">ConversationItemWithReference</a></code>
@@ -335,6 +341,8 @@ Types:
 - <code><a href="./src/resources/beta/realtime/realtime.ts">SessionCreatedEvent</a></code>
 - <code><a href="./src/resources/beta/realtime/realtime.ts">SessionUpdateEvent</a></code>
 - <code><a href="./src/resources/beta/realtime/realtime.ts">SessionUpdatedEvent</a></code>
+- <code><a href="./src/resources/beta/realtime/realtime.ts">TranscriptionSessionUpdate</a></code>
+- <code><a href="./src/resources/beta/realtime/realtime.ts">TranscriptionSessionUpdatedEvent</a></code>
 
 ### Sessions
 
@@ -347,6 +355,16 @@ Methods:
 
 - <code title="post /realtime/sessions">client.beta.realtime.sessions.<a href="./src/resources/beta/realtime/sessions.ts">create</a>({ ...params }) -> SessionCreateResponse</code>
 
+### TranscriptionSessions
+
+Types:
+
+- <code><a href="./src/resources/beta/realtime/transcription-sessions.ts">TranscriptionSession</a></code>
+
+Methods:
+
+- <code title="post /realtime/transcription_sessions">client.beta.realtime.transcriptionSessions.<a href="./src/resources/beta/realtime/transcription-sessions.ts">create</a>({ ...params }) -> TranscriptionSession</code>
+
 ## Assistants
 
 Types:

src/resources/audio/audio.ts

@@ -7,8 +7,14 @@ import * as TranscriptionsAPI from './transcriptions';
 import {
   Transcription,
   TranscriptionCreateParams,
+  TranscriptionCreateParamsNonStreaming,
+  TranscriptionCreateParamsStreaming,
   TranscriptionCreateResponse,
+  TranscriptionInclude,
   TranscriptionSegment,
+  TranscriptionStreamEvent,
+  TranscriptionTextDeltaEvent,
+  TranscriptionTextDoneEvent,
   TranscriptionVerbose,
   TranscriptionWord,
   Transcriptions,
@@ -28,11 +34,12 @@ export class Audio extends APIResource {
   speech: SpeechAPI.Speech = new SpeechAPI.Speech(this._client);
 }
 
-export type AudioModel = 'whisper-1';
+export type AudioModel = 'whisper-1' | 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe';
 
 /**
  * The format of the output, in one of these options: `json`, `text`, `srt`,
- * `verbose_json`, or `vtt`.
+ * `verbose_json`, or `vtt`. For `gpt-4o-transcribe` and `gpt-4o-mini-transcribe`,
+ * the only supported format is `json`.
  */
 export type AudioResponseFormat = 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt';
 
@@ -46,11 +53,17 @@ export declare namespace Audio {
   export {
     Transcriptions as Transcriptions,
     type Transcription as Transcription,
+    type TranscriptionInclude as TranscriptionInclude,
     type TranscriptionSegment as TranscriptionSegment,
+    type TranscriptionStreamEvent as TranscriptionStreamEvent,
+    type TranscriptionTextDeltaEvent as TranscriptionTextDeltaEvent,
+    type TranscriptionTextDoneEvent as TranscriptionTextDoneEvent,
     type TranscriptionVerbose as TranscriptionVerbose,
     type TranscriptionWord as TranscriptionWord,
     type TranscriptionCreateResponse as TranscriptionCreateResponse,
     type TranscriptionCreateParams as TranscriptionCreateParams,
+    type TranscriptionCreateParamsNonStreaming as TranscriptionCreateParamsNonStreaming,
+    type TranscriptionCreateParamsStreaming as TranscriptionCreateParamsStreaming,
   };
 
   export {

src/resources/audio/index.ts

@@ -5,11 +5,17 @@ export { Speech, type SpeechModel, type SpeechCreateParams } from './speech';
 export {
   Transcriptions,
   type Transcription,
+  type TranscriptionInclude,
   type TranscriptionSegment,
+  type TranscriptionStreamEvent,
+  type TranscriptionTextDeltaEvent,
+  type TranscriptionTextDoneEvent,
   type TranscriptionVerbose,
   type TranscriptionWord,
   type TranscriptionCreateResponse,
   type TranscriptionCreateParams,
+  type TranscriptionCreateParamsNonStreaming,
+  type TranscriptionCreateParamsStreaming,
 } from './transcriptions';
 export {
   Translations,

src/resources/audio/speech.ts

@@ -18,7 +18,7 @@ export class Speech extends APIResource {
   }
 }
 
-export type SpeechModel = 'tts-1' | 'tts-1-hd';
+export type SpeechModel = 'tts-1' | 'tts-1-hd' | 'gpt-4o-mini-tts';
 
 export interface SpeechCreateParams {
   /**
@@ -28,7 +28,7 @@ export interface SpeechCreateParams {
 
   /**
    * One of the available [TTS models](https://platform.openai.com/docs/models#tts):
-   * `tts-1` or `tts-1-hd`
+   * `tts-1`, `tts-1-hd` or `gpt-4o-mini-tts`.
    */
   model: (string & {}) | SpeechModel;
 
@@ -40,6 +40,12 @@ export interface SpeechCreateParams {
    */
   voice: 'alloy' | 'ash' | 'coral' | 'echo' | 'fable' | 'onyx' | 'nova' | 'sage' | 'shimmer';
 
+  /**
+   * Control the voice of your generated audio with additional instructions. Does not
+   * work with `tts-1` or `tts-1-hd`.
+   */
+  instructions?: string;
+
   /**
    * The format to audio in. Supported formats are `mp3`, `opus`, `aac`, `flac`,
    * `wav`, and `pcm`.