Merge pull request #254 from openai/erinkav/message-content

Add message content type

Author: erinkav-openai

openapi.yaml

@@ -3887,7 +3887,7 @@ paths:
                   "thread_abc123",
                   "msg_abc123"
                 );
-              
+
                 console.log(deletedMessage);
               }
           response: |
@@ -3896,7 +3896,7 @@ paths:
               "object": "thread.message.deleted",
               "deleted": true
             }
-      
+
 
   /threads/runs:
     post:
@@ -8592,7 +8592,7 @@ components:
           description: |
             The intended purpose of the uploaded file.
 
-            Use "assistants" for [Assistants](/docs/api-reference/assistants) and [Messages](/docs/api-reference/messages), "batch" for [Batch API](/docs/guides/batch), and "fine-tune" for [Fine-tuning](/docs/api-reference/fine-tuning).
+            Use "assistants" for [Assistants](/docs/api-reference/assistants) and [Message](/docs/api-reference/messages) files, "vision" for Assistants image file inputs, "batch" for [Batch API](/docs/guides/batch), and "fine-tune" for [Fine-tuning](/docs/api-reference/fine-tuning).
           type: string
           enum: ["assistants", "batch", "fine-tune"]
       required:
@@ -9225,7 +9225,7 @@ components:
           enum: ["file"]
         purpose:
           type: string
-          description: The intended purpose of the file. Supported values are `assistants`, `assistants_output`, `batch`, `batch_output`, `fine-tune`, and `fine-tune-results`.
+          description: The intended purpose of the file. Supported values are `assistants`, `assistants_output`, `batch`, `batch_output`, `fine-tune`, `fine-tune-results` and `vision`.
           enum:
             [
               "assistants",
@@ -9234,7 +9234,7 @@ components:
               "batch_output",
               "fine-tune",
               "fine-tune-results",
-
+              "vision"
             ]
         status:
           type: string
@@ -10961,6 +10961,7 @@ components:
           items:
             oneOf:
               - $ref: "#/components/schemas/MessageContentImageFileObject"
+              - $ref: "#/components/schemas/MessageContentImageUrlObject"
               - $ref: "#/components/schemas/MessageContentTextObject"
             x-oaiExpandable: true
         assistant_id:
@@ -11062,6 +11063,7 @@ components:
                 oneOf:
                   - $ref: "#/components/schemas/MessageDeltaContentImageFileObject"
                   - $ref: "#/components/schemas/MessageDeltaContentTextObject"
+                  - $ref: "#/components/schemas/MessageDeltaContentImageUrlObject"
                 x-oaiExpandable: true
       required:
         - id
@@ -11100,10 +11102,21 @@ components:
             - `user`: Indicates the message is sent by an actual user and should be used in most cases to represent user-generated messages.
             - `assistant`: Indicates the message is generated by the assistant. Use this value to insert messages from the assistant into the conversation.
         content:
-          type: string
-          minLength: 1
-          maxLength: 256000
-          description: The content of the message.
+          oneOf:
+              - type: string
+                description: The text contents of the message.
+                title: Text content
+              - type: array
+                description: An array of content parts with a defined type, each can be of type `text` or images can be passed with `image_url` or `image_file`. Image types are only supported on [Vision-compatible models](/docs/models/overview).
+                title: Array of content parts
+                items:
+                  oneOf:
+                    - $ref: "#/components/schemas/MessageContentImageFileObject"
+                    - $ref: "#/components/schemas/MessageContentImageUrlObject"
+                    - $ref: "#/components/schemas/MessageRequestContentTextObject"
+                  x-oaiExpandable: true
+                minItems: 1
+          x-oaiExpandable: true
         attachments:
           type: array
           items:
@@ -11194,8 +11207,13 @@ components:
           type: object
           properties:
             file_id:
-              description: The [File](/docs/api-reference/files) ID of the image in the message content.
+              description: The [File](/docs/api-reference/files) ID of the image in the message content. Set `purpose="vision"` when uploading the File if you need to later display the file content.
               type: string
+            detail:
+              type: string
+              description: Specifies the detail level of the image if specified by the user. `low` uses fewer tokens, you can opt in to high resolution using `high`.
+              enum: ["auto", "low", "high"]
+              default: "auto"
           required:
             - file_id
       required:
@@ -11218,8 +11236,67 @@ components:
           type: object
           properties:
             file_id:
-              description: The [File](/docs/api-reference/files) ID of the image in the message content.
+              description: The [File](/docs/api-reference/files) ID of the image in the message content. Set `purpose="vision"` when uploading the File if you need to later display the file content.
+              type: string
+            detail:
               type: string
+              description: Specifies the detail level of the image if specified by the user. `low` uses fewer tokens, you can opt in to high resolution using `high`.
+              enum: ["auto", "low", "high"]
+              default: "auto"
+      required:
+        - index
+        - type
+
+    MessageContentImageUrlObject:
+      title: Image URL
+      type: object
+      description: References an image URL in the content of a message.
+      properties:
+        type:
+          type: string
+          enum: ["image_url"]
+          description: The type of the content part.
+        image_url:
+          type: object
+          properties:
+            url:
+              type: string
+              description: "The external URL of the image, must be a supported image types: jpeg, jpg, png, gif, webp."
+              format: uri
+            detail:
+              type: string
+              description: Specifies the detail level of the image. `low` uses fewer tokens, you can opt in to high resolution using `high`. Default value is `auto`
+              enum: ["auto", "low", "high"]
+              default: "auto"
+          required:
+            - url
+      required:
+        - type
+        - image_url
+
+    MessageDeltaContentImageUrlObject:
+      title: Image URL
+      type: object
+      description: References an image URL in the content of a message.
+      properties:
+        index:
+          type: integer
+          description: The index of the content part in the message.
+        type:
+          description: Always `image_url`.
+          type: string
+          enum: ["image_url"]
+        image_url:
+          type: object
+          properties:
+            url:
+              description: "The URL of the image, must be a supported image types: jpeg, jpg, png, gif, webp."
+              type: string
+            detail:
+              type: string
+              description: Specifies the detail level of the image. `low` uses fewer tokens, you can opt in to high resolution using `high`.
+              enum: ["auto", "low", "high"]
+              default: "auto"
       required:
         - index
         - type
@@ -11253,6 +11330,22 @@ components:
         - type
         - text
 
+    MessageRequestContentTextObject:
+      title: Text
+      type: object
+      description: The text content that is part of a message.
+      properties:
+        type:
+          description: Always `text`.
+          type: string
+          enum: ["text"]
+        text:
+          type: string
+          description: Text content to be sent to the model
+      required:
+        - type
+        - text
+
     MessageContentTextAnnotationsFileCitationObject:
       title: File citation
       type: object