From a667c389eb557faea0b39e36f7e85a1c14635c0c Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 04:51:53 -0700 Subject: [PATCH 1/9] Switch to replicate.stream in Bun integration test --- integration/bun/index.test.ts | 2 +- integration/bun/index.ts | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/integration/bun/index.test.ts b/integration/bun/index.test.ts index e9fbaab..5c1cdce 100644 --- a/integration/bun/index.test.ts +++ b/integration/bun/index.test.ts @@ -19,5 +19,5 @@ import type { test("main", async () => { const output = await main(); - expect(output as any).toEqual("hello Brünnhilde Bun"); + expect(output).toContain("bun"); }); diff --git a/integration/bun/index.ts b/integration/bun/index.ts index 263a86e..97ddd04 100644 --- a/integration/bun/index.ts +++ b/integration/bun/index.ts @@ -5,12 +5,19 @@ const replicate = new Replicate({ }); export default async function main() { - return await replicate.run( - "replicate/hello-world:5c7d5dc6dd8bf75c1acaa8565735e7986bc5b66206b55cca93cb72c9bf15ccaa", - { - input: { - text: "Brünnhilde Bun", - }, + const model = "meta/llama-2-70b-chat"; + const options = { + input: { + prompt: "Write a poem about steam buns", + }, + }; + const output = []; + + for await (const { event, data } of replicate.stream(model, options)) { + if (event === "output") { + output.push(data); } - ); + } + + return output.join("").trim(); } From a2698d3f9cb936604515925fa8bebc59ea8c3863 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 05:07:32 -0700 Subject: [PATCH 2/9] Vendor @stardazed/streams-text-encoding --- .../text-decoder-stream.js | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 vendor/streams-text-encoding/text-decoder-stream.js diff --git a/vendor/streams-text-encoding/text-decoder-stream.js b/vendor/streams-text-encoding/text-decoder-stream.js new file mode 100644 index 0000000..c3f349c --- /dev/null +++ b/vendor/streams-text-encoding/text-decoder-stream.js @@ -0,0 +1,94 @@ +// Source: https://github.com/stardazed/sd-streams +// +// MIT License +// +// Copyright (c) 2018-Present @zenmumbler +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + +// /input.ts +var input_exports = {}; +__export(input_exports, { + TextDecoderStream: () => TextDecoderStream +}); +module.exports = __toCommonJS(input_exports); + +// http-url:https://unpkg.com/@stardazed/streams-text-encoding@1.0.2/dist/sd-streams-text-encoding.esm.js +var decDecoder = Symbol("decDecoder"); +var decTransform = Symbol("decTransform"); +var TextDecodeTransformer = class { + constructor(decoder) { + this.decoder_ = decoder; + } + transform(chunk, controller) { + if (!(chunk instanceof ArrayBuffer || ArrayBuffer.isView(chunk))) { + throw new TypeError("Input data must be a BufferSource"); + } + const text = this.decoder_.decode(chunk, { stream: true }); + if (text.length !== 0) { + controller.enqueue(text); + } + } + flush(controller) { + const text = this.decoder_.decode(); + if (text.length !== 0) { + controller.enqueue(text); + } + } +}; +var TextDecoderStream = class { + constructor(label, options) { + this[decDecoder] = new TextDecoder(label, options); + this[decTransform] = new TransformStream(new TextDecodeTransformer(this[decDecoder])); + } + get encoding() { + return this[decDecoder].encoding; + } + get fatal() { + return this[decDecoder].fatal; + } + get ignoreBOM() { + return this[decDecoder].ignoreBOM; + } + get readable() { + return this[decTransform].readable; + } + get writable() { + return this[decTransform].writable; + } +}; +var encEncoder = Symbol("encEncoder"); +var encTransform = Symbol("encTransform"); From a9566445fbad43e9cd2d557adffa1638f609e4f6 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 05:07:50 -0700 Subject: [PATCH 3/9] Modify vendored source of @stardazed/streams-text-encoding --- vendor/streams-text-encoding/text-decoder-stream.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vendor/streams-text-encoding/text-decoder-stream.js b/vendor/streams-text-encoding/text-decoder-stream.js index c3f349c..18c958e 100644 --- a/vendor/streams-text-encoding/text-decoder-stream.js +++ b/vendor/streams-text-encoding/text-decoder-stream.js @@ -1,4 +1,4 @@ -// Source: https://github.com/stardazed/sd-streams +// Adapted from https://github.com/stardazed/sd-streams // // MIT License // @@ -71,8 +71,9 @@ var TextDecodeTransformer = class { }; var TextDecoderStream = class { constructor(label, options) { - this[decDecoder] = new TextDecoder(label, options); - this[decTransform] = new TransformStream(new TextDecodeTransformer(this[decDecoder])); + const decoder = new TextDecoder(label || "utf-8", options || {}); + this[decDecoder] = decoder; + this[decTransform] = new TransformStream(new TextDecodeTransformer(decoder)); } get encoding() { return this[decDecoder].encoding; From 78ef9ac5113e38e74d0c92d9fe8e1ef3d45c4501 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 05:10:05 -0700 Subject: [PATCH 4/9] Use polyfill when TextDecoderStream isn't available --- integration/bun/index.ts | 2 +- lib/stream.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/integration/bun/index.ts b/integration/bun/index.ts index 97ddd04..05f1d62 100644 --- a/integration/bun/index.ts +++ b/integration/bun/index.ts @@ -8,7 +8,7 @@ export default async function main() { const model = "meta/llama-2-70b-chat"; const options = { input: { - prompt: "Write a poem about steam buns", + prompt: "Write a haiku about steam buns", }, }; const output = []; diff --git a/lib/stream.js b/lib/stream.js index cd9274c..b524206 100644 --- a/lib/stream.js +++ b/lib/stream.js @@ -4,6 +4,10 @@ const ApiError = require("./error"); const { EventSourceParserStream, } = require("../vendor/eventsource-parser/stream"); +const { TextDecoderStream } = + typeof global.TextDecoderStream === "undefined" + ? require("../vendor/streams-text-encoding/text-decoder-stream") + : global; /** * A server-sent event. From a58f770077ecc516e8eb26c84967c392b48d106d Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 05:16:08 -0700 Subject: [PATCH 5/9] Use globalThis instead of global --- lib/stream.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/stream.js b/lib/stream.js index b524206..c268ed6 100644 --- a/lib/stream.js +++ b/lib/stream.js @@ -5,9 +5,9 @@ const { EventSourceParserStream, } = require("../vendor/eventsource-parser/stream"); const { TextDecoderStream } = - typeof global.TextDecoderStream === "undefined" + typeof globalThis.TextDecoderStream === "undefined" ? require("../vendor/streams-text-encoding/text-decoder-stream") - : global; + : globalThis; /** * A server-sent event. From a722df3919e41ca3e15cb2a64d4ac789e3a32e92 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 05:21:33 -0700 Subject: [PATCH 6/9] Set 30s test timeout --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3dbba4..085ba5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,4 +139,4 @@ jobs: cd integration/${{ matrix.suite }} bun uninstall replicate bun install "file:../../${{ needs.build.outputs.tarball-name }}" - bun test + bun test --timeout 30000 From ec3049ce5709f70049d47069c4b7441462c98ac0 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 06:39:09 -0700 Subject: [PATCH 7/9] Add information about vendoring and patching TextDecoderStream polyfill --- README.md | 38 +++++++++++++------ .../text-decoder-stream.js | 2 +- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 255c8f1..ae184e7 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ app.get('/webhooks/replicate', async (c) => { const prediction = await c.req.json(); console.log(prediction); //=> {"id": "xyz", "status": "successful", ... } - + // Acknowledge the webhook. c.status(200); c.json({ok: true}); @@ -217,15 +217,15 @@ Run a model and await the result. Unlike [`replicate.prediction.create`](#replic const output = await replicate.run(identifier, options, progress); ``` -| name | type | description | -| ------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `identifier` | string | **Required**. The model version identifier in the format `{owner}/{name}:{version}`, for example `stability-ai/sdxl:8beff3369e81422112d93b89ca01426147de542cd4684c244b673b105188fe5f` | -| `options.input` | object | **Required**. An object with the model inputs. | -| `options.wait` | object | Options for waiting for the prediction to finish | -| `options.wait.interval` | number | Polling interval in milliseconds. Defaults to 500 | -| `options.webhook` | string | An HTTPS URL for receiving a webhook when the prediction has new output | -| `options.webhook_events_filter` | string[] | An array of events which should trigger [webhooks](https://replicate.com/docs/webhooks). Allowable values are `start`, `output`, `logs`, and `completed` | -| `options.signal` | object | An [AbortSignal](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) to cancel the prediction | +| name | type | description | +| ------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `identifier` | string | **Required**. The model version identifier in the format `{owner}/{name}:{version}`, for example `stability-ai/sdxl:8beff3369e81422112d93b89ca01426147de542cd4684c244b673b105188fe5f` | +| `options.input` | object | **Required**. An object with the model inputs. | +| `options.wait` | object | Options for waiting for the prediction to finish | +| `options.wait.interval` | number | Polling interval in milliseconds. Defaults to 500 | +| `options.webhook` | string | An HTTPS URL for receiving a webhook when the prediction has new output | +| `options.webhook_events_filter` | string[] | An array of events which should trigger [webhooks](https://replicate.com/docs/webhooks). Allowable values are `start`, `output`, `logs`, and `completed` | +| `options.signal` | object | An [AbortSignal](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) to cancel the prediction | | `progress` | function | Callback function that receives the prediction object as it's updated. The function is called when the prediction is created, each time it's updated while polling for completion, and when it's completed. | Throws `Error` if the prediction failed. @@ -246,7 +246,7 @@ Example that logs progress as the model is running: const model = "stability-ai/sdxl:8beff3369e81422112d93b89ca01426147de542cd4684c244b673b105188fe5f"; const input = { prompt: "a 19th century portrait of a raccoon gentleman wearing a suit" }; const onProgress = (prediction) => { - const last_log_line = prediction.logs.split("\n").pop() + const last_log_line = prediction.logs.split("\n").pop() console.log({id: prediction.id, log: last_log_line}) } const output = await replicate.run(model, { input }, onProgress) @@ -875,6 +875,20 @@ The `Replicate` constructor and all `replicate.*` methods are fully typed. We have a few dependencies that have been bundled into the vendor directory rather than adding external npm dependencies. -These have been generated using bundlejs.com and copied into the appropriate directory along with the license and repository information. +These have been generated using bundlejs.com and copied into the appropriate directory along with the license and repository information. * [eventsource-parser/stream](https://bundlejs.com/?bundle&q=eventsource-parser%40latest%2Fstream&config=%7B%22esbuild%22%3A%7B%22format%22%3A%22cjs%22%2C%22minify%22%3Afalse%2C%22platform%22%3A%22neutral%22%7D%7D) +* [streams-text-encoding/text-decoder-stream](https://bundlejs.com/?q=%40stardazed%2Fstreams-text-encoding&treeshake=%5B%7B+TextDecoderStream+%7D%5D&config=%7B%22esbuild%22%3A%7B%22format%22%3A%22cjs%22%2C%22minify%22%3Afalse%7D%7D) + +> [!NOTE] +> The vendored implementation of `TextDecoderStream` requires +> the following patch to be applied to the output of bundlejs.com: +> ```difff +> constructor(label, options) { +> - this[decDecoder] = new TextDecoder(label, options); +> - this[decTransform] = new TransformStream(new TextDecodeTransformer(this[decDecoder])); +> } +> + const decoder = new TextDecoder(label || "utf-8", options || {}); +> + this[decDecoder] = decoder; +> + this[decTransform] = new TransformStream(new TextDecodeTransformer(decoder)); +> ``` diff --git a/vendor/streams-text-encoding/text-decoder-stream.js b/vendor/streams-text-encoding/text-decoder-stream.js index 18c958e..f400709 100644 --- a/vendor/streams-text-encoding/text-decoder-stream.js +++ b/vendor/streams-text-encoding/text-decoder-stream.js @@ -1,4 +1,4 @@ -// Adapted from https://github.com/stardazed/sd-streams +// Adapted from https://github.com/stardazed/sd-streams // // MIT License // From 150834645225a3a9fb9b903884020fd6d79ccfa1 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 06:41:05 -0700 Subject: [PATCH 8/9] Use replicate/canary for streaming test --- integration/bun/index.test.ts | 2 +- integration/bun/index.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/integration/bun/index.test.ts b/integration/bun/index.test.ts index 5c1cdce..1357e5b 100644 --- a/integration/bun/index.test.ts +++ b/integration/bun/index.test.ts @@ -19,5 +19,5 @@ import type { test("main", async () => { const output = await main(); - expect(output).toContain("bun"); + expect(output).toContain("Brünnhilde Bun"); }); diff --git a/integration/bun/index.ts b/integration/bun/index.ts index 05f1d62..fc95399 100644 --- a/integration/bun/index.ts +++ b/integration/bun/index.ts @@ -5,10 +5,11 @@ const replicate = new Replicate({ }); export default async function main() { - const model = "meta/llama-2-70b-chat"; + const model = + "replicate/canary:30e22229542eb3f79d4f945dacb58d32001b02cc313ae6f54eef27904edf3272"; const options = { input: { - prompt: "Write a haiku about steam buns", + text: "Brünnhilde Bun", }, }; const output = []; From 8b9760eb4970e201f10b51b99279f0ec45ddd5cf Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Mon, 18 Mar 2024 06:43:12 -0700 Subject: [PATCH 9/9] Fix README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ae184e7..70a1a9c 100644 --- a/README.md +++ b/README.md @@ -883,12 +883,13 @@ These have been generated using bundlejs.com and copied into the appropriate dir > [!NOTE] > The vendored implementation of `TextDecoderStream` requires > the following patch to be applied to the output of bundlejs.com: -> ```difff +> +> ```diff > constructor(label, options) { > - this[decDecoder] = new TextDecoder(label, options); > - this[decTransform] = new TransformStream(new TextDecodeTransformer(this[decDecoder])); -> } > + const decoder = new TextDecoder(label || "utf-8", options || {}); > + this[decDecoder] = decoder; > + this[decTransform] = new TransformStream(new TextDecodeTransformer(decoder)); +> } > ```