diff --git a/README.md b/README.md index fbd837f..71ec84b 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ second parameter, with one of the following values: If an object with the given key is not found, `null` is returned. ```javascript -const entry = await blobs.get('some-key', { type: 'json' }) +const entry = await store.get('some-key', { type: 'json' }) console.log(entry) ``` @@ -209,7 +209,7 @@ second parameter, with one of the following values: If an object with the given key is not found, `null` is returned. ```javascript -const blob = await blobs.getWithMetadata('some-key', { type: 'json' }) +const blob = await store.getWithMetadata('some-key', { type: 'json' }) console.log(blob.data, blob.etag, blob.metadata) ``` @@ -223,7 +223,7 @@ const cachedETag = getFromMockCache('my-key') // Get entry from the blob store only if its ETag is different from the one you // have locally, which means the entry has changed since you last obtained it -const { data, etag, fresh } = await blobs.getWithMetadata('some-key', { etag: cachedETag }) +const { data, etag, fresh } = await store.getWithMetadata('some-key', { etag: cachedETag }) if (fresh) { // `data` is `null` because the local blob is fresh @@ -240,7 +240,7 @@ Creates an object with the given key and value. If an entry with the given key already exists, its value is overwritten. ```javascript -await blobs.set('some-key', 'This is a string value') +await store.set('some-key', 'This is a string value') ``` ### `setJSON(key: string, value: any, { metadata?: object }): Promise` @@ -250,7 +250,7 @@ Convenience method for creating a JSON-serialized object with the given key. If an entry with the given key already exists, its value is overwritten. ```javascript -await blobs.setJSON('some-key', { +await store.setJSON('some-key', { foo: 'bar', }) ``` @@ -260,9 +260,86 @@ await blobs.setJSON('some-key', { Deletes an object with the given key, if one exists. ```javascript -await blobs.delete('my-key') +await store.delete('my-key') ``` +### `list(options?: { cursor?: string, directories?: boolean, paginate?: boolean. prefix?: string }): Promise<{ blobs: BlobResult[], directories: string[] }>` + +Returns a list of blobs in a given store. + +```javascript +const { blobs } = await store.list() + +// [ { etag: 'etag1', key: 'some-key' }, { etag: 'etag2', key: 'another-key' } ] +console.log(blobs) +``` + +To filter down the entries that should be returned, an optional `prefix` parameter can be supplied. When used, only the +entries whose key starts with that prefix are returned. + +```javascript +const { blobs } = await store.list({ prefix: 'some' }) + +// [ { etag: 'etag1', key: 'some-key' } ] +console.log(blobs) +``` + +Optionally, you can choose to group blobs together under a common prefix and then browse them hierarchically when +listing a store, just like grouping files in a directory. To do this, use the `/` character in your keys to group them +into directories. + +Take the following list of keys as an example: + +``` +cats/garfield.jpg +cats/tom.jpg +mice/jerry.jpg +mice/mickey.jpg +pink-panther.jpg +``` + +By default, calling `store.list()` will return all five keys. + +```javascript +const { blobs } = await store.list() + +// [ +// { etag: "etag1", key: "cats/garfield.jpg" }, +// { etag: "etag2", key: "cats/tom.jpg" }, +// { etag: "etag3", key: "mice/jerry.jpg" }, +// { etag: "etag4", key: "mice/mickey.jpg" }, +// { etag: "etag5", key: "pink-panther.jpg" }, +// ] +console.log(blobs) +``` + +But if you want to list entries hierarchically, use the `directories` parameter. + +```javascript +const { blobs, directories } = await store.list({ directories: true }) + +// [ { etag: "etag1", key: "pink-panther.jpg" } ] +console.log(blobs) + +// [ "cats", "mice" ] +console.log(directories) +``` + +To drill down into a directory and get a list of its items, you can use the directory name as the `prefix` value. + +```javascript +const { blobs, directories } = await store.list({ directories: true, prefix: 'cats/' }) + +// [ { etag: "etag1", key: "cats/garfield.jpg" }, { etag: "etag2", key: "cats/tom.jpg" } ] +console.log(blobs) + +// [ ] +console.log(directories) +``` + +Note that we're only interested in entries under the `cats` directory, which is why we're using a trailing slash. +Without it, other keys like `catsuit` would also match. + ## Contributing Contributions are welcome! If you encounter any issues or have suggestions for improvements, please open an issue or diff --git a/src/backend/list.ts b/src/backend/list.ts new file mode 100644 index 0000000..17c14e6 --- /dev/null +++ b/src/backend/list.ts @@ -0,0 +1,12 @@ +export interface ListResponse { + blobs?: ListResponseBlob[] + directories?: string[] + next_cursor?: string +} + +export interface ListResponseBlob { + etag: string + last_modified: string + size: number + key: string +} diff --git a/src/client.ts b/src/client.ts index 9631d48..5c8a4a8 100644 --- a/src/client.ts +++ b/src/client.ts @@ -6,9 +6,10 @@ import { BlobInput, Fetcher, HTTPMethod } from './types.ts' interface MakeStoreRequestOptions { body?: BlobInput | null headers?: Record - key: string + key?: string metadata?: Metadata method: HTTPMethod + parameters?: Record storeName: string } @@ -20,6 +21,14 @@ export interface ClientOptions { token: string } +interface GetFinalRequestOptions { + key: string | undefined + metadata?: Metadata + method: string + parameters?: Record + storeName: string +} + export class Client { private apiURL?: string private edgeURL?: string @@ -41,7 +50,7 @@ export class Client { } } - private async getFinalRequest(storeName: string, key: string, method: string, metadata?: Metadata) { + private async getFinalRequest({ key, metadata, method, parameters = {}, storeName }: GetFinalRequestOptions) { const encodedMetadata = encodeMetadata(metadata) if (this.edgeURL) { @@ -53,38 +62,72 @@ export class Client { headers[METADATA_HEADER_EXTERNAL] = encodedMetadata } + const path = key ? `/${this.siteID}/${storeName}/${key}` : `/${this.siteID}/${storeName}` + const url = new URL(path, this.edgeURL) + + for (const key in parameters) { + url.searchParams.set(key, parameters[key]) + } + return { headers, - url: `${this.edgeURL}/${this.siteID}/${storeName}/${key}`, + url: url.toString(), } } - const apiURL = `${this.apiURL ?? 'https://api.netlify.com'}/api/v1/sites/${ - this.siteID - }/blobs/${key}?context=${storeName}` const apiHeaders: Record = { authorization: `Bearer ${this.token}` } + const url = new URL(`/api/v1/sites/${this.siteID}/blobs`, this.apiURL ?? 'https://api.netlify.com') + + for (const key in parameters) { + url.searchParams.set(key, parameters[key]) + } + + url.searchParams.set('context', storeName) + + if (key === undefined) { + return { + headers: apiHeaders, + url: url.toString(), + } + } + + url.pathname += `/${key}` if (encodedMetadata) { apiHeaders[METADATA_HEADER_EXTERNAL] = encodedMetadata } - const res = await this.fetch(apiURL, { headers: apiHeaders, method }) + const res = await this.fetch(url.toString(), { headers: apiHeaders, method }) if (res.status !== 200) { - throw new Error(`${method} operation has failed: API returned a ${res.status} response`) + throw new Error(`Netlify Blobs has generated an internal error: ${res.status} response`) } - const { url } = await res.json() + const { url: signedURL } = await res.json() const userHeaders = encodedMetadata ? { [METADATA_HEADER_INTERNAL]: encodedMetadata } : undefined return { headers: userHeaders, - url, + url: signedURL, } } - async makeRequest({ body, headers: extraHeaders, key, metadata, method, storeName }: MakeStoreRequestOptions) { - const { headers: baseHeaders = {}, url } = await this.getFinalRequest(storeName, key, method, metadata) + async makeRequest({ + body, + headers: extraHeaders, + key, + metadata, + method, + parameters, + storeName, + }: MakeStoreRequestOptions) { + const { headers: baseHeaders = {}, url } = await this.getFinalRequest({ + key, + metadata, + method, + parameters, + storeName, + }) const headers: Record = { ...baseHeaders, ...extraHeaders, @@ -106,17 +149,7 @@ export class Client { options.duplex = 'half' } - const res = await fetchAndRetry(this.fetch, url, options) - - if (res.status === 404 && method === HTTPMethod.GET) { - return null - } - - if (res.status !== 200 && res.status !== 304) { - throw new Error(`${method} operation has failed: store returned a ${res.status} response`) - } - - return res + return fetchAndRetry(this.fetch, url, options) } } diff --git a/src/list.test.ts b/src/list.test.ts new file mode 100644 index 0000000..bee82c7 --- /dev/null +++ b/src/list.test.ts @@ -0,0 +1,650 @@ +import { env, version as nodeVersion } from 'node:process' + +import semver from 'semver' +import { describe, test, expect, beforeAll, afterEach } from 'vitest' + +import { MockFetch } from '../test/mock_fetch.js' + +import { getStore } from './main.js' + +beforeAll(async () => { + if (semver.lt(nodeVersion, '18.0.0')) { + const nodeFetch = await import('node-fetch') + + // @ts-expect-error Expected type mismatch between native implementation and node-fetch + globalThis.fetch = nodeFetch.default + // @ts-expect-error Expected type mismatch between native implementation and node-fetch + globalThis.Request = nodeFetch.Request + // @ts-expect-error Expected type mismatch between native implementation and node-fetch + globalThis.Response = nodeFetch.Response + // @ts-expect-error Expected type mismatch between native implementation and node-fetch + globalThis.Headers = nodeFetch.Headers + } +}) + +afterEach(() => { + delete env.NETLIFY_BLOBS_CONTEXT +}) + +const siteID = '9a003659-aaaa-0000-aaaa-63d3720d8621' +const storeName = 'mystore' +const apiToken = 'some token' +const edgeToken = 'some other token' +const edgeURL = 'https://cloudfront.url' + +describe('list', () => { + describe('With API credentials', () => { + test('Lists blobs and handles pagination by default', async () => { + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag1', + key: 'key1', + size: 1, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag2', + key: 'key2', + size: 2, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: [], + next_cursor: 'cursor_1', + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?context=${storeName}`, + }) + .get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag3', + key: 'key3', + size: 3, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag4', + key: 'key4', + size: 4, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: [], + next_cursor: 'cursor_2', + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_1&context=${storeName}`, + }) + .get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag5', + key: 'key5', + size: 5, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: [], + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_2&context=${storeName}`, + }) + + globalThis.fetch = mockStore.fetch + + const store = getStore({ + name: 'mystore', + token: apiToken, + siteID, + }) + + const { blobs } = await store.list() + + expect(blobs).toEqual([ + { etag: 'etag1', key: 'key1' }, + { etag: 'etag2', key: 'key2' }, + { etag: 'etag3', key: 'key3' }, + { etag: 'etag4', key: 'key4' }, + { etag: 'etag5', key: 'key5' }, + ]) + + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('Accepts a `directories` parameter', async () => { + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag1', + key: 'key1', + size: 1, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag2', + key: 'key2', + size: 2, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir1'], + next_cursor: 'cursor_1', + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?directories=true&context=${storeName}`, + }) + .get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag3', + key: 'key3', + size: 3, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag4', + key: 'key4', + size: 4, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir2'], + next_cursor: 'cursor_2', + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_1&directories=true&context=${storeName}`, + }) + .get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag5', + key: 'key5', + size: 5, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir3'], + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_2&directories=true&context=${storeName}`, + }) + .get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag6', + key: 'key6', + size: 6, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: [], + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?prefix=dir2%2F&directories=true&context=${storeName}`, + }) + + globalThis.fetch = mockStore.fetch + + const store = getStore({ + name: 'mystore', + token: apiToken, + siteID, + }) + + const root = await store.list({ directories: true }) + + expect(root.blobs).toEqual([ + { etag: 'etag1', key: 'key1' }, + { etag: 'etag2', key: 'key2' }, + { etag: 'etag3', key: 'key3' }, + { etag: 'etag4', key: 'key4' }, + { etag: 'etag5', key: 'key5' }, + ]) + + expect(root.directories).toEqual(['dir1', 'dir2', 'dir3']) + + const directory = await store.list({ directories: true, prefix: `dir2/` }) + + expect(directory.blobs).toEqual([{ etag: 'etag6', key: 'key6' }]) + expect(directory.directories).toEqual([]) + + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('Accepts a `prefix` property', async () => { + const mockStore = new MockFetch().get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag1', + key: 'group/key1', + size: 1, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag2', + key: 'group/key2', + size: 2, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?prefix=group%2F&context=${storeName}`, + }) + + globalThis.fetch = mockStore.fetch + + const store = getStore({ + name: 'mystore', + token: apiToken, + siteID, + }) + + const { blobs } = await store.list({ + prefix: 'group/', + }) + + expect(blobs).toEqual([ + { etag: 'etag1', key: 'group/key1' }, + { etag: 'etag2', key: 'group/key2' }, + ]) + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('Paginates manually with `cursor` if `paginate: false`', async () => { + const mockStore = new MockFetch().get({ + headers: { authorization: `Bearer ${apiToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag3', + key: 'key3', + size: 3, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag4', + key: 'key4', + size: 4, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + next_cursor: 'cursor_2', + }), + ), + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs?cursor=cursor_1&context=${storeName}`, + }) + + globalThis.fetch = mockStore.fetch + + const store = getStore({ + name: 'mystore', + token: apiToken, + siteID, + }) + + const { blobs } = await store.list({ + cursor: 'cursor_1', + paginate: false, + }) + + expect(blobs).toEqual([ + { etag: 'etag3', key: 'key3' }, + { etag: 'etag4', key: 'key4' }, + ]) + expect(mockStore.fulfilled).toBeTruthy() + }) + }) + + describe('With edge credentials', () => { + test('Lists blobs and handles pagination by default', async () => { + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag1', + key: 'key1', + size: 1, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag2', + key: 'key2', + size: 2, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir1'], + next_cursor: 'cursor_1', + }), + ), + url: `${edgeURL}/${siteID}/${storeName}`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag3', + key: 'key3', + size: 3, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag4', + key: 'key4', + size: 4, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir2'], + next_cursor: 'cursor_2', + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_1`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag5', + key: 'key5', + size: 5, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir3'], + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_2`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag6', + key: 'key6', + size: 6, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: [], + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?prefix=dir2%2F`, + }) + + globalThis.fetch = mockStore.fetch + + const store = getStore({ + edgeURL, + name: storeName, + token: edgeToken, + siteID, + }) + + const root = await store.list() + + expect(root.blobs).toEqual([ + { etag: 'etag1', key: 'key1' }, + { etag: 'etag2', key: 'key2' }, + { etag: 'etag3', key: 'key3' }, + { etag: 'etag4', key: 'key4' }, + { etag: 'etag5', key: 'key5' }, + ]) + + // @ts-expect-error `directories` is not part of the return type + expect(root.directories).toBe(undefined) + + const directory = await store.list({ prefix: 'dir2/' }) + + expect(directory.blobs).toEqual([{ etag: 'etag6', key: 'key6' }]) + + // @ts-expect-error `directories` is not part of the return type + expect(directory.directories).toBe(undefined) + + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('Accepts a `directories` parameter', async () => { + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag1', + key: 'key1', + size: 1, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag2', + key: 'key2', + size: 2, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir1'], + next_cursor: 'cursor_1', + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?directories=true`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag3', + key: 'key3', + size: 3, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag4', + key: 'key4', + size: 4, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir2'], + next_cursor: 'cursor_2', + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_1&directories=true`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag5', + key: 'key5', + size: 5, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: ['dir3'], + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_2&directories=true`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag6', + key: 'key6', + size: 6, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + directories: [], + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?prefix=dir2%2F&directories=true`, + }) + + globalThis.fetch = mockStore.fetch + + const store = getStore({ + edgeURL, + name: storeName, + token: edgeToken, + siteID, + }) + + const root = await store.list({ directories: true }) + + expect(root.blobs).toEqual([ + { etag: 'etag1', key: 'key1' }, + { etag: 'etag2', key: 'key2' }, + { etag: 'etag3', key: 'key3' }, + { etag: 'etag4', key: 'key4' }, + { etag: 'etag5', key: 'key5' }, + ]) + + expect(root.directories).toEqual(['dir1', 'dir2', 'dir3']) + + const directory = await store.list({ directories: true, prefix: 'dir2/' }) + + expect(directory.blobs).toEqual([{ etag: 'etag6', key: 'key6' }]) + expect(directory.directories).toEqual([]) + + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('Accepts a `prefix` property', async () => { + const mockStore = new MockFetch().get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag1', + key: 'group/key1', + size: 1, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag2', + key: 'group/key2', + size: 2, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?prefix=group%2F`, + }) + + globalThis.fetch = mockStore.fetch + + const store = getStore({ + edgeURL, + name: storeName, + token: edgeToken, + siteID, + }) + + const { blobs } = await store.list({ + prefix: 'group/', + }) + + expect(blobs).toEqual([ + { etag: 'etag1', key: 'group/key1' }, + { etag: 'etag2', key: 'group/key2' }, + ]) + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('Paginates manually with `cursor` if `paginate: false`', async () => { + const mockStore = new MockFetch().get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response( + JSON.stringify({ + blobs: [ + { + etag: 'etag3', + key: 'key3', + size: 3, + last_modified: '2023-07-18T12:59:06Z', + }, + { + etag: 'etag4', + key: 'key4', + size: 4, + last_modified: '2023-07-18T12:59:06Z', + }, + ], + next_cursor: 'cursor_2', + }), + ), + url: `${edgeURL}/${siteID}/${storeName}?cursor=cursor_1`, + }) + + globalThis.fetch = mockStore.fetch + + const store = getStore({ + edgeURL, + name: storeName, + token: edgeToken, + siteID, + }) + + const { blobs } = await store.list({ + cursor: 'cursor_1', + paginate: false, + }) + + expect(blobs).toEqual([ + { etag: 'etag3', key: 'key3' }, + { etag: 'etag4', key: 'key4' }, + ]) + expect(mockStore.fulfilled).toBeTruthy() + }) + }) +}) diff --git a/src/main.test.ts b/src/main.test.ts index 450f817..97af08c 100644 --- a/src/main.test.ts +++ b/src/main.test.ts @@ -131,7 +131,7 @@ describe('get', () => { }) expect(async () => await blobs.get(key)).rejects.toThrowError( - 'get operation has failed: API returned a 401 response', + `Netlify Blobs has generated an internal error: 401 response`, ) expect(mockStore.fulfilled).toBeTruthy() }) @@ -157,7 +157,7 @@ describe('get', () => { }) await expect(async () => await blobs.get(key)).rejects.toThrowError( - 'get operation has failed: store returned a 401 response', + `Netlify Blobs has generated an internal error: 401 response`, ) expect(mockStore.fulfilled).toBeTruthy() @@ -233,7 +233,7 @@ describe('get', () => { }) await expect(async () => await blobs.get(key)).rejects.toThrowError( - 'get operation has failed: store returned a 401 response', + `Netlify Blobs has generated an internal error: 401 response`, ) expect(mockStore.fulfilled).toBeTruthy() @@ -592,7 +592,7 @@ describe('set', () => { }) expect(async () => await blobs.set(key, 'value')).rejects.toThrowError( - 'put operation has failed: API returned a 401 response', + `Netlify Blobs has generated an internal error: 401 response`, ) expect(mockStore.fulfilled).toBeTruthy() }) @@ -718,7 +718,7 @@ describe('set', () => { }) await expect(async () => await blobs.set(key, value)).rejects.toThrowError( - 'put operation has failed: store returned a 401 response', + `Netlify Blobs has generated an internal error: 401 response`, ) expect(mockStore.fulfilled).toBeTruthy() @@ -929,7 +929,7 @@ describe('delete', () => { }) expect(async () => await blobs.delete(key)).rejects.toThrowError( - 'delete operation has failed: API returned a 401 response', + `Netlify Blobs has generated an internal error: 401 response`, ) expect(mockStore.fulfilled).toBeTruthy() }) @@ -974,7 +974,7 @@ describe('delete', () => { }) await expect(async () => await blobs.delete(key)).rejects.toThrowError( - 'delete operation has failed: store returned a 401 response', + `Netlify Blobs has generated an internal error: 401 response`, ) expect(mockStore.fulfilled).toBeTruthy() @@ -1023,7 +1023,7 @@ describe('Deploy scope', () => { .get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), - url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=deploy:${deployID}`, + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=deploy%3A${deployID}`, }) .get({ response: new Response(value), @@ -1032,7 +1032,7 @@ describe('Deploy scope', () => { .get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), - url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=deploy:${deployID}`, + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=deploy%3A${deployID}`, }) .get({ response: new Response(value), @@ -1093,7 +1093,7 @@ describe('Deploy scope', () => { .get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), - url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=deploy:${deployID}`, + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=deploy%3A${deployID}`, }) .get({ response: new Response(value), @@ -1102,7 +1102,7 @@ describe('Deploy scope', () => { .get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), - url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=deploy:${deployID}`, + url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=deploy%3A${deployID}`, }) .get({ response: new Response(value), diff --git a/src/server.test.ts b/src/server.test.ts index 62e59c0..9765a27 100644 --- a/src/server.test.ts +++ b/src/server.test.ts @@ -105,7 +105,7 @@ describe('Local server', () => { }) await expect(async () => await blobs.get(key)).rejects.toThrowError( - 'get operation has failed: store returned a 403 response', + 'Netlify Blobs has generated an internal error: 403 response', ) await server.stop() diff --git a/src/store.ts b/src/store.ts index b3286e6..50ac421 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,6 +1,8 @@ +import { ListResponse, ListResponseBlob } from './backend/list.ts' import { Client } from './client.ts' import { decodeMetadata, Metadata } from './metadata.ts' import { BlobInput, HTTPMethod } from './types.ts' +import { BlobsInternalError } from './util.ts' interface BaseStoreOptions { client: Client @@ -26,6 +28,26 @@ interface GetWithMetadataResult { metadata: Metadata } +interface ListResult { + blobs: ListResultBlob[] +} + +interface ListResultWithDirectories extends ListResult { + directories: string[] +} + +interface ListResultBlob { + etag: string + key: string +} + +interface ListOptions { + cursor?: string + directories?: boolean + paginate?: boolean + prefix?: string +} + interface SetOptions { /** * Arbitrary metadata object to associate with an entry. Must be seralizable @@ -55,7 +77,11 @@ export class Store { } async delete(key: string) { - await this.client.makeRequest({ key, method: HTTPMethod.DELETE, storeName: this.name }) + const res = await this.client.makeRequest({ key, method: HTTPMethod.DELETE, storeName: this.name }) + + if (res.status !== 200 && res.status !== 404) { + throw new BlobsInternalError(res.status) + } } async get(key: string): Promise @@ -72,8 +98,12 @@ export class Store { const { type } = options ?? {} const res = await this.client.makeRequest({ key, method: HTTPMethod.GET, storeName: this.name }) - if (res === null) { - return res + if (res.status === 404) { + return null + } + + if (res.status !== 200) { + throw new BlobsInternalError(res.status) } if (type === undefined || type === 'text') { @@ -96,7 +126,7 @@ export class Store { return res.body } - throw new Error(`Invalid 'type' property: ${type}. Expected: arrayBuffer, blob, json, stream, or text.`) + throw new BlobsInternalError(res.status) } async getWithMetadata( @@ -145,6 +175,15 @@ export class Store { const { etag: requestETag, type } = options ?? {} const headers = requestETag ? { 'if-none-match': requestETag } : undefined const res = await this.client.makeRequest({ headers, key, method: HTTPMethod.GET, storeName: this.name }) + + if (res.status === 404) { + return null + } + + if (res.status !== 200 && res.status !== 304) { + throw new BlobsInternalError(res.status) + } + const responseETag = res?.headers.get('etag') ?? undefined let metadata: Metadata = {} @@ -157,10 +196,6 @@ export class Store { ) } - if (res === null) { - return null - } - const result: GetWithMetadataResult = { etag: responseETag, fresh: false, @@ -194,16 +229,46 @@ export class Store { throw new Error(`Invalid 'type' property: ${type}. Expected: arrayBuffer, blob, json, stream, or text.`) } + async list(options: ListOptions & { directories: true }): Promise + async list(options?: ListOptions & { directories?: false }): Promise + async list(options: ListOptions = {}): Promise { + const cursor = options.paginate === false ? options.cursor : undefined + const maxPages = options.paginate === false ? 1 : Number.POSITIVE_INFINITY + const res = await this.listAndPaginate({ + currentPage: 1, + directories: options.directories, + maxPages, + nextCursor: cursor, + prefix: options.prefix, + }) + const blobs = res.blobs?.map(Store.formatListResultBlob).filter(Boolean) as ListResultBlob[] + + if (options?.directories) { + return { + blobs, + directories: res.directories?.filter(Boolean) as string[], + } + } + + return { + blobs, + } + } + async set(key: string, data: BlobInput, { metadata }: SetOptions = {}) { Store.validateKey(key) - await this.client.makeRequest({ + const res = await this.client.makeRequest({ body: data, key, metadata, method: HTTPMethod.PUT, storeName: this.name, }) + + if (res.status !== 200) { + throw new BlobsInternalError(res.status) + } } async setJSON(key: string, data: unknown, { metadata }: SetOptions = {}) { @@ -214,7 +279,7 @@ export class Store { 'content-type': 'application/json', } - await this.client.makeRequest({ + const res = await this.client.makeRequest({ body: payload, headers, key, @@ -222,9 +287,24 @@ export class Store { method: HTTPMethod.PUT, storeName: this.name, }) + + if (res.status !== 200) { + throw new BlobsInternalError(res.status) + } } - static validateKey(key: string) { + private static formatListResultBlob(result: ListResponseBlob): ListResultBlob | null { + if (!result.key) { + return null + } + + return { + etag: result.etag, + key: result.key, + } + } + + private static validateKey(key: string) { if (key.startsWith('/') || !/^[\w%!.*'()/-]{1,600}$/.test(key)) { throw new Error( "Keys can only contain letters, numbers, percentage signs (%), exclamation marks (!), dots (.), asterisks (*), single quotes ('), parentheses (()), dashes (-) and underscores (_) up to a maximum of 600 characters. Keys can also contain forward slashes (/), but must not start with one.", @@ -232,7 +312,7 @@ export class Store { } } - static validateDeployID(deployID: string) { + private static validateDeployID(deployID: string) { // We could be stricter here and require a length of 24 characters, but the // CLI currently uses a deploy of `0` when running Netlify Dev, since there // is no actual deploy at that point. Let's go with a more loose validation @@ -242,7 +322,7 @@ export class Store { } } - static validateStoreName(name: string) { + private static validateStoreName(name: string) { if (name.startsWith('deploy:')) { throw new Error('Store name cannot start with the string `deploy:`, which is a reserved namespace.') } @@ -253,4 +333,69 @@ export class Store { ) } } + + private async listAndPaginate(options: { + accumulator?: ListResponse + directories?: boolean + currentPage: number + maxPages: number + nextCursor?: string + prefix?: string + }): Promise { + const { + accumulator = { blobs: [], directories: [] }, + currentPage, + directories, + maxPages, + nextCursor, + prefix, + } = options + + if (currentPage > maxPages || (currentPage > 1 && !nextCursor)) { + return accumulator + } + + const parameters: Record = {} + + if (nextCursor) { + parameters.cursor = nextCursor + } + + if (prefix) { + parameters.prefix = prefix + } + + if (directories) { + parameters.directories = 'true' + } + + const res = await this.client.makeRequest({ + method: HTTPMethod.GET, + parameters, + storeName: this.name, + }) + + if (res.status !== 200) { + throw new BlobsInternalError(res.status) + } + + try { + const current = (await res.json()) as ListResponse + const newAccumulator = { + ...current, + blobs: [...(accumulator.blobs || []), ...(current.blobs || [])], + directories: [...(accumulator.directories || []), ...(current.directories || [])], + } + + return this.listAndPaginate({ + accumulator: newAccumulator, + currentPage: currentPage + 1, + directories, + maxPages, + nextCursor: current.next_cursor, + }) + } catch (error: unknown) { + throw new Error(`'list()' has returned an internal error: ${error}`) + } + } } diff --git a/src/util.ts b/src/util.ts index 6fc2585..8394d0a 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,3 +1,11 @@ +export class BlobsInternalError extends Error { + constructor(statusCode: number) { + super(`Netlify Blobs has generated an internal error: ${statusCode} response`) + + this.name = 'BlobsInternalError' + } +} + export const isNodeError = (error: unknown): error is NodeJS.ErrnoException => error instanceof Error export type Logger = (...message: unknown[]) => void