feat: add support for metadata

Author: eduardoboucas

README.md

@@ -169,7 +169,7 @@ console.log(await store.get('my-key'))
 
 ## Store API reference
 
-### `get(key: string, { type: string }): Promise<any>`
+### `get(key: string, { type?: string }): Promise<any>`
 
 Retrieves an object with the given key.
 
@@ -191,6 +191,29 @@ const entry = await blobs.get('some-key', { type: 'json' })
 console.log(entry)
 ```
 
+### `getWithMetadata(key: string, { type?: string }): Promise<{ data: any, etag: string, metadata: object }>`
+
+Retrieves an object with the given key, the [ETag value](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag)
+for the entry, and any metadata that has been stored with the entry.
+
+Depending on the most convenient format for you to access the value, you may choose to supply a `type` property as a
+second parameter, with one of the following values:
+
+- `arrayBuffer`: Returns the entry as an
+  [`ArrayBuffer`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer)
+- `blob`: Returns the entry as a [`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob)
+- `json`: Parses the entry as JSON and returns the resulting object
+- `stream`: Returns the entry as a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream)
+- `text` (default): Returns the entry as a string of plain text
+
+If an object with the given key is not found, `null` is returned.
+
+```javascript
+const blob = await blobs.getWithMetadata('some-key', { type: 'json' })
+
+console.log(blob.data, blob.etag, blob.metadata)
+```
+
 ### `set(key: string, value: ArrayBuffer | Blob | ReadableStream | string): Promise<void>`
 
 Creates an object with the given key and value.

src/client.ts

@@ -1,11 +1,13 @@
 import { EnvironmentContext, getEnvironmentContext, MissingBlobsEnvironmentError } from './environment.ts'
+import { encodeMetadata, Metadata, METADATA_HEADER_EXTERNAL } from './metadata.ts'
 import { fetchAndRetry } from './retry.ts'
 import { BlobInput, Fetcher, HTTPMethod } from './types.ts'
 
 interface MakeStoreRequestOptions {
   body?: BlobInput | null
   headers?: Record<string, string>
   key: string
+  metadata?: Metadata
   method: HTTPMethod
   storeName: string
 }
@@ -33,21 +35,32 @@ export class Client {
     this.token = token
   }
 
-  private async getFinalRequest(storeName: string, key: string, method: string) {
+  private async getFinalRequest(storeName: string, key: string, method: string, metadata?: Metadata) {
     const encodedKey = encodeURIComponent(key)
 
     if (this.edgeURL) {
+      const headers: Record<string, string> = {
+        authorization: `Bearer ${this.token}`,
+      }
+
+      if (metadata) {
+        headers[METADATA_HEADER_EXTERNAL] = encodeMetadata(metadata)
+      }
+
       return {
-        headers: {
-          authorization: `Bearer ${this.token}`,
-        },
+        headers,
         url: `${this.edgeURL}/${this.siteID}/${storeName}/${encodedKey}`,
       }
     }
 
-    const apiURL = `${this.apiURL ?? 'https://api.netlify.com'}/api/v1/sites/${
+    let apiURL = `${this.apiURL ?? 'https://api.netlify.com'}/api/v1/sites/${
       this.siteID
     }/blobs/${encodedKey}?context=${storeName}`
+
+    if (metadata) {
+      apiURL += `&metadata=${encodeMetadata(metadata)}`
+    }
+
     const headers = { authorization: `Bearer ${this.token}` }
     const fetch = this.fetch ?? globalThis.fetch
     const res = await fetch(apiURL, { headers, method })
@@ -63,8 +76,8 @@ export class Client {
     }
   }
 
-  async makeRequest({ body, headers: extraHeaders, key, method, storeName }: MakeStoreRequestOptions) {
-    const { headers: baseHeaders = {}, url } = await this.getFinalRequest(storeName, key, method)
+  async makeRequest({ body, headers: extraHeaders, key, metadata, method, storeName }: MakeStoreRequestOptions) {
+    const { headers: baseHeaders = {}, url } = await this.getFinalRequest(storeName, key, method, metadata)
     const headers: Record<string, string> = {
       ...baseHeaders,
       ...extraHeaders,

src/main.test.ts

@@ -5,7 +5,7 @@ import semver from 'semver'
 import { describe, test, expect, beforeAll, afterEach } from 'vitest'
 
 import { MockFetch } from '../test/mock_fetch.js'
-import { streamToString } from '../test/util.js'
+import { base64Encode, streamToString } from '../test/util.js'
 
 import { MissingBlobsEnvironmentError } from './environment.js'
 import { getDeployStore, getStore } from './main.js'
@@ -164,34 +164,6 @@ describe('get', () => {
 
       expect(mockStore.fulfilled).toBeTruthy()
     })
-
-    test('Returns `null` when the blob entry contains an expiry date in the past', async () => {
-      const mockStore = new MockFetch()
-        .get({
-          headers: { authorization: `Bearer ${apiToken}` },
-          response: new Response(JSON.stringify({ url: signedURL })),
-          url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`,
-        })
-        .get({
-          response: new Response(value, {
-            headers: {
-              'x-nf-expires-at': (Date.now() - 1000).toString(),
-            },
-          }),
-          url: signedURL,
-        })
-
-      globalThis.fetch = mockStore.fetch
-
-      const blobs = getStore({
-        name: 'production',
-        token: apiToken,
-        siteID,
-      })
-
-      expect(await blobs.get(key)).toBeNull()
-      expect(mockStore.fulfilled).toBeTruthy()
-    })
   })
 
   describe('With edge credentials', () => {
@@ -318,6 +290,162 @@ describe('get', () => {
   })
 })
 
+describe('getWithMetadata', () => {
+  describe('With API credentials', () => {
+    test('Reads from the blob store and returns the etag and the metadata object', async () => {
+      const mockMetadata = {
+        name: 'Netlify',
+        cool: true,
+        functions: ['edge', 'serverless'],
+      }
+      const responseHeaders = {
+        etag: '123456789',
+        'x-amz-meta-user': `b64;${base64Encode(mockMetadata)}`,
+      }
+      const mockStore = new MockFetch()
+        .get({
+          headers: { authorization: `Bearer ${apiToken}` },
+          response: new Response(JSON.stringify({ url: signedURL })),
+          url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`,
+        })
+        .get({
+          response: new Response(value, { headers: responseHeaders }),
+          url: signedURL,
+        })
+        .get({
+          headers: { authorization: `Bearer ${apiToken}` },
+          response: new Response(JSON.stringify({ url: signedURL })),
+          url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`,
+        })
+        .get({
+          response: new Response(value, { headers: responseHeaders }),
+          url: signedURL,
+        })
+
+      globalThis.fetch = mockStore.fetch
+
+      const blobs = getStore({
+        name: 'production',
+        token: apiToken,
+        siteID,
+      })
+
+      const entry1 = await blobs.getWithMetadata(key)
+      expect(entry1.data).toBe(value)
+      expect(entry1.etag).toBe(responseHeaders.etag)
+      expect(entry1.metadata).toEqual(mockMetadata)
+
+      const entry2 = await blobs.getWithMetadata(key, { type: 'stream' })
+      expect(await streamToString(entry2.data as unknown as NodeJS.ReadableStream)).toBe(value)
+      expect(entry2.etag).toBe(responseHeaders.etag)
+      expect(entry2.metadata).toEqual(mockMetadata)
+
+      expect(mockStore.fulfilled).toBeTruthy()
+    })
+
+    test('Returns `null` when the pre-signed URL returns a 404', async () => {
+      const mockStore = new MockFetch()
+        .get({
+          headers: { authorization: `Bearer ${apiToken}` },
+          response: new Response(JSON.stringify({ url: signedURL })),
+          url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`,
+        })
+        .get({
+          response: new Response('Something went wrong', { status: 404 }),
+          url: signedURL,
+        })
+
+      globalThis.fetch = mockStore.fetch
+
+      const blobs = getStore({
+        name: 'production',
+        token: apiToken,
+        siteID,
+      })
+
+      expect(await blobs.getWithMetadata(key)).toBeNull()
+      expect(mockStore.fulfilled).toBeTruthy()
+    })
+
+    test('Throws when the metadata object cannot be parsed', async () => {
+      const responseHeaders = {
+        etag: '123456789',
+        'x-amz-meta-user': `b64;${base64Encode(`{"name": "Netlify", "cool`)}`,
+      }
+      const mockStore = new MockFetch()
+        .get({
+          headers: { authorization: `Bearer ${apiToken}` },
+          response: new Response(JSON.stringify({ url: signedURL })),
+          url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`,
+        })
+        .get({
+          response: new Response(value, { headers: responseHeaders }),
+          url: signedURL,
+        })
+
+      globalThis.fetch = mockStore.fetch
+
+      const blobs = getStore({
+        name: 'production',
+        token: apiToken,
+        siteID,
+      })
+
+      await expect(async () => await blobs.getWithMetadata(key)).rejects.toThrowError(
+        'An internal error occurred while trying to retrieve the metadata for an entry. Please try updating to the latest version of the Netlify Blobs client.',
+      )
+
+      expect(mockStore.fulfilled).toBeTruthy()
+    })
+  })
+
+  describe('With edge credentials', () => {
+    test('Reads from the blob store and returns the etag and the metadata object', async () => {
+      const mockMetadata = {
+        name: 'Netlify',
+        cool: true,
+        functions: ['edge', 'serverless'],
+      }
+      const responseHeaders = {
+        etag: '123456789',
+        'x-amz-meta-user': `b64;${base64Encode(mockMetadata)}`,
+      }
+      const mockStore = new MockFetch()
+        .get({
+          headers: { authorization: `Bearer ${edgeToken}` },
+          response: new Response(value, { headers: responseHeaders }),
+          url: `${edgeURL}/${siteID}/production/${key}`,
+        })
+        .get({
+          headers: { authorization: `Bearer ${edgeToken}` },
+          response: new Response(value, { headers: responseHeaders }),
+          url: `${edgeURL}/${siteID}/production/${key}`,
+        })
+
+      globalThis.fetch = mockStore.fetch
+
+      const blobs = getStore({
+        edgeURL,
+        name: 'production',
+        token: edgeToken,
+        siteID,
+      })
+
+      const entry1 = await blobs.getWithMetadata(key)
+      expect(entry1.data).toBe(value)
+      expect(entry1.etag).toBe(responseHeaders.etag)
+      expect(entry1.metadata).toEqual(mockMetadata)
+
+      const entry2 = await blobs.getWithMetadata(key, { type: 'stream' })
+      expect(await streamToString(entry2.data as unknown as NodeJS.ReadableStream)).toBe(value)
+      expect(entry2.etag).toBe(responseHeaders.etag)
+      expect(entry2.metadata).toEqual(mockMetadata)
+
+      expect(mockStore.fulfilled).toBeTruthy()
+    })
+  })
+})
+
 describe('set', () => {
   describe('With API credentials', () => {
     test('Writes to the blob store', async () => {
@@ -361,19 +489,23 @@ describe('set', () => {
       expect(mockStore.fulfilled).toBeTruthy()
     })
 
-    test('Accepts an `expiration` parameter', async () => {
-      const expiration = new Date(Date.now() + 15_000)
+    test('Accepts a `metadata` parameter', async () => {
+      const metadata = {
+        name: 'Netlify',
+        cool: true,
+        functions: ['edge', 'serverless'],
+      }
+      const encodedMetadata = `b64;${Buffer.from(JSON.stringify(metadata)).toString('base64')}`
       const mockStore = new MockFetch()
         .put({
           headers: { authorization: `Bearer ${apiToken}` },
           response: new Response(JSON.stringify({ url: signedURL })),
-          url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`,
+          url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production&metadata=${encodedMetadata}`,
         })
         .put({
           body: value,
           headers: {
             'cache-control': 'max-age=0, stale-while-revalidate=60',
-            'x-nf-expires-at': expiration.getTime().toString(),
           },
           response: new Response(null),
           url: signedURL,
@@ -387,7 +519,7 @@ describe('set', () => {
         siteID,
       })
 
-      await blobs.set(key, value, { expiration })
+      await blobs.set(key, value, { metadata })
 
       expect(mockStore.fulfilled).toBeTruthy()
     })
@@ -620,33 +752,34 @@ describe('setJSON', () => {
       expect(mockStore.fulfilled).toBeTruthy()
     })
 
-    test('Accepts an `expiration` parameter', async () => {
-      const expiration = new Date(Date.now() + 15_000)
-      const mockStore = new MockFetch()
-        .put({
-          headers: { authorization: `Bearer ${apiToken}` },
-          response: new Response(JSON.stringify({ url: signedURL })),
-          url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`,
-        })
-        .put({
-          body: JSON.stringify({ value }),
-          headers: {
-            'cache-control': 'max-age=0, stale-while-revalidate=60',
-            'x-nf-expires-at': expiration.getTime().toString(),
-          },
-          response: new Response(null),
-          url: signedURL,
-        })
+    test('Accepts a `metadata` parameter', async () => {
+      const metadata = {
+        name: 'Netlify',
+        cool: true,
+        functions: ['edge', 'serverless'],
+      }
+      const encodedMetadata = `b64;${Buffer.from(JSON.stringify(metadata)).toString('base64')}`
+      const mockStore = new MockFetch().put({
+        body: JSON.stringify({ value }),
+        headers: {
+          authorization: `Bearer ${edgeToken}`,
+          'cache-control': 'max-age=0, stale-while-revalidate=60',
+          'netlify-blobs-metadata': encodedMetadata,
+        },
+        response: new Response(null),
+        url: `${edgeURL}/${siteID}/production/${key}`,
+      })
 
       globalThis.fetch = mockStore.fetch
 
       const blobs = getStore({
+        edgeURL,
         name: 'production',
-        token: apiToken,
+        token: edgeToken,
         siteID,
       })
 
-      await blobs.setJSON(key, { value }, { expiration })
+      await blobs.setJSON(key, { value }, { metadata })
 
       expect(mockStore.fulfilled).toBeTruthy()
     })

src/metadata.ts

@@ -0,0 +1,37 @@
+import { Buffer } from 'node:buffer'
+
+export type Metadata = Record<string, unknown>
+
+const BASE64_PREFIX = 'b64;'
+export const METADATA_HEADER_INTERNAL = 'x-amz-meta-user'
+export const METADATA_HEADER_EXTERNAL = 'netlify-blobs-metadata'
+const METADATA_MAX_SIZE = 2 * 1024
+
+export const encodeMetadata = (metadata: Metadata) => {
+  const encodedObject = Buffer.from(JSON.stringify(metadata)).toString('base64')
+  const payload = `b64;${encodedObject}`
+
+  if (METADATA_HEADER_EXTERNAL.length + payload.length > METADATA_MAX_SIZE) {
+    throw new Error('Metadata object exceeds the maximum size')
+  }
+
+  return payload
+}
+
+export const decodeMetadata = (headers?: Headers): Metadata => {
+  if (!headers) {
+    return {}
+  }
+
+  const metadataHeader = headers.get(METADATA_HEADER_INTERNAL)
+
+  if (!metadataHeader || !metadataHeader.startsWith(BASE64_PREFIX)) {
+    return {}
+  }
+
+  const encodedData = metadataHeader.slice(BASE64_PREFIX.length)
+  const decodedData = Buffer.from(encodedData, 'base64').toString()
+  const metadata = JSON.parse(decodedData)
+
+  return metadata
+}

src/store.ts

@@ -1,4 +1,5 @@
 import { Client } from './client.ts'
+import { decodeMetadata, Metadata } from './metadata.ts'
 import { BlobInput, HTTPMethod } from './types.ts'
 
 interface BaseStoreOptions {
@@ -17,13 +18,14 @@ type StoreOptions = DeployStoreOptions | NamedStoreOptions
 
 interface SetOptions {
   /**
-   * Accepts an absolute date as a `Date` object, or a relative date as the
-   * number of seconds from the current date.
+   * Arbitrary metadata object to associate with an entry. Must be seralizable
+   * to JSON.
    */
-  expiration?: Date | number
+  metadata?: Metadata
 }
 
-const EXPIRY_HEADER = 'x-nf-expires-at'
+type BlobWithMetadata = { etag?: string } & { metadata: Metadata }
+type BlobResponseType = 'arrayBuffer' | 'blob' | 'json' | 'stream' | 'text'
 
 export class Store {
   private client: Client
@@ -34,26 +36,6 @@ export class Store {
     this.name = 'deployID' in options ? `deploy:${options.deployID}` : options.name
   }
 
-  private static getExpirationHeaders(expiration: Date | number | undefined): Record<string, string> {
-    if (typeof expiration === 'number') {
-      return {
-        [EXPIRY_HEADER]: (Date.now() + expiration).toString(),
-      }
-    }
-
-    if (expiration instanceof Date) {
-      return {
-        [EXPIRY_HEADER]: expiration.getTime().toString(),
-      }
-    }
-
-    if (expiration === undefined) {
-      return {}
-    }
-
-    throw new TypeError(`'expiration' value must be a number or a Date, ${typeof expiration} found.`)
-  }
-
   async delete(key: string) {
     await this.client.makeRequest({ key, method: HTTPMethod.DELETE, storeName: this.name })
   }
@@ -67,19 +49,10 @@ export class Store {
   async get(key: string, { type }: { type: 'text' }): Promise<string>
   async get(
     key: string,
-    options?: { type: 'arrayBuffer' | 'blob' | 'json' | 'stream' | 'text' },
+    options?: { type: BlobResponseType },
   ): Promise<ArrayBuffer | Blob | ReadableStream | string | null> {
     const { type } = options ?? {}
     const res = await this.client.makeRequest({ key, method: HTTPMethod.GET, storeName: this.name })
-    const expiration = res?.headers.get(EXPIRY_HEADER)
-
-    if (typeof expiration === 'string') {
-      const expirationTS = Number.parseInt(expiration)
-
-      if (!Number.isNaN(expirationTS) && expirationTS <= Date.now()) {
-        return null
-      }
-    }
 
     if (res === null) {
       return res
@@ -108,29 +81,93 @@ export class Store {
     throw new Error(`Invalid 'type' property: ${type}. Expected: arrayBuffer, blob, json, stream, or text.`)
   }
 
-  async set(key: string, data: BlobInput, { expiration }: SetOptions = {}) {
-    const headers = Store.getExpirationHeaders(expiration)
+  async getWithMetadata(key: string): Promise<{ data: string } & BlobWithMetadata>
+
+  async getWithMetadata(
+    key: string,
+    { type }: { type: 'arrayBuffer' },
+  ): Promise<{ data: ArrayBuffer } & BlobWithMetadata>
+
+  async getWithMetadata(key: string, { type }: { type: 'blob' }): Promise<{ data: Blob } & BlobWithMetadata>
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async getWithMetadata(key: string, { type }: { type: 'json' }): Promise<{ data: any } & BlobWithMetadata>
+
+  async getWithMetadata(key: string, { type }: { type: 'stream' }): Promise<{ data: ReadableStream } & BlobWithMetadata>
+
+  async getWithMetadata(key: string, { type }: { type: 'text' }): Promise<{ data: string } & BlobWithMetadata>
 
+  async getWithMetadata(
+    key: string,
+    options?: { type: BlobResponseType },
+  ): Promise<
+    | ({
+        data: ArrayBuffer | Blob | ReadableStream | string | null
+      } & BlobWithMetadata)
+    | null
+  > {
+    const { type } = options ?? {}
+    const res = await this.client.makeRequest({ key, method: HTTPMethod.GET, storeName: this.name })
+    const etag = res?.headers.get('etag') ?? undefined
+
+    let metadata: Metadata = {}
+
+    try {
+      metadata = decodeMetadata(res?.headers)
+    } catch {
+      throw new Error(
+        'An internal error occurred while trying to retrieve the metadata for an entry. Please try updating to the latest version of the Netlify Blobs client.',
+      )
+    }
+
+    if (res === null) {
+      return null
+    }
+
+    if (type === undefined || type === 'text') {
+      return { data: await res.text(), etag, metadata }
+    }
+
+    if (type === 'arrayBuffer') {
+      return { data: await res.arrayBuffer(), etag, metadata }
+    }
+
+    if (type === 'blob') {
+      return { data: await res.blob(), etag, metadata }
+    }
+
+    if (type === 'json') {
+      return { data: await res.json(), etag, metadata }
+    }
+
+    if (type === 'stream') {
+      return { data: res.body, etag, metadata }
+    }
+
+    throw new Error(`Invalid 'type' property: ${type}. Expected: arrayBuffer, blob, json, stream, or text.`)
+  }
+
+  async set(key: string, data: BlobInput, { metadata }: SetOptions = {}) {
     await this.client.makeRequest({
       body: data,
-      headers,
       key,
+      metadata,
       method: HTTPMethod.PUT,
       storeName: this.name,
     })
   }
 
-  async setJSON(key: string, data: unknown, { expiration }: SetOptions = {}) {
+  async setJSON(key: string, data: unknown, { metadata }: SetOptions = {}) {
     const payload = JSON.stringify(data)
     const headers = {
-      ...Store.getExpirationHeaders(expiration),
       'content-type': 'application/json',
     }
 
     await this.client.makeRequest({
       body: payload,
       headers,
       key,
+      metadata,
       method: HTTPMethod.PUT,
       storeName: this.name,
     })

test/util.ts

@@ -1,5 +1,11 @@
 import { Buffer } from 'node:buffer'
 
+export const base64Encode = (input: string | object) => {
+  const payload = typeof input === 'string' ? input : JSON.stringify(input)
+
+  return Buffer.from(payload).toString('base64')
+}
+
 export const streamToString = async function streamToString(stream: NodeJS.ReadableStream): Promise<string> {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const chunks: Array<any> = []