From f667c1b1d07c0391d0212e67c4f38b30b13fae01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Wed, 7 Feb 2024 16:24:43 +0000 Subject: [PATCH] feat: add `consistency` configuration property --- src/client.ts | 31 +++++++- src/consistency.ts | 11 +++ src/environment.ts | 1 + src/main.test.ts | 180 +++++++++++++++++++++++++++++++++++++++++++++ src/store.ts | 40 +++++++--- 5 files changed, 248 insertions(+), 15 deletions(-) create mode 100644 src/consistency.ts diff --git a/src/client.ts b/src/client.ts index 6ccd3a1..6ba1a6d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,3 +1,4 @@ +import { BlobsConsistencyError, ConsistencyMode } from './consistency.ts' import { EnvironmentContext, getEnvironmentContext, MissingBlobsEnvironmentError } from './environment.ts' import { encodeMetadata, Metadata, METADATA_HEADER_EXTERNAL, METADATA_HEADER_INTERNAL } from './metadata.ts' import { fetchAndRetry } from './retry.ts' @@ -5,6 +6,7 @@ import { BlobInput, Fetcher, HTTPMethod } from './types.ts' interface MakeStoreRequestOptions { body?: BlobInput | null + consistency?: ConsistencyMode headers?: Record key?: string metadata?: Metadata @@ -15,13 +17,16 @@ interface MakeStoreRequestOptions { export interface ClientOptions { apiURL?: string + consistency?: ConsistencyMode edgeURL?: string fetch?: Fetcher siteID: string token: string + uncachedEdgeURL?: string } interface GetFinalRequestOptions { + consistency?: ConsistencyMode key: string | undefined metadata?: Metadata method: string @@ -31,17 +36,21 @@ interface GetFinalRequestOptions { export class Client { private apiURL?: string + private consistency: ConsistencyMode private edgeURL?: string private fetch: Fetcher private siteID: string private token: string + private uncachedEdgeURL?: string - constructor({ apiURL, edgeURL, fetch, siteID, token }: ClientOptions) { + constructor({ apiURL, consistency, edgeURL, fetch, siteID, token, uncachedEdgeURL }: ClientOptions) { this.apiURL = apiURL + this.consistency = consistency ?? 'eventual' this.edgeURL = edgeURL this.fetch = fetch ?? globalThis.fetch this.siteID = siteID this.token = token + this.uncachedEdgeURL = uncachedEdgeURL if (!this.fetch) { throw new Error( @@ -50,10 +59,22 @@ export class Client { } } - private async getFinalRequest({ key, metadata, method, parameters = {}, storeName }: GetFinalRequestOptions) { + private async getFinalRequest({ + consistency: opConsistency, + key, + metadata, + method, + parameters = {}, + storeName, + }: GetFinalRequestOptions) { const encodedMetadata = encodeMetadata(metadata) + const consistency = opConsistency ?? this.consistency if (this.edgeURL) { + if (consistency === 'strong' && !this.uncachedEdgeURL) { + throw new BlobsConsistencyError() + } + const headers: Record = { authorization: `Bearer ${this.token}`, } @@ -63,7 +84,7 @@ export class Client { } const path = key ? `/${this.siteID}/${storeName}/${key}` : `/${this.siteID}/${storeName}` - const url = new URL(path, this.edgeURL) + const url = new URL(path, consistency === 'strong' ? this.uncachedEdgeURL : this.edgeURL) for (const key in parameters) { url.searchParams.set(key, parameters[key]) @@ -124,6 +145,7 @@ export class Client { async makeRequest({ body, + consistency, headers: extraHeaders, key, metadata, @@ -132,6 +154,7 @@ export class Client { storeName, }: MakeStoreRequestOptions) { const { headers: baseHeaders = {}, url } = await this.getFinalRequest({ + consistency, key, metadata, method, @@ -184,10 +207,12 @@ export const getClientOptions = ( const clientOptions = { apiURL: context.apiURL ?? options.apiURL, + consistency: options.consistency, edgeURL: context.edgeURL ?? options.edgeURL, fetch: options.fetch, siteID, token, + uncachedEdgeURL: context.uncachedEdgeURL ?? options.uncachedEdgeURL, } return clientOptions diff --git a/src/consistency.ts b/src/consistency.ts new file mode 100644 index 0000000..c4c41f3 --- /dev/null +++ b/src/consistency.ts @@ -0,0 +1,11 @@ +export type ConsistencyMode = 'eventual' | 'strong' + +export class BlobsConsistencyError extends Error { + constructor() { + super( + `Netlify Blobs has failed to perform a read using strong consistency because the environment has not been configured with a 'uncachedEdgeURL' property`, + ) + + this.name = 'BlobsConsistencyError' + } +} diff --git a/src/environment.ts b/src/environment.ts index c2524fb..9be36e2 100644 --- a/src/environment.ts +++ b/src/environment.ts @@ -17,6 +17,7 @@ export interface EnvironmentContext { edgeURL?: string siteID?: string token?: string + uncachedEdgeURL?: string } export const getEnvironmentContext = (): EnvironmentContext => { diff --git a/src/main.test.ts b/src/main.test.ts index 140f5d5..115b4ea 100644 --- a/src/main.test.ts +++ b/src/main.test.ts @@ -39,6 +39,7 @@ const apiToken = 'some token' const signedURL = 'https://signed.url/123456789' const edgeToken = 'some other token' const edgeURL = 'https://edge.netlify' +const uncachedEdgeURL = 'https://uncached.edge.netlify' describe('get', () => { describe('With API credentials', () => { @@ -1493,3 +1494,182 @@ describe(`getStore`, () => { ) }) }) + +describe('Consistency configuration', () => { + test('Respects the consistency mode supplied in the operation methods', async () => { + const mockMetadata = { + name: 'Netlify', + cool: true, + functions: ['edge', 'serverless'], + } + const headers = { + etag: '123456789', + 'x-amz-meta-user': `b64;${base64Encode(mockMetadata)}`, + } + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(value), + url: `${uncachedEdgeURL}/${siteID}/production/${key}`, + }) + .head({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(null, { headers }), + url: `${uncachedEdgeURL}/${siteID}/production/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(value, { headers }), + url: `${uncachedEdgeURL}/${siteID}/production/${key}`, + }) + + globalThis.fetch = mockStore.fetch + + const context = { + edgeURL, + siteID, + token: edgeToken, + uncachedEdgeURL, + } + + env.NETLIFY_BLOBS_CONTEXT = Buffer.from(JSON.stringify(context)).toString('base64') + + const blobs = getStore('production') + + const data = await blobs.get(key, { consistency: 'strong' }) + expect(data).toBe(value) + + const meta = await blobs.getMetadata(key, { consistency: 'strong' }) + expect(meta?.etag).toBe(headers.etag) + expect(meta?.metadata).toEqual(mockMetadata) + + const dataWithMeta = await blobs.getWithMetadata(key, { consistency: 'strong' }) + expect(dataWithMeta?.data).toBe(value) + expect(dataWithMeta?.etag).toBe(headers.etag) + expect(dataWithMeta?.metadata).toEqual(mockMetadata) + + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('Respects the consistency mode supplied in the store constructor', async () => { + const mockMetadata = { + name: 'Netlify', + cool: true, + functions: ['edge', 'serverless'], + } + const headers = { + etag: '123456789', + 'x-amz-meta-user': `b64;${base64Encode(mockMetadata)}`, + } + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(value), + url: `${uncachedEdgeURL}/${siteID}/production/${key}`, + }) + .head({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(null, { headers }), + url: `${uncachedEdgeURL}/${siteID}/production/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(value, { headers }), + url: `${uncachedEdgeURL}/${siteID}/production/${key}`, + }) + + globalThis.fetch = mockStore.fetch + + const blobs = getStore({ + consistency: 'strong', + edgeURL, + name: 'production', + token: edgeToken, + siteID, + uncachedEdgeURL, + }) + + const data = await blobs.get(key) + expect(data).toBe(value) + + const meta = await blobs.getMetadata(key) + expect(meta?.etag).toBe(headers.etag) + expect(meta?.metadata).toEqual(mockMetadata) + + const dataWithMeta = await blobs.getWithMetadata(key) + expect(dataWithMeta?.data).toBe(value) + expect(dataWithMeta?.etag).toBe(headers.etag) + expect(dataWithMeta?.metadata).toEqual(mockMetadata) + + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('The consistency mode from the operation methods takes precedence over the store configuration', async () => { + const mockMetadata = { + name: 'Netlify', + cool: true, + functions: ['edge', 'serverless'], + } + const headers = { + etag: '123456789', + 'x-amz-meta-user': `b64;${base64Encode(mockMetadata)}`, + } + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(value), + url: `${uncachedEdgeURL}/${siteID}/production/${key}`, + }) + .head({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(null, { headers }), + url: `${edgeURL}/${siteID}/production/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${edgeToken}` }, + response: new Response(value, { headers }), + url: `${edgeURL}/${siteID}/production/${key}`, + }) + + globalThis.fetch = mockStore.fetch + + const blobs = getStore({ + consistency: 'strong', + edgeURL, + name: 'production', + token: edgeToken, + siteID, + uncachedEdgeURL, + }) + + const data = await blobs.get(key) + expect(data).toBe(value) + + const meta = await blobs.getMetadata(key, { consistency: 'eventual' }) + expect(meta?.etag).toBe(headers.etag) + expect(meta?.metadata).toEqual(mockMetadata) + + const dataWithMeta = await blobs.getWithMetadata(key, { consistency: 'eventual' }) + expect(dataWithMeta?.data).toBe(value) + expect(dataWithMeta?.etag).toBe(headers.etag) + expect(dataWithMeta?.metadata).toEqual(mockMetadata) + + expect(mockStore.fulfilled).toBeTruthy() + }) + + test('Throws when strong consistency is used and no `uncachedEdgeURL` property has been defined', async () => { + const context = { + edgeURL, + siteID, + token: edgeToken, + } + + env.NETLIFY_BLOBS_CONTEXT = Buffer.from(JSON.stringify(context)).toString('base64') + + const store = getStore('productin') + + await expect(async () => await store.get('my-key', { consistency: 'strong' })).rejects.toThrowError( + "Netlify Blobs has failed to perform a read using strong consistency because the environment has not been configured with a 'uncachedEdgeURL' property", + ) + }) +}) diff --git a/src/store.ts b/src/store.ts index 1e0aa23..d003b82 100644 --- a/src/store.ts +++ b/src/store.ts @@ -2,12 +2,14 @@ import { Buffer } from 'node:buffer' import { ListResponse, ListResponseBlob } from './backend/list.ts' import { Client } from './client.ts' +import type { ConsistencyMode } from './consistency.ts' import { getMetadataFromResponse, Metadata } from './metadata.ts' import { BlobInput, HTTPMethod } from './types.ts' import { BlobsInternalError, collectIterator } from './util.ts' interface BaseStoreOptions { client: Client + consistency?: ConsistencyMode } interface DeployStoreOptions extends BaseStoreOptions { @@ -20,7 +22,12 @@ interface NamedStoreOptions extends BaseStoreOptions { export type StoreOptions = DeployStoreOptions | NamedStoreOptions +export interface GetOptions { + consistency?: ConsistencyMode +} + export interface GetWithMetadataOptions { + consistency?: ConsistencyMode etag?: string } @@ -57,10 +64,12 @@ export type BlobResponseType = 'arrayBuffer' | 'blob' | 'json' | 'stream' | 'tex export class Store { private client: Client + private consistency: ConsistencyMode private name: string constructor(options: StoreOptions) { this.client = options.client + this.consistency = options.consistency ?? 'eventual' if ('deployID' in options) { Store.validateDeployID(options.deployID) @@ -82,18 +91,19 @@ export class Store { } async get(key: string): Promise - async get(key: string, { type }: { type: 'arrayBuffer' }): Promise - async get(key: string, { type }: { type: 'blob' }): Promise + async get(key: string, opts: GetOptions): Promise + async get(key: string, { type }: GetOptions & { type: 'arrayBuffer' }): Promise + async get(key: string, { type }: GetOptions & { type: 'blob' }): Promise // eslint-disable-next-line @typescript-eslint/no-explicit-any - async get(key: string, { type }: { type: 'json' }): Promise - async get(key: string, { type }: { type: 'stream' }): Promise - async get(key: string, { type }: { type: 'text' }): Promise + async get(key: string, { type }: GetOptions & { type: 'json' }): Promise + async get(key: string, { type }: GetOptions & { type: 'stream' }): Promise + async get(key: string, { type }: GetOptions & { type: 'text' }): Promise async get( key: string, - options?: { type: BlobResponseType }, + options?: GetOptions & { type?: BlobResponseType }, ): Promise { - const { type } = options ?? {} - const res = await this.client.makeRequest({ key, method: HTTPMethod.GET, storeName: this.name }) + const { consistency, type } = options ?? {} + const res = await this.client.makeRequest({ consistency, key, method: HTTPMethod.GET, storeName: this.name }) if (res.status === 404) { return null @@ -126,8 +136,8 @@ export class Store { throw new BlobsInternalError(res.status) } - async getMetadata(key: string) { - const res = await this.client.makeRequest({ key, method: HTTPMethod.HEAD, storeName: this.name }) + async getMetadata(key: string, { consistency }: { consistency?: ConsistencyMode } = {}) { + const res = await this.client.makeRequest({ consistency, key, method: HTTPMethod.HEAD, storeName: this.name }) if (res.status === 404) { return null @@ -190,9 +200,15 @@ export class Store { } & GetWithMetadataResult) | null > { - const { etag: requestETag, type } = options ?? {} + const { consistency, etag: requestETag, type } = options ?? {} const headers = requestETag ? { 'if-none-match': requestETag } : undefined - const res = await this.client.makeRequest({ headers, key, method: HTTPMethod.GET, storeName: this.name }) + const res = await this.client.makeRequest({ + consistency, + headers, + key, + method: HTTPMethod.GET, + storeName: this.name, + }) if (res.status === 404) { return null