Skip to content

fix: fix metadata in local server #112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 14, 2023
Merged

fix: fix metadata in local server #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 70 additions & 68 deletions src/server.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,6 @@ const siteID = '9a003659-aaaa-0000-aaaa-63d3720d8621'
const token = 'my-very-secret-token'

test('Reads and writes from the file system', async () => {
const directory = await tmp.dir()
const server = new BlobsServer({
directory: directory.path,
token,
})
const { port } = await server.start()
const blobs = getStore({
edgeURL: `http://localhost:${port}`,
name: 'mystore',
token,
siteID,
})
const metadata = {
features: {
blobs: true,
Expand All @@ -51,27 +39,82 @@ test('Reads and writes from the file system', async () => {
name: 'Netlify',
}

await blobs.set('simple-key', 'value 1')
expect(await blobs.get('simple-key')).toBe('value 1')
// Store #1: Edge access
const directory1 = await tmp.dir()
const server1 = new BlobsServer({
directory: directory1.path,
token,
})
const { port: port1 } = await server1.start()
const store1 = getStore({
edgeURL: `http://localhost:${port1}`,
name: 'mystore1',
token,
siteID,
})

// Store #2: API access
const directory2 = await tmp.dir()
const server2 = new BlobsServer({
directory: directory2.path,
token,
})
const { port: port2 } = await server2.start()
const store2 = getStore({
apiURL: `http://localhost:${port2}`,
name: 'mystore2',
token,
siteID,
})

await blobs.set('simple-key', 'value 2', { metadata })
expect(await blobs.get('simple-key')).toBe('value 2')
for (const store of [store1, store2]) {
const list1 = await store.list()
expect(list1.blobs).toEqual([])
expect(list1.directories).toEqual([])

await blobs.set('parent/child', 'value 3')
expect(await blobs.get('parent/child')).toBe('value 3')
expect(await blobs.get('parent')).toBe(null)
await store.set('simple-key', 'value 1')
expect(await store.get('simple-key')).toBe('value 1')

const entry = await blobs.getWithMetadata('simple-key')
expect(entry?.metadata).toEqual(metadata)
await store.set('simple-key', 'value 2', { metadata })
expect(await store.get('simple-key')).toBe('value 2')

const entryMetadata = await blobs.getMetadata('simple-key')
expect(entryMetadata?.metadata).toEqual(metadata)
const list2 = await store.list()
expect(list2.blobs.length).toBe(1)
expect(list2.blobs[0].key).toBe('simple-key')
expect(list2.directories).toEqual([])

await blobs.delete('simple-key')
expect(await blobs.get('simple-key')).toBe(null)
await store.set('parent/child', 'value 3')
expect(await store.get('parent/child')).toBe('value 3')
expect(await store.get('parent')).toBe(null)

await server.stop()
await fs.rm(directory.path, { force: true, recursive: true })
const entry = await store.getWithMetadata('simple-key')
expect(entry?.metadata).toEqual(metadata)

const entryMetadata = await store.getMetadata('simple-key')
expect(entryMetadata?.metadata).toEqual(metadata)

const childEntryMetdata = await store.getMetadata('parent/child')
expect(childEntryMetdata?.metadata).toEqual({})

expect(await store.getWithMetadata('does-not-exist')).toBe(null)
expect(await store.getMetadata('does-not-exist')).toBe(null)

await store.delete('simple-key')
expect(await store.get('simple-key')).toBe(null)
expect(await store.getMetadata('simple-key')).toBe(null)
expect(await store.getWithMetadata('simple-key')).toBe(null)

const list3 = await store.list()
expect(list3.blobs.length).toBe(1)
expect(list3.blobs[0].key).toBe('parent/child')
expect(list3.directories).toEqual([])
}

await server1.stop()
await fs.rm(directory1.path, { force: true, recursive: true })

await server2.stop()
await fs.rm(directory2.path, { force: true, recursive: true })
})

test('Separates keys from different stores', async () => {
Expand Down Expand Up @@ -218,44 +261,3 @@ test('Lists entries', async () => {

expect(parachutesSongs2.directories).toEqual([])
})

test('Supports the API access interface', async () => {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We now have one single test with the same set of assertions testing both the API and Edge interfaces.

const directory = await tmp.dir()
const server = new BlobsServer({
directory: directory.path,
token,
})
const { port } = await server.start()
const blobs = getStore({
apiURL: `http://localhost:${port}`,
name: 'mystore',
token,
siteID,
})
const metadata = {
features: {
blobs: true,
functions: true,
},
name: 'Netlify',
}

await blobs.set('simple-key', 'value 1')
expect(await blobs.get('simple-key')).toBe('value 1')

await blobs.set('simple-key', 'value 2', { metadata })
expect(await blobs.get('simple-key')).toBe('value 2')

await blobs.set('parent/child', 'value 3')
expect(await blobs.get('parent/child')).toBe('value 3')
expect(await blobs.get('parent')).toBe(null)

const entry = await blobs.getWithMetadata('simple-key')
expect(entry?.metadata).toEqual(metadata)

await blobs.delete('simple-key')
expect(await blobs.get('simple-key')).toBe(null)

await server.stop()
await fs.rm(directory.path, { force: true, recursive: true })
})
109 changes: 78 additions & 31 deletions src/server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,28 +71,48 @@ export class BlobsServer {
}

async delete(req: http.IncomingMessage, res: http.ServerResponse) {
const apiMatch = this.parseAPIRequest(req)

if (apiMatch) {
return this.sendResponse(req, res, 200, JSON.stringify({ url: apiMatch.url.toString() }))
}

const url = new URL(req.url ?? '', this.address)
const { dataPath, key } = this.getLocalPaths(url)
const { dataPath, key, metadataPath } = this.getLocalPaths(url)

if (!dataPath || !key) {
return this.sendResponse(req, res, 400)
}

// Try to delete the metadata file, if one exists.
try {
await fs.rm(metadataPath, { force: true, recursive: true })
} catch {
// no-op
}

// Delete the data file.
try {
await fs.rm(dataPath, { recursive: true })
await fs.rm(dataPath, { force: true, recursive: true })
} catch (error: unknown) {
if (isNodeError(error) && error.code === 'ENOENT') {
return this.sendResponse(req, res, 404)
// An `ENOENT` error means we have tried to delete a key that doesn't
// exist, which shouldn't be treated as an error.
if (!isNodeError(error) || error.code !== 'ENOENT') {
return this.sendResponse(req, res, 500)
}

return this.sendResponse(req, res, 500)
}

return this.sendResponse(req, res, 200)
return this.sendResponse(req, res, 204)
}

async get(req: http.IncomingMessage, res: http.ServerResponse) {
const url = new URL(req.url ?? '', this.address)
const apiMatch = this.parseAPIRequest(req)
const url = apiMatch?.url ?? new URL(req.url ?? '', this.address)

if (apiMatch?.key) {
return this.sendResponse(req, res, 200, JSON.stringify({ url: apiMatch.url.toString() }))
}

const { dataPath, key, metadataPath, rootPath } = this.getLocalPaths(url)

if (!dataPath || !metadataPath) {
Expand Down Expand Up @@ -135,29 +155,29 @@ export class BlobsServer {
}

async head(req: http.IncomingMessage, res: http.ServerResponse) {
const url = new URL(req.url ?? '', this.address)
const url = this.parseAPIRequest(req)?.url ?? new URL(req.url ?? '', this.address)
const { dataPath, key, metadataPath } = this.getLocalPaths(url)

if (!dataPath || !metadataPath || !key) {
return this.sendResponse(req, res, 400)
}

const headers: Record<string, string> = {}

try {
const rawData = await fs.readFile(metadataPath, 'utf8')
const metadata = JSON.parse(rawData)
const encodedMetadata = encodeMetadata(metadata)

if (encodedMetadata) {
headers[METADATA_HEADER_INTERNAL] = encodedMetadata
res.setHeader(METADATA_HEADER_INTERNAL, encodedMetadata)
}
} catch (error) {
if (isNodeError(error) && (error.code === 'ENOENT' || error.code === 'ISDIR')) {
return this.sendResponse(req, res, 404)
}

this.logDebug('Could not read metadata file:', error)
}

for (const name in headers) {
res.setHeader(name, headers[name])
return this.sendResponse(req, res, 500)
}

res.end()
Expand All @@ -182,9 +202,13 @@ export class BlobsServer {
try {
await BlobsServer.walk({ directories, path: dataPath, prefix, rootPath, result })
} catch (error) {
this.logDebug('Could not perform list:', error)
// If the directory is not found, it just means there are no entries on
// the store, so that shouldn't be treated as an error.
if (!isNodeError(error) || error.code !== 'ENOENT') {
this.logDebug('Could not perform list:', error)

return this.sendResponse(req, res, 500)
return this.sendResponse(req, res, 500)
}
}

res.setHeader('content-type', 'application/json')
Expand All @@ -193,6 +217,12 @@ export class BlobsServer {
}

async put(req: http.IncomingMessage, res: http.ServerResponse) {
const apiMatch = this.parseAPIRequest(req)

if (apiMatch) {
return this.sendResponse(req, res, 200, JSON.stringify({ url: apiMatch.url.toString() }))
}

const url = new URL(req.url ?? '', this.address)
const { dataPath, key, metadataPath } = this.getLocalPaths(url)

Expand Down Expand Up @@ -263,19 +293,6 @@ export class BlobsServer {
return this.sendResponse(req, res, 403)
}

const apiURLMatch = req.url.match(API_URL_PATH)

// If this matches an API URL, return a signed URL.
if (apiURLMatch) {
const fullURL = new URL(req.url, this.address)
const storeName = fullURL.searchParams.get('context') ?? DEFAULT_STORE
const key = apiURLMatch.groups?.key as string
const siteID = apiURLMatch.groups?.site_id as string
const url = `${this.address}/${siteID}/${storeName}/${key}?signature=${this.tokenHash}`

return this.sendResponse(req, res, 200, JSON.stringify({ url }))
}

switch (req.method) {
case 'DELETE':
return this.delete(req, res)
Expand All @@ -295,8 +312,38 @@ export class BlobsServer {
}
}

/**
* Tries to parse a URL as being an API request and returns the different
* components, such as the store name, site ID, key, and signed URL.
*/
parseAPIRequest(req: http.IncomingMessage) {
if (!req.url) {
return null
}

const apiURLMatch = req.url.match(API_URL_PATH)

if (!apiURLMatch) {
return null
}

const fullURL = new URL(req.url, this.address)
const storeName = fullURL.searchParams.get('context') ?? DEFAULT_STORE
const key = apiURLMatch.groups?.key
const siteID = apiURLMatch.groups?.site_id as string
const urlPath = [siteID, storeName, key].filter(Boolean) as string[]
const url = new URL(`/${urlPath.join('/')}?signature=${this.tokenHash}`, this.address)

return {
key,
siteID,
storeName,
url,
}
}

sendResponse(req: http.IncomingMessage, res: http.ServerResponse, status: number, body?: string) {
this.logDebug(`${req.method} ${req.url}: ${status}`)
this.logDebug(`${req.method} ${req.url} ${status}`)

res.writeHead(status)
res.end(body)
Expand Down