diff --git a/.gitignore b/.gitignore index f7d5ab223..d97498a9c 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,5 @@ package.json.decrypt # they track the package.json version /modules/kms-keyring-browser/src/version.ts /modules/kms-keyring-node/src/version.ts -/modules/branch-keystore-node/src/version.ts \ No newline at end of file +/modules/branch-keystore-node/src/version.ts +/modules/integration-node/src/version.ts diff --git a/modules/integration-node/package.json b/modules/integration-node/package.json index f2d1ac8b4..1361652ce 100644 --- a/modules/integration-node/package.json +++ b/modules/integration-node/package.json @@ -2,6 +2,8 @@ "name": "@aws-crypto/integration-node", "version": "4.1.0", "scripts": { + "prepublishOnly": "npm run generate-version.ts; npm run build", + "generate-version.ts": "npx genversion --es6 src/version.ts", "build": "tsc -b tsconfig.json", "lint": "run-s lint-*", "lint-eslint": "eslint src/*.ts test/*.ts", @@ -24,7 +26,8 @@ "got": "^11.8.0", "stream-to-promise": "^3.0.0", "tslib": "^2.3.0", - "yargs": "^17.0.1" + "yargs": "^17.0.1", + "yazl": "^3.3.1" }, "sideEffects": false, "main": "./build/main/src/index.js", diff --git a/modules/integration-node/src/cli.ts b/modules/integration-node/src/cli.ts index 297e98e42..86566ff3c 100644 --- a/modules/integration-node/src/cli.ts +++ b/modules/integration-node/src/cli.ts @@ -47,7 +47,13 @@ const cli = yargs .option('decryptOracle', { alias: 'o', describe: 'a url to the decrypt oracle', - demandOption: true, + demandOption: false, + type: 'string', + }) + .option('decryptManifest', { + alias: 'd', + describe: 'a file path for to create a decrypt manifest zip file', + demandOption: false, type: 'string', }) ) @@ -103,15 +109,18 @@ const cli = yargs concurrency ) } else if (command === 'encrypt') { - const { manifestFile, keyFile, decryptOracle } = argv as unknown as { - manifestFile: string - keyFile: string - decryptOracle: string - } + const { manifestFile, keyFile, decryptOracle, decryptManifest } = + argv as unknown as { + manifestFile: string + keyFile: string + decryptOracle?: string + decryptManifest?: string + } result = await integrationEncryptTestVectors( manifestFile, keyFile, decryptOracle, + decryptManifest, tolerateFailures, testName, concurrency diff --git a/modules/integration-node/src/constants.ts b/modules/integration-node/src/constants.ts new file mode 100644 index 000000000..ac3284656 --- /dev/null +++ b/modules/integration-node/src/constants.ts @@ -0,0 +1,10 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const KEYS_MANIFEST_NAME_FILENAME = 'keys.json' +export const MANIFEST_NAME_FILENAME = 'manifest.json' +export const DECRYPT_MANIFEST_TYPE = 'awses-decrypt' +export const DECRYPT_MANIFEST_CLIENT_NAME = 'aws/aws-encryption-sdk-javascript' +export const MANIFEST_URI_PREFIX = 'file://' +export const MANIFEST_PLAINTEXT_PATH = 'plaintexts/' +export const MANIFEST_CIPHERTEXT_PATH = 'ciphertexts/' diff --git a/modules/integration-node/src/get_encrypt_test_iterator.ts b/modules/integration-node/src/get_encrypt_test_iterator.ts index 48f846920..0183c29b2 100644 --- a/modules/integration-node/src/get_encrypt_test_iterator.ts +++ b/modules/integration-node/src/get_encrypt_test_iterator.ts @@ -14,28 +14,51 @@ import { import { URL } from 'url' import { readFileSync } from 'fs' import got from 'got' +import { ZipFile } from 'yazl' +import { + KEYS_MANIFEST_NAME_FILENAME, + MANIFEST_PLAINTEXT_PATH, +} from './constants' export async function getEncryptTestVectorIterator( manifestFile: string, - keyFile: string + keyFile: string, + manifestZip?: ZipFile ) { const [manifest, keys]: [EncryptManifestList, KeyList] = await Promise.all([ getParsedJSON(manifestFile), getParsedJSON(keyFile), ]) - return _getEncryptTestVectorIterator(manifest, keys) + return _getEncryptTestVectorIterator(manifest, keys, manifestZip) } /* Just a simple more testable function */ export function _getEncryptTestVectorIterator( { tests, plaintexts }: EncryptManifestList, - { keys }: KeyList + keysManifest: KeyList, + manifestZip?: ZipFile ) { + if (manifestZip) { + // We assume that the keys manifest given for encrypt + // has all the keys required for decrypt. + manifestZip.addBuffer( + Buffer.from(JSON.stringify(keysManifest)), + `${KEYS_MANIFEST_NAME_FILENAME}` + ) + } + const { keys } = keysManifest const plaintextBytes: { [name: string]: Buffer } = {} Object.keys(plaintexts).forEach((name) => { plaintextBytes[name] = randomBytes(plaintexts[name]) + + if (manifestZip) { + manifestZip.addBuffer( + plaintextBytes[name], + `${MANIFEST_PLAINTEXT_PATH}${name}` + ) + } }) return (function* nextTest(): IterableIterator { @@ -60,6 +83,7 @@ export function _getEncryptTestVectorIterator( name, keysInfo, plainTextData: plaintextBytes[plaintext], + plaintextName: plaintext, encryptOp: { suiteId, frameLength, encryptionContext }, } } @@ -70,6 +94,7 @@ export interface EncryptTestVectorInfo { name: string keysInfo: KeyInfoTuple[] plainTextData: Buffer + plaintextName: string encryptOp: { suiteId: AlgorithmSuiteIdentifier frameLength: number diff --git a/modules/integration-node/src/integration_tests.ts b/modules/integration-node/src/integration_tests.ts index 4dc14c479..8793bccf6 100644 --- a/modules/integration-node/src/integration_tests.ts +++ b/modules/integration-node/src/integration_tests.ts @@ -6,6 +6,7 @@ import { TestVectorResult, parseIntegrationTestVectorsToTestVectorIterator, PositiveTestVectorInfo, + DecryptManifestList, } from '@aws-crypto/integration-vectors' import { EncryptTestVectorInfo, @@ -22,14 +23,27 @@ import { needs, DecryptOutput, } from '@aws-crypto/client-node' +import { version } from './version' import { URL } from 'url' import got from 'got' import streamToPromise from 'stream-to-promise' const { encrypt, decrypt, decryptUnsignedMessageStream } = buildClient( CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT ) +import { ZipFile } from 'yazl' +import { createWriteStream } from 'fs' +import { v4 } from 'uuid' import * as stream from 'stream' import * as util from 'util' +import { + DECRYPT_MANIFEST_CLIENT_NAME, + DECRYPT_MANIFEST_TYPE, + KEYS_MANIFEST_NAME_FILENAME, + MANIFEST_CIPHERTEXT_PATH, + MANIFEST_NAME_FILENAME, + MANIFEST_PLAINTEXT_PATH, + MANIFEST_URI_PREFIX, +} from './constants' const pipeline = util.promisify(stream.pipeline) const notSupportedEncryptMessages = [ @@ -111,11 +125,28 @@ async function testPositiveDecryptVector( } } -// This is only viable for small streams, if we start get get larger streams, an stream equality should get written +interface ProcessEncryptResults { + handleEncryptResult: HandleEncryptResult + // We need to have a done step to close the ZipFile. + done(): void + // The handleEncryptResult needs the ZipFile + // so that it can add ciphertexts and tests. + // But when we set up the encrypt manifest, + // we create plaintext files and have the keys manifest. + // This is a quick and dirty way to share the ZipFile + // between these two places. + manifestZip?: ZipFile +} + +interface HandleEncryptResult { + (encryptResult: Buffer, info: EncryptTestVectorInfo): Promise +} + export async function testEncryptVector( - { name, keysInfo, encryptOp, plainTextData }: EncryptTestVectorInfo, - decryptOracle: string + info: EncryptTestVectorInfo, + handleEncryptResult: HandleEncryptResult ): Promise { + const { name, keysInfo, encryptOp, plainTextData } = info try { const cmm = encryptMaterialsManagerNode(keysInfo) const { result: encryptResult } = await encrypt( @@ -124,7 +155,70 @@ export async function testEncryptVector( encryptOp ) - const decryptResponse = await got.post(decryptOracle, { + const result = await handleEncryptResult(encryptResult, info) + return { result, name } + } catch (err) { + return { result: false, name, err } + } +} + +// This isolates the logic on how to do both. +// Right now we only have 2 ways to handle results +// so this seems reasonable. +function composeEncryptResults( + decryptOracle?: string, + decryptManifest?: string +): ProcessEncryptResults { + if (!!decryptOracle && !!decryptManifest) { + const oracle = decryptOracleEncryptResults(decryptOracle) + const manifest = decryptionManifestEncryptResults(decryptManifest) + + return { + done() { + manifest.done() + oracle.done() + }, + + async handleEncryptResult( + encryptResult: Buffer, + info: EncryptTestVectorInfo + ): Promise { + return Promise.all([ + oracle.handleEncryptResult(encryptResult, info), + manifest.handleEncryptResult(encryptResult, info), + ]).then((results) => { + const [oracleResult, manifestResult] = results + return oracleResult && manifestResult + }) + }, + manifestZip: manifest.manifestZip, + } + } else if (decryptOracle) { + return decryptOracleEncryptResults(decryptOracle) + } else if (decryptManifest) { + return decryptionManifestEncryptResults(decryptManifest) + } + needs(false, 'unsupported') +} + +function decryptOracleEncryptResults( + decryptOracle: string +): ProcessEncryptResults { + const decryptOracleUrl = new URL(decryptOracle).toString() + return { + handleEncryptResult, + // There is nothing to do when the oracle is done + // since nothing is saved. + done: () => { + return null + }, + } + + async function handleEncryptResult( + encryptResult: Buffer, + info: EncryptTestVectorInfo + ): Promise { + const decryptResponse = await got.post(decryptOracleUrl, { headers: { 'Content-Type': 'application/octet-stream', Accept: 'application/octet-stream', @@ -134,10 +228,71 @@ export async function testEncryptVector( }) needs(decryptResponse.statusCode === 200, 'decrypt failure') const { body } = decryptResponse - const result = plainTextData.equals(body) - return { result, name } - } catch (err) { - return { result: false, name, err } + // This is only viable for small streams, + // if we start get get larger streams, + // a stream equality should get written + return info.plainTextData.equals(body) + } +} + +function decryptionManifestEncryptResults( + manifestPath: string +): ProcessEncryptResults { + const manifestZip = new ZipFile() + const manifest: DecryptManifestList = { + manifest: { + type: `${DECRYPT_MANIFEST_TYPE}`, + version: 2, + }, + client: { + name: `${DECRYPT_MANIFEST_CLIENT_NAME}`, + version, + }, + keys: `${MANIFEST_URI_PREFIX}${KEYS_MANIFEST_NAME_FILENAME}`, + tests: {}, + } + manifestZip.outputStream.pipe(createWriteStream(manifestPath)) + + return { + handleEncryptResult, + done: () => { + // All the tests have completed, + // so we write the manifest, + // as close the zip file. + manifestZip.addBuffer( + Buffer.from(JSON.stringify(manifest)), + `${MANIFEST_NAME_FILENAME}` + ) + manifestZip.end() + }, + manifestZip, + } + + async function handleEncryptResult( + encryptResult: Buffer, + info: EncryptTestVectorInfo + ): Promise { + const testName = v4() + + manifestZip.addBuffer( + encryptResult, + `${MANIFEST_CIPHERTEXT_PATH}${testName}` + ) + + manifest.tests[testName] = { + description: `Decrypt vector from ${info.name}`, + ciphertext: `${MANIFEST_URI_PREFIX}${MANIFEST_CIPHERTEXT_PATH}${testName}`, + 'master-keys': info.keysInfo.map((info) => info[0]), + result: { + output: { + plaintext: `${MANIFEST_URI_PREFIX}${MANIFEST_PLAINTEXT_PATH}${info.plaintextName}`, + }, + }, + } + + // These files are tested on decrypt + // so there is nothing to test at this point. + return true } } @@ -196,22 +351,42 @@ export async function integrationDecryptTestVectors( export async function integrationEncryptTestVectors( manifestFile: string, keyFile: string, - decryptOracle: string, + decryptOracle?: string, + decryptManifest?: string, tolerateFailures = 0, testName?: string, concurrency = 1 ): Promise { - const decryptOracleUrl = new URL(decryptOracle).toString() - const tests = await getEncryptTestVectorIterator(manifestFile, keyFile) + needs( + !!decryptOracle || !!decryptManifest, + 'Need to pass an oracle or manifest path.' + ) - return parallelTests(concurrency, tolerateFailures, runTest, tests) + const { done, handleEncryptResult, manifestZip } = composeEncryptResults( + decryptOracle, + decryptManifest + ) + + const tests = await getEncryptTestVectorIterator( + manifestFile, + keyFile, + manifestZip + ) + + return parallelTests(concurrency, tolerateFailures, runTest, tests).then( + (num) => { + // Do the output processing here + done() + return num + } + ) async function runTest(test: EncryptTestVectorInfo): Promise { if (testName) { if (test.name !== testName) return true } return handleTestResults( - await testEncryptVector(test, decryptOracleUrl), + await testEncryptVector(test, handleEncryptResult), notSupportedEncryptMessages ) } @@ -248,8 +423,14 @@ async function parallelTests< * we just process the value and ask for another. * Which will return done as true again. */ - if (!value && done) return _resolve(failureCount) - + if (!value && done) { + // We are done enqueueing work, + // but we need to wait until all that work is done + Promise.all([...queue]) + .then(() => _resolve(failureCount)) + .catch(console.log) + return + } /* I need to define the work to be enqueue * and a way to dequeue this work when complete. * A Set of promises works nicely. diff --git a/package-lock.json b/package-lock.json index a6fa95ca1..e3228ece5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -54,6 +54,7 @@ "@types/from2": "^2.3.0", "@types/mocha": "^9.0.0", "@types/node": "^16.9.1", + "@types/yazl": "^2.4.6", "@typescript-eslint/eslint-plugin": "^4.31.1", "@typescript-eslint/parser": "^4.31.1", "aws-sdk": "^2.1409.0", @@ -468,7 +469,8 @@ "got": "^11.8.0", "stream-to-promise": "^3.0.0", "tslib": "^2.3.0", - "yargs": "^17.0.1" + "yargs": "^17.0.1", + "yazl": "^3.3.1" }, "bin": { "integration-node": "build/main/src/cli.js" @@ -5480,6 +5482,16 @@ "@types/node": "*" } }, + "node_modules/@types/yazl": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/@types/yazl/-/yazl-2.4.6.tgz", + "integrity": "sha512-/ifFjQtcKaoZOjl5NNCQRR0fAKafB3Foxd7J/WvFPTMea46zekapcR30uzkwIkKAAuq5T6d0dkwz754RFH27hg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "4.33.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.33.0.tgz", @@ -21445,6 +21457,24 @@ "fd-slicer": "~1.1.0" } }, + "node_modules/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^1.0.0" + } + }, + "node_modules/yazl/node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/package.json b/package.json index 4d5a0ed07..4840bc9e6 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ "@types/from2": "^2.3.0", "@types/mocha": "^9.0.0", "@types/node": "^16.9.1", + "@types/yazl": "^2.4.6", "@typescript-eslint/eslint-plugin": "^4.31.1", "@typescript-eslint/parser": "^4.31.1", "aws-sdk": "^2.1409.0",