|
| 1 | +/*! |
| 2 | + * Copyright 2022 Google LLC. All Rights Reserved. |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + */ |
| 16 | + |
| 17 | +import yargs from 'yargs'; |
| 18 | +import {promises as fsp, rmSync} from 'fs'; |
| 19 | +import {Bucket, DownloadOptions, DownloadResponse, UploadOptions} from '../src'; |
| 20 | +import {performance} from 'perf_hooks'; |
| 21 | +// eslint-disable-next-line node/no-unsupported-features/node-builtins |
| 22 | +import {parentPort} from 'worker_threads'; |
| 23 | +import { |
| 24 | + BLOCK_SIZE_IN_BYTES, |
| 25 | + DEFAULT_PROJECT_ID, |
| 26 | + DEFAULT_NUMBER_OF_OBJECTS, |
| 27 | + DEFAULT_SMALL_FILE_SIZE_BYTES, |
| 28 | + DEFAULT_LARGE_FILE_SIZE_BYTES, |
| 29 | + NODE_DEFAULT_HIGHWATER_MARK_BYTES, |
| 30 | + generateRandomDirectoryStructure, |
| 31 | + getValidationType, |
| 32 | + performanceTestSetup, |
| 33 | + TestResult, |
| 34 | +} from './performanceUtils'; |
| 35 | +import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; |
| 36 | + |
| 37 | +const TEST_NAME_STRING = 'nodejs-perf-metrics-application'; |
| 38 | +const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics-shaffeeullah'; |
| 39 | + |
| 40 | +let bucket: Bucket; |
| 41 | + |
| 42 | +const checkType = getValidationType(); |
| 43 | + |
| 44 | +const argv = yargs(process.argv.slice(2)) |
| 45 | + .options({ |
| 46 | + bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, |
| 47 | + small: {type: 'number', default: DEFAULT_SMALL_FILE_SIZE_BYTES}, |
| 48 | + large: {type: 'number', default: DEFAULT_LARGE_FILE_SIZE_BYTES}, |
| 49 | + projectid: {type: 'string', default: DEFAULT_PROJECT_ID}, |
| 50 | + numobjects: {type: 'number', default: DEFAULT_NUMBER_OF_OBJECTS}, |
| 51 | + }) |
| 52 | + .parseSync(); |
| 53 | + |
| 54 | +/** |
| 55 | + * Main entry point. This function performs a test iteration and posts the message back |
| 56 | + * to the parent thread. |
| 57 | + */ |
| 58 | +async function main() { |
| 59 | + let result: TestResult = { |
| 60 | + op: '', |
| 61 | + objectSize: 0, |
| 62 | + appBufferSize: 0, |
| 63 | + libBufferSize: 0, |
| 64 | + crc32Enabled: false, |
| 65 | + md5Enabled: false, |
| 66 | + apiName: 'JSON', |
| 67 | + elapsedTimeUs: 0, |
| 68 | + cpuTimeUs: 0, |
| 69 | + status: '[OK]', |
| 70 | + }; |
| 71 | + |
| 72 | + ({bucket} = await performanceTestSetup(argv.projectid, argv.bucket)); |
| 73 | + |
| 74 | + switch (argv.testtype) { |
| 75 | + case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS: |
| 76 | + result = await performWriteTest(); |
| 77 | + break; |
| 78 | + case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS: |
| 79 | + result = await performReadTest(); |
| 80 | + break; |
| 81 | + // case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD: |
| 82 | + // result = await performLargeReadTest(); |
| 83 | + // break; |
| 84 | + default: |
| 85 | + break; |
| 86 | + } |
| 87 | + parentPort?.postMessage(result); |
| 88 | +} |
| 89 | + |
| 90 | +async function uploadInParallel( |
| 91 | + bucket: Bucket, |
| 92 | + paths: string[], |
| 93 | + options: UploadOptions |
| 94 | +) { |
| 95 | + const promises = []; |
| 96 | + for (const index in paths) { |
| 97 | + const path = paths[index]; |
| 98 | + const stat = await fsp.lstat(path); |
| 99 | + if (stat.isDirectory()) { |
| 100 | + continue; |
| 101 | + } |
| 102 | + options.destination = path; |
| 103 | + promises.push(bucket.upload(path, options)); |
| 104 | + } |
| 105 | + await Promise.all(promises).catch(console.error); |
| 106 | +} |
| 107 | + |
| 108 | +async function downloadInParallel(bucket: Bucket, options: DownloadOptions) { |
| 109 | + const promises: Promise<DownloadResponse>[] = []; |
| 110 | + const [files] = await bucket.getFiles(); |
| 111 | + files.forEach(file => { |
| 112 | + promises.push(file.download(options)); |
| 113 | + }); |
| 114 | + await Promise.all(promises).catch(console.error); |
| 115 | +} |
| 116 | + |
| 117 | +/** |
| 118 | + * Performs an iteration of the Write multiple objects test. |
| 119 | + * |
| 120 | + * @returns {Promise<TestResult>} Promise that resolves to a test result of an iteration. |
| 121 | + */ |
| 122 | +async function performWriteTest(): Promise<TestResult> { |
| 123 | + await bucket.deleteFiles(); //start clean |
| 124 | + |
| 125 | + const creationInfo = generateRandomDirectoryStructure( |
| 126 | + argv.numobjects, |
| 127 | + TEST_NAME_STRING, |
| 128 | + argv.small, |
| 129 | + argv.large |
| 130 | + ); |
| 131 | + |
| 132 | + const start = performance.now(); |
| 133 | + await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); |
| 134 | + const end = performance.now(); |
| 135 | + |
| 136 | + await bucket.deleteFiles(); //cleanup files |
| 137 | + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); |
| 138 | + |
| 139 | + const result: TestResult = { |
| 140 | + op: 'WRITE', |
| 141 | + objectSize: creationInfo.totalSizeInBytes, |
| 142 | + appBufferSize: BLOCK_SIZE_IN_BYTES, |
| 143 | + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, |
| 144 | + crc32Enabled: checkType === 'crc32c', |
| 145 | + md5Enabled: checkType === 'md5', |
| 146 | + apiName: 'JSON', |
| 147 | + elapsedTimeUs: Math.round((end - start) * 1000), |
| 148 | + cpuTimeUs: -1, |
| 149 | + status: '[OK]', |
| 150 | + }; |
| 151 | + return result; |
| 152 | +} |
| 153 | + |
| 154 | +/** |
| 155 | + * Performs an iteration of the read multiple objects test. |
| 156 | + * |
| 157 | + * @returns {Promise<TestResult>} Promise that resolves to an array of test results for the iteration. |
| 158 | + */ |
| 159 | +async function performReadTest(): Promise<TestResult> { |
| 160 | + await bucket.deleteFiles(); // start clean |
| 161 | + const creationInfo = generateRandomDirectoryStructure( |
| 162 | + argv.numobjects, |
| 163 | + TEST_NAME_STRING, |
| 164 | + argv.small, |
| 165 | + argv.large |
| 166 | + ); |
| 167 | + await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); |
| 168 | + |
| 169 | + const start = performance.now(); |
| 170 | + await downloadInParallel(bucket, {validation: checkType}); |
| 171 | + const end = performance.now(); |
| 172 | + |
| 173 | + const result: TestResult = { |
| 174 | + op: 'READ', |
| 175 | + objectSize: creationInfo.totalSizeInBytes, |
| 176 | + appBufferSize: BLOCK_SIZE_IN_BYTES, |
| 177 | + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, |
| 178 | + crc32Enabled: checkType === 'crc32c', |
| 179 | + md5Enabled: checkType === 'md5', |
| 180 | + apiName: 'JSON', |
| 181 | + elapsedTimeUs: Math.round((end - start) * 1000), |
| 182 | + cpuTimeUs: -1, |
| 183 | + status: '[OK]', |
| 184 | + }; |
| 185 | + |
| 186 | + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); |
| 187 | + await bucket.deleteFiles(); //cleanup |
| 188 | + return result; |
| 189 | +} |
| 190 | + |
| 191 | +main(); |
![gcf-owl-bot[bot]](https://avatars.githubusercontent.com/in/99011?v=4&size=40){kind=avatar}
0 commit comments