chore: batch rebuild clients script (#200)

Author: AllanZhengYP

CONTRIBUTING.md

@@ -58,12 +58,21 @@ Please be aware of the following notes prior to opening a pull request:
 3. Wherever possible, pull requests should contain tests as appropriate.
    Bugfixes should contain tests that exercise the corrected behavior (i.e., the
    test should fail without the bugfix and pass with it), and new features 
-   should be accompanied by tests exercising the feature.
-
-4. Pull requests that contain failing tests will not be merged until the test
-   failures are addressed. Pull requests that cause a significant drop in the
-   SDK's test coverage percentage are unlikely to be merged until tests have
-   been added.
+   should be accompanied by tests exercising the feature. Pull requests that
+   contain failing tests will not be merged until the test failures are addressed.
+   Pull requests that cause a significant drop in the SDK's test coverage
+   percentage are unlikely to be merged until tests have been added.
+
+4. Commit tile and message and pull request title and description must adhere to
+   [conventional commits][conventional commits]. Title must begin with `feat(module): title`,
+   `fix(module): title`, `docs(module): title`, `test(module): title`, `chore(module): title`.
+   Title should be lowercase and not period at the end of it. If the commit includes
+   a breaking change, the commit message must end with a single paragraph: `BREAKING 
+   CHANGE: a description of what broke` 
+
+5. After getting ready to open a pull request, make sure to run the `scripts/
+   rebuildClients.js` to re-generate all the service clients, and commit the 
+   change(if any) to a standalone commit following the guide above.
 
 ### Setup and Testing
 
@@ -129,4 +138,5 @@ node ./packages/package-generator/build/cli.js client --model models/dynamodb/20
 [pr]: https://github.com/aws/aws-sdk-js-v3/pulls
 [license]: http://aws.amazon.com/apache2.0/
 [cla]: http://en.wikipedia.org/wiki/Contributor_License_Agreement
-[AWS service models]: https://github.com/aws/aws-sdk-js-v3/tree/master/models
+[AWS service models]: https://github.com/aws/aws-sdk-js-v3/tree/master/models
+[conventional commits]: https://www.conventionalcommits.org/

scripts/rebuildClients.js

@@ -0,0 +1,97 @@
+const yargs = require('yargs');
+const path = require('path');
+const fs = require('fs');
+const execSync = require('child_process').execSync;
+const ServiceModelFinder = require('./utils/ModelFinder').ServiceModelFinder;
+const clientNameRegex = require('./utils/constants').clientNameRegex;
+
+/**
+ * This script will scan packages directory, recognize service client packages and re-generate them from models
+ * If this command fails with cannot find model \'clientModuleIdentifier\', please run npm bootstrap && npm test first. 
+ */
+const models = yargs
+    .alias('m', 'models')
+    .string('m')
+    .default('m', path.join('..', '..', 'models'))
+    .describe('m', 'the directory of models')
+    .help()
+    .coerce('m', (directory) => {
+        return path.normalize(path.join(__filename, path.normalize(directory)));
+    })
+    .argv
+    .models;
+
+console.info('models directory: ', models);
+
+const existingServiceClients = grabExistingClients();
+console.info('existing service clients: ', existingServiceClients.map(item => path.parse(item).base));
+console.info('generating service clients...');
+const serviceModelFinder = new ServiceModelFinder(models);
+for (const serviceClient of existingServiceClients) {
+    const clientName = path.parse(serviceClient).base;
+    const models = serviceModelFinder.findModelsForService(clientName);
+    const [_, serviceId, runtime] = clientNameRegex.exec(clientName);
+    console.info(`generating ${runtime} client from model at ${models.service}`)
+    generateClient(models, runtime);
+}
+
+console.log('done!');
+
+
+function grabExistingClients() {
+    const packagesDir = path.join(path.dirname(__dirname), 'packages');
+    const clientPackagesPaths = [];
+    for (const package of fs.readdirSync(packagesDir)) {
+        const packageDir = path.join(packagesDir, package);
+        if (fs.statSync(packageDir).isDirectory() && isClientPackage(packageDir)) {
+            clientPackagesPaths.push(packageDir);
+        }
+    }
+    return clientPackagesPaths;
+}
+
+function isClientPackage(directory) {
+    const baseName = path.parse(directory).base;
+    if (!clientNameRegex.test(baseName)) return false;
+    const clientFiles = [
+        {base: 'commands', isFile: false},
+        {base: 'model', isFile: false},
+        {base: 'types', isFile: false},
+        {base: `(\\w+)Client\\.ts`, isFile: true},
+        {base: `(\\w+)Configuration\\.ts`, isFile: true},
+        {base: `index\\.ts`, isFile: true},
+    ]
+    try {
+        const files = fs.readdirSync(directory);
+        for (const clientFilePattern of clientFiles) {
+            const matchedFile = arrayFindPattern(files, clientFilePattern.base);
+            if (
+                !matchedFile &&
+                fs.statSync(path.join(directory, matchedFile)).isFile() !== clientFilePattern.isFile
+            ) {
+                return false;
+            }
+
+        }
+    } catch(e) {
+        return false;
+    }
+    return true;
+}
+
+function arrayFindPattern(array, pattern) {
+    return array.find((item) => {
+        const matched = new RegExp(pattern).exec(item);
+        //RegExp.exec() returns null if no matched
+        return Boolean(matched)
+    })
+}
+
+function generateClient(models, runtime) {
+    const command = `node packages/package-generator/build/cli.js client --m ${models.service} ${models.smoke ? `--s ${models.smoke}` : ''} --r ${runtime}`;
+    const packageRoot = path.dirname(__dirname);
+    const log = execSync(command, {cwd: packageRoot});
+    console.info(log.toString());
+}
+
+module.exports.clientNameRegex = clientNameRegex;

scripts/utils/ModelFinder.js

@@ -0,0 +1,61 @@
+const fs = require('fs');
+const path = require('path');
+const clientNameRegex = require('./constants').clientNameRegex;
+const clientModuleIdentifier = require('../../packages/package-generator/build/clientModuleIdentifier').clientModuleIdentifier;
+
+class ServiceModelFinder {
+    constructor(models) {
+        this.latestModels = this.fetchLatestModels(models);
+        this.loadedModelCache = new Map();
+    }
+
+    fetchLatestModels(modelsDir) {
+        const serviceModelDirs = [];
+        for (const modelName of fs.readdirSync(modelsDir)) {
+            const modelDir = path.join(modelsDir, modelName);
+            if (!fs.statSync(modelDir).isDirectory()) continue;
+            const versions = fs.readdirSync(modelDir).filter(
+                version => fs.statSync(path.join(modelDir, version)).isDirectory()
+            );
+            if (!versions || versions.length === 0) {
+                throw new Error(`No api version found in ${modelDir}`);
+            }
+            const latestVersion = versions.sort().reverse()[0];
+            const versionDir = path.join(modelDir, latestVersion);
+            const serviceModels = fs.readdirSync(versionDir);
+            if (serviceModels.find(modelName => modelName === 'service-2.json')) {
+                serviceModelDirs.push(versionDir);
+            }
+        }
+        return serviceModelDirs;
+    }
+
+    /**
+     * Fetch the directory of model and smoke test for given service name.
+     * @param {string} service service client package name. like: client-sqs-node
+     * @returns {object} looks like {service: string, smoke: string};
+     */
+    findModelsForService(packageName) {
+        const [_, service, runtime] = clientNameRegex.exec(packageName);
+        if (this.loadedModelCache.has(`client-${service}`)) {
+            return this.loadedModelCache.get(`client-${service}`);
+        }
+        for (const latestModel of this.latestModels.slice(this.loadedModelCache.size)) {
+            const modelDir = path.join(latestModel, 'service-2.json');
+            const smokeDir = path.join(latestModel, 'smoke.json');
+            const {metadata} = JSON.parse(fs.readFileSync(modelDir).toString());
+            const universalClientName = clientModuleIdentifier(metadata);
+            const loadedModel = {service: modelDir};
+            if (fs.existsSync(smokeDir)) {
+                loadedModel.smoke = smokeDir;
+            }
+            this.loadedModelCache.set(universalClientName, loadedModel);
+            if (universalClientName === `client-${service}`) {
+                return loadedModel;
+            }
+        }
+        throw new Error(`No model found for ${packageName}`);
+    }
+}
+
+module.exports.ServiceModelFinder = ServiceModelFinder;

scripts/utils/constants.js

@@ -0,0 +1,3 @@
+module.exports = {
+    clientNameRegex: /^client-(\w+-?\w+)-(node|browser|universal)$/
+}