make it easier to run all integration tests from local data

Author: trxcllnt

js/bin/integration.js

@@ -17,6 +17,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+var fs = require('fs');
+var glob = require('glob');
 var path = require('path');
 var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`));
 var child_process = require(`child_process`);
@@ -29,12 +31,14 @@ var optionList = [
     {
         type: String,
         name: 'arrow', alias: 'a',
-        description: 'The Arrow file to read/write'
+        multiple: true, defaultValue: [],
+        description: 'The Arrow file[s] to read/write'
     },
     {
         type: String,
         name: 'json', alias: 'j',
-        description: 'The JSON file to read/write'
+        multiple: true, defaultValue: [],
+        description: 'The JSON file[s] to read/write'
     }
 ];
 
@@ -66,20 +70,60 @@ function print_usage() {
     process.exit(1);
 }
 
-if (!argv.arrow || !argv.json || !argv.mode) {
+let jsonPaths = argv.json;
+let arrowPaths = argv.arrow;
+
+if (!argv.mode) {
+    return print_usage();
+}
+
+let mode = argv.mode.toUpperCase();
+if (mode === 'VALIDATE' && !jsonPaths.length) {
+    jsonPaths = glob.sync(path.resolve(__dirname, `../test/data/json/`, `*.json`));
+    if (!arrowPaths.length) {
+        [jsonPaths, arrowPaths] = jsonPaths.reduce(([jsonPaths, arrowPaths], jsonPath) => {
+            const { name } = path.parse(jsonPath);
+            for (const source of ['cpp', 'java']) {
+                for (const format of ['file', 'stream']) {
+                    const arrowPath = path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`);
+                    if (fs.existsSync(arrowPath)) {
+                        jsonPaths.push(jsonPath);
+                        arrowPaths.push(arrowPath);
+                        console.log('-j', jsonPath, '-a', arrowPath, '\\');
+                    }
+                }
+            }
+            return [jsonPaths, arrowPaths];
+        }, [[], []]);
+    }
+} else if (!jsonPaths.length) {
     return print_usage();
 }
 
-switch (argv.mode.toUpperCase()) {
+switch (mode) {
     case 'VALIDATE':
+        const args = [`test`, `-i`].concat(argv._unknown || []);
+        jsonPaths.forEach((p, i) => {
+            args.push('-j', p, '-a', arrowPaths[i]);
+        });
         child_process.spawnSync(
-            gulp,
-            [`test`, `-i`].concat(process.argv.slice(2)),
+            gulp, args,
             {
                 cwd: path.resolve(__dirname, '..'),
                 stdio: ['ignore', 'inherit', 'inherit']
             }
         );
+        // for (let i = -1, n = jsonPaths.length; ++i < n;) {
+        //     const jsonPath = jsonPaths[i];
+        //     const arrowPath = arrowPaths[i];
+        //     child_process.spawnSync(
+        //         gulp, args.concat(['-j', jsonPath, '-a', arrowPath]),
+        //         {
+        //             cwd: path.resolve(__dirname, '..'),
+        //             stdio: ['ignore', 'inherit', 'inherit']
+        //         }
+        //     );
+        // }
         break;
     default:
         print_usage();

js/gulp/argv.js

@@ -22,13 +22,11 @@ const argv = require(`command-line-args`)([
     { name: `target`, type: String, defaultValue: `` },
     { name: `module`, type: String, defaultValue: `` },
     { name: `coverage`, type: Boolean, defaultValue: false },
-    { name: `json_file`, alias: `j`, type: String, defaultValue: null },
-    { name: `arrow_file`, alias: `a`, type: String, defaultValue: null },
     { name: `integration`, alias: `i`, type: Boolean, defaultValue: false },
     { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
     { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] },
-    { name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] },
-    { name: `formats`, alias: `f`, type: String, multiple: true, defaultValue: [`file`, `stream`] },
+    { name: `json_files`, alias: `j`, type: String, multiple: true, defaultValue: [] },
+    { name: `arrow_files`, alias: `a`, type: String, multiple: true, defaultValue: [] },
 ], { partial: true });
 
 const { targets, modules } = argv;

js/gulp/test-task.js

@@ -48,11 +48,9 @@ const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function
     opts.env = { ...opts.env,
         TEST_TARGET: target,
         TEST_MODULE: format,
-        JSON_PATH: argv.json_file,
-        ARROW_PATH: argv.arrow_file,
         TEST_TS_SOURCE: !!argv.coverage,
-        TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]),
-        TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]),
+        JSON_PATHS: JSON.stringify(Array.isArray(argv.json_files) ? argv.json_files : [argv.json_files]),
+        ARROW_PATHS: JSON.stringify(Array.isArray(argv.arrow_files) ? argv.arrow_files : [argv.arrow_files]),
     };
     return !debug ?
         child_process.spawn(jest, args, opts) :

js/package.json

@@ -62,8 +62,8 @@
   "devDependencies": {
     "@std/esm": "0.19.7",
     "@types/flatbuffers": "1.6.5",
-    "@types/glob": "5.0.34",
-    "@types/jest": "22.0.1",
+    "@types/glob": "5.0.35",
+    "@types/jest": "22.1.0",
     "@types/node": "9.3.0",
     "ast-types": "0.10.1",
     "benchmark": "2.1.4",
@@ -78,9 +78,9 @@
     "gulp-rename": "1.2.2",
     "gulp-sourcemaps": "2.6.3",
     "gulp-transform-js-ast": "1.0.2",
-    "gulp-typescript": "3.2.3",
+    "gulp-typescript": "3.2.4",
     "ix": "2.3.4",
-    "jest": "22.1.3",
+    "jest": "22.1.4",
     "jest-environment-node-debug": "2.0.0",
     "json": "9.0.6",
     "lerna": "2.7.1",

js/test/integration/validate-tests.ts

@@ -18,28 +18,41 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-if (!process.env.JSON_PATH || !process.env.ARROW_PATH) {
-    throw new Error('Integration tests need paths to both json and arrow files');
-}
+import Arrow from '../Arrow';
+import { zip } from 'ix/iterable/zip';
+import { toArray } from 'ix/iterable/toarray';
 
-const jsonPath = path.resolve(process.env.JSON_PATH + '');
-const arrowPath = path.resolve(process.env.ARROW_PATH + '');
+/* tslint:disable */
+const { parse: bignumJSONParse } = require('json-bignum');
 
-if (!fs.existsSync(jsonPath) || !fs.existsSync(arrowPath)) {
-    throw new Error('Integration tests need both json and arrow files to exist');
-}
+const { Table, read } = Arrow;
 
-/* tslint:disable */
-const { parse } = require('json-bignum');
+if (!process.env.JSON_PATHS || !process.env.ARROW_PATHS) {
+    throw new Error('Integration tests need paths to both json and arrow files');
+}
 
-const jsonData = parse(fs.readFileSync(jsonPath, 'utf8'));
-const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)];
+function resolvePathArgs(paths: string) {
+    let pathsArray = JSON.parse(paths) as string | string[];
+    return (Array.isArray(pathsArray) ? pathsArray : [pathsArray])
+        .map((p) => path.resolve(p))
+        .map((p) => {
+            if (fs.existsSync(p)) {
+                return p;
+            }
+            console.warn(`Could not find file "${p}"`);
+            return undefined;
+        });
+}
 
-import Arrow from '../Arrow';
-import { zip } from 'ix/iterable/zip';
-import { toArray } from 'ix/iterable/toarray';
+const getOrReadFileBuffer = ((cache: any) => function getFileBuffer(path: string, ...args: any[]) {
+    return cache[path] || (cache[path] = fs.readFileSync(path, ...args));
+})({});
 
-const { Table, read } = Arrow;
+const jsonAndArrowPaths = toArray(zip(
+    resolvePathArgs(process.env.JSON_PATHS!),
+    resolvePathArgs(process.env.ARROW_PATHS!)
+))
+.filter(([p1, p2]) => p1 !== undefined && p2 !== undefined) as [string, string][];
 
 expect.extend({
     toEqualVector(v1: any, v2: any) {
@@ -66,7 +79,7 @@ expect.extend({
 
         for (let i = -1, n = props.length; ++i < n;) {
             const prop = props[i];
-            if (this.utils.stringify(v1[prop]) !== this.utils.stringify(v2[prop])) {
+            if (`${v1[prop]}` !== `${v2[prop]}`) {
                 propsFailures.push(`${prop}: ${format(v1[prop], v2[prop], ' !== ')}`);
             }
         }
@@ -98,35 +111,43 @@ expect.extend({
 });
 
 describe(`Integration`, () => {
-    testReaderIntegration();
-    testTableFromBuffersIntegration();
+    for (const [jsonFilePath, arrowFilePath] of jsonAndArrowPaths) {
+        let { name, dir } = path.parse(arrowFilePath);
+        dir = dir.split(path.sep).slice(-2).join(path.sep);
+        const json = bignumJSONParse(getOrReadFileBuffer(jsonFilePath, 'utf8'));
+        const arrowBuffer = getOrReadFileBuffer(arrowFilePath) as Uint8Array;
+        describe(path.join(dir, name), () => {
+            testReaderIntegration(json, arrowBuffer);
+            testTableFromBuffersIntegration(json, arrowBuffer);
+        });
+    }
 });
 
-function testReaderIntegration() {
-    test(`json and arrow buffers report the same values`, () => {
-        debugger;
+function testReaderIntegration(jsonData: any, arrowBuffer: Uint8Array) {
+    test(`json and arrow record batches report the same values`, () => {
         expect.hasAssertions();
         const jsonRecordBatches = toArray(read(jsonData));
-        const binaryRecordBatches = toArray(read(arrowBuffers));
+        const binaryRecordBatches = toArray(read(arrowBuffer));
         for (const [jsonRecordBatch, binaryRecordBatch] of zip(jsonRecordBatches, binaryRecordBatches)) {
             expect(jsonRecordBatch.length).toEqual(binaryRecordBatch.length);
             expect(jsonRecordBatch.numCols).toEqual(binaryRecordBatch.numCols);
             for (let i = -1, n = jsonRecordBatch.numCols; ++i < n;) {
+                (jsonRecordBatch.columns[i] as any).name = jsonRecordBatch.schema.fields[i].name;
                 (expect(jsonRecordBatch.columns[i]) as any).toEqualVector(binaryRecordBatch.columns[i]);
             }
         }
     });
 }
 
-function testTableFromBuffersIntegration() {
-    test(`json and arrow buffers report the same values`, () => {
-        debugger;
+function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
+    test(`json and arrow tables report the same values`, () => {
         expect.hasAssertions();
         const jsonTable = Table.from(jsonData);
-        const binaryTable = Table.from(arrowBuffers);
+        const binaryTable = Table.from(arrowBuffer);
         expect(jsonTable.length).toEqual(binaryTable.length);
         expect(jsonTable.numCols).toEqual(binaryTable.numCols);
         for (let i = -1, n = jsonTable.numCols; ++i < n;) {
+            (jsonTable.columns[i] as any).name = jsonTable.schema.fields[i].name;
             (expect(jsonTable.columns[i]) as any).toEqualVector(binaryTable.columns[i]);
         }
     });