Generating Schemas for Particle Accelerator with schema2kotlin (#4829)

## Summary
Modifies schema2kotlin tool to add generated schemas to Entity_Specs. Schemas can be accessed from the specs statically as a .schema property.
This PR also introduces a SchemaRegistry. All generated schemas are automatically registered in the registry.

Currently, collections of references are not supported.

## Changelog

* First iteration of schema gen on entities

* packaging in a schema registry

* Generation adds to schema registry

* excluding wasm from generation

* Improved shape of registry

* minor adjustments

* sigh lint

* Added test for mapOf

* no difference w/ wasm golden

* rm import

* rm unused field

* grooming schema registry

* addField arguments are more ergonomic

* ktlint fix

* EntitySpec Interface can return a schema

* fixed build

* revising from feedback

* Fixed indentation

* Potential presub fix: added dep to rule

* Potential Fix: create alias in 3rd party

* Found right place to assign deps

* Need to handle wasm use case

* Treat wasm, jvm separately.

* rm alias for wasm

* Cleaner sdk solution: export core/data

* rm whitespace, ran buildifier

Author: alxmrs

java/arcs/core/data/SchemaRegistry.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2020 Google LLC.
+ *
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt
+ *
+ * Code distributed by Google as part of this project is also subject to an additional IP rights
+ * grant found at
+ * http://polymer.github.io/PATENTS.txt
+ */
+package arcs.core.data
+
+typealias SchemaHash = String
+
+/**
+ * A registry for generated [Schema]s.
+ */
+object SchemaRegistry {
+    private val schemas = mutableMapOf<SchemaHash, Schema>()
+
+    /** Store a [Schema] in the registry. */
+    fun register(schema: Schema) {
+        schemas[schema.hash] = schema
+    }
+
+    /** Given a [SchemaHash], return the [Schema] for that hash, if it exists. */
+    operator fun get(hash: SchemaHash) = schemas[hash]
+}

java/arcs/core/storage/api/BUILD

@@ -12,7 +12,7 @@ arcs_kt_library(
     srcs = glob(["*.kt"]),
     deps = [
         "//java/arcs/core/common",
-        "//java/arcs/core/data:rawentity",
+        "//java/arcs/core/data",
         "//java/arcs/core/data/util:data-util",
         "//third_party/kotlin/kotlinx_coroutines",
     ],

java/arcs/core/storage/api/Entity.kt

@@ -13,6 +13,7 @@ package arcs.core.storage.api
 
 import arcs.core.common.Referencable
 import arcs.core.data.RawEntity
+import arcs.core.data.Schema
 import arcs.core.data.util.ReferencablePrimitive
 import kotlin.IllegalArgumentException
 import kotlin.reflect.KClass
@@ -38,6 +39,9 @@ interface EntitySpec<T : Entity> {
      * TODO: replace this with kotlinx.serialization
      */
     fun deserialize(data: RawEntity): T
+
+    /** Returns the corresponding [Schema] for the specified [Entity]. */
+    fun schema(): Schema
 }
 
 /**

java/arcs/sdk/BUILD

@@ -18,6 +18,7 @@ arcs_kt_jvm_library(
     ]),
     visibility = ["//visibility:public"],
     exports = [
+        "//java/arcs/core/data",
         "//java/arcs/core/data:rawentity",
         "//java/arcs/core/data/util:data-util",
         "//java/arcs/core/host",

src/tools/schema2base.ts

@@ -15,8 +15,16 @@ import {Runtime} from '../runtime/runtime.js';
 import {SchemaGraph, SchemaNode} from './schema2graph.js';
 import {ParticleSpec} from '../runtime/particle-spec.js';
 
+export type AddFieldOptions = Readonly<{
+  field: string;
+  typeChar: string;
+  isOptional?: boolean;
+  refClassName?: string;
+  isCollection?: boolean;
+}>;
+
 export interface ClassGenerator {
-  addField(field: string, typeChar: string, isOptional: boolean, refClassName: string|null): void;
+  addField(opts: AddFieldOptions): void;
   generate(schemaHash: string, fieldCount: number): string;
 }
 
@@ -78,14 +86,20 @@ export abstract class Schema2Base {
         for (const [field, descriptor] of fields) {
           if (descriptor.kind === 'schema-primitive') {
             if (['Text', 'URL', 'Number', 'Boolean'].includes(descriptor.type)) {
-              generator.addField(field, descriptor.type[0], false, null);
+              generator.addField({field, typeChar: descriptor.type[0]});
             } else {
               throw new Error(`Schema type '${descriptor.type}' for field '${field}' is not supported`);
             }
           } else if (descriptor.kind === 'schema-reference') {
-            generator.addField(field, 'R', false, node.refs.get(field).name);
+            generator.addField({field, typeChar: 'R', refClassName: node.refs.get(field).name});
           } else if (descriptor.kind === 'schema-collection' && descriptor.schema.kind === 'schema-reference') {
             // TODO: support collections of references
+          } else if (descriptor.kind === 'schema-collection') {
+            const schema = descriptor.schema;
+            if (!['Text', 'URL', 'Number', 'Boolean'].includes(schema.type)) {
+              throw new Error(`Schema type '${schema.type}' for field '${field}' is not supported`);
+            }
+            generator.addField({field, typeChar: schema.type[0], isCollection: true});
           } else {
             throw new Error(`Schema kind '${descriptor.kind}' for field '${field}' is not supported`);
           }

src/tools/schema2cpp.ts

@@ -7,7 +7,7 @@
  * subject to an additional IP rights grant found at
  * http://polymer.github.io/PATENTS.txt
  */
-import {Schema2Base, ClassGenerator} from './schema2base.js';
+import {Schema2Base, ClassGenerator, AddFieldOptions} from './schema2base.js';
 import {SchemaNode} from './schema2graph.js';
 import {ParticleSpec} from '../runtime/particle-spec.js';
 import {Type} from '../runtime/type.js';
@@ -113,7 +113,7 @@ class CppGenerator implements ClassGenerator {
 
   constructor(readonly node: SchemaNode, readonly namespace: string) {}
 
-  addField(field: string, typeChar: string, isOptional: boolean, refClassName: string|null) {
+  addField({field, typeChar, refClassName, isOptional = false, isCollection = false}: AddFieldOptions) {
     const fixed = fixName(field);
     const valid = `${field}_valid_`;
     let {type, defaultVal, isString} = typeMap[typeChar];

src/tools/schema2kotlin.ts

@@ -7,7 +7,7 @@
  * subject to an additional IP rights grant found at
  * http://polymer.github.io/PATENTS.txt
  */
-import {Schema2Base, ClassGenerator} from './schema2base.js';
+import {Schema2Base, ClassGenerator, AddFieldOptions} from './schema2base.js';
 import {SchemaNode} from './schema2graph.js';
 import {ParticleSpec} from '../runtime/particle-spec.js';
 import minimist from 'minimist';
@@ -28,10 +28,10 @@ const keywords = [
 ];
 
 const typeMap = {
-  'T': {type: 'String',  decodeFn: 'decodeText()', defaultVal: `""`},
-  'U': {type: 'String',  decodeFn: 'decodeText()', defaultVal: `""`},
-  'N': {type: 'Double',  decodeFn: 'decodeNum()',  defaultVal: '0.0'},
-  'B': {type: 'Boolean', decodeFn: 'decodeBool()', defaultVal: 'false'},
+  'T': {type: 'String',  decodeFn: 'decodeText()', defaultVal: `""`, schemaType: 'FieldType.Text'},
+  'U': {type: 'String',  decodeFn: 'decodeText()', defaultVal: `""`, schemaType: 'FieldType.Text'},
+  'N': {type: 'Double',  decodeFn: 'decodeNum()',  defaultVal: '0.0', schemaType: 'FieldType.Number'},
+  'B': {type: 'Boolean', decodeFn: 'decodeBool()', defaultVal: 'false', schemaType: 'FieldType.Boolean'},
 };
 
 export class Schema2Kotlin extends Schema2Base {
@@ -54,7 +54,14 @@ package ${this.scope}
 // Current implementation doesn't support references or optional field detection
 
 import arcs.sdk.*
-${this.opts.wasm ? 'import arcs.sdk.wasm.*' : 'import arcs.core.storage.api.toPrimitiveValue\nimport arcs.core.data.RawEntity\nimport arcs.core.data.util.toReferencable\nimport arcs.core.data.util.ReferencablePrimitive'}
+${this.opts.wasm ?
+      `import arcs.sdk.wasm.*` :
+      `\
+import arcs.sdk.Entity
+import arcs.core.data.*
+import arcs.core.data.util.toReferencable
+import arcs.core.data.util.ReferencablePrimitive
+import arcs.core.storage.api.toPrimitiveValue`}
 `;
   }
 
@@ -128,7 +135,7 @@ abstract class Abstract${particleName} : ${this.opts.wasm ? 'WasmParticleImpl' :
   }
 }
 
-class KotlinGenerator implements ClassGenerator {
+export class KotlinGenerator implements ClassGenerator {
   fields: string[] = [];
   fieldVals: string[] = [];
   setFields: string[] = [];
@@ -142,11 +149,13 @@ class KotlinGenerator implements ClassGenerator {
   fieldSerializes: string[] = [];
   fieldDeserializes: string[] = [];
   fieldsForToString: string[] = [];
+  singletonSchemaFields: string[] = [];
+  collectionSchemaFields: string[] = [];
 
   constructor(readonly node: SchemaNode, private readonly opts: minimist.ParsedArgs) {}
 
   // TODO: allow optional fields in kotlin
-  addField(field: string, typeChar: string, isOptional: boolean, refClassName: string|null) {
+  addField({field, typeChar, refClassName, isOptional = false, isCollection = false}: AddFieldOptions) {
     // TODO: support reference types in kotlin
     if (typeChar === 'R') return;
 
@@ -179,6 +188,46 @@ class KotlinGenerator implements ClassGenerator {
     this.fieldSerializes.push(`"${field}" to ${fixed}.toReferencable()`);
     this.fieldDeserializes.push(`${fixed} = data.singletons["${fixed}"].toPrimitiveValue(${type}::class, ${defaultVal})`);
     this.fieldsForToString.push(`${fixed} = $${fixed}`);
+    if (isCollection) {
+      this.collectionSchemaFields.push(`"${field}" to ${typeMap[typeChar].schemaType}`);
+    } else {
+      this.singletonSchemaFields.push(`"${field}" to ${typeMap[typeChar].schemaType}`);
+    }
+  }
+
+  mapOf(items: string[]): string {
+    switch (items.length) {
+      case 0:
+        return `emptyMap()`;
+      case 1:
+        return `mapOf(${items[0]})`;
+      default:
+        return `\
+mapOf(
+${this.leftPad(items.join(',\n'), 4)}
+)`;
+    }
+
+  }
+
+  createSchema(schemaHash: string): string {
+    const schemaNames = this.node.schema.names.map(n => `SchemaName("${n}")`);
+    return `\
+Schema(
+    listOf(${schemaNames.join(',\n' + ' '.repeat(8))}),
+    SchemaFields(
+        singletons = ${this.leftPad(this.mapOf(this.singletonSchemaFields), 8, true)},
+        collections = ${this.leftPad(this.mapOf(this.collectionSchemaFields), 8, true)}
+    ),
+    "${schemaHash}"
+)`;
+  }
+
+  leftPad(input: string, indent: number, skipFirst: boolean = false) {
+    return input
+      .split('\n')
+      .map((line: string, idx: number) => (idx === 0 && skipFirst) ? line : ' '.repeat(indent) + line)
+      .join('\n');
   }
 
   generate(schemaHash: string, fieldCount: number): string {
@@ -254,7 +303,18 @@ ${this.opts.wasm ? `
 }
 
 class ${name}_Spec() : ${this.getType('EntitySpec')}<${name}> {
+${this.opts.wasm ? '' : `\
 
+    companion object {
+        val schema = ${this.leftPad(this.createSchema(schemaHash), 8, true)}
+        
+        init {
+            SchemaRegistry.register(schema)
+        }
+    }
+    
+    override fun schema() = schema
+`}
     override fun create() = ${name}()
     ${!this.opts.wasm ? `
     override fun deserialize(data: RawEntity): ${name} {

src/tools/tests/goldens/generated-schemas.jvm.kt

@@ -9,10 +9,11 @@ package arcs.sdk
 // Current implementation doesn't support references or optional field detection
 
 import arcs.sdk.*
-import arcs.core.storage.api.toPrimitiveValue
-import arcs.core.data.RawEntity
+import arcs.sdk.Entity
+import arcs.core.data.*
 import arcs.core.data.util.toReferencable
 import arcs.core.data.util.ReferencablePrimitive
+import arcs.core.storage.api.toPrimitiveValue
 
 class GoldInternal1() : Entity {
 
@@ -71,6 +72,23 @@ class GoldInternal1() : Entity {
 
 class GoldInternal1_Spec() : EntitySpec<GoldInternal1> {
 
+    companion object {
+        val schema = Schema(
+            listOf(),
+            SchemaFields(
+                singletons = mapOf("val" to FieldType.Text),
+                collections = emptyMap()
+            ),
+            "485712110d89359a3e539dac987329cd2649d889"
+        )
+
+        init {
+            SchemaRegistry.register(schema)
+        }
+    }
+
+    override fun schema() = schema
+
     override fun create() = GoldInternal1()
 
     override fun deserialize(data: RawEntity): GoldInternal1 {
@@ -179,6 +197,28 @@ class Gold_Data() : Entity {
 
 class Gold_Data_Spec() : EntitySpec<Gold_Data> {
 
+    companion object {
+        val schema = Schema(
+            listOf(),
+            SchemaFields(
+                singletons = mapOf(
+                    "num" to FieldType.Number,
+                    "txt" to FieldType.Text,
+                    "lnk" to FieldType.Text,
+                    "flg" to FieldType.Boolean
+                ),
+                collections = emptyMap()
+            ),
+            "d8058d336e472da47b289eafb39733f77eadb111"
+        )
+
+        init {
+            SchemaRegistry.register(schema)
+        }
+    }
+
+    override fun schema() = schema
+
     override fun create() = Gold_Data()
 
     override fun deserialize(data: RawEntity): Gold_Data {

src/tools/tests/schema2kotlin-test.ts

@@ -0,0 +1,41 @@
+/**
+ * @license
+ * Copyright (c) 2020 Google Inc. All rights reserved.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt
+ * Code distributed by Google as part of this project is also
+ * subject to an additional IP rights grant found at
+ * http://polymer.github.io/PATENTS.txt
+ */
+
+
+import {assert} from '../../platform/chai-node.js';
+import {KotlinGenerator} from '../schema2kotlin.js';
+import {SchemaNode} from '../schema2graph.js';
+import {Schema} from '../../runtime/schema.js';
+
+
+describe('schema2wasm', () => {
+  describe('kotlin-generator', () => {
+    const ktGen = new KotlinGenerator(new SchemaNode(new Schema([], {}), 'dummyNode'), {arg: '', _: []});
+    it('when no items are present, it creates an empty map', () => {
+      const actual = ktGen.mapOf([]);
+
+      assert.strictEqual('emptyMap()', actual);
+    });
+    it('when one item is present, it creates a single-line map', () => {
+      const actual = ktGen.mapOf([`"a" to "b"`]);
+
+      assert.strictEqual('mapOf("a" to "b")', actual);
+    });
+    it('when multiple items are present, it creates a multi-line map', () => {
+      const actual = ktGen.mapOf([`"a" to "b"`, `"b" to "c"`]);
+
+      assert.strictEqual(`\
+mapOf(
+    "a" to "b",
+    "b" to "c"
+)`, actual);
+    });
+  });
+});

src/tools/tests/schema2wasm-test.ts

@@ -9,9 +9,8 @@
  */
 import {assert} from '../../platform/chai-web.js';
 import {Manifest} from '../../runtime/manifest.js';
-import {Schema} from '../../runtime/schema.js';
 import {Dictionary} from '../../runtime/hot.js';
-import {Schema2Base, ClassGenerator} from '../schema2base.js';
+import {Schema2Base, ClassGenerator, AddFieldOptions} from '../schema2base.js';
 import {SchemaNode} from '../schema2graph.js';
 import {Schema2Cpp} from '../schema2cpp.js';
 import {Schema2Kotlin} from '../schema2kotlin.js';
@@ -32,7 +31,7 @@ class Schema2Mock extends Schema2Base {
     const collector = {count: 0, adds: []};
     this.res[node.name] = collector;
     return {
-      addField(field: string, typeChar: string, isOptional: boolean, refClassName: string|null) {
+      addField({field, typeChar, isOptional, refClassName}: AddFieldOptions) {
         const refInfo = refClassName ? `<${refClassName}>` : '';
         collector.adds.push(field + ':' + typeChar + refInfo + (isOptional ? '?' : ''));
       },