recipe2plan 1/n: Macros, dummy generation, recipe resolution. (#4809)

## Summary
This PR introduces the shape of recipe2plan. Here, I create a sigh script and macros that specify how this code generation will fit with our build system. So far, the core of the feature, recipe2plan.ts, resolves recipes from the input manifest. This incremental step does not create or assign storage keys, as this depends on in-flight CLs from @mmandlis.

On deck are: fleshing out code-generation (Plan objects), increase test coverage, and assigning storage keys.

Down the road, we will: Adjust rules to output protos, delegating codegen to Kotlin / KotlinPoet.

## Changelog
* creating structure of ts r2p script

* Created fast CLI rapper for r2p script

* WIP figuring out ways to resolve a recipe

* fix sp

* Passing lint, still WIP

* Build macros for recipe2plan work

* Lightweight iteration on recipe2plan, stubbed out tests

* Revised method to find corresponding create handles

* Added type info

* Added method to get all handles to manifest + get all handles by Id

* fix: no flatMap

* - currently, fails since it cannot find associated stores for handles.

* uncleaned, but working recipe resolution

* Cleaned up for r2p, pt 1

* Fixed test, added TODO

* Fixed build rule, simplified runtime

* tools/sigh lint

* Impl suggestsions for r2p

* Improved build rules

* Removed flatMap, added TODOs and link to GH issue

* fix, bad comparison op

* implemented more review suggestions

* rm generator

* renamed to tryResolve

* nested unit tests

* updated test name

* added semicolon

Author: alxmrs

java/arcs/core/data/testdata/BUILD

@@ -1,5 +1,6 @@
 load(
     "//third_party/java/arcs/build_defs:build_defs.bzl",
+    "arcs_kt_plan",
     "arcs_manifest_proto",
 )
 
@@ -19,3 +20,9 @@ arcs_manifest_proto(
     src = "Example.arcs",
     visibility = ["//visibility:public"],
 )
+
+arcs_kt_plan(
+    name = "example_plan",
+    src = "Example.arcs",
+    visibility = ["//visibility:public"],
+)

src/runtime/manifest.ts

@@ -173,7 +173,10 @@ export class Manifest {
   get allRecipes() {
     return [...new Set(this._findAll(manifest => manifest._recipes))];
   }
-
+  get allHandles() {
+    // TODO(#4820) Update `reduce` to use flatMap
+    return this.allRecipes.reduce((acc, x) => acc.concat(x.handles), []);
+  }
   get activeRecipe() {
     return this._recipes.find(recipe => recipe.annotation === 'active');
   }
@@ -305,41 +308,58 @@ export class Manifest {
   findStoresByType(type: Type, options = {tags: <string[]>[], subtype: false}): UnifiedStore[] {
     const tags = options.tags || [];
     const subtype = options.subtype || false;
-    function typePredicate(store: UnifiedStore) {
-      const resolvedType = type.resolvedType();
-      if (!resolvedType.isResolved()) {
-        return (type instanceof CollectionType) === (store.type instanceof CollectionType) &&
-          (type instanceof BigCollectionType) === (store.type instanceof BigCollectionType);
-      }
-
-      if (subtype) {
-        const [left, right] = Type.unwrapPair(store.type, resolvedType);
-        if (left instanceof EntityType && right instanceof EntityType) {
-          return left.entitySchema.isAtleastAsSpecificAs(right.entitySchema);
-        }
-        return false;
-      }
-
-      return TypeChecker.compareTypes({type: store.type}, {type});
-    }
     function tagPredicate(manifest: Manifest, store: UnifiedStore) {
       return tags.filter(tag => !manifest.storeTags.get(store).includes(tag)).length === 0;
     }
-
-    const stores = [...this._findAll(manifest => manifest._stores.filter(store => typePredicate(store) && tagPredicate(manifest, store)))];
+    const stores = [...this._findAll(manifest =>
+      manifest._stores.filter(store => this.typesMatch(store, type, subtype) && tagPredicate(manifest, store)))];
 
     // Quick check that a new handle can fulfill the type contract.
     // Rewrite of this method tracked by https://github.com/PolymerLabs/arcs/issues/1636.
     return stores.filter(s => !!Handle.effectiveType(
       type, [{type: s.type, direction: (s.type instanceof InterfaceType) ? 'hosts' : 'reads writes'}]));
   }
+  findHandlesByType(type: Type, options = {tags: <string[]>[], fates: <string[]>[], subtype: false}): Handle[] {
+    const tags = options.tags || [];
+    const subtype = options.subtype || false;
+    const fates = options.fates || [];
+    function hasAllTags(handle: Handle) {
+      return tags.every(tag => handle.tags.includes(tag));
+    }
+    function matchesFate(handle: Handle) {
+      return fates === [] || fates.includes(handle.fate);
+    }
+    // TODO(#4820) Update `reduce` to use flatMap
+    return [...this.allRecipes
+      .reduce((acc, r) => acc.concat(r.handles), [])
+      .filter(h => this.typesMatch(h, type, subtype) && hasAllTags(h) && matchesFate(h))];
+  }
+  findHandlesById(id: string): Handle[] {
+    return this.allHandles.filter(h => h.id === id);
+  }
   findInterfaceByName(name: string) {
     return this._find(manifest => manifest._interfaces.find(iface => iface.name === name));
   }
-
   findRecipesByVerb(verb: string) {
     return [...this._findAll(manifest => manifest._recipes.filter(recipe => recipe.verbs.includes(verb)))];
   }
+  private typesMatch(candidate: {type: Type}, type: Type, checkSubtype: boolean) {
+    const resolvedType = type.resolvedType();
+    if (!resolvedType.isResolved()) {
+      return (type instanceof CollectionType) === (candidate.type instanceof CollectionType) &&
+        (type instanceof BigCollectionType) === (candidate.type instanceof BigCollectionType);
+    }
+
+    if (checkSubtype) {
+      const [left, right] = Type.unwrapPair(candidate.type, resolvedType);
+      if (left instanceof EntityType && right instanceof EntityType) {
+        return left.entitySchema.isAtleastAsSpecificAs(right.entitySchema);
+      }
+      return false;
+    }
+
+    return TypeChecker.compareTypes({type: candidate.type}, {type});
+  }
 
   generateID(subcomponent?: string): Id {
     return this.idGenerator.newChildId(this.id, subcomponent);

src/runtime/recipe/recipe-resolver.ts

@@ -211,7 +211,7 @@ export class RecipeResolver {
   // Attempts to run basic resolution on the given recipe. Returns a new
   // instance of the recipe normalized and resolved if possible. Returns null if
   // normalization or attempting to resolve slot connection fails.
-  async resolve(recipe: Recipe, options?: IsValidOptions) {
+  async resolve(recipe: Recipe, options?: IsValidOptions): Promise<Recipe | null> {
     recipe = recipe.clone();
     if (!recipe.normalize(options)) {
       console.warn(`could not normalize a recipe: ${

src/tools/recipe2plan-cli.ts

@@ -0,0 +1,73 @@
+/**
+ * @license
+ * Copyright (c) 2020 Google Inc. All rights reserved.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt
+ * Code distributed by Google as part of this project is also
+ * subject to an additional IP rights grant found at
+ * http://polymer.github.io/PATENTS.txt
+ */
+import minimist from 'minimist';
+import fs from 'fs';
+import path from 'path';
+import {Runtime} from '../runtime/runtime.js';
+import {recipe2plan} from './recipe2plan.js';
+
+const opts = minimist(process.argv.slice(2), {
+  string: ['outdir', 'outfile'],
+  alias: {d: 'outdir', f: 'outfile'},
+  default: {outdir: '.'}
+});
+
+if (opts.help || opts._.length === 0) {
+  console.log(`
+Usage
+  $ tools/sigh recipe2plan [options] path/to/manifest.arcs
+
+Description
+  Generates Kotlin plans from recipes in a manifest. 
+
+Options
+  --outfile, -f output filename; required
+  --outdir, -d  output directory; defaults to '.'
+  --help        usage info
+`);
+  process.exit(0);
+}
+
+if (!opts.outfile) {
+  console.error(`Parameter --outfile is required.`);
+  process.exit(1);
+}
+
+// TODO(alxr): Support generation from multiple manifests
+if (opts._.length > 1) {
+  console.error(`Only a single manifest is allowed`);
+  process.exit(1);
+}
+
+if (opts._.some((file) => !file.endsWith('.arcs'))) {
+  console.error(`Only Arcs manifests ('*.arcs') are allowed.`);
+  process.exit(1);
+}
+
+async function main() {
+  try {
+    Runtime.init('../..');
+    fs.mkdirSync(opts.outdir, {recursive: true});
+
+    const plans = await recipe2plan(opts._[0]);
+
+    const outPath = path.join(opts.outdir, opts.outfile);
+    console.log(outPath);
+
+    const outFile = fs.openSync(outPath, 'w');
+    fs.writeSync(outFile, plans);
+    fs.closeSync(outFile);
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+}
+
+void main();

src/tools/recipe2plan.ts

@@ -0,0 +1,42 @@
+/**
+ * @license
+ * Copyright 2020 Google LLC.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt
+ * Code distributed by Google as part of this project is also
+ * subject to an additional IP rights grant found at
+ * http://polymer.github.io/PATENTS.txt
+ */
+import {Runtime} from '../runtime/runtime.js';
+import {Recipe} from '../runtime/recipe/recipe.js';
+import {StorageKeyRecipeResolver} from './storage-key-recipe-resolver.js';
+
+
+/**
+ * Generates Kotlin Plans from recipes in an arcs manifest.
+ *
+ * @param path path/to/manifest.arcs
+ * @return Generated Kotlin code.
+ */
+export async function recipe2plan(path: string): Promise<string> {
+  const manifest = await Runtime.parseFile(path);
+
+  const recipes = await (new StorageKeyRecipeResolver(manifest)).resolve();
+
+  const plans = await generatePlans(recipes);
+
+  return plans.join('\n');
+}
+
+
+/**
+ * Converts each resolved recipes into a Kotlin Plan class.
+ *
+ * @param resolutions A series of resolved recipes.
+ * @return List of generated Kotlin plans
+ */
+async function generatePlans(resolutions: Recipe[]): Promise<string[]> {
+  // TODO Implement
+  return [''];
+}
+

src/tools/storage-key-recipe-resolver.ts

@@ -0,0 +1,128 @@
+/**
+ * @license
+ * Copyright 2020 Google LLC.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt
+ * Code distributed by Google as part of this project is also
+ * subject to an additional IP rights grant found at
+ * http://polymer.github.io/PATENTS.txt
+ */
+import {Runtime} from '../runtime/runtime.js';
+import {Manifest} from '../runtime/manifest.js';
+import {Loader} from '../platform/loader-web.js';
+import {IsValidOptions, Recipe, RecipeComponent} from '../runtime/recipe/recipe.js';
+import {ramDiskStorageKeyPrefixForTest} from '../runtime/testing/handle-for-test.js';
+import {Arc} from '../runtime/arc.js';
+import {RecipeResolver} from '../runtime/recipe/recipe-resolver.js';
+import {CapabilitiesResolver} from '../runtime/capabilities-resolver.js';
+import {Store} from '../runtime/storageNG/store.js';
+import {Exists} from '../runtime/storageNG/drivers/driver.js';
+
+/**
+ * Responsible for resolving recipes with storage keys.
+ */
+export class StorageKeyRecipeResolver {
+  private readonly runtime: Runtime;
+
+  constructor(context: Manifest) {
+    const loader = new Loader();
+    this.runtime = new Runtime({loader, context});
+  }
+
+  /**
+   * Produces resolved recipes with storage keys.
+   *
+   * TODO(alxr): Apply to long-running recipes appropriately.
+   * @throws Error if recipe fails to resolve on first or second pass.
+   * @yields Resolved recipes with storage keys
+   */
+  async resolve(): Promise<Recipe[]> {
+    const recipes = [];
+    for (const recipe of this.runtime.context.allRecipes) {
+      const arc = this.runtime.newArc(this.getArcId(recipe), ramDiskStorageKeyPrefixForTest());
+      const opts = {errors: new Map<Recipe | RecipeComponent, string>()};
+      const resolved = await this.tryResolve(recipe, arc, opts);
+      if (!resolved) {
+        throw Error(`Recipe ${recipe.name} failed to resolve:\n${[...opts.errors.values()].join('\n')}`);
+      }
+      this.createStoresForCreateHandles(resolved, arc);
+      if (!resolved.isResolved()) {
+        throw Error(`Recipe ${resolved.name} did not properly resolve!\n${resolved.toString({showUnresolved: true})}`);
+      }
+      recipes.push(resolved);
+    }
+    return recipes;
+  }
+
+  /**
+   * Resolves unresolved recipe or normalizes resolved recipe.
+   *
+   * @param recipe long-running or ephemeral recipe
+   * @param arc Arc is associated with input recipe
+   * @param opts contains `errors` map for reporting.
+   */
+  async tryResolve(recipe: Recipe, arc: Arc, opts?: IsValidOptions): Promise<Recipe | null> {
+    const normalized = recipe.clone();
+    normalized.normalize();
+    if (normalized.isResolved()) return normalized;
+
+    return await (new RecipeResolver(arc).resolve(recipe, opts));
+  }
+
+  /** Returns the arcId from annotations on the recipe when present. */
+  getArcId(recipe: Recipe): string | null {
+    for (const trigger of recipe.triggers) {
+      for (const [key, val] of trigger) {
+        if (key === 'arcId') {
+          return val;
+        }
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Create stores with keys for all create handles.
+   *
+   * @param recipe should be long running.
+   * @param arc Arc is associated with current recipe.
+   */
+  createStoresForCreateHandles(recipe: Recipe, arc: Arc) {
+    const resolver = new CapabilitiesResolver({arcId: arc.id});
+    for (const createHandle of recipe.handles.filter(h => h.fate === 'create')) {
+      const storageKey = ramDiskStorageKeyPrefixForTest()(arc.id); // TODO(#4818) create the storage keys.
+      const store = new Store({storageKey, exists: Exists.MayExist, type: createHandle.type, id: createHandle.id});
+      arc.context.registerStore(store, createHandle.tags);
+    }
+  }
+
+  /**
+   * TODO(#4818) method to match `map` and `copy` fated handles with storage keys from `create` handles.
+   *
+   * @throws when a mapped handle is associated with too many stores (ambiguous mapping).
+   * @throws when a mapped handle isn't associated with any store (no matches found).
+   * @throws when handle is mapped to a handle from an ephemeral recipe.
+   * @param recipe long-running or ephemeral recipe
+   */
+  matchKeysToHandles(recipe: Recipe) {
+    recipe.handles
+      .filter(h => h.fate === 'map' || h.fate === 'copy')
+      .forEach(handle => {
+        const matches = this.runtime.context.findHandlesById(handle.id)
+          .filter(h => h.fate === 'create');
+
+        if (matches.length === 0) {
+          throw Error(`No matching handles found for ${handle.localName}.`);
+        } else if (matches.length > 1) {
+          throw Error(`More than one handle found for ${handle.localName}.`);
+        }
+
+        const match = matches[0];
+        if (!match.recipe.isLongRunning) {
+          throw Error(`Handle ${handle.localName} mapped to ephemeral handle ${match.localName}.`);
+        }
+
+        handle.storageKey = match.storageKey;
+      });
+  }
+}