Merge pull request #9685 from github/redsun82/swift-code-reorg

Swift: split up generated C++ code

Author: redsun82

.github/workflows/swift-codegen.yml

@@ -29,8 +29,8 @@ jobs:
           extra_args: swift-codegen --all-files
       - name: Generate C++ files
         run: |
-          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-headers
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-cpp-files
       - uses: actions/upload-artifact@v3
         with:
-          name: swift-generated-headers
-          path: swift-generated-headers/*.h
+          name: swift-generated-cpp-files
+          path: swift-generated-cpp-files/**

swift/codegen/generators/cppgen.py

@@ -12,6 +12,7 @@
 """
 
 import functools
+import pathlib
 from typing import Dict
 
 import inflection
@@ -70,12 +71,20 @@ def _get_class(self, name: str) -> cpp.Class:
         )
 
     def get_classes(self):
-        inheritance_graph = {k: cls.bases for k, cls in self._classmap.items()}
-        return [self._get_class(cls) for cls in toposort_flatten(inheritance_graph)]
+        grouped = {pathlib.Path(): {}}
+        for k, cls in self._classmap.items():
+            grouped.setdefault(cls.dir, {}).update({k: cls})
+        ret = {}
+        for dir, map in grouped.items():
+            inheritance_graph = {k: {b for b in cls.bases if b in map} for k, cls in map.items()}
+            ret[dir] = [self._get_class(cls) for cls in toposort_flatten(inheritance_graph)]
+        return ret
 
 
 def generate(opts, renderer):
     assert opts.cpp_output
     processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes})
     out = opts.cpp_output
-    renderer.render(cpp.ClassList(processor.get_classes(), opts.schema), out / f"TrapClasses.h")
+    for dir, classes in processor.get_classes().items():
+        include_parent = (dir != pathlib.Path())
+        renderer.render(cpp.ClassList(classes, opts.schema, include_parent), out / dir / "TrapClasses")

swift/codegen/generators/dbschemegen.py

@@ -35,6 +35,7 @@ def cls_to_dbscheme(cls: schema.Class):
     """ Yield all dbscheme entities needed to model class `cls` """
     if cls.derived:
         yield Union(dbtype(cls.name), (dbtype(c) for c in cls.derived))
+    dir = cls.dir if cls.dir != pathlib.Path() else None
     # output a table specific to a class only if it is a leaf class or it has 1-to-1 properties
     # Leaf classes need a table to bind the `@` ids
     # 1-to-1 properties are added to a class specific table
@@ -49,7 +50,8 @@ def cls_to_dbscheme(cls: schema.Class):
                 Column("id", type=dbtype(cls.name), binding=binding),
             ] + [
                 Column(f.name, dbtype(f.type)) for f in cls.properties if f.is_single
-            ]
+            ],
+            dir=dir,
         )
     # use property-specific tables for 1-to-many and 1-to-at-most-1 properties
     for f in cls.properties:
@@ -61,7 +63,8 @@ def cls_to_dbscheme(cls: schema.Class):
                     Column("id", type=dbtype(cls.name)),
                     Column("index", type="int"),
                     Column(inflection.singularize(f.name), dbtype(f.type)),
-                ]
+                ],
+                dir=dir,
             )
         elif f.is_optional:
             yield Table(
@@ -71,6 +74,7 @@ def cls_to_dbscheme(cls: schema.Class):
                     Column("id", type=dbtype(cls.name)),
                     Column(f.name, dbtype(f.type)),
                 ],
+                dir=dir,
             )
         elif f.is_predicate:
             yield Table(
@@ -79,6 +83,7 @@ def cls_to_dbscheme(cls: schema.Class):
                 columns=[
                     Column("id", type=dbtype(cls.name)),
                 ],
+                dir=dir,
             )
 
 

swift/codegen/generators/trapgen.py

@@ -12,6 +12,7 @@
 """
 
 import logging
+import pathlib
 
 import inflection
 from toposort import toposort_flatten
@@ -71,17 +72,18 @@ def generate(opts, renderer):
     tag_graph = {}
     out = opts.cpp_output
 
-    traps = []
+    traps = {pathlib.Path(): []}
     for e in dbscheme.iterload(opts.dbscheme):
         if e.is_table:
-            traps.append(get_trap(e))
+            traps.setdefault(e.dir, []).append(get_trap(e))
         elif e.is_union:
             tag_graph.setdefault(e.lhs, set())
             for d in e.rhs:
                 tag_graph.setdefault(d.type, set()).add(e.lhs)
 
-    renderer.render(cpp.TrapList(traps, opts.dbscheme),
-                    out / f"TrapEntries.h")
+    for dir, entries in traps.items():
+        dir = dir or pathlib.Path()
+        renderer.render(cpp.TrapList(entries, opts.dbscheme), out / dir / "TrapEntries")
 
     tags = []
     for index, tag in enumerate(toposort_flatten(tag_graph)):
@@ -91,4 +93,4 @@ def generate(opts, renderer):
             index=index,
             id=tag,
         ))
-    renderer.render(cpp.TagList(tags, opts.dbscheme), out / f"TrapTags.h")
+    renderer.render(cpp.TagList(tags, opts.dbscheme), out / "TrapTags")

swift/codegen/lib/cpp.py

@@ -99,14 +99,15 @@ def has_bases(self):
 @dataclass
 class TrapList:
     template: ClassVar = 'trap_traps'
-
+    extensions = ["h", "cpp"]
     traps: List[Trap]
     source: str
 
 
 @dataclass
 class TagList:
     template: ClassVar = 'trap_tags'
+    extensions = ["h"]
 
     tags: List[Tag]
     source: str
@@ -143,6 +144,8 @@ def single_fields(self):
 @dataclass
 class ClassList:
     template: ClassVar = "cpp_classes"
+    extensions: ClassVar = ["h", "cpp"]
 
     classes: List[Class]
     source: str
+    include_parent: bool = False

swift/codegen/lib/dbscheme.py

@@ -1,6 +1,7 @@
 """ dbscheme format representation """
 
 import logging
+import pathlib
 import re
 from dataclasses import dataclass
 from typing import ClassVar, List
@@ -64,6 +65,7 @@ class Table(Decl):
     name: str
     columns: List[Column]
     keyset: KeySet = None
+    dir: pathlib.Path = None
 
     def __post_init__(self):
         if self.columns:
@@ -108,13 +110,15 @@ class Scheme:
 class Re:
     entity = re.compile(
         "(?m)"
-        r"(?:^#keyset\[(?P<tablekeys>[\w\s,]+)\][\s\n]*)?^(?P<table>\w+)\((?P<tablebody>[^\)]*)\);?"
+        r"(?:^#keyset\[(?P<tablekeys>[\w\s,]+)\][\s\n]*)?^(?P<table>\w+)\("
+        r"(?:\s*//dir=(?P<tabledir>\S*))?(?P<tablebody>[^\)]*)"
+        r"\);?"
         "|"
         r"^(?P<union>@\w+)\s*=\s*(?P<unionbody>@\w+(?:\s*\|\s*@\w+)*)\s*;?"
     )
     field = re.compile(r"(?m)[\w\s]*\s(?P<field>\w+)\s*:\s*(?P<type>@?\w+)(?P<ref>\s+ref)?")
     key = re.compile(r"@\w+")
-    comment = re.compile(r"(?m)(?s)/\*.*?\*/|//[^\n]*$")
+    comment = re.compile(r"(?m)(?s)/\*.*?\*/|//(?!dir=)[^\n]*$")  # lookahead avoid ignoring metadata like //dir=foo
 
 
 def get_column(match):
@@ -133,6 +137,7 @@ def get_table(match):
         name=match["table"],
         columns=[get_column(f) for f in Re.field.finditer(match["tablebody"])],
         keyset=keyset,
+        dir=pathlib.PosixPath(match["tabledir"]) if match["tabledir"] else None,
     )
 
 

swift/codegen/lib/render.py

@@ -28,18 +28,23 @@ def render(self, data, output: pathlib.Path):
 
         `data` must have a `template` attribute denoting which template to use from the template directory.
 
-        If the file is unchanged, then no write is performed (and `done_something` remains unchanged)
-
-        If `guard_base` is provided, it must be a path at the root of `output` and a header guard will be injected in
-        the template based off of the relative path of `output` in `guard_base`
+        Optionally, `data` can also have an `extensions` attribute denoting list of file extensions: they will all be
+        appended to the template name with an underscore and be generated in turn.
         """
         mnemonic = type(data).__name__
         output.parent.mkdir(parents=True, exist_ok=True)
-        data = self._r.render_name(data.template, data, generator=self._generator)
-        with open(output, "w") as out:
-            out.write(data)
-        log.debug(f"generated {mnemonic} {output.name}")
-        self.written.add(output)
+        extensions = getattr(data, "extensions", [None])
+        for ext in extensions:
+            output_filename = output
+            template = data.template
+            if ext:
+                output_filename = output_filename.with_suffix(f".{ext}")
+                template += f"_{ext}"
+            contents = self._r.render_name(template, data, generator=self._generator)
+            with open(output_filename, "w") as out:
+                out.write(contents)
+            log.debug(f"{mnemonic}: generated {output.name}")
+            self.written.add(output_filename)
 
     def cleanup(self, existing):
         """ Remove files in `existing` for which no `render` has been called """

swift/codegen/templates/cpp_classes_cpp.mustache

@@ -0,0 +1,37 @@
+// generated by {{generator}} from {{source}}
+// clang-format off
+#include "./TrapClasses.h"
+
+namespace codeql {
+{{#classes}}
+
+void {{name}}::emit({{^final}}TrapLabel<{{name}}Tag> id, {{/final}}std::ostream& out) const {
+  {{#trap_name}}
+  out << {{.}}Trap{id{{#single_fields}}, {{field_name}}{{/single_fields}}} << '\n';
+  {{/trap_name}}
+  {{#bases}}
+  {{ref.name}}::emit(id, out);
+  {{/bases}}
+  {{#fields}}
+  {{#is_predicate}}
+  if ({{field_name}}) out << {{trap_name}}Trap{id} << '\n';
+  {{/is_predicate}}
+  {{#is_optional}}
+  {{^is_repeated}}
+  if ({{field_name}}) out << {{trap_name}}Trap{id, *{{field_name}}} << '\n';
+  {{/is_repeated}}
+  {{/is_optional}}
+  {{#is_repeated}}
+  for (auto i = 0u; i < {{field_name}}.size(); ++i) {
+    {{^is_optional}}
+    out << {{trap_name}}Trap{id, i, {{field_name}}[i]};
+    {{/is_optional}}
+    {{#is_optional}}
+    if ({{field_name}}[i]) out << {{trap_name}}Trap{id, i, *{{field_name}}[i]};
+    {{/is_optional}}
+  }
+  {{/is_repeated}}
+  {{/fields}}
+}
+{{/classes}}
+}

swift/codegen/templates/cpp_classes.mustache renamed to swift/codegen/templates/cpp_classes_h.mustache

@@ -9,6 +9,9 @@
 #include "swift/extractor/trap/TrapLabel.h"
 #include "swift/extractor/trap/TrapTagTraits.h"
 #include "./TrapEntries.h"
+{{#include_parent}}
+#include "../TrapClasses.h"
+{{/include_parent}}
 
 namespace codeql {
 {{#classes}}
@@ -31,34 +34,7 @@ struct {{name}}{{#has_bases}} : {{#bases}}{{^first}}, {{/first}}{{ref.name}}{{/b
   {{/final}}
 
  protected:
-  void emit({{^final}}TrapLabel<{{name}}Tag> id, {{/final}}std::ostream& out) const {
-    {{#trap_name}}
-    out << {{.}}Trap{id{{#single_fields}}, {{field_name}}{{/single_fields}}} << '\n';
-    {{/trap_name}}
-    {{#bases}}
-    {{ref.name}}::emit(id, out);
-    {{/bases}}
-    {{#fields}}
-    {{#is_predicate}}
-    if ({{field_name}}) out << {{trap_name}}Trap{id} << '\n';
-    {{/is_predicate}}
-    {{#is_optional}}
-    {{^is_repeated}}
-    if ({{field_name}}) out << {{trap_name}}Trap{id, *{{field_name}}} << '\n';
-    {{/is_repeated}}
-    {{/is_optional}}
-    {{#is_repeated}}
-    for (auto i = 0u; i < {{field_name}}.size(); ++i) {
-      {{^is_optional}}
-      out << {{trap_name}}Trap{id, i, {{field_name}}[i]};
-      {{/is_optional}}
-      {{#is_optional}}
-      if ({{field_name}}[i]) out << {{trap_name}}Trap{id, i, *{{field_name}}[i]};
-      {{/is_optional}}
-    }
-    {{/is_repeated}}
-    {{/fields}}
-  }
+  void emit({{^final}}TrapLabel<{{name}}Tag> id, {{/final}}std::ostream& out) const;
 };
 
 template <>

swift/codegen/templates/dbscheme.mustache

@@ -18,7 +18,7 @@
 {{#keyset}}
 #keyset[{{#ids}}{{^first}}, {{/first}}{{id}}{{/ids}}]
 {{/keyset}}
-{{name}}({{#columns}}{{^first}},{{/first}}
+{{name}}({{#dir}}  //dir={{.}}{{/dir}}{{#columns}}{{^first}},{{/first}}
   {{lhstype}} {{name}}: {{rhstype}}{{/columns}}
 );
 {{/is_table}}

swift/codegen/templates/trap_tags.mustache renamed to swift/codegen/templates/trap_tags_h.mustache

@@ -0,0 +1,15 @@
+// generated by {{generator}} from {{source}}
+// clang-format off
+#include "./TrapEntries.h"
+
+namespace codeql {
+{{#traps}}
+
+// {{table_name}}
+std::ostream &operator<<(std::ostream &out, const {{name}}Trap &e) {
+  out << "{{table_name}}("{{#fields}}{{^first}} << ", "{{/first}}
+      << {{#get_streamer}}e.{{field_name}}{{/get_streamer}}{{/fields}} << ")";
+  return out;
+}
+{{/traps}}
+}

swift/codegen/templates/trap_traps_cpp.mustache

@@ -7,28 +7,19 @@
 
 #include "swift/extractor/trap/TrapLabel.h"
 #include "swift/extractor/trap/TrapTagTraits.h"
-#include "./TrapTags.h"
+#include "swift/extractor/trap/generated/TrapTags.h"
 
 namespace codeql {
 {{#traps}}
 
 // {{table_name}}
 struct {{name}}Trap {
-  static constexpr bool is_binding = {{#id}}true{{/id}}{{^id}}false{{/id}};
-{{#id}}
-  {{type}} getBoundLabel() const { return {{field_name}}; }
-{{/id}}
-
 {{#fields}}
   {{type}} {{field_name}}{};
 {{/fields}}
 };
 
-inline std::ostream &operator<<(std::ostream &out, const {{name}}Trap &e) {
-  out << "{{table_name}}("{{#fields}}{{^first}} << ", "{{/first}}
-      << {{#get_streamer}}e.{{field_name}}{{/get_streamer}}{{/fields}} << ")";
-  return out;
-}
+std::ostream &operator<<(std::ostream &out, const {{name}}Trap &e);
 {{#id}}
 
 namespace detail {