Swift: split generated C++ files by directory

Author: redsun82

swift/codegen/generators/cppgen.py

@@ -12,6 +12,7 @@
 """
 
 import functools
+import pathlib
 from typing import Dict
 
 import inflection
@@ -70,12 +71,20 @@ def _get_class(self, name: str) -> cpp.Class:
         )
 
     def get_classes(self):
-        inheritance_graph = {k: cls.bases for k, cls in self._classmap.items()}
-        return [self._get_class(cls) for cls in toposort_flatten(inheritance_graph)]
+        grouped = {pathlib.Path(): {}}
+        for k, cls in self._classmap.items():
+            grouped.setdefault(cls.dir, {}).update({k: cls})
+        ret = {}
+        for dir, map in grouped.items():
+            inheritance_graph = {k: {b for b in cls.bases if b in map} for k, cls in map.items()}
+            ret[dir] = [self._get_class(cls) for cls in toposort_flatten(inheritance_graph)]
+        return ret
 
 
 def generate(opts, renderer):
     assert opts.cpp_output
     processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes})
     out = opts.cpp_output
-    renderer.render(cpp.ClassList(processor.get_classes(), opts.schema), out / "TrapClasses")
+    for dir, classes in processor.get_classes().items():
+        include_parent = (dir != pathlib.Path())
+        renderer.render(cpp.ClassList(classes, opts.schema, include_parent), out / dir / "TrapClasses")

swift/codegen/generators/dbschemegen.py

@@ -35,6 +35,7 @@ def cls_to_dbscheme(cls: schema.Class):
     """ Yield all dbscheme entities needed to model class `cls` """
     if cls.derived:
         yield Union(dbtype(cls.name), (dbtype(c) for c in cls.derived))
+    dir = cls.dir if cls.dir != pathlib.Path() else None
     # output a table specific to a class only if it is a leaf class or it has 1-to-1 properties
     # Leaf classes need a table to bind the `@` ids
     # 1-to-1 properties are added to a class specific table
@@ -49,7 +50,8 @@ def cls_to_dbscheme(cls: schema.Class):
                         Column("id", type=dbtype(cls.name), binding=binding),
                     ] + [
                         Column(f.name, dbtype(f.type)) for f in cls.properties if f.is_single
-                    ]
+                    ],
+            dir=dir,
         )
     # use property-specific tables for 1-to-many and 1-to-at-most-1 properties
     for f in cls.properties:
@@ -61,7 +63,8 @@ def cls_to_dbscheme(cls: schema.Class):
                     Column("id", type=dbtype(cls.name)),
                     Column("index", type="int"),
                     Column(inflection.singularize(f.name), dbtype(f.type)),
-                ]
+                ],
+                dir=dir,
             )
         elif f.is_optional:
             yield Table(
@@ -71,6 +74,7 @@ def cls_to_dbscheme(cls: schema.Class):
                     Column("id", type=dbtype(cls.name)),
                     Column(f.name, dbtype(f.type)),
                 ],
+                dir=dir,
             )
         elif f.is_predicate:
             yield Table(
@@ -79,6 +83,7 @@ def cls_to_dbscheme(cls: schema.Class):
                 columns=[
                     Column("id", type=dbtype(cls.name)),
                 ],
+                dir=dir,
             )
 
 

swift/codegen/generators/trapgen.py

@@ -12,6 +12,7 @@
 """
 
 import logging
+import pathlib
 
 import inflection
 from toposort import toposort_flatten
@@ -71,16 +72,18 @@ def generate(opts, renderer):
     tag_graph = {}
     out = opts.cpp_output
 
-    traps = []
+    traps = {pathlib.Path(): []}
     for e in dbscheme.iterload(opts.dbscheme):
         if e.is_table:
-            traps.append(get_trap(e))
+            traps.setdefault(e.dir, []).append(get_trap(e))
         elif e.is_union:
             tag_graph.setdefault(e.lhs, set())
             for d in e.rhs:
                 tag_graph.setdefault(d.type, set()).add(e.lhs)
 
-    renderer.render(cpp.TrapList(traps, opts.dbscheme), out / "TrapEntries")
+    for dir, entries in traps.items():
+        dir = dir or pathlib.Path()
+        renderer.render(cpp.TrapList(entries, opts.dbscheme), out / dir / "TrapEntries")
 
     tags = []
     for index, tag in enumerate(toposort_flatten(tag_graph)):

swift/codegen/lib/cpp.py

@@ -148,3 +148,4 @@ class ClassList:
 
     classes: List[Class]
     source: str
+    include_parent: bool = False

swift/codegen/lib/dbscheme.py

@@ -1,6 +1,7 @@
 """ dbscheme format representation """
 
 import logging
+import pathlib
 import re
 from dataclasses import dataclass
 from typing import ClassVar, List
@@ -64,6 +65,7 @@ class Table(Decl):
     name: str
     columns: List[Column]
     keyset: KeySet = None
+    dir: pathlib.Path = None
 
     def __post_init__(self):
         if self.columns:
@@ -108,13 +110,15 @@ class Scheme:
 class Re:
     entity = re.compile(
         "(?m)"
-        r"(?:^#keyset\[(?P<tablekeys>[\w\s,]+)\][\s\n]*)?^(?P<table>\w+)\((?P<tablebody>[^\)]*)\);?"
+        r"(?:^#keyset\[(?P<tablekeys>[\w\s,]+)\][\s\n]*)?^(?P<table>\w+)\("
+        r"(?:\s*//dir=(?P<tabledir>\S*))?(?P<tablebody>[^\)]*)"
+        r"\);?"
         "|"
         r"^(?P<union>@\w+)\s*=\s*(?P<unionbody>@\w+(?:\s*\|\s*@\w+)*)\s*;?"
     )
     field = re.compile(r"(?m)[\w\s]*\s(?P<field>\w+)\s*:\s*(?P<type>@?\w+)(?P<ref>\s+ref)?")
     key = re.compile(r"@\w+")
-    comment = re.compile(r"(?m)(?s)/\*.*?\*/|//[^\n]*$")
+    comment = re.compile(r"(?m)(?s)/\*.*?\*/|//(?!dir=)[^\n]*$")  # lookahead avoid ignoring metadata like //dir=foo
 
 
 def get_column(match):
@@ -133,6 +137,7 @@ def get_table(match):
         name=match["table"],
         columns=[get_column(f) for f in Re.field.finditer(match["tablebody"])],
         keyset=keyset,
+        dir=pathlib.PosixPath(match["tabledir"]) if match["tabledir"] else None,
     )
 
 

swift/codegen/templates/cpp_classes_h.mustache

@@ -9,6 +9,9 @@
 #include "swift/extractor/trap/TrapLabel.h"
 #include "swift/extractor/trap/TrapTagTraits.h"
 #include "./TrapEntries.h"
+{{#include_parent}}
+#include "../TrapClasses.h"
+{{/include_parent}}
 
 namespace codeql {
 {{#classes}}

swift/codegen/templates/dbscheme.mustache

@@ -18,7 +18,7 @@
 {{#keyset}}
 #keyset[{{#ids}}{{^first}}, {{/first}}{{id}}{{/ids}}]
 {{/keyset}}
-{{name}}({{#columns}}{{^first}},{{/first}}
+{{name}}({{#dir}}  //dir={{.}}{{/dir}}{{#columns}}{{^first}},{{/first}}
   {{lhstype}} {{name}}: {{rhstype}}{{/columns}}
 );
 {{/is_table}}

swift/codegen/templates/trap_traps_h.mustache

@@ -7,7 +7,7 @@
 
 #include "swift/extractor/trap/TrapLabel.h"
 #include "swift/extractor/trap/TrapTagTraits.h"
-#include "./TrapTags.h"
+#include "swift/extractor/trap/generated/TrapTags.h"
 
 namespace codeql {
 {{#traps}}

swift/codegen/test/test_cppgen.py

@@ -8,16 +8,27 @@
 
 
 @pytest.fixture
-def generate(opts, renderer, input):
+def generate_grouped(opts, renderer, input):
     opts.cpp_output = output_dir
 
     def ret(classes):
         input.classes = classes
         generated = run_generation(cppgen.generate, opts, renderer)
-        assert set(generated) == {output_dir / "TrapClasses"}
-        generated = generated[output_dir / "TrapClasses"]
-        assert isinstance(generated, cpp.ClassList)
-        return generated.classes
+        for f, g in generated.items():
+            assert isinstance(g, cpp.ClassList), f
+            assert g.include_parent is (f.parent != output_dir)
+            assert f.name == "TrapClasses", f
+        return {str(f.parent.relative_to(output_dir)): g.classes for f, g in generated.items()}
+
+    return ret
+
+
+@pytest.fixture
+def generate(generate_grouped):
+    def ret(classes):
+        generated = generate_grouped(classes)
+        assert set(generated) == {"."}
+        return generated["."]
 
     return ret
 
@@ -88,10 +99,12 @@ def test_class_with_field(generate, type, expected, property_cls, optional, repe
 
 def test_class_with_predicate(generate):
     assert generate([
-        schema.Class(name="MyClass", properties=[schema.PredicateProperty("prop")]),
+        schema.Class(name="MyClass", properties=[
+            schema.PredicateProperty("prop")]),
     ]) == [
                cpp.Class(name="MyClass",
-                         fields=[cpp.Field("prop", "bool", trap_name="MyClassProp", is_predicate=True)],
+                         fields=[
+                             cpp.Field("prop", "bool", trap_name="MyClassProp", is_predicate=True)],
                          trap_name="MyClasses",
                          final=True)
            ]
@@ -136,5 +149,21 @@ def test_class_with_keyword_field(generate, name):
            ]
 
 
+def test_classes_with_dirs(generate_grouped):
+    cbase = cpp.Class(name="CBase")
+    assert generate_grouped([
+        schema.Class(name="A"),
+        schema.Class(name="B", dir=pathlib.Path("foo")),
+        schema.Class(name="C", bases={"CBase"}, dir=pathlib.Path("bar")),
+        schema.Class(name="CBase", derived={"C"}, dir=pathlib.Path("bar")),
+        schema.Class(name="D", dir=pathlib.Path("foo/bar/baz")),
+    ]) == {
+               ".": [cpp.Class(name="A", trap_name="As", final=True)],
+               "foo": [cpp.Class(name="B", trap_name="Bs", final=True)],
+               "bar": [cbase, cpp.Class(name="C", bases=[cbase], trap_name="Cs", final=True)],
+               "foo/bar/baz": [cpp.Class(name="D", trap_name="Ds", final=True)],
+           }
+
+
 if __name__ == '__main__':
     sys.exit(pytest.main([__file__] + sys.argv[1:]))

swift/codegen/test/test_dbscheme.py

@@ -114,6 +114,18 @@ def test_load_table_with_multiple_columns(load):
     ]
 
 
+def test_load_table_with_multiple_columns_and_dir(load):
+    columns = ",\n".join(c for c, _ in expected_columns)
+    expected = [deepcopy(e) for _, e in expected_columns]
+    assert load(f"""
+foos(     //dir=foo/bar/baz
+{columns}
+);
+""") == [
+        dbscheme.Table(name="foos", columns=expected, dir=pathlib.Path("foo/bar/baz"))
+    ]
+
+
 def test_load_multiple_table_with_columns(load):
     tables = [f"table{i}({col});" for i, (col, _) in enumerate(expected_columns)]
     expected = [dbscheme.Table(name=f"table{i}", columns=[deepcopy(e)]) for i, (_, e) in enumerate(expected_columns)]