Merge pull request #9034 from redsun82/swift-cpp-gen

Swift: add structured C++ generated classes

Author: redsun82

swift/codegen/BUILD.bazel

@@ -1,5 +1,17 @@
 load("@swift_codegen_deps//:requirements.bzl", "requirement")
 
+filegroup(
+    name = "schema",
+    srcs = ["schema.yml"],
+    visibility = ["//swift:__subpackages__"],
+)
+
+filegroup(
+    name = "schema_includes",
+    srcs = glob(["*.dbscheme"]),
+    visibility = ["//swift:__subpackages__"],
+)
+
 py_binary(
     name = "codegen",
     srcs = glob(
@@ -15,6 +27,17 @@ py_binary(
 py_binary(
     name = "trapgen",
     srcs = ["trapgen.py"],
+    data = ["//swift/codegen/templates:trap"],
+    visibility = ["//swift:__subpackages__"],
+    deps = [
+        "//swift/codegen/lib",
+        requirement("toposort"),
+    ],
+)
+
+py_binary(
+    name = "cppgen",
+    srcs = ["cppgen.py"],
     data = ["//swift/codegen/templates:cpp"],
     visibility = ["//swift:__subpackages__"],
     deps = [

swift/codegen/cppgen.py

@@ -0,0 +1,69 @@
+import functools
+from typing import Dict
+
+import inflection
+from toposort import toposort_flatten
+
+from swift.codegen.lib import cpp, generator, schema
+
+
+def _get_type(t: str, trap_affix: str) -> str:
+    if t == "string":
+        return "std::string"
+    if t == "boolean":
+        return "bool"
+    if t[0].isupper():
+        return f"{trap_affix}Label<{t}Tag>"
+    return t
+
+
+def _get_field(cls: schema.Class, p: schema.Property, trap_affix: str) -> cpp.Field:
+    trap_name = None
+    if not p.is_single:
+        trap_name = inflection.pluralize(inflection.camelize(f"{cls.name}_{p.name}"))
+    args = dict(
+        name=p.name + ("_" if p.name in cpp.cpp_keywords else ""),
+        type=_get_type(p.type, trap_affix),
+        is_optional=p.is_optional,
+        is_repeated=p.is_repeated,
+        trap_name=trap_name,
+    )
+    args.update(cpp.get_field_override(p.name))
+    return cpp.Field(**args)
+
+
+class Processor:
+    def __init__(self, data: Dict[str, schema.Class], trap_affix: str):
+        self._classmap = data
+        self._trap_affix = trap_affix
+
+    @functools.lru_cache(maxsize=None)
+    def _get_class(self, name: str) -> cpp.Class:
+        cls = self._classmap[name]
+        trap_name = None
+        if not cls.derived or any(p.is_single for p in cls.properties):
+            trap_name = inflection.pluralize(cls.name)
+        return cpp.Class(
+            name=name,
+            bases=[self._get_class(b) for b in cls.bases],
+            fields=[_get_field(cls, p, self._trap_affix) for p in cls.properties],
+            final=not cls.derived,
+            trap_name=trap_name,
+        )
+
+    def get_classes(self):
+        inheritance_graph = {k: cls.bases for k, cls in self._classmap.items()}
+        return [self._get_class(cls) for cls in toposort_flatten(inheritance_graph)]
+
+
+def generate(opts, renderer):
+    processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes}, opts.trap_affix)
+    out = opts.cpp_output
+    renderer.render(cpp.ClassList(processor.get_classes(), opts.cpp_namespace, opts.trap_affix,
+                                  opts.cpp_include_dir), out / f"{opts.trap_affix}Classes.h")
+
+
+tags = ("cpp", "schema")
+
+if __name__ == "__main__":
+    generator.run()

swift/codegen/dbschemegen.py

@@ -38,37 +38,37 @@ def cls_to_dbscheme(cls: schema.Class):
         )
     # use property-specific tables for 1-to-many and 1-to-at-most-1 properties
     for f in cls.properties:
-        if f.is_optional:
+        if f.is_repeated:
             yield Table(
-                keyset=KeySet(["id"]),
+                keyset=KeySet(["id", "index"]),
                 name=inflection.tableize(f"{cls.name}_{f.name}"),
                 columns=[
                     Column("id", type=dbtype(cls.name)),
-                    Column(f.name, dbtype(f.type)),
-                ],
+                    Column("index", type="int"),
+                    Column(inflection.singularize(f.name), dbtype(f.type)),
+                ]
             )
-        elif f.is_repeated:
+        elif f.is_optional:
             yield Table(
-                keyset=KeySet(["id", "index"]),
+                keyset=KeySet(["id"]),
                 name=inflection.tableize(f"{cls.name}_{f.name}"),
                 columns=[
                     Column("id", type=dbtype(cls.name)),
-                    Column("index", type="int"),
-                    Column(inflection.singularize(f.name), dbtype(f.type)),
-                ]
+                    Column(f.name, dbtype(f.type)),
+                ],
             )
 
 
 def get_declarations(data: schema.Schema):
     return [d for cls in data.classes for d in cls_to_dbscheme(cls)]
 
 
-def get_includes(data: schema.Schema, include_dir: pathlib.Path):
+def get_includes(data: schema.Schema, include_dir: pathlib.Path, swift_dir: pathlib.Path):
     includes = []
     for inc in data.includes:
         inc = include_dir / inc
         with open(inc) as inclusion:
-            includes.append(SchemeInclude(src=inc.relative_to(paths.swift_dir), data=inclusion.read()))
+            includes.append(SchemeInclude(src=inc.relative_to(swift_dir), data=inclusion.read()))
     return includes
 
 
@@ -78,8 +78,8 @@ def generate(opts, renderer):
 
     data = schema.load(input)
 
-    dbscheme = Scheme(src=input.relative_to(paths.swift_dir),
-                      includes=get_includes(data, include_dir=input.parent),
+    dbscheme = Scheme(src=input.relative_to(opts.swift_dir),
+                      includes=get_includes(data, include_dir=input.parent, swift_dir=opts.swift_dir),
                       declarations=get_declarations(data))
 
     renderer.render(dbscheme, out)

swift/codegen/lib/cpp.py

@@ -1,3 +1,4 @@
+import re
 from dataclasses import dataclass, field
 from typing import List, ClassVar
 
@@ -14,13 +15,35 @@
                 "typeid", "typename", "union", "unsigned", "using", "virtual", "void", "volatile", "wchar_t", "while",
                 "xor", "xor_eq"}
 
+_field_overrides = [
+    (re.compile(r"(start|end)_(line|column)|index|width|num_.*"), {"type": "unsigned"}),
+    (re.compile(r"(.*)_"), lambda m: {"name": m[1]}),
+]
+
+
+def get_field_override(field: str):
+    for r, o in _field_overrides:
+        m = r.fullmatch(field)
+        if m:
+            return o(m) if callable(o) else o
+    return {}
+
 
 @dataclass
 class Field:
     name: str
     type: str
+    is_optional: bool = False
+    is_repeated: bool = False
+    trap_name: str = None
     first: bool = False
 
+    def __post_init__(self):
+        if self.is_optional:
+            self.type = f"std::optional<{self.type}>"
+        if self.is_repeated:
+            self.type = f"std::vector<{self.type}>"
+
     @property
     def cpp_name(self):
         if self.name in cpp_keywords:
@@ -36,6 +59,12 @@ def get_streamer(self):
         else:
             return lambda x: x
 
+    @property
+    def is_single(self):
+        return not (self.is_optional or self.is_repeated)
+
+
+
 
 @dataclass
 class Trap:
@@ -74,13 +103,55 @@ def has_bases(self):
 
 @dataclass
 class TrapList:
-    template: ClassVar = 'cpp_traps'
+    template: ClassVar = 'trap_traps'
 
-    traps: List[Trap] = field(default_factory=list)
+    traps: List[Trap]
+    namespace: str
+    trap_affix: str
+    include_dir: str
 
 
 @dataclass
 class TagList:
-    template: ClassVar = 'cpp_tags'
+    template: ClassVar = 'trap_tags'
+
+    tags: List[Tag]
+    namespace: str
+
+
+@dataclass
+class ClassBase:
+    ref: 'Class'
+    first: bool = False
+
+
+@dataclass
+class Class:
+    name: str
+    bases: List[ClassBase] = field(default_factory=list)
+    final: bool = False
+    fields: List[Field] = field(default_factory=list)
+    trap_name: str = None
+
+    def __post_init__(self):
+        self.bases = [ClassBase(c) for c in sorted(self.bases, key=lambda cls: cls.name)]
+        if self.bases:
+            self.bases[0].first = True
+
+    @property
+    def has_bases(self):
+        return bool(self.bases)
+
+    @property
+    def single_fields(self):
+        return [f for f in self.fields if f.is_single]
+
+
+@dataclass
+class ClassList:
+    template: ClassVar = "cpp_classes"
 
-    tags: List[Tag] = field(default_factory=list)
+    classes: List[Class]
+    namespace: str
+    trap_affix: str
+    include_dir: str

swift/codegen/lib/generator.py

@@ -3,26 +3,30 @@
 import argparse
 import logging
 import sys
+from typing import Set
 
-from . import options, render
+from . import options, render, paths
 
 
-def _parse(tags):
+def _parse(tags: Set[str]) -> argparse.Namespace:
     parser = argparse.ArgumentParser()
     for opt in options.get(tags):
         opt.add_to(parser)
-    ret = parser.parse_args()
-    log_level = logging.DEBUG if ret.verbose else logging.INFO
-    logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
-    return ret
+    return parser.parse_args()
 
 
-def run(*modules):
+def run(*modules, **kwargs):
     """ run generation functions in specified in `modules`, or in current module by default
     """
     if modules:
-        opts = _parse({t for m in modules for t in m.tags})
+        if kwargs:
+            opts = argparse.Namespace(**kwargs)
+        else:
+            opts = _parse({t for m in modules for t in m.tags})
+        log_level = logging.DEBUG if opts.verbose else logging.INFO
+        logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
+        exe_path = paths.exe_file.relative_to(opts.swift_dir)
         for m in modules:
-            m.generate(opts, render.Renderer())
+            m.generate(opts, render.Renderer(exe_path))
     else:
-        run(sys.modules["__main__"])
+        run(sys.modules["__main__"], **kwargs)

swift/codegen/lib/options.py

@@ -10,12 +10,18 @@
 
 def _init_options():
     Option("--verbose", "-v", action="store_true")
+    Option("--swift-dir", type=_abspath, default=paths.swift_dir)
     Option("--schema", tags=["schema"], type=_abspath, default=paths.swift_dir / "codegen/schema.yml")
     Option("--dbscheme", tags=["dbscheme"], type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme")
     Option("--ql-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
     Option("--ql-stub-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
+    Option("--ql-format", tags=["ql"], action="store_true", default=True)
+    Option("--no-ql-format", tags=["ql"], action="store_false", dest="ql_format")
     Option("--codeql-binary", tags=["ql"], default="codeql")
-    Option("--trap-output", tags=["trap"], type=_abspath, required=True)
+    Option("--cpp-output", tags=["cpp"], type=_abspath, required=True)
+    Option("--cpp-namespace", tags=["cpp"], default="codeql")
+    Option("--trap-affix", tags=["cpp"], default="Trap")
+    Option("--cpp-include-dir", tags=["cpp"], required=True)
 
 
 def _abspath(x):

swift/codegen/lib/paths.py

@@ -15,7 +15,4 @@
 lib_dir = swift_dir / 'codegen' / 'lib'
 templates_dir = swift_dir / 'codegen' / 'templates'
 
-try:
-    exe_file = pathlib.Path(sys.argv[0]).resolve().relative_to(swift_dir)
-except ValueError:
-    exe_file = pathlib.Path(sys.argv[0]).name
+exe_file = pathlib.Path(sys.argv[0]).resolve()

swift/codegen/lib/ql.py

@@ -8,7 +8,6 @@
 @dataclass
 class Param:
     param: str
-    type: str = None
     first: bool = False
 
 
@@ -19,15 +18,11 @@ class Property:
     tablename: str
     tableparams: List[Param]
     plural: str = None
-    params: List[Param] = field(default_factory=list)
     first: bool = False
     local_var: str = "x"
+    is_optional: bool = False
 
     def __post_init__(self):
-        if self.params:
-            self.params[0].first = True
-        while self.local_var in (p.param for p in self.params):
-            self.local_var += "_"
         assert self.tableparams
         if self.type_is_class:
             self.tableparams = [x if x != "result" else self.local_var for x in self.tableparams]
@@ -43,6 +38,10 @@ def indefinite_article(self):
     def type_is_class(self):
         return self.type[0].isupper()
 
+    @property
+    def is_repeated(self):
+        return bool(self.plural)
+
 
 @dataclass
 class Class: