Swift: use IPA layer in generated classes

Author: redsun82

swift/codegen/generators/cppgen.py

@@ -83,7 +83,7 @@ def get_classes(self):
 
 def generate(opts, renderer):
     assert opts.cpp_output
-    processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes})
+    processor = Processor(schema.load(opts.schema).classes)
     out = opts.cpp_output
     for dir, classes in processor.get_classes().items():
         include_parent = (dir != pathlib.Path())

swift/codegen/generators/dbschemegen.py

@@ -88,7 +88,7 @@ def cls_to_dbscheme(cls: schema.Class):
 
 
 def get_declarations(data: schema.Schema):
-    return [d for cls in data.classes for d in cls_to_dbscheme(cls)]
+    return [d for cls in data.classes.values() for d in cls_to_dbscheme(cls)]
 
 
 def get_includes(data: schema.Schema, include_dir: pathlib.Path, swift_dir: pathlib.Path):

swift/codegen/generators/qlgen.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# TODO this should probably be split in different generators now: ql, qltest, maybe qlipa
 
 import logging
 import pathlib
@@ -8,6 +9,7 @@
 import itertools
 
 import inflection
+from toposort import toposort_flatten
 
 from swift.codegen.lib import schema, ql
 
@@ -27,55 +29,55 @@ class ModifiedStubMarkedAsGeneratedError(Error):
     pass
 
 
-def get_ql_property(cls: schema.Class, prop: schema.Property):
-    common_args = dict(
+def get_ql_property(cls: schema.Class, source: schema.Class, prop: schema.Property) -> ql.Property:
+    args = dict(
         type=prop.type if not prop.is_predicate else "predicate",
         qltest_skip="qltest_skip" in prop.pragmas,
         is_child=prop.is_child,
         is_optional=prop.is_optional,
         is_predicate=prop.is_predicate,
     )
     if prop.is_single:
-        return ql.Property(
-            **common_args,
+        args.update(
             singular=inflection.camelize(prop.name),
-            tablename=inflection.tableize(cls.name),
-            tableparams=[
-                "this"] + ["result" if p is prop else "_" for p in cls.properties if p.is_single],
+            tablename=inflection.tableize(source.name),
+            tableparams=["this"] + ["result" if p is prop else "_" for p in source.properties if p.is_single],
         )
     elif prop.is_repeated:
-        return ql.Property(
-            **common_args,
+        args.update(
             singular=inflection.singularize(inflection.camelize(prop.name)),
             plural=inflection.pluralize(inflection.camelize(prop.name)),
-            tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
+            tablename=inflection.tableize(f"{source.name}_{prop.name}"),
             tableparams=["this", "index", "result"],
         )
     elif prop.is_optional:
-        return ql.Property(
-            **common_args,
+        args.update(
             singular=inflection.camelize(prop.name),
-            tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
+            tablename=inflection.tableize(f"{source.name}_{prop.name}"),
             tableparams=["this", "result"],
         )
     elif prop.is_predicate:
-        return ql.Property(
-            **common_args,
-            singular=inflection.camelize(
-                prop.name, uppercase_first_letter=False),
-            tablename=inflection.underscore(f"{cls.name}_{prop.name}"),
+        args.update(
+            singular=inflection.camelize(prop.name, uppercase_first_letter=False),
+            tablename=inflection.underscore(f"{source.name}_{prop.name}"),
             tableparams=["this"],
         )
+    else:
+        raise ValueError(f"unknown property kind for {prop.name} from {source.name}")
+    return ql.Property(**args)
 
 
-def get_ql_class(cls: schema.Class):
+def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]):
     pragmas = {k: True for k in cls.pragmas if k.startswith("ql")}
     return ql.Class(
         name=cls.name,
         bases=cls.bases,
         final=not cls.derived,
-        properties=[get_ql_property(cls, p) for p in cls.properties],
+
+        properties=[get_ql_property(cls, cls, p) for p in cls.properties],
         dir=cls.dir,
+        has_db_id=not cls.ipa or cls.ipa.from_class,
+        ipa=bool(cls.ipa),
         **pragmas,
     )
 
@@ -92,11 +94,11 @@ def _to_db_type(x: str) -> str:
 def get_ql_ipa_class(cls: schema.Class):
     if cls.derived:
         return ql.Ipa.NonFinalClass(name=cls.name, derived=sorted(cls.derived))
-    if cls.ipa and cls.ipa.from_class:
+    if cls.ipa and cls.ipa.from_class is not None:
         source = cls.ipa.from_class
         _final_db_class_lookup.setdefault(source, ql.Ipa.FinalClassDb(source)).subtract_type(cls.name)
         return ql.Ipa.FinalClassIpaFrom(name=cls.name, type=_to_db_type(source))
-    if cls.ipa and cls.ipa.on_arguments:
+    if cls.ipa and cls.ipa.on_arguments is not None:
         return ql.Ipa.FinalClassIpaOn(name=cls.name,
                                       params=[ql.Ipa.Param(k, _to_db_type(v)) for k, v in cls.ipa.on_arguments.items()])
     return _final_db_class_lookup.setdefault(cls.name, ql.Ipa.FinalClassDb(cls.name))
@@ -136,7 +138,6 @@ def _is_generated_stub(file):
         line_threshold = 5
         first_lines = list(itertools.islice(contents, line_threshold))
         if len(first_lines) == line_threshold or not _generated_stub_re.match("".join(first_lines)):
-            print("".join(first_lines))
             raise ModifiedStubMarkedAsGeneratedError(
                 f"{file.name} stub was modified but is still marked as generated")
         return True
@@ -154,23 +155,28 @@ def format(codeql, files):
         log.debug(line.strip())
 
 
-def _get_all_properties(cls: ql.Class, lookup: typing.Dict[str, ql.Class]) -> typing.Iterable[
-        typing.Tuple[ql.Class, ql.Property]]:
-    for b in cls.bases:
+def _get_all_properties(cls: schema.Class, lookup: typing.Dict[str, schema.Class],
+                        already_seen: typing.Optional[typing.Set[int]] = None) -> \
+        typing.Iterable[typing.Tuple[schema.Class, schema.Property]]:
+    # deduplicate using ids
+    if already_seen is None:
+        already_seen = set()
+    for b in sorted(cls.bases):
         base = lookup[b]
-        for item in _get_all_properties(base, lookup):
+        for item in _get_all_properties(base, lookup, already_seen):
             yield item
     for p in cls.properties:
-        yield cls, p
+        if id(p) not in already_seen:
+            already_seen.add(id(p))
+            yield cls, p
 
 
-def _get_all_properties_to_be_tested(cls: ql.Class, lookup: typing.Dict[str, ql.Class]) -> typing.Iterable[
-        ql.PropertyForTest]:
-    # deduplicate using id
-    already_seen = set()
+def _get_all_properties_to_be_tested(cls: schema.Class, lookup: typing.Dict[str, schema.Class]) -> \
+        typing.Iterable[ql.PropertyForTest]:
     for c, p in _get_all_properties(cls, lookup):
-        if not (c.qltest_skip or p.qltest_skip or id(p) in already_seen):
-            already_seen.add(id(p))
+        if not ("qltest_skip" in c.pragmas or "qltest_skip" in p.pragmas):
+            # TODO here operations are duplicated, but should be better if we split ql and qltest generation
+            p = get_ql_property(cls, c, p)
             yield ql.PropertyForTest(p.getter, p.type, p.is_single, p.is_predicate, p.is_repeated)
 
 
@@ -184,17 +190,18 @@ def _partition(l, pred):
     return map(list, _partition_iter(l, pred))
 
 
-def _is_in_qltest_collapsed_hierachy(cls: ql.Class, lookup: typing.Dict[str, ql.Class]):
-    return cls.qltest_collapse_hierarchy or _is_under_qltest_collapsed_hierachy(cls, lookup)
+def _is_in_qltest_collapsed_hierachy(cls: schema.Class, lookup: typing.Dict[str, schema.Class]):
+    return "qltest_collapse_hierarchy" in cls.pragmas or _is_under_qltest_collapsed_hierachy(cls, lookup)
 
 
-def _is_under_qltest_collapsed_hierachy(cls: ql.Class, lookup: typing.Dict[str, ql.Class]):
-    return not cls.qltest_uncollapse_hierarchy and any(
+def _is_under_qltest_collapsed_hierachy(cls: schema.Class, lookup: typing.Dict[str, schema.Class]):
+    return "qltest_uncollapse_hierarchy" not in cls.pragmas and any(
         _is_in_qltest_collapsed_hierachy(lookup[b], lookup) for b in cls.bases)
 
 
-def _should_skip_qltest(cls: ql.Class, lookup: typing.Dict[str, ql.Class]):
-    return cls.qltest_skip or not (cls.final or cls.qltest_collapse_hierarchy) or _is_under_qltest_collapsed_hierachy(
+def _should_skip_qltest(cls: schema.Class, lookup: typing.Dict[str, schema.Class]):
+    return "qltest_skip" in cls.pragmas or not (
+            cls.final or "qltest_collapse_hierarchy" in cls.pragmas) or _is_under_qltest_collapsed_hierachy(
         cls, lookup)
 
 
@@ -211,12 +218,14 @@ def generate(opts, renderer):
     existing |= {q for q in test_out.rglob(missing_test_source_filename)}
 
     data = schema.load(input)
-    data.classes.sort(key=lambda cls: (cls.dir, cls.name))
+    inheritance_graph = {name: cls.bases for name, cls in data.classes.items()}
+    input_classes = [data.classes[name] for name in toposort_flatten(inheritance_graph)]
 
-    classes = [get_ql_class(cls) for cls in data.classes]
-    lookup = {cls.name: cls for cls in classes}
+    classes = [get_ql_class(cls, data.classes) for cls in input_classes]
     imports = {}
 
+    renderer.render(ql.DbClasses(cls for cls in classes if not cls.ipa), out / "Db.qll")
+
     for c in classes:
         imports[c.name] = get_import(stub_out / c.path, opts.swift_dir)
 
@@ -237,17 +246,17 @@ def generate(opts, renderer):
     renderer.render(ql.GetParentImplementation(
         classes), out / 'GetImmediateParent.qll')
 
-    for c in classes:
-        if _should_skip_qltest(c, lookup):
+    for c in input_classes:
+        if _should_skip_qltest(c, data.classes):
             continue
-        test_dir = test_out / c.path
+        test_dir = test_out / c.dir / c.name
         test_dir.mkdir(parents=True, exist_ok=True)
         if not any(test_dir.glob("*.swift")):
-            log.warning(f"no test source in {c.path}")
+            log.warning(f"no test source in {c.dir / c.name}")
             renderer.render(ql.MissingTestInstructions(),
                             test_dir / missing_test_source_filename)
             continue
-        total_props, partial_props = _partition(_get_all_properties_to_be_tested(c, lookup),
+        total_props, partial_props = _partition(_get_all_properties_to_be_tested(c, data.classes),
                                                 lambda p: p.is_single or p.is_predicate)
         renderer.render(ql.ClassTester(class_name=c.name,
                                        properties=total_props), test_dir / f"{c.name}.ql")
@@ -258,16 +267,16 @@ def generate(opts, renderer):
     final_ipa_types = []
     non_final_ipa_types = []
     constructor_imports = []
-    for cls in data.classes:
+    for cls in input_classes:
         ipa_type = get_ql_ipa_class(cls)
         if ipa_type.is_final:
             final_ipa_types.append(ipa_type)
-            if ipa_type.is_ipa:
+            if ipa_type.is_ipa_from or (ipa_type.is_ipa_on and ipa_type.has_params):
                 stub_file = stub_out / cls.dir / f"{cls.name}Constructor.qll"
                 if not stub_file.is_file() or _is_generated_stub(stub_file):
                     renderer.render(ql.Ipa.ConstructorStub(ipa_type), stub_file)
                 constructor_imports.append(get_import(stub_file, opts.swift_dir))
-        else:
+        elif cls.name != schema.root_class_name:
             non_final_ipa_types.append(ipa_type)
 
     renderer.render(ql.Ipa.Types(schema.root_class_name, final_ipa_types, non_final_ipa_types), out / "IpaTypes.qll")

swift/codegen/lib/ql.py

@@ -14,7 +14,7 @@
 
 import pathlib
 from dataclasses import dataclass, field
-from typing import List, ClassVar, Union
+from typing import List, ClassVar, Union, Optional
 
 import inflection
 
@@ -28,21 +28,18 @@ class Param:
 @dataclass
 class Property:
     singular: str
-    type: str = None
-    tablename: str = None
+    type: Optional[str] = None
+    tablename: Optional[str] = None
     tableparams: List[Param] = field(default_factory=list)
-    plural: str = None
+    plural: Optional[str] = None
     first: bool = False
-    local_var: str = "x"
     is_optional: bool = False
     is_predicate: bool = False
     is_child: bool = False
     qltest_skip: bool = False
 
     def __post_init__(self):
         if self.tableparams:
-            if self.type_is_class:
-                self.tableparams = [x if x != "result" else self.local_var for x in self.tableparams]
             self.tableparams = [Param(x) for x in self.tableparams]
             self.tableparams[0].first = True
 
@@ -82,24 +79,27 @@ class Class:
     qltest_skip: bool = False
     qltest_collapse_hierarchy: bool = False
     qltest_uncollapse_hierarchy: bool = False
+    has_db_id: bool = False
+    ipa: bool = False
 
     def __post_init__(self):
         self.bases = sorted(self.bases)
         if self.properties:
             self.properties[0].first = True
 
     @property
-    def db_id(self):
-        return "@" + inflection.underscore(self.name)
-
-    @property
-    def root(self):
+    def root(self) -> bool:
         return not self.bases
 
     @property
-    def path(self):
+    def path(self) -> pathlib.Path:
         return self.dir / self.name
 
+    @property
+    def db_id(self) -> Optional[str]:
+        if self.has_db_id:
+            return "@" + inflection.underscore(self.name)
+
 
 @dataclass
 class Stub:
@@ -109,6 +109,13 @@ class Stub:
     base_import: str
 
 
+@dataclass
+class DbClasses:
+    template: ClassVar = 'ql_db'
+
+    classes: List[Class] = field(default_factory=list)
+
+
 @dataclass
 class ImportList:
     template: ClassVar = 'ql_imports'
@@ -126,7 +133,7 @@ class GetParentImplementation:
 @dataclass
 class PropertyForTest:
     getter: str
-    type: str = None
+    type: Optional[str] = None
     is_single: bool = False
     is_predicate: bool = False
     is_repeated: bool = False
@@ -198,6 +205,10 @@ def __post_init__(self):
             if self.params:
                 self.params[0].first = True
 
+        @property
+        def has_params(self) -> bool:
+            return bool(self.params)
+
     @dataclass
     class FinalClassDb(FinalClass):
         is_db: ClassVar = True