Merge pull request #8735 from redsun82/swift-dbscheme-gen

Swift: dbscheme generator

Author: AlexDenisov

.pre-commit-config.yaml

@@ -2,28 +2,41 @@
 # See https://pre-commit.com/hooks.html for more hooks
 exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
 
--   repo: local
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v13.0.1
     hooks:
-    -   id: codeql-format
+      - id: clang-format
+        files: ^swift/.*\.(h|c|cpp)$
+
+  - repo: local
+    hooks:
+      - id: codeql-format
         name: Fix QL file formatting
         files: \.qll?$
         language: system
         entry: codeql query format --in-place
 
-    -   id: sync-files
+      - id: sync-files
         name: Fix files required to be identical
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
 
-    -   id: qhelp
+      - id: qhelp
         name: Check query help generation
         files: \.qhelp$
         language: system
         entry: python3 misc/scripts/check-qhelp.py
+
+      - id: swift-codegen
+        name: Run Swift checked in code generation
+        files: ^swift/(codegen/|.*/generated/|ql/lib/swift\.dbscheme$)
+        language: system
+        entry: bazel run //swift/codegen
+        pass_filenames: false

swift/README.md

@@ -4,6 +4,32 @@ The Swift codeql package is an experimental and unsupported work in progress.
 
 ## Usage
 
-Run `bazel run //swift:create-extractor-pack`, which will install `swift/extractor-pack`.
+Run
+
+```bash
+bazel run //swift:create-extractor-pack
+```
+
+which will install `swift/extractor-pack`.
+
 Using `--search-path=swift/extractor-pack` will then pick up the Swift extractor. You can also use
 `--search-path=swift`, as the extractor pack is mentioned in `swift/.codeqlmanifest.json`.
+
+Notice you can run `bazel run :create-extractor-pack` if you already are in the `swift` directory.
+
+## Code generation
+
+Make sure to install the [pip requirements](./codegen/requirements.txt) via
+
+```bash
+python3 -m pip install -r codegen/requirements.txt
+```
+
+Run
+
+```bash
+bazel run //swift/codegen
+```
+
+to update generated files. This can be shortened to
+`bazel run codegen` if you are in the `swift` directory.

swift/codegen/BUILD.bazel

@@ -0,0 +1,4 @@
+py_binary(
+    name = "codegen",
+    srcs = glob(["**/*.py"]),
+)

swift/codegen/codegen.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+""" Driver script to run all checked in code generation """
+
+from lib import generator
+import dbschemegen
+
+if __name__ == "__main__":
+    generator.run(dbschemegen.generate)

swift/codegen/dbschemegen.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+import pathlib
+
+import inflection
+
+from lib import paths, schema, generator
+from lib.dbscheme import *
+
+log = logging.getLogger(__name__)
+
+
+def dbtype(typename):
+    """ translate a type to a dbscheme counterpart, using `@lower_underscore` format for classes """
+    if typename[0].isupper():
+        return "@" + inflection.underscore(typename)
+    return typename
+
+
+def cls_to_dbscheme(cls: schema.Class):
+    """ Yield all dbscheme entities needed to model class `cls` """
+    if cls.derived:
+        yield DbUnion(dbtype(cls.name), (dbtype(c) for c in cls.derived))
+    # output a table specific to a class only if it is a leaf class or it has 1-to-1 properties
+    # Leaf classes need a table to bind the `@` ids
+    # 1-to-1 properties are added to a class specific table
+    # in other cases, separate tables are used for the properties, and a class specific table is unneeded
+    if not cls.derived or any(f.is_single for f in cls.properties):
+        binding = not cls.derived
+        keyset = DbKeySet(["id"]) if cls.derived else None
+        yield DbTable(
+            keyset=keyset,
+            name=inflection.tableize(cls.name),
+            columns=[
+                        DbColumn("id", type=dbtype(cls.name), binding=binding),
+                    ] + [
+                        DbColumn(f.name, dbtype(f.type)) for f in cls.properties if f.is_single
+                    ]
+        )
+    # use property-specific tables for 1-to-many and 1-to-at-most-1 properties
+    for f in cls.properties:
+        if f.is_optional:
+            yield DbTable(
+                keyset=DbKeySet(["id"]),
+                name=inflection.tableize(f"{cls.name}_{f.name}"),
+                columns=[
+                    DbColumn("id", type=dbtype(cls.name)),
+                    DbColumn(f.name, dbtype(f.type)),
+                ],
+            )
+        elif f.is_repeated:
+            yield DbTable(
+                keyset=DbKeySet(["id", "index"]),
+                name=inflection.tableize(f"{cls.name}_{f.name}"),
+                columns=[
+                    DbColumn("id", type=dbtype(cls.name)),
+                    DbColumn("index", type="int"),
+                    DbColumn(inflection.singularize(f.name), dbtype(f.type)),
+                ]
+            )
+
+
+def get_declarations(data: schema.Schema):
+    return [d for cls in data.classes.values() for d in cls_to_dbscheme(cls)]
+
+
+def get_includes(data: schema.Schema, include_dir: pathlib.Path):
+    includes = []
+    for inc in data.includes:
+        inc = include_dir / inc
+        with open(inc) as inclusion:
+            includes.append(DbSchemeInclude(src=inc.relative_to(paths.swift_dir), data=inclusion.read()))
+    return includes
+
+
+def generate(opts, renderer):
+    input = opts.schema.resolve()
+    out = opts.dbscheme.resolve()
+
+    with open(input) as src:
+        data = schema.load(src)
+
+    dbscheme = DbScheme(src=input.relative_to(paths.swift_dir),
+                        includes=get_includes(data, include_dir=input.parent),
+                        declarations=get_declarations(data))
+
+    renderer.render(dbscheme, out)
+
+
+if __name__ == "__main__":
+    generator.run(generate, tags=["schema", "dbscheme"])

swift/codegen/lib/dbscheme.py

@@ -0,0 +1,104 @@
+""" dbscheme format representation """
+
+import logging
+from dataclasses import dataclass
+from typing import ClassVar, List
+
+log = logging.getLogger(__name__)
+
+dbscheme_keywords = {"case", "boolean", "int", "string", "type"}
+
+
+@dataclass
+class DbColumn:
+    schema_name: str
+    type: str
+    binding: bool = False
+    first: bool = False
+
+    @property
+    def name(self):
+        if self.schema_name in dbscheme_keywords:
+            return self.schema_name + "_"
+        return self.schema_name
+
+    @property
+    def lhstype(self):
+        if self.type[0] == "@":
+            return "unique int" if self.binding else "int"
+        return self.type
+
+    @property
+    def rhstype(self):
+        if self.type[0] == "@" and self.binding:
+            return self.type
+        return self.type + " ref"
+
+
+@dataclass
+class DbKeySetId:
+    id: str
+    first: bool = False
+
+
+@dataclass
+class DbKeySet:
+    ids: List[DbKeySetId]
+
+    def __post_init__(self):
+        assert self.ids
+        self.ids = [DbKeySetId(x) for x in self.ids]
+        self.ids[0].first = True
+
+
+class DbDecl:
+    is_table = False
+    is_union = False
+
+
+@dataclass
+class DbTable(DbDecl):
+    is_table: ClassVar = True
+
+    name: str
+    columns: List[DbColumn]
+    keyset: DbKeySet = None
+
+    def __post_init__(self):
+        if self.columns:
+            self.columns[0].first = True
+
+
+@dataclass
+class DbUnionCase:
+    type: str
+    first: bool = False
+
+
+@dataclass
+class DbUnion(DbDecl):
+    is_union: ClassVar = True
+
+    lhs: str
+    rhs: List[DbUnionCase]
+
+    def __post_init__(self):
+        assert self.rhs
+        self.rhs = [DbUnionCase(x) for x in self.rhs]
+        self.rhs.sort(key=lambda c: c.type)
+        self.rhs[0].first = True
+
+
+@dataclass
+class DbSchemeInclude:
+    src: str
+    data: str
+
+
+@dataclass
+class DbScheme:
+    template: ClassVar = 'dbscheme'
+
+    src: str
+    includes: List[DbSchemeInclude]
+    declarations: List[DbDecl]

swift/codegen/lib/generator.py

@@ -0,0 +1,29 @@
+""" generator script scaffolding """
+
+import argparse
+import logging
+import sys
+
+from . import options, render
+
+
+def _parse(tags):
+    parser = argparse.ArgumentParser()
+    for opt in options.get(tags):
+        opt.add_to(parser)
+    ret = parser.parse_args()
+    log_level = logging.DEBUG if ret.verbose else logging.INFO
+    logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
+    return ret
+
+
+def run(*generators, tags=None):
+    """ run generation functions in `generators`, parsing options tagged with `tags` (all if unspecified)
+
+    `generators` should be callables taking as input an option namespace and a `render.Renderer` instance
+    """
+    opts = _parse(tags)
+    renderer = render.Renderer(dryrun=opts.check)
+    for g in generators:
+        g(opts, renderer)
+    sys.exit(1 if opts.check and renderer.done_something else 0)

swift/codegen/lib/options.py

@@ -0,0 +1,48 @@
+""" generator options, categorized by tags """
+
+import argparse
+import collections
+import pathlib
+from typing import Tuple
+
+from . import paths
+
+
+def _init_options():
+    Option("--check", "-c", action="store_true")
+    Option("--verbose", "-v", action="store_true")
+    Option("--schema", tags=["schema"], type=pathlib.Path, default=paths.swift_dir / "codegen/schema.yml")
+    Option("--dbscheme", tags=["dbscheme"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/swift.dbscheme")
+
+
+_options = collections.defaultdict(list)
+
+
+class Option:
+    def __init__(self, *args, tags=None, **kwargs):
+        tags = tags or []
+        self.args = args
+        self.kwargs = kwargs
+        if tags:
+            for t in tags:
+                _options[t].append(self)
+        else:
+            _options["*"].append(self)
+
+    def add_to(self, parser: argparse.ArgumentParser):
+        parser.add_argument(*self.args, **self.kwargs)
+
+
+_init_options()
+
+
+def get(tags: Tuple[str]):
+    """ get options marked by `tags`
+
+    Return all options if tags is falsy. Options tagged by wildcard '*' are always returned
+    """
+    if not tags:
+        return (o for tagged_opts in _options.values() for o in tagged_opts)
+    else:
+        # use specifically tagged options + those tagged with wildcard *
+        return (o for tag in ('*',) + tags for o in _options[tag])

swift/codegen/lib/paths.py

@@ -0,0 +1,17 @@
+""" module providing useful filesystem paths """
+
+import pathlib
+import sys
+import os
+
+try:
+    _workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY'])  # <- means we are using bazel run
+    swift_dir = _workspace_dir / 'swift'
+    lib_dir = swift_dir / 'codegen' / 'lib'
+except KeyError:
+    _this_file = pathlib.Path(__file__).resolve()
+    swift_dir = _this_file.parents[2]
+    lib_dir = _this_file.parent
+
+
+exe_file = pathlib.Path(sys.argv[0]).resolve()