Swift: merge codegen and cppcodegen

Python code was simplified, and now a `--generate` option can be used
to drive what can be generated.

The extractor pack creation now will use an internally generated
dbscheme. This should be the same as the checked in one, but doing so
allows `bazel run create-extractor-pack` and `bazel run codegen` to be
run independently from one another, while previously the former had to
follow the latter in case of a schema change. This is the change that
triggered the above simplification, as in order for the two dbscheme
files to be identical, the first `// generated` line had to state the
same generator script.

Author: redsun82

.github/workflows/swift-codegen.yml

@@ -25,7 +25,7 @@ jobs:
           git diff --exit-code --stat HEAD
       - name: Generate C++ files
         run: |
-          bazel run //swift/codegen:cppcodegen -- --cpp-output=$PWD/swift-generated-headers
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-headers
       - uses: actions/upload-artifact@v3
         with:
           name: swift-generated-headers

swift/BUILD.bazel

@@ -3,17 +3,11 @@ load("@rules_pkg//:install.bzl", "pkg_install")
 load("//:defs.bzl", "codeql_platform")
 load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles")
 
-filegroup(
-    name = "dbscheme",
-    srcs = ["ql/lib/swift.dbscheme"],
-    visibility = ["//visibility:public"],
-)
-
 pkg_files(
     name = "dbscheme_files",
     srcs = [
         "ql/lib/swift.dbscheme.stats",
-        ":dbscheme",
+        "//swift/extractor/trap:generated_dbscheme",
     ],
 )
 

swift/codegen/BUILD.bazel

@@ -15,15 +15,6 @@ filegroup(
 py_binary(
     name = "codegen",
     srcs = ["codegen.py"],
-    visibility = ["//swift/codegen/test:__pkg__"],
-    deps = ["//swift/codegen/generators"],
-)
-
-# as opposed to the above, that is meant to only be run with bazel run,
-# we need to be precise with data dependencies of this which is meant be run during build
-py_binary(
-    name = "cppcodegen",
-    srcs = ["cppcodegen.py"],
     data = [
         ":schema",
         ":schema_includes",

swift/codegen/codegen.py

@@ -1,7 +1,49 @@
 #!/usr/bin/env python3
-""" Driver script to run all checked in code generation """
+""" Driver script to run all code generation """
+
+import argparse
+import logging
+import pathlib
+import sys
+import importlib
+import types
+import typing
+
+from swift.codegen.lib import render, paths
+from swift.codegen.generators import generate
+
+
+def _parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser()
+    p.add_argument("--generate", type=lambda x: x.split(","), default=["dbscheme", "ql"])
+    p.add_argument("--verbose", "-v", action="store_true")
+    p.add_argument("--swift-dir", type=_abspath, default=paths.swift_dir)
+    p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "codegen/schema.yml")
+    p.add_argument("--dbscheme", type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme")
+    p.add_argument("--ql-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
+    p.add_argument("--ql-stub-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
+    p.add_argument("--ql-format", action="store_true", default=True)
+    p.add_argument("--no-ql-format", action="store_false", dest="ql_format")
+    p.add_argument("--codeql-binary", default="codeql")
+    p.add_argument("--cpp-output", type=_abspath)
+    p.add_argument("--cpp-namespace", default="codeql")
+    p.add_argument("--trap-affix", default="Trap")
+    p.add_argument("--cpp-include-dir", default="swift/extractor/trap")
+    return p.parse_args()
+
+
+def _abspath(x: str) -> pathlib.Path:
+    return pathlib.Path(x).resolve()
+
+
+def run():
+    opts = _parse_args()
+    log_level = logging.DEBUG if opts.verbose else logging.INFO
+    logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
+    exe_path = paths.exe_file.relative_to(opts.swift_dir)
+    for target in opts.generate:
+        generate(target, opts, render.Renderer(exe_path))
 
-from swift.codegen.generators import generator, dbschemegen, qlgen
 
 if __name__ == "__main__":
-    generator.run(dbschemegen, qlgen)
+    run()

swift/codegen/cppcodegen.py

@@ -0,0 +1,6 @@
+from . import dbschemegen, qlgen, trapgen, cppgen
+
+
+def generate(target, opts, renderer):
+    module = globals()[f"{target}gen"]
+    module.generate(opts, renderer)

swift/codegen/generators/__init__.py

@@ -5,7 +5,6 @@
 from toposort import toposort_flatten
 
 from swift.codegen.lib import cpp, schema
-from swift.codegen.generators import generator
 
 
 def _get_type(t: str, trap_affix: str) -> str:
@@ -64,13 +63,8 @@ def get_classes(self):
 
 
 def generate(opts, renderer):
+    assert opts.cpp_output
     processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes}, opts.trap_affix)
     out = opts.cpp_output
     renderer.render(cpp.ClassList(processor.get_classes(), opts.cpp_namespace, opts.trap_affix,
                                   opts.cpp_include_dir, opts.schema), out / f"{opts.trap_affix}Classes.h")
-
-
-tags = ("cpp", "schema")
-
-if __name__ == "__main__":
-    generator.run()

swift/codegen/generators/cppgen.py

@@ -4,7 +4,6 @@
 import inflection
 
 from swift.codegen.lib import schema
-from swift.codegen.generators import generator
 from swift.codegen.lib.dbscheme import *
 
 log = logging.getLogger(__name__)
@@ -93,9 +92,3 @@ def generate(opts, renderer):
                       declarations=get_declarations(data))
 
     renderer.render(dbscheme, out)
-
-
-tags = ("schema", "dbscheme")
-
-if __name__ == "__main__":
-    generator.run()