Swift: added trapgen

This checks in the trapgen script generating trap entries in C++.

The codegen suite has been slightly reorganized, moving the templates
directory up one level and chopping everything into smaller bazel
packages. Running tests is now done via
```
bazel run //swift/codegen/test
```

With respect to the PoC, the nested `codeql::trap` namespace has been
dropped in favour of a `Trap` prefix (or suffix in case of entries)
within the `codeql` namespace. Also, generated C++ code is not checked
in in git any more, and generated during build. Finally, labels get
printed in hex in the trap file.

`TrapLabel` is for the moment only default-constructible, so only one
single label is possible. `TrapArena`, that is responsible for creating
disjoint labels will come in a later commit.

Author: redsun82

.github/workflows/swift-codegen.yml

@@ -13,18 +13,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
       - uses: actions/setup-python@v3
         with:
           python-version: '~3.8'
           cache: 'pip'
-      - uses: ./.github/actions/fetch-codeql
-      - uses: bazelbuild/setup-bazelisk@v2
-      - name: Install dependencies
+      - name: Install python dependencies
         run: |
           pip install -r swift/codegen/requirements.txt
       - name: Run unit tests
         run: |
-          bazel test //swift/codegen:tests --test_output=errors
+          bazel test //swift/codegen/test --test_output=errors
       - name: Check that code was generated
         run: |
           bazel run //swift/codegen

.github/workflows/swift-qltest.yml

@@ -29,6 +29,13 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
       - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '~3.8'
+          cache: 'pip'
+      - name: Install python dependencies
+        run: |
+          pip install -r codegen/requirements.txt
       - name: Build Swift extractor
         run: |
           bazel run //swift:create-extractor-pack

.pre-commit-config.yaml

@@ -47,5 +47,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //swift/codegen:tests
+        entry: bazel test //swift/codegen/test
         pass_filenames: false

swift/.gitignore

@@ -1 +1,5 @@
-extractor-pack
+# directory created by bazel run //swift:create-extractor-pack
+/extractor-pack
+
+# output files created by running tests
+*.o

swift/BUILD.bazel

@@ -2,11 +2,17 @@ load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files"
 load("@rules_pkg//:install.bzl", "pkg_install")
 load("//:defs.bzl", "codeql_platform")
 
-pkg_files(
+filegroup(
     name = "dbscheme",
+    srcs = ["ql/lib/swift.dbscheme"],
+    visibility = ["//visibility:public"],
+)
+
+pkg_files(
+    name = "dbscheme_files",
     srcs = [
-        "ql/lib/swift.dbscheme",
         "ql/lib/swift.dbscheme.stats",
+        ":dbscheme",
     ],
 )
 
@@ -25,7 +31,7 @@ pkg_files(
 pkg_filegroup(
     name = "extractor-pack-generic",
     srcs = [
-        ":dbscheme",
+        ":dbscheme_files",
         ":manifest",
         ":qltest",
     ],
@@ -58,7 +64,7 @@ pkg_filegroup(
     name = "extractor-pack-arch",
     srcs = [
         ":extractor",
-        ":swift-test-sdk-arch"
+        ":swift-test-sdk-arch",
     ],
     visibility = ["//visibility:public"],
 )

swift/codegen/BUILD.bazel

@@ -1,31 +1,16 @@
 py_binary(
     name = "codegen",
-    srcs = glob([
-        "lib/*.py",
-        "*.py",
-    ]),
+    srcs = glob(["*.py"]),
+    visibility = ["//swift/codegen/test:__pkg__"],
+    deps = ["//swift/codegen/lib"],
 )
 
-py_library(
-    name = "test_utils",
-    testonly = True,
-    srcs = ["test/utils.py"],
-    deps = [":codegen"],
-)
-
-[
-    py_test(
-        name = src[len("test/"):-len(".py")],
-        size = "small",
-        srcs = [src],
-        deps = [
-            ":codegen",
-            ":test_utils",
-        ],
-    )
-    for src in glob(["test/test_*.py"])
-]
-
-test_suite(
-    name = "tests",
+# as opposed to the above, that is meant to only be run with bazel run,
+# we need to be precise with data dependencies of this which is meant be run during build
+py_binary(
+    name = "trapgen",
+    srcs = ["trapgen.py"],
+    data = ["//swift/codegen/templates:cpp"],
+    visibility = ["//swift:__subpackages__"],
+    deps = ["//swift/codegen/lib"],
 )

swift/codegen/codegen.py

@@ -6,4 +6,4 @@
 import qlgen
 
 if __name__ == "__main__":
-    generator.run(dbschemegen.generate, qlgen.generate)
+    generator.run(dbschemegen, qlgen)

swift/codegen/dbschemegen.py

@@ -79,11 +79,13 @@ def generate(opts, renderer):
     data = schema.load(input)
 
     dbscheme = Scheme(src=input.relative_to(paths.swift_dir),
-                        includes=get_includes(data, include_dir=input.parent),
-                        declarations=get_declarations(data))
+                      includes=get_includes(data, include_dir=input.parent),
+                      declarations=get_declarations(data))
 
     renderer.render(dbscheme, out)
 
 
+tags = ("schema", "dbscheme")
+
 if __name__ == "__main__":
-    generator.run(generate, tags=["schema", "dbscheme"])
+    generator.run()

swift/codegen/lib/BUILD.bazel

@@ -0,0 +1,5 @@
+py_library(
+    name = "lib",
+    srcs = glob(["*.py"]),
+    visibility = ["//swift/codegen:__subpackages__"],
+)

swift/codegen/lib/cpp.py

@@ -0,0 +1,83 @@
+from dataclasses import dataclass, field
+from typing import List, ClassVar
+
+# taken from https://en.cppreference.com/w/cpp/keyword
+cpp_keywords = {"alignas", "alignof", "and", "and_eq", "asm", "atomic_cancel", "atomic_commit", "atomic_noexcept",
+                "auto", "bitand", "bitor", "bool", "break", "case", "catch", "char", "char8_t", "char16_t", "char32_t",
+                "class", "compl", "concept", "const", "consteval", "constexpr", "constinit", "const_cast", "continue",
+                "co_await", "co_return", "co_yield", "decltype", "default", "delete", "do", "double", "dynamic_cast",
+                "else", "enum", "explicit", "export", "extern", "false", "float", "for", "friend", "goto", "if",
+                "inline", "int", "long", "mutable", "namespace", "new", "noexcept", "not", "not_eq", "nullptr",
+                "operator", "or", "or_eq", "private", "protected", "public", "reflexpr", "register", "reinterpret_cast",
+                "requires", "return", "short", "signed", "sizeof", "static", "static_assert", "static_cast", "struct",
+                "switch", "synchronized", "template", "this", "thread_local", "throw", "true", "try", "typedef",
+                "typeid", "typename", "union", "unsigned", "using", "virtual", "void", "volatile", "wchar_t", "while",
+                "xor", "xor_eq"}
+
+
+@dataclass
+class Field:
+    name: str
+    type: str
+    first: bool = False
+
+    def cpp_name(self):
+        if self.name in cpp_keywords:
+            return self.name + "_"
+        return self.name
+
+    def stream(self):
+        if self.type == "std::string":
+            return lambda x: f"trapQuoted({x})"
+        elif self.type == "bool":
+            return lambda x: f'({x} ? "true" : "false")'
+        else:
+            return lambda x: x
+
+
+@dataclass
+class Trap:
+    table_name: str
+    name: str
+    fields: List[Field]
+    id: Field = None
+
+    def __post_init__(self):
+        assert self.fields
+        self.fields[0].first = True
+
+
+@dataclass
+class TagBase:
+    base: str
+    first: bool = False
+
+
+@dataclass
+class Tag:
+    name: str
+    bases: List[TagBase]
+    index: int
+    id: str
+
+    def __post_init__(self):
+        if self.bases:
+            self.bases = [TagBase(b) for b in self.bases]
+            self.bases[0].first = True
+
+    def has_bases(self):
+        return bool(self.bases)
+
+
+@dataclass
+class TrapList:
+    template: ClassVar = 'cpp_traps'
+
+    traps: List[Trap] = field(default_factory=list)
+
+
+@dataclass
+class TagList:
+    template: ClassVar = 'cpp_tags'
+
+    tags: List[Tag] = field(default_factory=list)