Merge pull request #266 from github/dbartol/refactor-packs

Refactor Ruby into library and query packs

Author: aibaars

.codeqlmanifest.json

@@ -1,5 +1,6 @@
 {
     "provide": [
+        "ql/lib/qlpack.yml",
         "ql/src/qlpack.yml",
         "ql/consistency-queries/qlpack.yml",
         "ql/test/qlpack.yml",

.github/workflows/build.yml

@@ -47,17 +47,17 @@ jobs:
         run: cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: target/release/ruby-generator --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: ruby.dbscheme
-          path: ql/src/ruby.dbscheme
+          path: ql/lib/ruby.dbscheme
       - uses: actions/upload-artifact@v2
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: TreeSitter.qll
-          path: ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+          path: ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         with:
           name: extractor-${{ matrix.os }}
@@ -82,8 +82,10 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
       - name: Build Query Pack
         run: |
+          codeql/codeql pack create ql/lib --output target/packs
+          codeql/codeql pack install ql/src
           codeql/codeql pack create ql/src --output target/packs
-          PACK_FOLDER=$(readlink -f target/packs/github/codeql-ruby/*)
+          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - name: Compile with previous CodeQL versions
@@ -126,7 +128,7 @@ jobs:
           path: osx64
       - run: |
           mkdir -p ruby
-          cp -r codeql-extractor.yml tools ql/src/ruby.dbscheme.stats ruby/
+          cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
           cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
           cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
@@ -204,26 +206,20 @@ jobs:
           echo "| 2 |" > "test.expected"
           echo 'name: sample-tests
           version: 0.0.0
-          libraryPathDependencies:
-          - codeql-ruby
+          dependencies:
+            codeql/ruby-all: 0.0.1
           extractor: ruby
           tests: .
           ' > qlpack.yml
-          echo '- qlpack: codeql-ruby
-          - include:
-              kind:
-              - problem
-              - path-problem
-          ' > suite.qls
       - name: Run QL test
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" .
+          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
       - name: Create database
         shell: bash
         run: |
           "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
       - name: Analyze database
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database suite.qls
+          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/dataset_measure.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches: [main]
     paths:
-      - ql/src/ruby.dbscheme
+      - ql/lib/ruby.dbscheme
   pull_request:
     branches: [main]
     paths:
-      - ql/src/ruby.dbscheme
+      - ql/lib/ruby.dbscheme
   workflow_dispatch:
 
 jobs:
@@ -73,8 +73,8 @@ jobs:
           path: stats
       - run: |
           python -m pip install --user lxml
-          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/src/ruby.dbscheme.stats --normalise ruby_tokeninfo
+          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
       - uses: actions/upload-artifact@v2
         with:
           name: ruby.dbscheme.stats
-          path: ql/src/ruby.dbscheme.stats
+          path: ql/lib/ruby.dbscheme.stats

.github/workflows/qltest.yml

@@ -31,14 +31,18 @@ jobs:
       - name: Build Extractor
         run: env "PATH=$PATH:${{ github.workspace }}/codeql" scripts/create-extractor-pack.sh
       - name: Run QL tests
-        run: codeql/codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/consistency-queries ql/test
+        run: |
+          codeql/codeql pack install ql/test
+          codeql/codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --additional-packs "${HOME}/.codeql/packages/codeql/suite-helpers/0.0.1" --consistency-queries ql/consistency-queries ql/test
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql/codeql query format --check-only
       - name: Check QL compilation
-        run: codeql/codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}" "ql/src" "ql/examples"
+        run: |
+          codeql/codeql pack install ql/src
+          codeql/codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}" "ql/src" "ql/examples"
       - name: Check DB upgrade scripts
         run: |
           echo >empty.trap
-          codeql/codeql dataset import -S ql/src/upgrades/initial/ruby.dbscheme testdb empty.trap
-          codeql/codeql dataset upgrade testdb --additional-packs ql/src/upgrades
-          diff -q testdb/ruby.dbscheme ql/src/ruby.dbscheme
+          codeql/codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
+          codeql/codeql dataset upgrade testdb --additional-packs ql/lib/upgrades
+          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme

.gitignore

@@ -5,3 +5,4 @@ extractor-pack
 ql/test/**/*.testproj
 ql/test/**/*.actual
 ql/test/**/CONSISTENCY
+.codeql

Makefile

@@ -21,16 +21,16 @@ FILES=codeql-extractor.yml\
       tools/autobuild.sh\
       tools/qltest.sh\
       tools/autobuild.cmd\
-      ql/src/ruby.dbscheme.stats\
-      ql/src/ruby.dbscheme
+      ql/lib/ruby.dbscheme.stats\
+      ql/lib/ruby.dbscheme
 
 BIN_FILES=target/release/ruby-extractor$(EXE) target/release/ruby-autobuilder$(EXE)
 
 extractor-common:
 	rm -rf build
 	mkdir build
 	mkdir build/codeql-extractor-ruby
-	cp codeql-extractor.yml ql/src/ruby.dbscheme ql/src/ruby.dbscheme.stats build/codeql-extractor-ruby
+	cp codeql-extractor.yml ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats build/codeql-extractor-ruby
 	cp -r tools build/codeql-extractor-ruby/
 
 .PHONY:	tools
@@ -45,8 +45,8 @@ target/release/%$(EXE):
 
 dbscheme:
 	cargo build --bin ruby-generator
-	cargo run -p ruby-generator -- --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
-	codeql query format -i ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+	cargo run -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+	codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 .PHONY:	extractor
 extractor:	$(FILES) $(BIN_FILES)
@@ -61,7 +61,7 @@ extractor:	$(FILES) $(BIN_FILES)
 	cp tools/autobuild.sh extractor-pack/tools/autobuild.sh
 	cp tools/qltest.sh extractor-pack/tools/qltest.sh
 	cp tools/autobuild.cmd extractor-pack/tools/autobuild.cmd
-	cp ql/src/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
-	cp ql/src/ruby.dbscheme extractor-pack/ruby.dbscheme
+	cp ql/lib/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
+	cp ql/lib/ruby.dbscheme extractor-pack/ruby.dbscheme
 	cp target/release/ruby-extractor$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
 	cp target/release/ruby-autobuilder$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/autobuilder$(EXE)

README.md

@@ -12,13 +12,13 @@ cargo build --release
 
 ## Generating the database schema and QL library
 
-The generated `ql/src/ruby.dbscheme` and `ql/src/codeql_ruby/ast/internal/TreeSitter.qll` files are included in the repository, but they can be re-generated as follows:
+The generated `ql/lib/ruby.dbscheme` and `ql/lib/codeql/ruby/ast/internal/TreeSitter.qll` files are included in the repository, but they can be re-generated as follows:
 
 ```bash
 # Run the generator
-cargo run --release -p ruby-generator -- --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+cargo run --release -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 # Then auto-format the QL library
-codeql query format -i ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 ```
 
 ## Building a CodeQL database for a Ruby program

doc/prepare-db-upgrade.md

@@ -1,6 +1,6 @@
 # Upgrading the Ruby database schema
 
-The schema (`ql/src/ruby.dbscheme`) is automatically generated from tree-sitter's `node-types.json`. When the tree-sitter grammar changes, the database schema is likely to change as well, and we need to write an upgrade script. This document explains how to do that.
+The schema (`ql/lib/ruby.dbscheme`) is automatically generated from tree-sitter's `node-types.json`. When the tree-sitter grammar changes, the database schema is likely to change as well, and we need to write an upgrade script. This document explains how to do that.
 
 ## Process Overview
 
@@ -81,11 +81,11 @@ Upgrade scripts can be a little bit fiddly, so it's essential that you test them
 
 To create the upgrade directory manually, without using `scripts/prepare-db-upgrade.sh`:
 
-1. Get a hash of the old `.dbscheme` file, from just before your changes. You can do this by checking out the code prior to your changes and running `git hash-object ql/src/ruby.dbscheme`
+1. Get a hash of the old `.dbscheme` file, from just before your changes. You can do this by checking out the code prior to your changes and running `git hash-object ql/lib/ruby.dbscheme`
 
-2. Go back to your branch and create an upgrade directory with that hash as its name, for example: `mkdir ql/src/upgrades/454f1e15151422355049dc4f1f0486a03baeffef`
+2. Go back to your branch and create an upgrade directory with that hash as its name, for example: `mkdir ql/lib/upgrades/454f1e15151422355049dc4f1f0486a03baeffef`
 
 3. Copy the old `.dbscheme` file to that directory, using the name old.dbscheme.
-   `cp ql/src/ruby.dbscheme ql/src/upgrades/454f1e15151422355049dc4f1f0486a03baeffef/old.dbscheme`
+   `cp ql/lib/ruby.dbscheme ql/lib/upgrades/454f1e15151422355049dc4f1f0486a03baeffef/old.dbscheme`
 
 4. Put a copy of your new `.dbscheme` file in that directory and create an `upgrade.properties` file (as described above).

ql/consistency-queries/AstConsistency.ql

@@ -1,5 +1,5 @@
-import codeql_ruby.AST
-import codeql_ruby.ast.internal.Synthesis
+import codeql.ruby.AST
+import codeql.ruby.ast.internal.Synthesis
 
 private string getAPrimaryQlClass(AstNode node) {
   result = node.getAPrimaryQlClass()

ql/consistency-queries/CfgConsistency.ql

@@ -1 +1 @@
-import codeql_ruby.controlflow.internal.Consistency
+import codeql.ruby.controlflow.internal.Consistency