Support JSON column data prefixes

When we store data in JSON columns, if there is data we wrap it in `{
"data": theData }`. We do this because it could be an array, a primitive
or an object. Arrays and primitives can't be stored in a JSON column.

This hides this from the user because it’s an implementation detail and
means their expectation doesn’t match how we store

Author: matt-aitken

apps/webapp/app/v3/querySchemas.ts

@@ -330,6 +330,7 @@ export const runsSchema: TableSchema = {
     // For JSON columns, NULL checks are transformed to check for empty object '{}'
     // So `error IS NULL` becomes `error = '{}'` and `error IS NOT NULL` becomes `error != '{}'`
     // textColumn uses the pre-materialized text columns for better performance
+    // dataPrefix handles the internal {"data": ...} wrapper transparently
     output: {
       name: "output",
       ...column("JSON", {
@@ -338,6 +339,7 @@ export const runsSchema: TableSchema = {
       }),
       nullValue: "'{}'", // Transform NULL checks to compare against empty object
       textColumn: "output_text", // Use output_text for full JSON value queries
+      dataPrefix: "data", // Internal data is wrapped in {"data": ...}
     },
     error: {
       name: "error",
@@ -348,6 +350,7 @@ export const runsSchema: TableSchema = {
       }),
       nullValue: "'{}'", // Transform NULL checks to compare against empty object
       textColumn: "error_text", // Use error_text for full JSON value queries
+      dataPrefix: "data", // Internal data is wrapped in {"data": ...}
     },
 
     // Tags & versions

internal-packages/tsql/src/query/printer.test.ts

@@ -842,6 +842,156 @@ describe("ClickHousePrinter", () => {
     });
   });
 
+  describe("dataPrefix for JSON columns", () => {
+    // Create a schema with JSON columns that have dataPrefix set
+    const dataPrefixSchema: TableSchema = {
+      name: "runs",
+      clickhouseName: "trigger_dev.task_runs_v2",
+      columns: {
+        id: { name: "id", ...column("String") },
+        output: {
+          name: "output",
+          ...column("JSON"),
+          nullValue: "'{}'",
+          dataPrefix: "data",
+        },
+        error: {
+          name: "error",
+          ...column("JSON"),
+          nullValue: "'{}'",
+          dataPrefix: "data",
+        },
+        status: { name: "status", ...column("String") },
+        organization_id: { name: "organization_id", ...column("String") },
+        project_id: { name: "project_id", ...column("String") },
+        environment_id: { name: "environment_id", ...column("String") },
+      },
+      tenantColumns: {
+        organizationId: "organization_id",
+        projectId: "project_id",
+        environmentId: "environment_id",
+      },
+    };
+
+    function createDataPrefixContext() {
+      const schema = createSchemaRegistry([dataPrefixSchema]);
+      return createPrinterContext({
+        organizationId: "org_test",
+        projectId: "proj_test",
+        environmentId: "env_test",
+        schema,
+      });
+    }
+
+    describe("SELECT clause", () => {
+      it("should inject dataPrefix into JSON subfield path", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery("SELECT output.message FROM runs", ctx);
+
+        // Should transform output.message to output.data.message
+        expect(sql).toContain("output.data.message");
+      });
+
+      it("should generate clean alias without dataPrefix", () => {
+        const ctx = createDataPrefixContext();
+        const { sql, columns } = printQuery("SELECT output.message FROM runs", ctx);
+
+        // Alias should be output_message, not output_data_message
+        expect(sql).toContain("AS output_message");
+        expect(sql).not.toContain("AS output_data_message");
+        expect(columns).toContainEqual(
+          expect.objectContaining({ name: "output_message" })
+        );
+      });
+
+      it("should handle nested paths with dataPrefix", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery("SELECT output.user.name FROM runs", ctx);
+
+        // Should transform output.user.name to output.data.user.name
+        expect(sql).toContain("output.data.user.name");
+        // Alias should be output_user_name
+        expect(sql).toContain("AS output_user_name");
+      });
+
+      it("should work with multiple JSON columns with dataPrefix", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery("SELECT output.msg, error.code FROM runs", ctx);
+
+        expect(sql).toContain("output.data.msg");
+        expect(sql).toContain("error.data.code");
+        expect(sql).toContain("AS output_msg");
+        expect(sql).toContain("AS error_code");
+      });
+
+      it("should not affect bare JSON column selection", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery("SELECT output FROM runs", ctx);
+
+        // Bare column should not have dataPrefix injected
+        expect(sql).not.toContain("output.data");
+        expect(sql).toMatch(/SELECT\s+output[\s,]/);
+      });
+    });
+
+    describe("WHERE clause", () => {
+      it("should inject dataPrefix into WHERE comparison", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery(
+          "SELECT id FROM runs WHERE output.status = 'success'",
+          ctx
+        );
+
+        // Should transform output.status to output.data.status
+        expect(sql).toContain("output.data.status");
+      });
+
+      it("should inject dataPrefix into LIKE comparison", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery(
+          "SELECT id FROM runs WHERE error.message LIKE '%failed%'",
+          ctx
+        );
+
+        expect(sql).toContain("error.data.message");
+      });
+    });
+
+    describe("GROUP BY clause", () => {
+      it("should inject dataPrefix into GROUP BY", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery(
+          "SELECT output.type, count() AS cnt FROM runs GROUP BY output.type",
+          ctx
+        );
+
+        // Should inject dataPrefix in both SELECT and GROUP BY
+        expect(sql).toContain("output.data.type");
+        expect(sql).toContain("GROUP BY output.data.type");
+      });
+    });
+
+    describe("edge cases", () => {
+      it("should not affect columns without dataPrefix", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery("SELECT status FROM runs", ctx);
+
+        // Regular column should not be affected
+        expect(sql).toContain("status");
+        expect(sql).not.toContain("status.data");
+      });
+
+      it("should work with explicit alias on JSON subfield", () => {
+        const ctx = createDataPrefixContext();
+        const { sql } = printQuery("SELECT output.message AS msg FROM runs", ctx);
+
+        // Should inject dataPrefix but use user's alias
+        expect(sql).toContain("output.data.message");
+        expect(sql).toContain("AS msg");
+      });
+    });
+  });
+
   describe("ORDER BY clauses", () => {
     it("should print ORDER BY ASC", () => {
       const { sql } = printQuery("SELECT * FROM task_runs ORDER BY created_at ASC");

internal-packages/tsql/src/query/printer.ts

@@ -657,10 +657,10 @@ export class ClickHousePrinter {
           // If so, add an alias to preserve the nice column name (dots → underscores)
           const isJsonSubfield = this.isJsonSubfieldAccess(field.chain);
           if (isJsonSubfield) {
-            // Build the alias using underscores (e.g., "error_data_name")
-            const aliasName = field.chain
-              .filter((p): p is string => typeof p === "string")
-              .join("_");
+            // Build the alias using underscores, excluding any dataPrefix
+            // e.g., output.message -> "output_message" (not "output_data_message")
+            const dataPrefix = this.getDataPrefixForField(field.chain);
+            const aliasName = this.buildAliasWithoutDataPrefix(field.chain, dataPrefix);
             sqlResult = `${visited} AS ${this.printIdentifier(aliasName)}`;
             // Override output name for metadata
             effectiveOutputName = aliasName;
@@ -2125,8 +2125,11 @@ export class ClickHousePrinter {
       return `(${virtualExpression})`;
     }
 
+    // Inject dataPrefix for JSON columns if needed (e.g., output.message -> output.data.message)
+    const chainWithPrefix = this.injectDataPrefix(node.chain);
+
     // Try to resolve column names through table context
-    const resolvedChain = this.resolveFieldChain(node.chain);
+    const resolvedChain = this.resolveFieldChain(chainWithPrefix);
 
     // Print each chain element
     let result = resolvedChain.map((part) => this.printIdentifierOrIndex(part)).join(".");
@@ -2273,6 +2276,81 @@ export class ClickHousePrinter {
     return this.getTextColumnForField((expr as Field).chain);
   }
 
+  /**
+   * Get the dataPrefix for a field chain if the root column has one defined.
+   * Returns null if the column doesn't have a dataPrefix or if this isn't a subfield access.
+   */
+  private getDataPrefixForField(chain: Array<string | number>): string | null {
+    if (chain.length < 2) return null; // Need at least column.subfield
+
+    const firstPart = chain[0];
+    if (typeof firstPart !== "string") return null;
+
+    // Check if first part is a table alias (table.column.subfield)
+    const tableSchema = this.tableContexts.get(firstPart);
+    if (tableSchema) {
+      // Qualified: table.column.subfield - need at least 3 parts
+      if (chain.length < 3) return null;
+      const columnName = chain[1];
+      if (typeof columnName !== "string") return null;
+      const columnSchema = tableSchema.columns[columnName];
+      return columnSchema?.dataPrefix ?? null;
+    }
+
+    // Unqualified: column.subfield
+    const columnSchema = this.resolveFieldToColumnSchema([firstPart]);
+    return columnSchema?.dataPrefix ?? null;
+  }
+
+  /**
+   * Inject dataPrefix into a field chain if the root column has one defined.
+   * e.g., [output, message] -> [output, data, message] when dataPrefix is "data"
+   * Returns the original chain if no dataPrefix applies.
+   */
+  private injectDataPrefix(chain: Array<string | number>): Array<string | number> {
+    const dataPrefix = this.getDataPrefixForField(chain);
+    if (!dataPrefix) return chain;
+
+    const firstPart = chain[0];
+    if (typeof firstPart !== "string") return chain;
+
+    // Check if first part is a table alias
+    const tableSchema = this.tableContexts.get(firstPart);
+    if (tableSchema) {
+      // Qualified: table.column.subfield -> table.column.dataPrefix.subfield
+      // [table, column, subfield] -> [table, column, dataPrefix, subfield]
+      return [chain[0], chain[1], dataPrefix, ...chain.slice(2)];
+    }
+
+    // Unqualified: column.subfield -> column.dataPrefix.subfield
+    // [column, subfield] -> [column, dataPrefix, subfield]
+    return [chain[0], dataPrefix, ...chain.slice(1)];
+  }
+
+  /**
+   * Build an alias name for a field chain, excluding the dataPrefix if present.
+   * e.g., [output, message] with dataPrefix "data" -> "output_message"
+   * This gives users clean column names without the internal data wrapper.
+   */
+  private buildAliasWithoutDataPrefix(
+    chain: Array<string | number>,
+    dataPrefix: string | null
+  ): string {
+    // Filter to just string parts and join with underscores
+    const parts = chain.filter((p): p is string => typeof p === "string");
+
+    if (dataPrefix) {
+      // Remove the dataPrefix from the parts (it's an implementation detail)
+      const prefixIndex = parts.indexOf(dataPrefix);
+      if (prefixIndex > 0) {
+        // Only remove if it's not the first element (column name)
+        parts.splice(prefixIndex, 1);
+      }
+    }
+
+    return parts.join("_");
+  }
+
   /**
    * Resolve a field chain to its column schema (if it references a known column)
    */

internal-packages/tsql/src/query/schema.ts

@@ -230,6 +230,26 @@ export interface ColumnSchema {
    * ```
    */
   textColumn?: string;
+  /**
+   * Prefix path for JSON column data access.
+   *
+   * When set, user paths like `output.message` are automatically transformed
+   * to `output.data.message` in the actual query, and result aliases exclude
+   * the prefix (e.g., `output_message` instead of `output_data_message`).
+   *
+   * This is useful when JSON data is stored wrapped in a container object
+   * (e.g., `{"data": actualData}`) to handle arrays and primitives.
+   *
+   * @example
+   * ```typescript
+   * {
+   *   name: "output",
+   *   type: "JSON",
+   *   dataPrefix: "data",  // output.message → output.data.message
+   * }
+   * ```
+   */
+  dataPrefix?: string;
 }
 
 /**