SELECT * only returns the core columns but shows info

Author: matt-aitken

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx

@@ -40,6 +40,7 @@ import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Badge } from "~/components/primitives/Badge";
 import { Button } from "~/components/primitives/Buttons";
+import { Callout } from "~/components/primitives/Callout";
 import { CopyableText } from "~/components/primitives/CopyableText";
 import { Header2, Header3 } from "~/components/primitives/Headers";
 import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
@@ -172,23 +173,29 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   );
   if (!canAccess) {
     return typedjson(
-      { error: "Unauthorized", rows: null, columns: null, stats: null },
+      { error: "Unauthorized", rows: null, columns: null, stats: null, hiddenColumns: null },
       { status: 403 }
     );
   }
 
   const project = await findProjectBySlug(organizationSlug, projectParam, user.id);
   if (!project) {
     return typedjson(
-      { error: "Project not found", rows: null, columns: null, stats: null },
+      { error: "Project not found", rows: null, columns: null, stats: null, hiddenColumns: null },
       { status: 404 }
     );
   }
 
   const environment = await findEnvironmentBySlug(project.id, envParam, user.id);
   if (!environment) {
     return typedjson(
-      { error: "Environment not found", rows: null, columns: null, stats: null },
+      {
+        error: "Environment not found",
+        rows: null,
+        columns: null,
+        stats: null,
+        hiddenColumns: null,
+      },
       { status: 404 }
     );
   }
@@ -206,6 +213,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
         rows: null,
         columns: null,
         stats: null,
+        hiddenColumns: null,
       },
       { status: 400 }
     );
@@ -232,7 +240,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
 
     if (error) {
       return typedjson(
-        { error: error.message, rows: null, columns: null, stats: null },
+        { error: error.message, rows: null, columns: null, stats: null, hiddenColumns: null },
         { status: 400 }
       );
     }
@@ -242,11 +250,12 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
       rows: result.rows,
       columns: result.columns,
       stats: result.stats,
+      hiddenColumns: result.hiddenColumns ?? null,
     });
   } catch (err) {
     const errorMessage = err instanceof Error ? err.message : "Unknown error executing query";
     return typedjson(
-      { error: errorMessage, rows: null, columns: null, stats: null },
+      { error: errorMessage, rows: null, columns: null, stats: null, hiddenColumns: null },
       { status: 500 }
     );
   }
@@ -464,11 +473,25 @@ export default function Page() {
                         </Button>
                       </div>
                     ) : results?.rows && results?.columns ? (
-                      <TSQLResultsTable
-                        rows={results.rows}
-                        columns={results.columns}
-                        prettyFormatting={prettyFormatting}
-                      />
+                      <div className="flex h-full flex-col overflow-hidden">
+                        {results.hiddenColumns && results.hiddenColumns.length > 0 && (
+                          <Callout variant="warning" className="m-2 shrink-0 text-sm">
+                            <code>SELECT *</code> doesn't return all columns because it's slow. The
+                            following columns are not shown:{" "}
+                            <span className="font-mono text-xs">
+                              {results.hiddenColumns.join(", ")}
+                            </span>
+                            . Specify them explicitly to include them.
+                          </Callout>
+                        )}
+                        <div className="min-h-0 flex-1 overflow-hidden">
+                          <TSQLResultsTable
+                            rows={results.rows}
+                            columns={results.columns}
+                            prettyFormatting={prettyFormatting}
+                          />
+                        </div>
+                      </div>
                     ) : (
                       <Paragraph variant="small" className="p-4 text-text-dimmed">
                         Run a query to see results here.

internal-packages/clickhouse/src/client/tsql.ts

@@ -76,6 +76,11 @@ export interface TSQLQuerySuccess<T> {
   rows: T[];
   columns: OutputColumnMetadata[];
   stats: QueryStats;
+  /**
+   * Columns that were hidden when SELECT * was used.
+   * Only populated when SELECT * is transformed to core columns only.
+   */
+  hiddenColumns?: string[];
 }
 
 /**
@@ -114,7 +119,7 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
 
   try {
     // 1. Compile the TSQL query to ClickHouse SQL
-    const { sql, params, columns } = compileTSQL(options.query, {
+    const { sql, params, columns, hiddenColumns } = compileTSQL(options.query, {
       organizationId: options.organizationId,
       projectId: options.projectId,
       environmentId: options.environmentId,
@@ -145,17 +150,20 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
 
     const { rows, stats } = result;
 
+    // Build the result, including hiddenColumns if present
+    const baseResult = { columns, stats, hiddenColumns };
+
     // 3. Transform result values if enabled
     if (shouldTransformValues && rows) {
       const transformedRows = transformResults(
         rows as Record<string, unknown>[],
         options.tableSchema,
         { fieldMappings: options.fieldMappings }
       );
-      return [null, { rows: transformedRows as z.output<TOut>[], columns, stats }];
+      return [null, { rows: transformedRows as z.output<TOut>[], ...baseResult }];
     }
 
-    return [null, { rows: rows ?? [], columns, stats }];
+    return [null, { rows: rows ?? [], ...baseResult }];
   } catch (error) {
     const errorMessage = error instanceof Error ? error.message : "Unknown error";
 

internal-packages/tsql/src/query/printer.ts

@@ -70,6 +70,11 @@ export interface PrintResult {
   params: Record<string, unknown>;
   /** Metadata for each column in the SELECT clause, in order */
   columns: OutputColumnMetadata[];
+  /**
+   * Columns that were hidden when SELECT * was used.
+   * Only populated when SELECT * is transformed to core columns only.
+   */
+  hiddenColumns?: string[];
 }
 
 /**
@@ -109,6 +114,8 @@ export class ClickHousePrinter {
   private outputColumns: OutputColumnMetadata[] = [];
   /** Whether we're currently processing GROUP BY expressions */
   private inGroupByContext = false;
+  /** Columns hidden when SELECT * is expanded to core columns only */
+  private hiddenColumns: string[] = [];
 
   constructor(
     private context: PrinterContext,
@@ -122,12 +129,17 @@ export class ClickHousePrinter {
    */
   print(node: SelectQuery | SelectSetQuery): PrintResult {
     this.outputColumns = [];
+    this.hiddenColumns = [];
     const sql = this.visit(node);
-    return {
+    const result: PrintResult = {
       sql,
       params: this.context.getParams(),
       columns: this.outputColumns,
     };
+    if (this.hiddenColumns.length > 0) {
+      result.hiddenColumns = this.hiddenColumns;
+    }
+    return result;
   }
 
   /**
@@ -605,7 +617,8 @@ export class ClickHousePrinter {
   }
 
   /**
-   * Expand SELECT * to all selectable columns from all tables in context
+   * Expand SELECT * to core columns only from all tables in context.
+   * Non-core columns are tracked in hiddenColumns for user notification.
    */
   private expandAllTableColumns(collectMetadata: boolean): string[] {
     const results: string[] = [];
@@ -615,7 +628,8 @@ export class ClickHousePrinter {
       const tableColumns = this.getSelectableColumnsFromSchema(
         tableSchema,
         tableAlias,
-        collectMetadata
+        collectMetadata,
+        true // onlyCoreColumns - SELECT * only returns core columns
       );
       results.push(...tableColumns);
     }
@@ -629,7 +643,8 @@ export class ClickHousePrinter {
   }
 
   /**
-   * Expand table.* to all selectable columns from a specific table
+   * Expand table.* to core columns only from a specific table.
+   * Non-core columns are tracked in hiddenColumns for user notification.
    */
   private expandTableColumns(tableAlias: string, collectMetadata: boolean): string[] {
     const tableSchema = this.tableContexts.get(tableAlias);
@@ -639,30 +654,51 @@ export class ClickHousePrinter {
       return [`${this.printIdentifier(tableAlias)}.*`];
     }
 
-    return this.getSelectableColumnsFromSchema(tableSchema, tableAlias, collectMetadata);
+    return this.getSelectableColumnsFromSchema(
+      tableSchema,
+      tableAlias,
+      collectMetadata,
+      true // onlyCoreColumns - table.* only returns core columns
+    );
   }
 
   /**
-   * Get all selectable columns from a table schema as SQL strings
+   * Get selectable columns from a table schema as SQL strings
    *
    * @param tableSchema - The table schema
    * @param tableAlias - The alias used for the table in the query (for table-qualified columns)
    * @param collectMetadata - Whether to collect column metadata
+   * @param onlyCoreColumns - If true, only return core columns and track hidden columns (but falls back to all columns if no core columns are defined)
    * @returns Array of SQL column strings
    */
   private getSelectableColumnsFromSchema(
     tableSchema: TableSchema,
     tableAlias: string,
-    collectMetadata: boolean
+    collectMetadata: boolean,
+    onlyCoreColumns = false
   ): string[] {
     const results: string[] = [];
 
+    // Check if any core columns exist - if not, we'll return all columns as a fallback
+    const hasCoreColumns = Object.values(tableSchema.columns).some(
+      (col) => col.coreColumn === true && col.selectable !== false
+    );
+
+    // Only filter to core columns if the schema defines some core columns
+    const shouldFilterToCoreOnly = onlyCoreColumns && hasCoreColumns;
+
     for (const [columnName, columnSchema] of Object.entries(tableSchema.columns)) {
       // Skip non-selectable columns
       if (columnSchema.selectable === false) {
         continue;
       }
 
+      // If filtering to core columns only, skip non-core and track them
+      if (shouldFilterToCoreOnly && !columnSchema.coreColumn) {
+        this.hiddenColumns.push(columnName);
+        continue;
+      }
+
       // Build the SQL for this column
       let sqlResult: string;
 

internal-packages/tsql/src/query/validator.ts

@@ -209,17 +209,16 @@ function validateSelectQuery(node: SelectQuery, context: ValidationContext): voi
         coreColumns.push(...tableCoreColumns);
       }
 
-      // Build suggestion message
-      let suggestionMsg = "SELECT * will be far slower than selecting specific columns. ";
+      // Build info message about SELECT * behavior
+      let suggestionMsg = "SELECT * doesn't return all columns.";
       if (coreColumns.length > 0) {
-        suggestionMsg += `Consider selecting specific columns, e.g.: ${coreColumns.join(", ")}`;
-      } else {
-        suggestionMsg += "Consider selecting only the columns you need.";
+        suggestionMsg += `It will return: ${coreColumns.join(", ")}. `;
       }
+      suggestionMsg += "Specify columns explicitly to include other columns.";
 
       context.issues.push({
         message: suggestionMsg,
-        severity: "warning",
+        severity: "info",
         type: "select_star",
         suggestedColumns: coreColumns.length > 0 ? coreColumns : undefined,
       });