feat: remote environment support

This commit reorganizes the `web-codegen-scorer` to support remote
environments. A remote environment is similar to the existing concept of
environments, with the exception that the lifecycle of an environment
can be managed in a hosted standalone server within a e.g. corporate
network.

The server would then provide additional features to the
web-codegen-scorer, like:

- different models for file generation
- different execution sandboxes for building and serving an app (e.g.
  consider a framework like Wiz that is internal to Google)

In practice, a remote environment exposes all of the important internal
hooks to advanced users, so that they can be fully in charge of:

- file generation via LLMs
- building generated apps
- repairing generated apps
- serving generated apps

Most users will never have to deal with this, but the architecture is highly
beneficial for further separation of concerns in the codebase, plus potentially
paving the way to support different languages (if we intend to do so), because
the logic for testing a "served app" is easy to disable with these changes.

Author: devversion

examples/environments/angular/config.js

@@ -1,3 +1,5 @@
+// @ts-check
+
 import { getBuiltInRatings } from 'web-codegen-scorer';
 
 /** @type {import("web-codegen-scorer").EnvironmentConfig} */
@@ -9,4 +11,5 @@ export default {
   generationSystemPrompt: './system-instructions.md',
   executablePrompts: ['../../prompts/**/*.md'],
   packageManager: 'npm',
+  buildCommand: '',
 };

examples/environments/remote_env/config.js

@@ -0,0 +1,12 @@
+import { getBuiltInRatings } from 'web-codegen-scorer';
+
+/** @type {import("web-codegen-scorer").EnvironmentConfig} */
+export default {
+  displayName: 'Remote Env (example)',
+  clientSideFramework: 'angular',
+  sourceDirectory: './project',
+  ratings: getBuiltInRatings(),
+  generationSystemPrompt: './system-instructions.md',
+  executablePrompts: ['../../prompts/**/*.md'],
+  packageManager: 'npm',
+};

report-app/src/app/pages/report-viewer/report-viewer.html

@@ -257,22 +257,22 @@ <h2>Generated applications</h2>
               {{ result.promptDef.name }}
 
               <div class="status-badge-group">
-                @let initialBuild = result.attemptDetails[0].buildResult;
-                @let repairBuild =
+                @let initialAttempt = result.attemptDetails[0];
+                @let repairAttempt =
                   result.attemptDetails.length > 1
-                    ? result.attemptDetails[1].buildResult
+                    ? result.attemptDetails[1]
                     : null;
-                @let finalBuild = repairBuild ?? initialBuild;
+                @let finalAttempt = result.attemptDetails.at(-1)!;
 
-                @if (finalBuild.runtimeErrors) {
+                @if (finalAttempt.serveTestingResult?.runtimeErrors) {
                   <span class="status-badge error">Runtime error</span>
                 }
 
-                @if (repairBuild?.status === 'error') {
+                @if (repairAttempt?.buildResult?.status === 'error') {
                   <span class="status-badge error">Build after repair</span>
                 }
 
-                @if (initialBuild.status === 'error') {
+                @if (initialAttempt?.buildResult?.status === 'error') {
                   <span class="status-badge error">Initial build failed</span>
                 }
               </div>
@@ -354,7 +354,8 @@ <h5>
                 <h4>Additional info</h4>
                 @for (attempt of result.attemptDetails; track attempt) {
                   @let isBuilt = attempt.buildResult.status === 'success';
-                  @let axeViolations = attempt.buildResult.axeViolations;
+                  @let axeViolations =
+                    attempt.serveTestingResult?.axeViolations;
                   @let hasAxeViolations =
                     axeViolations && axeViolations.length > 0;
 
@@ -501,12 +502,12 @@ <h5>Response</h5>
                 }
               </div>
 
-              @if (finalBuild.runtimeErrors) {
+              @let finalRuntimeErrors =
+                finalAttempt.serveTestingResult?.runtimeErrors;
+              @if (finalRuntimeErrors) {
                 <div class="app-details-section">
                   <h4>Runtime errors</h4>
-                  <pre class="callout warn code">{{
-                    finalBuild.runtimeErrors
-                  }}</pre>
+                  <pre class="callout warn code">{{ finalRuntimeErrors }}</pre>
                 </div>
               }
 

report-app/src/app/pages/report-viewer/report-viewer.ts

@@ -12,7 +12,7 @@ import {
   viewChild,
 } from '@angular/core';
 import { NgxJsonViewerModule } from 'ngx-json-viewer';
-import { BuildErrorType } from '../../../../../runner/builder/builder-types';
+import { BuildErrorType } from '../../../../../runner/workers/builder/builder-types';
 import {
   AssessmentResult,
   IndividualAssessment,
@@ -237,7 +237,7 @@ export class ReportViewer {
   });
 
   protected getScreenshotUrl(result: AssessmentResult): string | null {
-    return result.build.screenshotPngUrl ?? null;
+    return result.finalAttempt.serveTestingResult?.screenshotPngUrl ?? null;
   }
 
   protected isLoading = this.reportsFetcher.isLoadingSingleReport;

report-app/src/app/shared/debugging-zip.ts

@@ -1,4 +1,4 @@
-import { BuildResultStatus } from '../../../../runner/builder/builder-types';
+import { BuildResultStatus } from '../../../../runner/workers/builder/builder-types';
 import {
   AssessmentResult,
   RunInfo,
@@ -31,11 +31,11 @@ export async function createPromptDebuggingZip(
   zip.file('generated-files.md', generatedFiles);
 
   let errors = ``;
-  if (app.build.runtimeErrors) {
-    errors += `## Runtime errors\n${app.build.runtimeErrors}\n`;
+  if (app.finalAttempt.serveTestingResult?.runtimeErrors) {
+    errors += `## Runtime errors\n${app.finalAttempt.serveTestingResult?.runtimeErrors}\n`;
   }
-  if (app.build.status === BuildResultStatus.ERROR) {
-    errors += `## Build error\n  ${app.build.message}`;
+  if (app.finalAttempt.buildResult.status === BuildResultStatus.ERROR) {
+    errors += `## Build error\n  ${app.finalAttempt.buildResult.message}`;
   }
 
   zip.file('errors.md', errors);

runner/builder/builder-types.ts

@@ -18,6 +18,7 @@ import {
 import { DirectorySnapshot } from './directory-snapshot.js';
 import { LlmResponseFile } from '../../shared-interfaces.js';
 import { UserFacingError } from '../../utils/errors.js';
+import assert from 'assert';
 
 const SUPPORTED_MODELS = [
   'gemini-2.5-pro',
@@ -45,6 +46,19 @@ export class GeminiCliRunner implements LlmRunner {
     options: LlmGenerateFilesRequestOptions
   ): Promise<LlmGenerateFilesResponse> {
     const { context, model } = options;
+
+    // TODO: Consider removing these assertions when we have better types here.
+    // These fields are always set when running in a local environment, and this
+    // is a requirement for selecting the `gemini-cli` runner.
+    assert(
+      context.buildCommand,
+      'Expected a `buildCommand` to be set in the LLM generate request context'
+    );
+    assert(
+      context.packageManager,
+      'Expected a `packageManager` to be set in the LLM generate request context'
+    );
+
     const ignoreFilePath = join(context.directory, '.geminiignore');
     const instructionFilePath = join(context.directory, 'GEMINI.md');
     const settingsDir = join(context.directory, '.gemini');

runner/codegen/gemini-cli/gemini-cli-runner.ts

@@ -98,14 +98,24 @@ export interface LlmGenerateFilesContext {
   /**
    * Combined system instructions and prompt for the environments
    * where the two can't be provided separately.
+   *
+   * TODO(crisbeto): Can we explain the reason for this better?
    */
   combinedPrompt: string;
   /** Directory in which the generation will occur. */
   directory: string;
-  /** Command that the LLM can use to verify that the build works. */
-  buildCommand: string;
-  /** Package manager that the LLM can use. */
-  packageManager: string;
+  /**
+   * Command that the LLM can use to verify that the build works.
+   *
+   * Can be `undefined` for remote environments.
+   */
+  buildCommand: string | undefined;
+  /**
+   * Package manager that the LLM can use.
+   *
+   * Can be `undefined` for remote environments.
+   */
+  packageManager: string | undefined;
   /** All available package managers supported by the runner. */
   possiblePackageManagers: string[];
 }

runner/codegen/llm-runner.ts

@@ -0,0 +1,55 @@
+import z from 'zod';
+import { ratingSchema } from '../ratings/rating-types.js';
+import { MultiStepPrompt } from './multi-step-prompt.js';
+import { mcpServerOptionsSchema } from '../codegen/llm-runner.js';
+import { getPossiblePackageManagers } from './environment-config.js';
+
+export const baseEnvironmentConfigSchema = z.strictObject({
+  /** Display name for the environment. */
+  displayName: z.string(),
+  /**
+   * Optional unique ID for the environment.
+   * If one isn't provided, it will be computed from the `displayName`.
+   */
+  id: z.string().optional(),
+  /** ID of the client-side framework used within the environment. */
+  clientSideFramework: z.string(),
+  /** Ratings to run when evaluating the environment. */
+  ratings: z.array(ratingSchema),
+  /** Path to the prompt used by the LLM for generating files. */
+  generationSystemPrompt: z.string(),
+  /**
+   * Path to the prompt used by the LLM for repairing builds or failures.
+   *
+   * If unset or `null`, the eval tool will use its default repair instructions.
+   */
+  repairSystemPrompt: z.union([z.string(), z.null()]).optional(),
+  /**
+   * Path to the prompt used by the LLM for editing.
+   *
+   * Prompts running after the initial generation are considered as editing (e.g. multi step prompts).
+   * If `null`, the eval tool will use the generation prompt for edits.
+   */
+  editingSystemPrompt: z.union([z.string(), z.null()]).optional(),
+  /** Prompts that should be sent to the LLM and written into the output. */
+  executablePrompts: z.array(
+    z.union([
+      z.string(),
+      z.strictObject({
+        path: z.string(),
+        name: z.string().optional(),
+        ratings: z.array(ratingSchema).optional(),
+      }),
+      z.custom<MultiStepPrompt>((data) => data instanceof MultiStepPrompt),
+    ])
+  ),
+  /**
+   * ID of the fullstack framework used within the environment.
+   * If omitted, it will default to the `clientSideFramework`.
+   */
+  fullStackFramework: z.string().optional(),
+  /** Path to the prompt to use when rating code. */
+  codeRatingPrompt: z.string().optional(),
+  /** When enabled, the system prompts for this environment won't be included in the report. */
+  classifyPrompts: z.boolean().optional(),
+});