Moved isOOMRunError and added SIBABRT condition

Author: matt-aitken

apps/webapp/app/v3/services/alerts/deliverAlert.server.ts

@@ -13,6 +13,7 @@ import {
   RunFailedWebhook,
   DeploymentFailedWebhook,
   DeploymentSuccessWebhook,
+  isOOMRunError,
 } from "@trigger.dev/core/v3";
 import assertNever from "assert-never";
 import { subtle } from "crypto";
@@ -375,7 +376,7 @@ export class DeliverAlertService extends BaseService {
                     idempotencyKey: alert.taskRun.idempotencyKey ?? undefined,
                     tags: alert.taskRun.runTags,
                     error,
-                    isOutOfMemoryError: isOOMError(error),
+                    isOutOfMemoryError: isOOMRunError(error),
                     machine: alert.taskRun.machinePreset ?? "Unknown",
                     dashboardUrl: `${env.APP_ORIGIN}${v3RunPath(
                       alert.project.organization,

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -11,6 +11,7 @@ import {
   TaskRunSuccessfulExecutionResult,
   flattenAttributes,
   isManualOutOfMemoryError,
+  isOOMRunError,
   sanitizeError,
   shouldRetryError,
   taskRunErrorEnhancer,
@@ -255,7 +256,7 @@ export class CompleteAttemptService extends BaseService {
 
     let retriableError = shouldRetryError(taskRunErrorEnhancer(completion.error));
     let isOOMRetry = false;
-    let isOOMAttempt = isOOMError(completion.error);
+    let isOOMAttempt = isOOMRunError(completion.error);
     let isOnMaxOOMMachine = false;
     let oomMachine: MachinePresetName | undefined;
 
@@ -738,45 +739,6 @@ async function findAttempt(prismaClient: PrismaClientOrTransaction, friendlyId:
   });
 }
 
-export function isOOMError(error: TaskRunError) {
-  if (error.type === "INTERNAL_ERROR") {
-    if (
-      error.code === "TASK_PROCESS_OOM_KILLED" ||
-      error.code === "TASK_PROCESS_MAYBE_OOM_KILLED"
-    ) {
-      return true;
-    }
-
-    // For the purposes of retrying on a larger machine, we're going to treat this is an OOM error.
-    // This is what they look like if we're executing using k8s. They then get corrected later, but it's too late.
-    // {"code": "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE", "type": "INTERNAL_ERROR", "message": "Process exited with code -1 after signal SIGKILL."}
-    if (
-      error.code === "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE" &&
-      error.message &&
-      error.message.includes("SIGKILL") &&
-      error.message.includes("-1")
-    ) {
-      return true;
-    }
-  }
-
-  if (error.type === "BUILT_IN_ERROR") {
-    // ffmpeg also does weird stuff
-    // { "name": "Error", "type": "BUILT_IN_ERROR", "message": "ffmpeg was killed with signal SIGKILL" }
-    if (error.message && error.message.includes("ffmpeg was killed with signal SIGKILL")) {
-      return true;
-    }
-  }
-
-  // Special `OutOfMemoryError` for doing a manual OOM kill.
-  // Useful if a native library does an OOM but doesn't actually crash the run and you want to manually
-  if (isManualOutOfMemoryError(error)) {
-    return true;
-  }
-
-  return false;
-}
-
 function exitRun(runId: string) {
   socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
     version: "v1",

packages/core/src/v3/errors.ts

@@ -76,6 +76,58 @@ export function isManualOutOfMemoryError(error: TaskRunError) {
   return false;
 }
 
+export function isOOMRunError(error: TaskRunError) {
+  if (error.type === "INTERNAL_ERROR") {
+    if (
+      error.code === "TASK_PROCESS_OOM_KILLED" ||
+      error.code === "TASK_PROCESS_MAYBE_OOM_KILLED"
+    ) {
+      return true;
+    }
+
+    // For the purposes of retrying on a larger machine, we're going to treat this is an OOM error.
+    // This is what they look like if we're executing using k8s. They then get corrected later, but it's too late.
+    // {"code": "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE", "type": "INTERNAL_ERROR", "message": "Process exited with code -1 after signal SIGKILL."}
+    if (
+      error.code === "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE" &&
+      error.message &&
+      error.message.includes("-1")
+    ) {
+      if (error.message.includes("SIGKILL")) {
+        return true;
+      }
+
+      if (error.message.includes("SIGABRT") && error.stackTrace) {
+        const oomIndicators = [
+          "JavaScript heap out of memory",
+          "Reached heap limit",
+          "FATAL ERROR: Reached heap limit Allocation failed",
+        ];
+
+        if (oomIndicators.some((indicator) => error.stackTrace!.includes(indicator))) {
+          return true;
+        }
+      }
+    }
+  }
+
+  if (error.type === "BUILT_IN_ERROR") {
+    // ffmpeg also does weird stuff
+    // { "name": "Error", "type": "BUILT_IN_ERROR", "message": "ffmpeg was killed with signal SIGKILL" }
+    if (error.message && error.message.includes("ffmpeg was killed with signal SIGKILL")) {
+      return true;
+    }
+  }
+
+  // Special `OutOfMemoryError` for doing a manual OOM kill.
+  // Useful if a native library does an OOM but doesn't actually crash the run and you want to manually
+  if (isManualOutOfMemoryError(error)) {
+    return true;
+  }
+
+  return false;
+}
+
 export class TaskPayloadParsedError extends Error {
   public readonly cause: unknown;
 
@@ -562,6 +614,8 @@ const findSignalInMessage = (message?: string, truncateLength = 100) => {
     return "SIGSEGV";
   } else if (trunc.includes("SIGKILL")) {
     return "SIGKILL";
+  } else if (trunc.includes("SIGABRT")) {
+    return "SIGABRT";
   } else {
     return;
   }
@@ -587,6 +641,10 @@ export function taskRunErrorEnhancer(error: TaskRunError): EnhanceError<TaskRunE
               return {
                 ...getPrettyTaskRunError("TASK_PROCESS_MAYBE_OOM_KILLED"),
               };
+            case "SIGABRT":
+              return {
+                ...getPrettyTaskRunError("TASK_PROCESS_MAYBE_OOM_KILLED"),
+              };
             default:
               return {
                 ...getPrettyTaskRunError("TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE"),
@@ -636,6 +694,10 @@ export function taskRunErrorEnhancer(error: TaskRunError): EnhanceError<TaskRunE
             return {
               ...getPrettyTaskRunError("TASK_PROCESS_MAYBE_OOM_KILLED"),
             };
+          case "SIGABRT":
+            return {
+              ...getPrettyTaskRunError("TASK_PROCESS_MAYBE_OOM_KILLED"),
+            };
           default: {
             return {
               ...getPrettyTaskRunError("TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE"),
@@ -689,6 +751,11 @@ export function exceptionEventEnhancer(
               ...exception,
               ...getPrettyExceptionEvent("TASK_PROCESS_MAYBE_OOM_KILLED"),
             };
+          case "SIGABRT":
+            return {
+              ...exception,
+              ...getPrettyExceptionEvent("TASK_PROCESS_MAYBE_OOM_KILLED"),
+            };
           default:
             return exception;
         }