Make VM throw StackOverflowError on call-stack overflow and enable compiled test to verify recovery

vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java

@@ -343,6 +343,7 @@ public void updateAllDependencies() {
         dependsClassesInterfaces.clear();
         exportsClassesInterfaces.clear();
         dependsClassesInterfaces.add("java_lang_NullPointerException");
+        dependsClassesInterfaces.add("java_lang_StackOverflowError");
         setBaseClass(baseClass);
         if (isAnnotation) {
             dependsClassesInterfaces.add("java_lang_annotation_Annotation");

vm/ByteCodeTranslator/src/nativeMethods.m

@@ -25,6 +25,7 @@
 #include "java_util_HashMap.h"
 #include "java_util_HashMap_Entry.h"
 #include "java_lang_NullPointerException.h"
+#include "java_lang_StackOverflowError.h"
 #include "java_lang_Class.h"
 #include "java_lang_System.h"
 
@@ -1550,9 +1551,12 @@ void initMethodStack(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT __cn1ThisObject, int
         THROW_NULL_POINTER_EXCEPTION();
     }
 #endif
+    if(threadStateData->callStackOffset >= CN1_MAX_STACK_CALL_DEPTH - 1) {
+        throwException(threadStateData, __NEW_INSTANCE_java_lang_StackOverflowError(threadStateData));
+        return;
+    }
     memset(&threadStateData->threadObjectStack[threadStateData->threadObjectStackOffset], 0, sizeof(struct elementStruct) * (localsStackSize + stackSize));
     threadStateData->threadObjectStackOffset += localsStackSize + stackSize;
-    CODENAME_ONE_ASSERT(threadStateData->callStackOffset < CN1_MAX_STACK_CALL_DEPTH - 1);
     threadStateData->callStackClass[threadStateData->callStackOffset] = classNameId;
     threadStateData->callStackMethod[threadStateData->callStackOffset] = methodNameId;
     threadStateData->callStackOffset++;

vm/JavaAPI/src/java/lang/StackOverflowError.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Codename One designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *  
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ * 
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * 
+ * Please contact Codename One through http://www.codenameone.com/ if you 
+ * need additional information or have any questions.
+ */
+
+package java.lang;
+/**
+ * Thrown when a stack overflow occurs because an application recurses too deeply.
+ * Since: JDK1.0, CLDC 1.0
+ */
+public class StackOverflowError extends java.lang.VirtualMachineError{
+    /**
+     * Constructs a StackOverflowError with no detail message.
+     */
+    public StackOverflowError(){
+    }
+
+    /**
+     * Constructs a StackOverflowError with the specified detail message.
+     * s - the detail message.
+     */
+    public StackOverflowError(java.lang.String s){
+        super(s);
+    }
+
+}

vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetIntegrationTest.java

@@ -42,6 +42,9 @@ void generatesRunnableHelloWorldUsingCleanTarget(CompilerHelper.CompilerConfig c
         Files.write(sourceDir.resolve("java/lang/Exception.java"), javaLangExceptionSource().getBytes(StandardCharsets.UTF_8));
         Files.write(sourceDir.resolve("java/lang/RuntimeException.java"), javaLangRuntimeExceptionSource().getBytes(StandardCharsets.UTF_8));
         Files.write(sourceDir.resolve("java/lang/NullPointerException.java"), javaLangNullPointerExceptionSource().getBytes(StandardCharsets.UTF_8));
+        Files.write(sourceDir.resolve("java/lang/Error.java"), javaLangErrorSource().getBytes(StandardCharsets.UTF_8));
+        Files.write(sourceDir.resolve("java/lang/VirtualMachineError.java"), javaLangVirtualMachineErrorSource().getBytes(StandardCharsets.UTF_8));
+        Files.write(sourceDir.resolve("java/lang/StackOverflowError.java"), javaLangStackOverflowErrorSource().getBytes(StandardCharsets.UTF_8));
         Files.write(sourceDir.resolve("native_hello.c"), nativeHelloSource().getBytes(StandardCharsets.UTF_8));
 
         List<String> compileArgs = new java.util.ArrayList<>();
@@ -82,6 +85,9 @@ void generatesRunnableHelloWorldUsingCleanTarget(CompilerHelper.CompilerConfig c
         compileArgs.add(sourceDir.resolve("java/lang/Exception.java").toString());
         compileArgs.add(sourceDir.resolve("java/lang/RuntimeException.java").toString());
         compileArgs.add(sourceDir.resolve("java/lang/NullPointerException.java").toString());
+        compileArgs.add(sourceDir.resolve("java/lang/Error.java").toString());
+        compileArgs.add(sourceDir.resolve("java/lang/VirtualMachineError.java").toString());
+        compileArgs.add(sourceDir.resolve("java/lang/StackOverflowError.java").toString());
 
         int compileResult = CompilerHelper.compile(config.jdkHome, compileArgs);
         assertEquals(0, compileResult, "HelloWorld.java should compile with " + config);
@@ -426,6 +432,24 @@ static String javaLangNullPointerExceptionSource() {
                 "}\n";
     }
 
+    static String javaLangErrorSource() {
+        return "package java.lang;\n" +
+                "public class Error extends Throwable {\n" +
+                "}\n";
+    }
+
+    static String javaLangVirtualMachineErrorSource() {
+        return "package java.lang;\n" +
+                "public class VirtualMachineError extends Error {\n" +
+                "}\n";
+    }
+
+    static String javaLangStackOverflowErrorSource() {
+        return "package java.lang;\n" +
+                "public class StackOverflowError extends VirtualMachineError {\n" +
+                "}\n";
+    }
+
     static String javaLangLongSource() {
         return "package java.lang;\n" +
                 "public final class Long extends Number {\n" +

vm/tests/src/test/java/com/codename1/tools/translator/CompilerHelper.java

@@ -350,8 +350,21 @@ public static boolean compileAndRun(String code, String expectedOutput) throws E
                 "    }\n" +
                 "}\n" +
                 "\n" +
-                "void throwException(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT obj) {\n" +
-                "    (void)obj;\n" +
+                "void throwException(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT exceptionArg) {\n" +
+                "    java_lang_Throwable_fillInStack__(threadStateData, exceptionArg);\n" +
+                "    threadStateData->exception = exceptionArg;\n" +
+                "    threadStateData->tryBlockOffset--;\n" +
+                "    while (threadStateData->tryBlockOffset >= 0) {\n" +
+                "        if (threadStateData->blocks[threadStateData->tryBlockOffset].monitor != 0) {\n" +
+                "            monitorExitBlock(threadStateData, threadStateData->blocks[threadStateData->tryBlockOffset].monitor);\n" +
+                "            continue;\n" +
+                "        } else if (threadStateData->blocks[threadStateData->tryBlockOffset].exceptionClass <= 0 || instanceofFunction(threadStateData->blocks[threadStateData->tryBlockOffset].exceptionClass, exceptionArg->__codenameOneParentClsReference->classId)) {\n" +
+                "            int off = threadStateData->tryBlockOffset;\n" +
+                "            longjmp(threadStateData->blocks[off].destination, 1);\n" +
+                "            return;\n" +
+                "        }\n" +
+                "        threadStateData->tryBlockOffset--;\n" +
+                "    }\n" +
                 "    exit(1);\n" +
                 "}\n" +
                 "\n" +
@@ -368,7 +381,7 @@ public static boolean compileAndRun(String code, String expectedOutput) throws E
                 "// This suggests we are in a mode where objects are ints? No, that's likely for old CLDC/C++ target?\n" +
                 "// Or maybe `ExecutorApp` is configured with `none` or `ios` but generated headers use this?\n" +
                 "// Let's try to match the signature from the error message.\n" +
-                "int instanceofFunction(int sourceClass, int destId) { return 0; }\n" +
+                "int instanceofFunction(int sourceClass, int destId) { return 1; }\n" +
                 "\n" +
                 // "struct clazz class__java_lang_Class = {0};\n" +
                 // "struct clazz class__java_lang_String = {0};\n" +
@@ -378,6 +391,7 @@ public static boolean compileAndRun(String code, String expectedOutput) throws E
                 "void java_lang_Object_wait___long_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT me, JAVA_LONG ms, JAVA_INT ns) {}\n" +
                 "void java_lang_Object_notifyAll__(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT me) {}\n" +
                 "void java_lang_Object_notify__(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT me) {}\n" +
+                "void java_lang_Throwable_fillInStack__(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT me) { (void)me; }\n" +
                 "JAVA_OBJECT java_lang_Thread_currentThread___R_java_lang_Thread(CODENAME_ONE_THREAD_STATE) {\n" +
                 "    return JAVA_NULL; // Simplification\n" +
                 "}\n" +
@@ -524,6 +538,15 @@ public static boolean compileAndRun(String code, String expectedOutput) throws E
                 "    }\n" +
                 "    printf(\"\\n\");\n" +
                 "}\n";
+            content +=
+                "void test_Main_prepareOverflow__(CODENAME_ONE_THREAD_STATE) {\n" +
+                "    threadStateData->callStackOffset = CN1_MAX_STACK_CALL_DEPTH - 1;\n" +
+                "    threadStateData->threadObjectStackOffset = 0;\n" +
+                "}\n" +
+                "void test_Main_resetOverflow__(CODENAME_ONE_THREAD_STATE) {\n" +
+                "    threadStateData->callStackOffset = 0;\n" +
+                "    threadStateData->threadObjectStackOffset = 0;\n" +
+                "}\n";
 
             java.nio.file.Files.write(stubs, content.getBytes(java.nio.charset.StandardCharsets.UTF_8));
 

vm/tests/src/test/java/com/codename1/tools/translator/LambdaIntegrationTest.java

@@ -12,6 +12,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -109,6 +110,22 @@ void translatesLambdaBytecodeToLLVMExecutable(String targetVersion) throws Excep
         );
         assertEquals(0, compileResult, "LambdaApp should compile");
 
+        Files.walk(javaApiDir)
+                .forEach(source -> {
+                    try {
+                        Path destination = classesDir.resolve(javaApiDir.relativize(source));
+                        if (Files.isDirectory(source)) {
+                            if (!Files.exists(destination)) {
+                                Files.createDirectory(destination);
+                            }
+                        } else {
+                            Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING);
+                        }
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+
         Files.copy(nativeReport, classesDir.resolve("native_report.c"));
 
         Path outputDir = Files.createTempDirectory("lambda-integration-output");

vm/tests/src/test/java/com/codename1/tools/translator/LockIntegrationTest.java

@@ -189,6 +189,9 @@ private void writeMockJavaClasses(Path sourceDir) throws Exception {
         Files.write(lang.resolve("NullPointerException.java"), "package java.lang; public class NullPointerException extends RuntimeException { public NullPointerException() {} }".getBytes(StandardCharsets.UTF_8));
         Files.write(lang.resolve("IllegalMonitorStateException.java"), "package java.lang; public class IllegalMonitorStateException extends RuntimeException { public IllegalMonitorStateException() {} }".getBytes(StandardCharsets.UTF_8));
         Files.write(lang.resolve("IllegalArgumentException.java"), "package java.lang; public class IllegalArgumentException extends RuntimeException { public IllegalArgumentException(String s) {} }".getBytes(StandardCharsets.UTF_8));
+        Files.write(lang.resolve("Error.java"), "package java.lang; public class Error extends Throwable { public Error() {} public Error(String s) {} }".getBytes(StandardCharsets.UTF_8));
+        Files.write(lang.resolve("VirtualMachineError.java"), "package java.lang; public class VirtualMachineError extends Error { public VirtualMachineError() {} public VirtualMachineError(String s) { super(s); } }".getBytes(StandardCharsets.UTF_8));
+        Files.write(lang.resolve("StackOverflowError.java"), "package java.lang; public class StackOverflowError extends VirtualMachineError { public StackOverflowError() {} public StackOverflowError(String s) { super(s); } }".getBytes(StandardCharsets.UTF_8));
 
         // java.io.Serializable
         Files.write(io.resolve("Serializable.java"), "package java.io; public interface Serializable {}".getBytes(StandardCharsets.UTF_8));

vm/tests/src/test/java/com/codename1/tools/translator/ReadWriteLockIntegrationTest.java

@@ -231,6 +231,9 @@ private void writeMockJavaClasses(Path sourceDir) throws Exception {
         Files.write(lang.resolve("IllegalMonitorStateException.java"), "package java.lang; public class IllegalMonitorStateException extends RuntimeException { public IllegalMonitorStateException() {} }".getBytes(StandardCharsets.UTF_8));
         Files.write(lang.resolve("IllegalArgumentException.java"), "package java.lang; public class IllegalArgumentException extends RuntimeException { public IllegalArgumentException(String s) {} }".getBytes(StandardCharsets.UTF_8));
         Files.write(lang.resolve("UnsupportedOperationException.java"), "package java.lang; public class UnsupportedOperationException extends RuntimeException { public UnsupportedOperationException() {} }".getBytes(StandardCharsets.UTF_8));
+        Files.write(lang.resolve("Error.java"), "package java.lang; public class Error extends Throwable { public Error() {} public Error(String s) {} }".getBytes(StandardCharsets.UTF_8));
+        Files.write(lang.resolve("VirtualMachineError.java"), "package java.lang; public class VirtualMachineError extends Error { public VirtualMachineError() {} public VirtualMachineError(String s) { super(s); } }".getBytes(StandardCharsets.UTF_8));
+        Files.write(lang.resolve("StackOverflowError.java"), "package java.lang; public class StackOverflowError extends VirtualMachineError { public StackOverflowError() {} public StackOverflowError(String s) { super(s); } }".getBytes(StandardCharsets.UTF_8));
 
         // java.io.Serializable
         Files.write(io.resolve("Serializable.java"), "package java.io; public interface Serializable {}".getBytes(StandardCharsets.UTF_8));

vm/tests/src/test/java/com/codename1/tools/translator/StackOverflowHandlingTest.java

@@ -0,0 +1,88 @@
+package com.codename1.tools.translator;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.regex.Pattern;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class StackOverflowHandlingTest {
+    @Test
+    void stackOverflowIsCatchableInCompiledApp() throws Exception {
+        String code = "package test;\n" +
+                "public class Main {\n" +
+                "    private static native void print(String s);\n" +
+                "    private static native void prepareOverflow();\n" +
+                "    private static native void resetOverflow();\n" +
+                "    private static int triggerOverflow() {\n" +
+                "        return 1;\n" +
+                "    }\n" +
+                "    private static int safeRecurse(int depth) {\n" +
+                "        if (depth <= 0) {\n" +
+                "            return 0;\n" +
+                "        }\n" +
+                "        return 1 + safeRecurse(depth - 1);\n" +
+                "    }\n" +
+                "    public static void main(String[] args) {\n" +
+                "        boolean overflowed = false;\n" +
+                "        try {\n" +
+                "            prepareOverflow();\n" +
+                "            triggerOverflow();\n" +
+                "        } catch (StackOverflowError err) {\n" +
+                "            overflowed = true;\n" +
+                "            resetOverflow();\n" +
+                "        }\n" +
+                "        if (overflowed && safeRecurse(10) == 10) {\n" +
+                "            print(\"OVERFLOW_RECOVER\");\n" +
+                "        }\n" +
+                "    }\n" +
+                "}";
+        assertTrue(CompilerHelper.compileAndRun(code, "OVERFLOW_RECOVER"),
+                "Compiled app should catch StackOverflowError and continue execution");
+    }
+
+    @Test
+    void initMethodStackThrowsStackOverflowError() throws Exception {
+        Path nativeMethods = Paths.get("..", "ByteCodeTranslator", "src", "nativeMethods.m")
+                .toAbsolutePath()
+                .normalize();
+        String content = new String(Files.readAllBytes(nativeMethods), StandardCharsets.UTF_8);
+
+        assertTrue(content.contains("__NEW_INSTANCE_java_lang_StackOverflowError"),
+                "nativeMethods.m should reference StackOverflowError allocation");
+        assertTrue(content.contains("throwException(threadStateData, __NEW_INSTANCE_java_lang_StackOverflowError"),
+                "nativeMethods.m should throw StackOverflowError on overflow");
+        assertFalse(content.contains("CODENAME_ONE_ASSERT(threadStateData->callStackOffset < CN1_MAX_STACK_CALL_DEPTH - 1)"),
+                "nativeMethods.m should not assert on call stack overflow");
+
+        Pattern earlyReturn = Pattern.compile(
+                "StackOverflowError\\(threadStateData\\)\\);\\s*return;",
+                Pattern.MULTILINE
+        );
+        assertTrue(earlyReturn.matcher(content).find(),
+                "nativeMethods.m should return immediately after throwing StackOverflowError");
+    }
+
+    @Test
+    void javaApiContainsVmErrors() throws Exception {
+        Path stackOverflowError = Paths.get("..", "JavaAPI", "src", "java", "lang", "StackOverflowError.java")
+                .toAbsolutePath()
+                .normalize();
+        Path virtualMachineError = Paths.get("..", "JavaAPI", "src", "java", "lang", "VirtualMachineError.java")
+                .toAbsolutePath()
+                .normalize();
+
+        String stackOverflowContent = new String(Files.readAllBytes(stackOverflowError), StandardCharsets.UTF_8);
+        String virtualMachineContent = new String(Files.readAllBytes(virtualMachineError), StandardCharsets.UTF_8);
+
+        assertTrue(stackOverflowContent.contains("extends java.lang.VirtualMachineError"),
+                "StackOverflowError should extend VirtualMachineError");
+        assertTrue(virtualMachineContent.contains("class VirtualMachineError"),
+                "VirtualMachineError should be present in JavaAPI");
+    }
+}