refactor exception handling

Author: Bob Strahan

lib/idp_common_pkg/idp_common/classification/service.py

@@ -1168,6 +1168,7 @@ def classify_document(self, document: Document) -> Document:
             all_page_results = list(cached_page_classifications.values())
             combined_metering = {}
             errors_lock = threading.Lock()  # Thread safety for error collection
+            failed_page_exceptions = {}  # Store original exceptions for failed pages
 
             # Determine which pages need classification
             pages_to_classify = {}
@@ -1240,16 +1241,36 @@ def classify_document(self, document: Document) -> Document:
                                 combined_metering, page_metering
                             )
                         except Exception as e:
+                            # Capture exception details in the document object instead of raising
                             error_msg = f"Error classifying page {page_id}: {str(e)}"
                             logger.error(error_msg)
                             with errors_lock:
                                 document.errors.append(error_msg)
+                                # Store the original exception for later use
+                                failed_page_exceptions[page_id] = e
+
                             # Mark page as unclassified on error
                             if page_id in document.pages:
                                 document.pages[page_id].classification = "unclassified"
                                 document.pages[page_id].confidence = 0.0
-                            # raise exception to enable client retries
-                            raise
+
+            # Store failed page exceptions in document metadata for caller to access
+            if failed_page_exceptions:
+                # Store the first encountered exception as the primary failure cause
+                first_exception = next(iter(failed_page_exceptions.values()))
+                document.metadata = document.metadata or {}
+                document.metadata["failed_page_exceptions"] = {
+                    page_id: {
+                        "exception_type": type(exc).__name__,
+                        "exception_message": str(exc),
+                        "exception_class": exc.__class__.__module__
+                        + "."
+                        + exc.__class__.__name__,
+                    }
+                    for page_id, exc in failed_page_exceptions.items()
+                }
+                # Store the primary exception for easy access by caller
+                document.metadata["primary_exception"] = first_exception
             else:
                 logger.info(
                     f"All {len(cached_page_classifications)} page classifications found in cache"
@@ -1301,7 +1322,7 @@ def classify_document(self, document: Document) -> Document:
             )
 
         except Exception as e:
-            # Cache successful page classifications before raising exception
+            # Cache successful page classifications before handling exception
             if pages_to_classify:
                 successful_results = [
                     r
@@ -1317,6 +1338,9 @@ def classify_document(self, document: Document) -> Document:
             document = self._update_document_status(
                 document, success=False, error_message=error_msg
             )
+            # Store the exception in metadata for caller to access
+            document.metadata = document.metadata or {}
+            document.metadata["primary_exception"] = e
             # raise exception to enable client retries
             raise
 

patterns/pattern-2/src/classification_function/index.py

@@ -66,13 +66,36 @@ def handler(event, context):
     # Classify the document - the service will update the Document directly
     document = service.classify_document(document)
 
-    # Check if document processing failed
-    if document.status == Status.FAILED:
+    # Check if document processing failed or has pages that failed to classify
+    failed_page_exceptions = None
+    primary_exception = None
+    
+    # Check for failed page exceptions in metadata
+    if document.metadata and "failed_page_exceptions" in document.metadata:
+        failed_page_exceptions = document.metadata["failed_page_exceptions"]
+        primary_exception = document.metadata.get("primary_exception")
+        
+        # Log details about failed pages
+        logger.error(f"Document {document.id} has {len(failed_page_exceptions)} pages that failed to classify:")
+        for page_id, exc_info in failed_page_exceptions.items():
+            logger.error(f"  Page {page_id}: {exc_info['exception_type']} - {exc_info['exception_message']}")
+    
+    # Check if document processing completely failed or has critical page failures
+    if document.status == Status.FAILED or failed_page_exceptions:
         error_message = f"Classification failed for document {document.id}"
+        if failed_page_exceptions:
+            error_message += f" - {len(failed_page_exceptions)} pages failed to classify"
+        
         logger.error(error_message)
         # Update document status in AppSync before raising exception
         appsync_service.update_document(document)
-        raise Exception(error_message)
+        
+        # Raise the original exception type if available, otherwise raise generic exception
+        if primary_exception:
+            logger.error(f"Re-raising original exception: {type(primary_exception).__name__}")
+            raise primary_exception
+        else:
+            raise Exception(error_message)
 
     t1 = time.time()
     logger.info(f"Time taken for classification: {t1-t0:.2f} seconds")

patterns/pattern-3/src/classification_function/index.py

@@ -72,12 +72,36 @@ def handler(event, context):
     # Classify the document - the service will update the Document directly
     document = service.classify_document(document)
 
-    # Check if document processing failed
-    if document.status == Status.FAILED:
+    # Check if document processing failed or has pages that failed to classify
+    failed_page_exceptions = None
+    primary_exception = None
+    
+    # Check for failed page exceptions in metadata
+    if document.metadata and "failed_page_exceptions" in document.metadata:
+        failed_page_exceptions = document.metadata["failed_page_exceptions"]
+        primary_exception = document.metadata.get("primary_exception")
+        
+        # Log details about failed pages
+        logger.error(f"Document {document.id} has {len(failed_page_exceptions)} pages that failed to classify:")
+        for page_id, exc_info in failed_page_exceptions.items():
+            logger.error(f"  Page {page_id}: {exc_info['exception_type']} - {exc_info['exception_message']}")
+    
+    # Check if document processing completely failed or has critical page failures
+    if document.status == Status.FAILED or failed_page_exceptions:
         error_message = f"Classification failed for document {document.id}"
+        if failed_page_exceptions:
+            error_message += f" - {len(failed_page_exceptions)} pages failed to classify"
+        
         logger.error(error_message)
+        # Update document status in AppSync before raising exception
         appsync_service.update_document(document)
-        raise Exception(error_message)
+        
+        # Raise the original exception type if available, otherwise raise generic exception
+        if primary_exception:
+            logger.error(f"Re-raising original exception: {type(primary_exception).__name__}")
+            raise primary_exception
+        else:
+            raise Exception(error_message)
 
     t1 = time.time()
     logger.info(f"Time taken for classification: {t1-t0:.2f} seconds")