Add --verbose flag for detailed error reporting in publish.py

- Add --verbose/-v flag to enable detailed error output for debugging
- Implement comprehensive error logging with log_verbose() and log_error_details() methods
- Track build errors in build_errors list for summary reporting
- Show detailed SAM build/package command output and error messages in verbose mode
- Add error summary with preview for non-verbose mode
- Include exception tracebacks in verbose mode for better debugging
- Update unit tests to handle new verbose parameter and build_errors attribute
- Resolves issue where build failures only showed generic 'Failed to build' message

Example usage:
  python publish.py bucket prefix region --verbose

This addresses the issue where SAM build failures like Python version mismatches
were not visible to users, making debugging difficult.

Author: Taniya Mathur

lib/idp_common_pkg/tests/unit/test_publish.py

@@ -49,14 +49,15 @@ def test_init_default_values(self):
         assert publisher.bucket is None
         assert publisher.prefix_and_version is None
         assert publisher.version is None
-        assert publisher.public_sample_udop_model == ""
-        assert publisher.public is False
-        assert publisher.main_template == "idp-main.yaml"
-        assert publisher.use_container_flag == ""
-        assert publisher.stat_cmd is None
-        assert publisher.s3_client is None
-        assert publisher.cf_client is None
-        assert publisher._print_lock is not None
+        assert publisher.verbose is False
+        assert publisher.build_errors == []
+
+    def test_init_verbose_mode(self):
+        """Test that IDPPublisher initializes correctly with verbose mode enabled"""
+        publisher = IDPPublisher(verbose=True)
+
+        assert publisher.verbose is True
+        assert publisher.build_errors == []
 
 
 @pytest.mark.unit

publish.py

@@ -39,15 +39,55 @@
 
 
 class IDPPublisher:
-    def __init__(self):
+    def __init__(self, verbose=False):
         self.console = Console()
+        self.verbose = verbose
         self.bucket_basename = None
         self.prefix = None
         self.region = None
         self.acl = None
         self.bucket = None
         self.prefix_and_version = None
         self.version = None
+        self.build_errors = []  # Track build errors for verbose reporting
+
+    def log_verbose(self, message, style="dim"):
+        """Log verbose messages if verbose mode is enabled"""
+        if self.verbose:
+            self.console.print(f"[{style}]{message}[/{style}]")
+
+    def log_error_details(self, component, error_output):
+        """Log detailed error information and store for summary"""
+        error_info = {
+            "component": component,
+            "error": error_output
+        }
+        self.build_errors.append(error_info)
+        
+        if self.verbose:
+            self.console.print(f"[red]❌ {component} build failed:[/red]")
+            self.console.print(f"[red]{error_output}[/red]")
+        else:
+            self.console.print(f"[red]❌ {component} build failed (use --verbose for details)[/red]")
+
+    def print_error_summary(self):
+        """Print summary of all build errors"""
+        if not self.build_errors:
+            return
+            
+        self.console.print("\n[red]❌ Build Error Summary:[/red]")
+        for i, error_info in enumerate(self.build_errors, 1):
+            self.console.print(f"\n[red]{i}. {error_info['component']}:[/red]")
+            if self.verbose:
+                self.console.print(f"[red]{error_info['error']}[/red]")
+            else:
+                # Show first few lines of error for non-verbose mode
+                error_lines = error_info['error'].strip().split('\n')
+                preview_lines = error_lines[:3]  # Show first 3 lines
+                for line in preview_lines:
+                    self.console.print(f"[red]  {line}[/red]")
+                if len(error_lines) > 3:
+                    self.console.print(f"[dim]  ... ({len(error_lines) - 3} more lines, use --verbose for full output)[/dim]")
         self.public_sample_udop_model = ""
         self.public = False
         self.main_template = "idp-main.yaml"
@@ -579,23 +619,29 @@ def build_and_package_template(self, directory):
                         shutil.rmtree(cache_dir)
 
                 sam_build_start = time.time()
+                self.log_verbose(f"Running SAM build command: {' '.join(cmd)}")
                 result = subprocess.run(
                     cmd, cwd=abs_directory, capture_output=True, text=True
                 )
                 sam_build_time = time.time() - sam_build_start
 
                 if result.returncode != 0:
                     # If cached build fails, try without cache
+                    self.log_verbose(f"Cached build failed for {directory}, retrying without cache")
                     self.console.print(
                         f"[yellow]Cached build failed for {directory}, retrying without cache[/yellow]"
                     )
                     cmd_no_cache = [c for c in cmd if c != "--cached"]
                     sam_build_start = time.time()
+                    self.log_verbose(f"Running SAM build command (no cache): {' '.join(cmd_no_cache)}")
                     result = subprocess.run(
                         cmd_no_cache, cwd=abs_directory, capture_output=True, text=True
                     )
                     sam_build_time = time.time() - sam_build_start
                     if result.returncode != 0:
+                        # Log detailed error information
+                        error_output = f"STDOUT:\n{result.stdout}\n\nSTDERR:\n{result.stderr}"
+                        self.log_error_details(f"SAM build for {directory}", error_output)
                         return False
 
                 # Package the template
@@ -620,12 +666,16 @@ def build_and_package_template(self, directory):
                 ]
 
                 sam_package_start = time.time()
+                self.log_verbose(f"Running SAM package command: {' '.join(cmd)}")
                 result = subprocess.run(
                     cmd, cwd=abs_directory, capture_output=True, text=True
                 )
                 sam_package_time = time.time() - sam_package_start
 
                 if result.returncode != 0:
+                    # Log detailed error information
+                    error_output = f"STDOUT:\n{result.stdout}\n\nSTDERR:\n{result.stderr}"
+                    self.log_error_details(f"SAM package for {directory}", error_output)
                     return False
 
                 # Log timing information
@@ -726,6 +776,11 @@ def build_patterns_concurrently(self, max_workers=None):
                         progress.update(main_task, completed=completed)
 
                     except Exception as e:
+                        # Log detailed error information
+                        import traceback
+                        error_output = f"Exception: {str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+                        self.log_error_details(f"Pattern {pattern} build exception", error_output)
+                        
                         progress.update(
                             pattern_tasks[pattern],
                             description=f"[red]{pattern}[/red] - Error: {str(e)[:30]}...",
@@ -815,6 +870,11 @@ def build_options_concurrently(self, max_workers=None):
                         progress.update(main_task, completed=completed)
 
                     except Exception as e:
+                        # Log detailed error information
+                        import traceback
+                        error_output = f"Exception: {str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+                        self.log_error_details(f"Option {option} build exception", error_output)
+                        
                         progress.update(
                             option_tasks[option],
                             description=f"[red]{option}[/red] - Error: {str(e)[:30]}...",
@@ -1020,8 +1080,11 @@ def build_main_template(self, webui_zipfile):
                     self.prefix_and_version,
                 ]
 
-                result = subprocess.run(cmd)
+                self.log_verbose(f"Running main template SAM package command: {' '.join(cmd)}")
+                result = subprocess.run(cmd, capture_output=True, text=True)
                 if result.returncode != 0:
+                    error_output = f"STDOUT:\n{result.stdout}\n\nSTDERR:\n{result.stderr}"
+                    self.log_error_details("Main template SAM package", error_output)
                     self.console.print("[red]Error packaging main template[/red]")
                     sys.exit(1)
             finally:
@@ -1171,9 +1234,12 @@ def run(self, args):
             patterns_time = time.time() - patterns_start
 
             if not patterns_success:
+                self.print_error_summary()
                 self.console.print(
                     "[red]❌ Error: Failed to build one or more patterns[/red]"
                 )
+                if not self.verbose:
+                    self.console.print("[dim]Use --verbose flag for detailed error information[/dim]")
                 sys.exit(1)
 
             # Build options concurrently
@@ -1185,9 +1251,12 @@ def run(self, args):
             options_time = time.time() - options_start
 
             if not options_success:
+                self.print_error_summary()
                 self.console.print(
                     "[red]❌ Error: Failed to build one or more options[/red]"
                 )
+                if not self.verbose:
+                    self.console.print("[dim]Use --verbose flag for detailed error information[/dim]")
                 sys.exit(1)
 
             total_build_time = time.time() - start_time
@@ -1244,6 +1313,9 @@ def main(
         "--max-workers",
         help="Maximum number of concurrent workers (default: auto-detect)",
     ),
+    verbose: bool = typer.Option(
+        False, "--verbose", "-v", help="Enable verbose output for debugging"
+    ),
 ):
     """
     [bold cyan]GenAI IDP Publisher[/bold cyan]
@@ -1262,7 +1334,11 @@ def main(
         if max_workers is not None:
             args.extend(["--max-workers", str(max_workers)])
 
-        publisher = IDPPublisher()
+        publisher = IDPPublisher(verbose=verbose)
+        
+        if verbose:
+            console.print("[dim]Verbose mode enabled - detailed error output will be shown[/dim]")
+        
         publisher.run(args)
 
     except KeyboardInterrupt: