Skip to content

Commit 5238e8f

Browse files
author
Bob Strahan
committed
Defend against float conversion exceptions
1 parent c6e7b9c commit 5238e8f

File tree

2 files changed

+98
-63
lines changed

2 files changed

+98
-63
lines changed

‎lib/idp_common_pkg/idp_common/assessment/service.py‎

Lines changed: 2 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -858,20 +858,11 @@ def process_document_section(self, document: Document, section_id: str) -> Docum
858858
# Create a default assessment structure
859859
default_assessment = {
860860
"confidence": 0.5,
861-
"confidence_reason": f"LLM returned list format for non-list attribute '{attr_name}'. Using default confidence.",
861+
"confidence_reason": f"LLM returned list format for non-list attribute '{attr_name}'. Using default confidence (0.5) and threshold ({attr_threshold}).",
862862
"confidence_threshold": attr_threshold,
863863
}
864864
enhanced_assessment_data[attr_name] = default_assessment
865865

866-
# Add alert since we're using default confidence
867-
if 0.5 < attr_threshold:
868-
confidence_threshold_alerts.append(
869-
{
870-
"attribute_name": attr_name,
871-
"confidence": 0.5,
872-
"confidence_threshold": attr_threshold,
873-
}
874-
)
875866
else:
876867
# Handle other unexpected types
877868
logger.warning(
@@ -882,21 +873,11 @@ def process_document_section(self, document: Document, section_id: str) -> Docum
882873
# Create a default assessment structure
883874
default_assessment = {
884875
"confidence": 0.5,
885-
"confidence_reason": f"LLM returned unexpected type {type(attr_assessment)} for attribute '{attr_name}'. Using default confidence.",
876+
"confidence_reason": f"LLM returned unexpected type {type(attr_assessment)} for attribute '{attr_name}'. Using default confidence (0.5) and threshold ({attr_threshold}).",
886877
"confidence_threshold": attr_threshold,
887878
}
888879
enhanced_assessment_data[attr_name] = default_assessment
889880

890-
# Add alert since we're using default confidence
891-
if 0.5 < attr_threshold:
892-
confidence_threshold_alerts.append(
893-
{
894-
"attribute_name": attr_name,
895-
"confidence": 0.5,
896-
"confidence_threshold": attr_threshold,
897-
}
898-
)
899-
900881
# Update the existing extraction result with enhanced assessment data
901882
extraction_data["explainability_info"] = [enhanced_assessment_data]
902883
extraction_data["metadata"] = extraction_data.get("metadata", {})

‎patterns/pattern-3/template.yaml‎

Lines changed: 96 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,19 @@ Resources:
377377
model:
378378
type: string
379379
description: Model identifier
380-
enum: ["us.amazon.nova-lite-v1:0", "us.amazon.nova-pro-v1:0", "us.amazon.nova-premier-v1:0", "us.anthropic.claude-3-haiku-20240307-v1:0", "us.anthropic.claude-3-5-sonnet-20241022-v2:0", "us.anthropic.claude-3-7-sonnet-20250219-v1:0", "us.anthropic.claude-sonnet-4-20250514-v1:0", "us.anthropic.claude-opus-4-20250514-v1:0"]
380+
enum:
381+
- !If
382+
- HasCustomExtractionModelARN
383+
- !Ref CustomExtractionModelARN
384+
- !Ref AWS::NoValue
385+
- "us.amazon.nova-lite-v1:0"
386+
- "us.amazon.nova-pro-v1:0"
387+
- "us.amazon.nova-premier-v1:0"
388+
- "us.anthropic.claude-3-haiku-20240307-v1:0"
389+
- "us.anthropic.claude-3-5-sonnet-20241022-v2:0"
390+
- "us.anthropic.claude-3-7-sonnet-20250219-v1:0"
391+
- "us.anthropic.claude-sonnet-4-20250514-v1:0"
392+
- "us.anthropic.claude-opus-4-20250514-v1:0"
381393
order: 1
382394
temperature:
383395
type: number
@@ -404,49 +416,50 @@ Resources:
404416
order: 6
405417
task_prompt:
406418
type: string
407-
description: Task prompt - supports placeholders {DOCUMENT_CLASS} (replaced with the detected class label), {ATTRIBUTE_NAMES_AND_DESCRIPTIONS} (replaced with the attribute names and descriptions for the detected class), and {DOCUMENT_TEXT} (replaced by the OCR output). Optionally use <<CACHEPOINT>> to separate static and dynamic elements of prompt for Bedrock prompt caching.
419+
description: Task prompt - supports placeholders {DOCUMENT_CLASS} (replaced with the detected class label), {ATTRIBUTE_NAMES_AND_DESCRIPTIONS} (replaced with the attribute names and descriptions for the detected class), {FEW_SHOT_EXAMPLES} (replaced by classPrompt and image data from examples in class definitions), {DOCUMENT_TEXT} (replaced by the OCR output), and {DOCUMENT_IMAGE} (replaced by the page image attachments for each page). Optionally use <<CACHEPOINT>> to separate static and dynamic elements of prompt for Bedrock prompt caching.
408420
order: 7
409-
evaluation:
421+
assessment:
410422
order: 5
411423
type: object
412-
sectionLabel: Evaluation Inference
424+
sectionLabel: Assessment Inference
413425
properties:
414-
llm_method:
415-
type: object
416-
properties:
417-
model:
418-
type: string
419-
description: Bedrock model ID
420-
enum: ["us.amazon.nova-lite-v1:0", "us.amazon.nova-pro-v1:0", "us.amazon.nova-premier-v1:0", "us.anthropic.claude-3-haiku-20240307-v1:0", "us.anthropic.claude-3-5-sonnet-20241022-v2:0", "us.anthropic.claude-3-7-sonnet-20250219-v1:0", "us.anthropic.claude-sonnet-4-20250514-v1:0", "us.anthropic.claude-opus-4-20250514-v1:0"]
421-
order: 1
422-
temperature:
423-
type: number
424-
description: Sampling temperature
425-
default: 0.0
426-
order: 2
427-
top_k:
428-
type: number
429-
description: Sampling Top K
430-
default: 250
431-
order: 3
432-
top_p:
433-
type: number
434-
description: Sampling Top P
435-
order: 4
436-
max_tokens:
437-
type: number
438-
description: Max tokens
439-
order: 5
440-
system_prompt:
441-
type: string
442-
format: textarea
443-
description: System prompt for LLM evaluation
444-
order: 6
445-
task_prompt:
446-
type: string
447-
format: textarea
448-
description: Task prompt for LLM evaluation - supports parameters {DOCUMENT_CLASS}, {ATTRIBUTE_NAME}, {ATTRIBUTE_DESCRIPTION}, {EXPECTED_VALUE} and {ACTUAL_VALUE}
449-
order: 7
426+
default_confidence_threshold:
427+
type: number
428+
description: Default confidence threshold for all attributes (0.0 to 1.0). If an attribute doesn't have its own threshold, this default will be used for confidence threshold alerts.
429+
minimum: 0
430+
maximum: 1
431+
order: 1
432+
model:
433+
type: string
434+
description: Bedrock model ID
435+
enum: ["us.amazon.nova-lite-v1:0", "us.amazon.nova-pro-v1:0", "us.amazon.nova-premier-v1:0", "us.anthropic.claude-3-haiku-20240307-v1:0", "us.anthropic.claude-3-5-sonnet-20241022-v2:0", "us.anthropic.claude-3-7-sonnet-20250219-v1:0", "us.anthropic.claude-sonnet-4-20250514-v1:0", "us.anthropic.claude-opus-4-20250514-v1:0"]
436+
order: 2
437+
temperature:
438+
type: number
439+
description: Sampling temperature
440+
order: 3
441+
top_k:
442+
type: number
443+
description: Sampling Top K
444+
order: 4
445+
top_p:
446+
type: number
447+
description: Sampling Top P
448+
order: 5
449+
max_tokens:
450+
type: number
451+
description: Max tokens
452+
order: 6
453+
system_prompt:
454+
type: string
455+
format: textarea
456+
description: System prompt
457+
order: 7
458+
task_prompt:
459+
type: string
460+
format: textarea
461+
description: Task prompt - supports placeholders {DOCUMENT_TEXT} (markdown text), {OCR_TEXT_CONFIDENCE} (OCR text blocks with confidence), {DOCUMENT_CLASS}, {ATTRIBUTE_NAMES_AND_DESCRIPTIONS}, {EXTRACTION_RESULTS} and {DOCUMENT_IMAGE}. Use <<CACHEPOINT>> to separate static and dynamic elements of prompt for Bedrock prompt caching.
462+
order: 8
450463
summarization:
451464
order: 6
452465
type: object
@@ -483,10 +496,51 @@ Resources:
483496
format: textarea
484497
description: Task prompt - supports parameter {DOCUMENT_TEXT}. Optionally use <<CACHEPOINT>> to separate static and dynamic elements of prompt for Bedrock prompt caching.
485498
order: 7
486-
pricing:
499+
evaluation:
487500
order: 7
501+
type: object
502+
sectionLabel: Evaluation Inference
503+
properties:
504+
llm_method:
505+
type: object
506+
properties:
507+
model:
508+
type: string
509+
description: Bedrock model ID
510+
enum: ["us.amazon.nova-lite-v1:0", "us.amazon.nova-pro-v1:0", "us.amazon.nova-premier-v1:0", "us.anthropic.claude-3-haiku-20240307-v1:0", "us.anthropic.claude-3-5-sonnet-20241022-v2:0", "us.anthropic.claude-3-7-sonnet-20250219-v1:0", "us.anthropic.claude-sonnet-4-20250514-v1:0", "us.anthropic.claude-opus-4-20250514-v1:0"]
511+
order: 1
512+
temperature:
513+
type: number
514+
description: Sampling temperature
515+
default: 0.0
516+
order: 2
517+
top_k:
518+
type: number
519+
description: Sampling Top K
520+
default: 250
521+
order: 3
522+
top_p:
523+
type: number
524+
description: Sampling Top P
525+
order: 4
526+
max_tokens:
527+
type: number
528+
description: Max tokens
529+
order: 5
530+
system_prompt:
531+
type: string
532+
format: textarea
533+
description: System prompt for LLM evaluation
534+
order: 6
535+
task_prompt:
536+
type: string
537+
format: textarea
538+
description: Task prompt for LLM evaluation - supports placeholders {DOCUMENT_CLASS}, {ATTRIBUTE_NAME}, {ATTRIBUTE_DESCRIPTION}, {EXPECTED_VALUE} and {ACTUAL_VALUE}
539+
order: 7
540+
pricing:
541+
order: 8
488542
type: array
489-
sectionLabel: Pricing Estimates - check https://aws.amazon.com/bedrock/pricing/ & https://aws.amazon.com/textract/pricing/
543+
sectionLabel: Pricing (Estimates only - check https://aws.amazon.com/bedrock/pricing/ & https://aws.amazon.com/textract/pricing/)
490544
listLabel: Services
491545
itemLabel: Service/API
492546
columns: 2

0 commit comments

Comments
 (0)