Skip to content

Commit 28a79dc

Browse files
author
Taniya Mathur
committed
refactor: remove duplicate S3 upload code in publish.py
- Extract S3 upload logic into helper methods _upload_template_to_s3() and _check_and_upload_template() - Simplify set_public_acls() method to raise single exception - Use loop for template uploads to handle main and versioned templates with same logic - Reduce code duplication and improve maintainability
1 parent 05593e5 commit 28a79dc

File tree

1 file changed

+60
-190
lines changed

1 file changed

+60
-190
lines changed

‎publish.py‎

Lines changed: 60 additions & 190 deletions
Original file line numberDiff line numberDiff line change
@@ -1376,6 +1376,37 @@ def package_ui(self):
13761376

13771377
return zipfile_name
13781378

1379+
def _upload_template_to_s3(self, template_path, s3_key, description):
1380+
"""Helper method to upload template to S3 with error handling"""
1381+
self.console.print(f"[cyan]Uploading {description} to S3: {s3_key}[/cyan]")
1382+
try:
1383+
self.s3_client.upload_file(template_path, self.bucket, s3_key)
1384+
self.console.print(f"[green]✅ {description} uploaded successfully[/green]")
1385+
except Exception as e:
1386+
self.console.print(f"[red]Failed to upload {description}: {e}[/red]")
1387+
sys.exit(1)
1388+
1389+
def _check_and_upload_template(self, template_path, s3_key, description):
1390+
"""Helper method to check if template exists in S3 and upload if missing"""
1391+
try:
1392+
self.s3_client.head_object(Bucket=self.bucket, Key=s3_key)
1393+
self.console.print(f"[green]✅ {description} already exists in S3[/green]")
1394+
except ClientError as e:
1395+
if e.response["Error"]["Code"] == "404":
1396+
self.console.print(
1397+
f"[yellow]{description} missing from S3, uploading: {s3_key}[/yellow]"
1398+
)
1399+
if not os.path.exists(template_path):
1400+
self.console.print(
1401+
f"[red]Error: No template to upload at {template_path}[/red]"
1402+
)
1403+
sys.exit(1)
1404+
self._upload_template_to_s3(template_path, s3_key, description)
1405+
else:
1406+
self.console.print(
1407+
f"[yellow]Could not check {description} existence: {e}[/yellow]"
1408+
)
1409+
13791410
def build_main_template(self, webui_zipfile, components_needing_rebuild):
13801411
"""Build and package main template with smart detection"""
13811412
self.console.print("[bold cyan]BUILDING main[/bold cyan]")
@@ -1490,142 +1521,32 @@ def build_main_template(self, webui_zipfile, components_needing_rebuild):
14901521
else:
14911522
self.console.print("[green]✅ Main template is up to date[/green]")
14921523

1493-
# Upload main template based on whether build was required
1494-
final_template_key = f"{self.prefix}/{self.main_template}"
1495-
versioned_template_key = f"{self.prefix}/{self.main_template.replace('.yaml', f'_{self.version}.yaml')}"
1524+
# Upload templates
14961525
packaged_template_path = ".aws-sam/idp-main.yaml"
1526+
templates = [
1527+
(f"{self.prefix}/{self.main_template}", "Main template"),
1528+
(
1529+
f"{self.prefix}/{self.main_template.replace('.yaml', f'_{self.version}.yaml')}",
1530+
"Versioned main template",
1531+
),
1532+
]
14971533

1498-
if main_needs_build:
1499-
# Main was rebuilt, so upload the new template
1500-
if not os.path.exists(packaged_template_path):
1501-
self.console.print(
1502-
f"[red]Error: Packaged template not found at {packaged_template_path}[/red]"
1503-
)
1504-
sys.exit(1)
1505-
1506-
self.console.print(
1507-
f"[cyan]Uploading main template to S3: {final_template_key}[/cyan]"
1508-
)
1509-
try:
1510-
self.s3_client.upload_file(
1511-
packaged_template_path,
1512-
self.bucket,
1513-
final_template_key,
1514-
)
1515-
self.console.print(
1516-
"[green]✅ Main template uploaded successfully[/green]"
1517-
)
1518-
except Exception as e:
1519-
self.console.print(f"[red]Failed to upload main template: {e}[/red]")
1520-
sys.exit(1)
1521-
1522-
# Also upload versioned copy
1523-
self.console.print(
1524-
f"[cyan]Uploading versioned main template to S3: {versioned_template_key}[/cyan]"
1525-
)
1526-
try:
1527-
self.s3_client.upload_file(
1528-
packaged_template_path,
1529-
self.bucket,
1530-
versioned_template_key,
1531-
)
1532-
self.console.print(
1533-
"[green]✅ Versioned main template uploaded successfully[/green]"
1534-
)
1535-
except Exception as e:
1536-
self.console.print(
1537-
f"[red]Failed to upload versioned main template: {e}[/red]"
1538-
)
1539-
sys.exit(1)
1540-
else:
1541-
# Main was not rebuilt, check if template exists in S3
1542-
try:
1543-
self.s3_client.head_object(Bucket=self.bucket, Key=final_template_key)
1544-
self.console.print(
1545-
"[green]✅ Main template already exists in S3[/green]"
1546-
)
1547-
except self.s3_client.exceptions.NoSuchKey:
1548-
self.console.print(
1549-
f"[yellow]Main template missing from S3, uploading: {final_template_key}[/yellow]"
1550-
)
1534+
for s3_key, description in templates:
1535+
if main_needs_build:
15511536
if not os.path.exists(packaged_template_path):
15521537
self.console.print(
1553-
f"[red]Error: No packaged template to upload at {packaged_template_path}[/red]"
1538+
f"[red]Error: Packaged template not found at {packaged_template_path}[/red]"
15541539
)
15551540
sys.exit(1)
1556-
try:
1557-
self.s3_client.upload_file(
1558-
packaged_template_path,
1559-
self.bucket,
1560-
final_template_key,
1561-
)
1562-
self.console.print(
1563-
"[green]✅ Main template uploaded successfully[/green]"
1564-
)
1565-
1566-
# Also upload versioned copy
1567-
self.console.print(
1568-
f"[cyan]Uploading versioned main template to S3: {versioned_template_key}[/cyan]"
1569-
)
1570-
self.s3_client.upload_file(
1571-
packaged_template_path,
1572-
self.bucket,
1573-
versioned_template_key,
1574-
)
1575-
self.console.print(
1576-
"[green]✅ Versioned main template uploaded successfully[/green]"
1577-
)
1578-
except Exception as e:
1579-
self.console.print(
1580-
f"[red]Failed to upload main template: {e}[/red]"
1581-
)
1582-
sys.exit(1)
1583-
except Exception as e:
1584-
self.console.print(
1585-
f"[yellow]Could not check S3 template existence: {e}[/yellow]"
1586-
)
1587-
self.console.print("[yellow]Proceeding without upload[/yellow]")
1588-
1589-
# Check versioned template separately
1590-
try:
1591-
self.s3_client.head_object(
1592-
Bucket=self.bucket, Key=versioned_template_key
1593-
)
1594-
self.console.print(
1595-
"[green]✅ Versioned main template already exists in S3[/green]"
1541+
self._upload_template_to_s3(packaged_template_path, s3_key, description)
1542+
else:
1543+
self._check_and_upload_template(
1544+
packaged_template_path, s3_key, description
15961545
)
1597-
except ClientError as e:
1598-
if e.response["Error"]["Code"] == "404":
1599-
self.console.print(
1600-
f"[yellow]Versioned template missing from S3, uploading: {versioned_template_key}[/yellow]"
1601-
)
1602-
if not os.path.exists(packaged_template_path):
1603-
self.console.print(
1604-
f"[red]Error: No packaged template to upload at {packaged_template_path}[/red]"
1605-
)
1606-
sys.exit(1)
1607-
try:
1608-
self.s3_client.upload_file(
1609-
packaged_template_path,
1610-
self.bucket,
1611-
versioned_template_key,
1612-
)
1613-
self.console.print(
1614-
"[green]✅ Versioned main template uploaded successfully[/green]"
1615-
)
1616-
except Exception as upload_e:
1617-
self.console.print(
1618-
f"[red]Failed to upload versioned template: {upload_e}[/red]"
1619-
)
1620-
sys.exit(1)
1621-
else:
1622-
self.console.print(
1623-
f"[yellow]Could not check versioned template existence: {e}[/yellow]"
1624-
)
16251546

16261547
# Validate the template
16271548
template_url = (
1628-
f"https://s3.{self.region}.amazonaws.com/{self.bucket}/{final_template_key}"
1549+
f"https://s3.{self.region}.amazonaws.com/{self.bucket}/{templates[0][0]}"
16291550
)
16301551
self.console.print(f"[cyan]Validating template: {template_url}[/cyan]")
16311552

@@ -1964,84 +1885,33 @@ def set_public_acls(self):
19641885
return
19651886

19661887
total_files = len(objects)
1967-
successful_acls = 0
1968-
failed_acls = 0
1969-
19701888
self.console.print(f"[cyan]Setting ACLs on {total_files} files...[/cyan]")
19711889

19721890
for i, obj in enumerate(objects, 1):
1973-
try:
1974-
self.s3_client.put_object_acl(
1975-
Bucket=self.bucket, Key=obj["Key"], ACL="public-read"
1891+
self.s3_client.put_object_acl(
1892+
Bucket=self.bucket, Key=obj["Key"], ACL="public-read"
1893+
)
1894+
if i % 10 == 0 or i == total_files:
1895+
self.console.print(
1896+
f"[cyan]Progress: {i}/{total_files} files processed[/cyan]"
19761897
)
1977-
successful_acls += 1
1978-
if i % 10 == 0 or i == total_files: # Progress every 10 files
1979-
self.console.print(
1980-
f"[cyan]Progress: {i}/{total_files} files processed[/cyan]"
1981-
)
1982-
except ClientError as e:
1983-
failed_acls += 1
1984-
if "AccessDenied" in str(e) or "BlockPublicAcls" in str(e):
1985-
# Don't spam with individual permission error messages
1986-
pass
1987-
else:
1988-
self.console.print(
1989-
f"[yellow]Warning: Could not set ACL for {obj['Key']}: {e}[/yellow]"
1990-
)
19911898

1992-
# Also set ACL for main template files (only if they exist)
1899+
# Set ACL for main template files
19931900
main_template_keys = [
19941901
f"{self.prefix}/{self.main_template}",
19951902
f"{self.prefix}/{self.main_template.replace('.yaml', f'_{self.version}.yaml')}",
19961903
]
19971904

19981905
for key in main_template_keys:
1999-
try:
2000-
# Check if the key exists first
2001-
self.s3_client.head_object(Bucket=self.bucket, Key=key)
2002-
# If it exists, set the ACL
2003-
self.s3_client.put_object_acl(
2004-
Bucket=self.bucket, Key=key, ACL="public-read"
2005-
)
2006-
successful_acls += 1
2007-
except ClientError as e:
2008-
if e.response["Error"]["Code"] == "404":
2009-
# Key doesn't exist, skip silently
2010-
continue
2011-
else:
2012-
failed_acls += 1
2013-
if "AccessDenied" not in str(
2014-
e
2015-
) and "BlockPublicAcls" not in str(e):
2016-
self.console.print(
2017-
f"[yellow]Warning: Could not set ACL for {key}: {e}[/yellow]"
2018-
)
2019-
2020-
# Report accurate final status
2021-
if failed_acls == 0:
2022-
self.console.print("[green]✅ Public ACLs set successfully[/green]")
2023-
elif successful_acls == 0:
2024-
self.console.print(
2025-
f"[red]❌ Could not set public ACLs on any files ({failed_acls} failed)[/red]"
2026-
)
2027-
self.console.print(
2028-
"[yellow]Files uploaded successfully but are NOT publicly accessible[/yellow]"
2029-
)
2030-
self.console.print(
2031-
"[yellow]This is usually due to 'Block Public Access' settings or insufficient permissions[/yellow]"
2032-
)
2033-
else:
2034-
self.console.print(
2035-
f"[yellow]⚠️ Partial ACL success: {successful_acls} succeeded, {failed_acls} failed[/yellow]"
1906+
self.s3_client.head_object(Bucket=self.bucket, Key=key)
1907+
self.s3_client.put_object_acl(
1908+
Bucket=self.bucket, Key=key, ACL="public-read"
20361909
)
20371910

1911+
self.console.print("[green]✅ Public ACLs set successfully[/green]")
1912+
20381913
except Exception as e:
2039-
self.console.print(
2040-
f"[yellow]Warning: Failed to set public ACLs: {e}[/yellow]"
2041-
)
2042-
self.console.print(
2043-
"[yellow]Files were uploaded successfully but may not be publicly accessible[/yellow]"
2044-
)
1914+
raise Exception(f"Failed to set public ACLs: {e}")
20451915

20461916
def run(self, args):
20471917
"""Main execution method"""

0 commit comments

Comments
 (0)