From a13565f7792e451ca795ae32b328c0ef839551ee Mon Sep 17 00:00:00 2001 From: ASPactores Date: Sun, 21 Sep 2025 22:45:41 +0800 Subject: [PATCH 1/2] feat: add initial excel export script --- .isort.cfg | 2 +- .../pycon_registrations/pycon_registration.py | 12 ++ backend/pyproject.toml | 4 + .../scripts/export_registrations_to_excel.py | 21 +++ backend/usecase/export_data_usecase.py | 124 +++++++++++++++++ backend/uv.lock | 125 ++++++++++++++++++ 6 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 backend/scripts/export_registrations_to_excel.py create mode 100644 backend/usecase/export_data_usecase.py diff --git a/.isort.cfg b/.isort.cfg index 47fb658a..19526fd3 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -known_third_party = aws,boto3,botocore,constants,controller,dotenv,external_gateway,fastapi,fastapi_cloudauth,lambda_decorators,lambdawarmer,mangum,model,pydantic,pynamodb,pytz,repository,requests,starlette,typing_extensions,ulid,usecase,utils +known_third_party = PIL,aws,boto3,botocore,constants,controller,dotenv,external_gateway,fastapi,fastapi_cloudauth,lambda_decorators,lambdawarmer,mangum,model,openpyxl,pandas,pydantic,pynamodb,pytz,repository,requests,starlette,typing_extensions,ulid,usecase,utils diff --git a/backend/model/pycon_registrations/pycon_registration.py b/backend/model/pycon_registrations/pycon_registration.py index 8b8c8584..789594fa 100644 --- a/backend/model/pycon_registrations/pycon_registration.py +++ b/backend/model/pycon_registrations/pycon_registration.py @@ -128,3 +128,15 @@ class Config: class PyconRegistrationPatch(PyconRegistration): class Config: extra = Extra.ignore + + +class PyconExportData(BaseModel): + firstName: str = Field(..., title='First Name') + lastName: str = Field(..., title='Last Name') + nickname: str = Field(..., title='Nickname') + jobTitle: str = Field(..., title='Job Title', description='Your current job title or role in tech') + email: EmailStr = Field(..., title='Email') + contactNumber: str = Field(..., title='Contact Number') + organization: str = Field(..., title='Affiliated Company or Organization') + ticketType: TicketTypes = Field(title='Ticket Type') + idURL: Optional[HttpUrl] = Field(None, title='ID URL') diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 6b591c5a..b913ef19 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,6 +17,10 @@ dependencies = [ "python-dateutil==2.9.0.post0", "requests==2.32.3", "pytz==2024.2", + "polars>=1.33.1", + "openpyxl>=3.1.5", + "pandas>=2.3.2", + "pillow>=11.3.0", ] [dependency-groups] diff --git a/backend/scripts/export_registrations_to_excel.py b/backend/scripts/export_registrations_to_excel.py new file mode 100644 index 00000000..76000506 --- /dev/null +++ b/backend/scripts/export_registrations_to_excel.py @@ -0,0 +1,21 @@ +import argparse +import os + +from dotenv import load_dotenv + +script_dir = os.path.dirname(os.path.abspath(__file__)) +parser = argparse.ArgumentParser() +parser.add_argument( + '--env-file', type=str, default=os.path.join(script_dir, '..', '.env'), help='Path to the .env file' +) +parser.add_argument('--event-id', type=str, required=True, help='Event ID') +parser.add_argument('--file-name', type=str, required=True, help='Output Excel file name') +args = parser.parse_args() +load_dotenv(dotenv_path=args.env_file) + +if __name__ == '__main__': + from usecase.export_data_usecase import ExportDataUsecase + + usecase = ExportDataUsecase() + + response = usecase.export_registrations_to_excel(event_id=args.event_id, file_name=args.file_name) diff --git a/backend/usecase/export_data_usecase.py b/backend/usecase/export_data_usecase.py new file mode 100644 index 00000000..ad1dd4fd --- /dev/null +++ b/backend/usecase/export_data_usecase.py @@ -0,0 +1,124 @@ +import os +from http import HTTPStatus +from io import BytesIO +from pathlib import Path + +import pandas as pd +import requests +from fastapi.responses import JSONResponse +from model.pycon_registrations.pycon_registration import PyconExportData +from model.registrations.registration import Registration +from openpyxl import load_workbook +from openpyxl.drawing.image import Image +from PIL import Image as PilImage +from repository.registrations_repository import RegistrationsRepository +from usecase.pycon_registration_usecase import PyconRegistrationUsecase +from utils.logger import logger + + +class ExportDataUsecase: + def __init__(self): + self.__registrations_repository = RegistrationsRepository() + self.__pycon_registration_usecase = PyconRegistrationUsecase() + + def export_registrations_to_excel(self, event_id: str, file_name: str): + reg_status, registration, reg_message = self.__registrations_repository.query_registrations(event_id=event_id) + + if reg_status != HTTPStatus.OK: + return JSONResponse(status_code=reg_status, content={'message': reg_message}) + + registration_with_presigned_url = [ + self.__pycon_registration_usecase.collect_pre_signed_url_pycon(registration=reg) for reg in registration + ] + + export_data_dicts = [ + { + 'firstName': reg.firstName, + 'lastName': reg.lastName, + 'nickname': reg.nickname, + 'jobTitle': reg.jobTitle, + 'email': reg.email, + 'contactNumber': reg.contactNumber, + 'organization': reg.organization, + 'ticketType': str(reg.ticketType), + 'idURL': reg.imageIdUrl, + } + for reg in registration_with_presigned_url + ] + + column_mapping = {} + for field_name, field_info in PyconExportData.__fields__.items(): + if field_name != 'idURL': + column_mapping[field_name] = field_info.field_info.title + + df = pd.DataFrame(export_data_dicts) + + df_to_excel = df.drop('idURL', axis=1) + + df_to_excel['ID Image'] = '' + column_mapping['ID Image'] = 'ID Image' + + df_to_excel.rename(columns=column_mapping, inplace=True) + + output_file_name = Path(file_name).with_suffix('.xlsx').name + output_path = os.path.join(os.getcwd(), output_file_name) + + try: + with pd.ExcelWriter(output_path, engine='openpyxl') as writer: + df_to_excel.to_excel(writer, sheet_name='Registrations', index=False) + + workbook = writer.book + worksheet = writer.sheets['Registrations'] + + FIXED_IMAGE_WIDTH = 400 + + image_column_idx = df_to_excel.columns.get_loc('ID Image') + 1 + image_column_letter = chr(65 + image_column_idx - 1) + worksheet.column_dimensions[image_column_letter].width = FIXED_IMAGE_WIDTH * 0.15 + + for index, row in df.iterrows(): + if row['idURL']: + try: + response = requests.get(row['idURL']) + + if response.status_code == HTTPStatus.OK: + image_content = response.content + + with PilImage.open(BytesIO(image_content)) as pil_img: + original_width, original_height = pil_img.size + + new_height = int((FIXED_IMAGE_WIDTH / original_width) * original_height) + + image_stream_for_openpyxl = BytesIO(image_content) + + img = Image(image_stream_for_openpyxl) + img.width = FIXED_IMAGE_WIDTH + img.height = new_height + + row_height_in_points = new_height * 0.75 + worksheet.row_dimensions[index + 2].height = row_height_in_points + + cell = f'{image_column_letter}{index + 2}' + worksheet.add_image(img, cell) + + else: + logger.error( + f"Failed to download image from {row['idURL']}. Status code: {response.status_code}" + ) + worksheet.cell( + row=index + 2, column=image_column_idx + ).value = f'Error: {response.status_code}' + + except Exception as e: + logger.error(f"An error occurred while processing image from {row['idURL']}: {e}") + worksheet.cell(row=index + 2, column=image_column_idx).value = f'Error: {str(e)}' + + logger.info(f'Successfully exported data to {output_path}') + return JSONResponse(status_code=HTTPStatus.OK, content={'message': f'Data exported to {output_file_name}'}) + + except Exception as e: + logger.error(f'An error occurred during Excel export: {e}') + return JSONResponse( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + content={'message': f'An error occurred during Excel export: {e}'}, + ) diff --git a/backend/uv.lock b/backend/uv.lock index 50d126b4..1d78e468 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -349,6 +349,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/ee/bf0adb559ad3c786f12bcbc9296b3f5675f529199bef03e2df281fa1fadb/email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631", size = 33521 }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059 }, +] + [[package]] name = "fastapi" version = "0.96.0" @@ -637,6 +646,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314 }, ] +[[package]] +name = "numpy" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/45/e80d203ef6b267aa29b22714fb558930b27960a0c5ce3c19c999232bb3eb/numpy-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ffc4f5caba7dfcbe944ed674b7eef683c7e94874046454bb79ed7ee0236f59d", size = 21259253 }, + { url = "https://files.pythonhosted.org/packages/52/18/cf2c648fccf339e59302e00e5f2bc87725a3ce1992f30f3f78c9044d7c43/numpy-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7e946c7170858a0295f79a60214424caac2ffdb0063d4d79cb681f9aa0aa569", size = 14450980 }, + { url = "https://files.pythonhosted.org/packages/93/fb/9af1082bec870188c42a1c239839915b74a5099c392389ff04215dcee812/numpy-2.3.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cd4260f64bc794c3390a63bf0728220dd1a68170c169088a1e0dfa2fde1be12f", size = 5379709 }, + { url = "https://files.pythonhosted.org/packages/75/0f/bfd7abca52bcbf9a4a65abc83fe18ef01ccdeb37bfb28bbd6ad613447c79/numpy-2.3.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f0ddb4b96a87b6728df9362135e764eac3cfa674499943ebc44ce96c478ab125", size = 6913923 }, + { url = "https://files.pythonhosted.org/packages/79/55/d69adad255e87ab7afda1caf93ca997859092afeb697703e2f010f7c2e55/numpy-2.3.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afd07d377f478344ec6ca2b8d4ca08ae8bd44706763d1efb56397de606393f48", size = 14589591 }, + { url = "https://files.pythonhosted.org/packages/10/a2/010b0e27ddeacab7839957d7a8f00e91206e0c2c47abbb5f35a2630e5387/numpy-2.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc92a5dedcc53857249ca51ef29f5e5f2f8c513e22cfb90faeb20343b8c6f7a6", size = 16938714 }, + { url = "https://files.pythonhosted.org/packages/1c/6b/12ce8ede632c7126eb2762b9e15e18e204b81725b81f35176eac14dc5b82/numpy-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7af05ed4dc19f308e1d9fc759f36f21921eb7bbfc82843eeec6b2a2863a0aefa", size = 16370592 }, + { url = "https://files.pythonhosted.org/packages/b4/35/aba8568b2593067bb6a8fe4c52babb23b4c3b9c80e1b49dff03a09925e4a/numpy-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:433bf137e338677cebdd5beac0199ac84712ad9d630b74eceeb759eaa45ddf30", size = 18884474 }, + { url = "https://files.pythonhosted.org/packages/45/fa/7f43ba10c77575e8be7b0138d107e4f44ca4a1ef322cd16980ea3e8b8222/numpy-2.3.3-cp311-cp311-win32.whl", hash = "sha256:eb63d443d7b4ffd1e873f8155260d7f58e7e4b095961b01c91062935c2491e57", size = 6599794 }, + { url = "https://files.pythonhosted.org/packages/0a/a2/a4f78cb2241fe5664a22a10332f2be886dcdea8784c9f6a01c272da9b426/numpy-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:ec9d249840f6a565f58d8f913bccac2444235025bbb13e9a4681783572ee3caa", size = 13088104 }, + { url = "https://files.pythonhosted.org/packages/79/64/e424e975adbd38282ebcd4891661965b78783de893b381cbc4832fb9beb2/numpy-2.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:74c2a948d02f88c11a3c075d9733f1ae67d97c6bdb97f2bb542f980458b257e7", size = 10460772 }, + { url = "https://files.pythonhosted.org/packages/b8/f2/7e0a37cfced2644c9563c529f29fa28acbd0960dde32ece683aafa6f4949/numpy-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1e02c7159791cd481e1e6d5ddd766b62a4d5acf8df4d4d1afe35ee9c5c33a41e", size = 21131019 }, + { url = "https://files.pythonhosted.org/packages/1a/7e/3291f505297ed63831135a6cc0f474da0c868a1f31b0dd9a9f03a7a0d2ed/numpy-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:dca2d0fc80b3893ae72197b39f69d55a3cd8b17ea1b50aa4c62de82419936150", size = 14376288 }, + { url = "https://files.pythonhosted.org/packages/bf/4b/ae02e985bdeee73d7b5abdefeb98aef1207e96d4c0621ee0cf228ddfac3c/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:99683cbe0658f8271b333a1b1b4bb3173750ad59c0c61f5bbdc5b318918fffe3", size = 5305425 }, + { url = "https://files.pythonhosted.org/packages/8b/eb/9df215d6d7250db32007941500dc51c48190be25f2401d5b2b564e467247/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d9d537a39cc9de668e5cd0e25affb17aec17b577c6b3ae8a3d866b479fbe88d0", size = 6819053 }, + { url = "https://files.pythonhosted.org/packages/57/62/208293d7d6b2a8998a4a1f23ac758648c3c32182d4ce4346062018362e29/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8596ba2f8af5f93b01d97563832686d20206d303024777f6dfc2e7c7c3f1850e", size = 14420354 }, + { url = "https://files.pythonhosted.org/packages/ed/0c/8e86e0ff7072e14a71b4c6af63175e40d1e7e933ce9b9e9f765a95b4e0c3/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1ec5615b05369925bd1125f27df33f3b6c8bc10d788d5999ecd8769a1fa04db", size = 16760413 }, + { url = "https://files.pythonhosted.org/packages/af/11/0cc63f9f321ccf63886ac203336777140011fb669e739da36d8db3c53b98/numpy-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2e267c7da5bf7309670523896df97f93f6e469fb931161f483cd6882b3b1a5dc", size = 12971844 }, +] + +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910 }, +] + [[package]] name = "packaging" version = "25.0" @@ -646,6 +693,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 }, ] +[[package]] +name = "pandas" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/8e/0e90233ac205ad182bd6b422532695d2b9414944a280488105d598c70023/pandas-2.3.2.tar.gz", hash = "sha256:ab7b58f8f82706890924ccdfb5f48002b83d2b5a3845976a9fb705d36c34dcdb", size = 4488684 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/59/f3e010879f118c2d400902d2d871c2226cef29b08c09fb8dc41111730400/pandas-2.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1333e9c299adcbb68ee89a9bb568fc3f20f9cbb419f1dd5225071e6cddb2a743", size = 11563308 }, + { url = "https://files.pythonhosted.org/packages/38/18/48f10f1cc5c397af59571d638d211f494dba481f449c19adbd282aa8f4ca/pandas-2.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:76972bcbd7de8e91ad5f0ca884a9f2c477a2125354af624e022c49e5bd0dfff4", size = 10820319 }, + { url = "https://files.pythonhosted.org/packages/95/3b/1e9b69632898b048e223834cd9702052bcf06b15e1ae716eda3196fb972e/pandas-2.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b98bdd7c456a05eef7cd21fd6b29e3ca243591fe531c62be94a2cc987efb5ac2", size = 11790097 }, + { url = "https://files.pythonhosted.org/packages/8b/ef/0e2ffb30b1f7fbc9a588bd01e3c14a0d96854d09a887e15e30cc19961227/pandas-2.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d81573b3f7db40d020983f78721e9bfc425f411e616ef019a10ebf597aedb2e", size = 12397958 }, + { url = "https://files.pythonhosted.org/packages/23/82/e6b85f0d92e9afb0e7f705a51d1399b79c7380c19687bfbf3d2837743249/pandas-2.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e190b738675a73b581736cc8ec71ae113d6c3768d0bd18bffa5b9a0927b0b6ea", size = 13225600 }, + { url = "https://files.pythonhosted.org/packages/e8/f1/f682015893d9ed51611948bd83683670842286a8edd4f68c2c1c3b231eef/pandas-2.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c253828cb08f47488d60f43c5fc95114c771bbfff085da54bfc79cb4f9e3a372", size = 13879433 }, + { url = "https://files.pythonhosted.org/packages/a7/e7/ae86261695b6c8a36d6a4c8d5f9b9ede8248510d689a2f379a18354b37d7/pandas-2.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:9467697b8083f9667b212633ad6aa4ab32436dcbaf4cd57325debb0ddef2012f", size = 11336557 }, +] + [[package]] name = "pathspec" version = "0.12.1" @@ -655,6 +723,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, ] +[[package]] +name = "pillow" +version = "11.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/d0d6dea55cd152ce3d6767bb38a8fc10e33796ba4ba210cbab9354b6d238/pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523", size = 47113069 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/26/77f8ed17ca4ffd60e1dcd220a6ec6d71210ba398cfa33a13a1cd614c5613/pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722", size = 5316531 }, + { url = "https://files.pythonhosted.org/packages/cb/39/ee475903197ce709322a17a866892efb560f57900d9af2e55f86db51b0a5/pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288", size = 4686560 }, + { url = "https://files.pythonhosted.org/packages/d5/90/442068a160fd179938ba55ec8c97050a612426fae5ec0a764e345839f76d/pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d", size = 5870978 }, + { url = "https://files.pythonhosted.org/packages/13/92/dcdd147ab02daf405387f0218dcf792dc6dd5b14d2573d40b4caeef01059/pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494", size = 7641168 }, + { url = "https://files.pythonhosted.org/packages/6e/db/839d6ba7fd38b51af641aa904e2960e7a5644d60ec754c046b7d2aee00e5/pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58", size = 5973053 }, + { url = "https://files.pythonhosted.org/packages/f2/2f/d7675ecae6c43e9f12aa8d58b6012683b20b6edfbdac7abcb4e6af7a3784/pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f", size = 6640273 }, + { url = "https://files.pythonhosted.org/packages/45/ad/931694675ede172e15b2ff03c8144a0ddaea1d87adb72bb07655eaffb654/pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e", size = 6082043 }, + { url = "https://files.pythonhosted.org/packages/3a/04/ba8f2b11fc80d2dd462d7abec16351b45ec99cbbaea4387648a44190351a/pillow-11.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:932c754c2d51ad2b2271fd01c3d121daaa35e27efae2a616f77bf164bc0b3e94", size = 6715516 }, + { url = "https://files.pythonhosted.org/packages/48/59/8cd06d7f3944cc7d892e8533c56b0acb68399f640786313275faec1e3b6f/pillow-11.3.0-cp311-cp311-win32.whl", hash = "sha256:b4b8f3efc8d530a1544e5962bd6b403d5f7fe8b9e08227c6b255f98ad82b4ba0", size = 6274768 }, + { url = "https://files.pythonhosted.org/packages/f1/cc/29c0f5d64ab8eae20f3232da8f8571660aa0ab4b8f1331da5c2f5f9a938e/pillow-11.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1a992e86b0dd7aeb1f053cd506508c0999d710a8f07b4c791c63843fc6a807ac", size = 6986055 }, + { url = "https://files.pythonhosted.org/packages/c6/df/90bd886fabd544c25addd63e5ca6932c86f2b701d5da6c7839387a076b4a/pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd", size = 2423079 }, + { url = "https://files.pythonhosted.org/packages/9e/e3/6fa84033758276fb31da12e5fb66ad747ae83b93c67af17f8c6ff4cc8f34/pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6", size = 5270566 }, + { url = "https://files.pythonhosted.org/packages/5b/ee/e8d2e1ab4892970b561e1ba96cbd59c0d28cf66737fc44abb2aec3795a4e/pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438", size = 4654618 }, + { url = "https://files.pythonhosted.org/packages/f2/6d/17f80f4e1f0761f02160fc433abd4109fa1548dcfdca46cfdadaf9efa565/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3", size = 4874248 }, + { url = "https://files.pythonhosted.org/packages/de/5f/c22340acd61cef960130585bbe2120e2fd8434c214802f07e8c03596b17e/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c", size = 6583963 }, + { url = "https://files.pythonhosted.org/packages/31/5e/03966aedfbfcbb4d5f8aa042452d3361f325b963ebbadddac05b122e47dd/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361", size = 4957170 }, + { url = "https://files.pythonhosted.org/packages/cc/2d/e082982aacc927fc2cab48e1e731bdb1643a1406acace8bed0900a61464e/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7", size = 5581505 }, + { url = "https://files.pythonhosted.org/packages/34/e7/ae39f538fd6844e982063c3a5e4598b8ced43b9633baa3a85ef33af8c05c/pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8", size = 6984598 }, +] + [[package]] name = "platformdirs" version = "4.3.8" @@ -673,6 +767,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, ] +[[package]] +name = "polars" +version = "1.33.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/da/8246f1d69d7e49f96f0c5529057a19af1536621748ef214bbd4112c83b8e/polars-1.33.1.tar.gz", hash = "sha256:fa3fdc34eab52a71498264d6ff9b0aa6955eb4b0ae8add5d3cb43e4b84644007", size = 4822485 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/79/c51e7e1d707d8359bcb76e543a8315b7ae14069ecf5e75262a0ecb32e044/polars-1.33.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3881c444b0f14778ba94232f077a709d435977879c1b7d7bd566b55bd1830bb5", size = 39132875 }, + { url = "https://files.pythonhosted.org/packages/f8/15/1094099a1b9cb4fbff58cd8ed3af8964f4d22a5b682ea0b7bb72bf4bc3d9/polars-1.33.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:29200b89c9a461e6f06fc1660bc9c848407640ee30fe0e5ef4947cfd49d55337", size = 35638783 }, + { url = "https://files.pythonhosted.org/packages/8d/b9/9ac769e4d8e8f22b0f2e974914a63dd14dec1340cd23093de40f0d67d73b/polars-1.33.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:444940646e76342abaa47f126c70e3e40b56e8e02a9e89e5c5d1c24b086db58a", size = 39742297 }, + { url = "https://files.pythonhosted.org/packages/7a/26/4c5da9f42fa067b2302fe62bcbf91faac5506c6513d910fae9548fc78d65/polars-1.33.1-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:094a37d06789286649f654f229ec4efb9376630645ba8963b70cb9c0b008b3e1", size = 36684940 }, + { url = "https://files.pythonhosted.org/packages/06/a6/dc535da476c93b2efac619e04ab81081e004e4b4553352cd10e0d33a015d/polars-1.33.1-cp39-abi3-win_amd64.whl", hash = "sha256:c9781c704432a2276a185ee25898aa427f39a904fbe8fde4ae779596cdbd7a9e", size = 39456676 }, + { url = "https://files.pythonhosted.org/packages/cb/4e/a4300d52dd81b58130ccadf3873f11b3c6de54836ad4a8f32bac2bd2ba17/polars-1.33.1-cp39-abi3-win_arm64.whl", hash = "sha256:c3cfddb3b78eae01a218222bdba8048529fef7e14889a71e33a5198644427642", size = 35445171 }, +] + [[package]] name = "pre-commit" version = "3.3.3" @@ -1087,6 +1195,10 @@ dependencies = [ { name = "lambda-decorators" }, { name = "lambda-warmer-py" }, { name = "mangum" }, + { name = "openpyxl" }, + { name = "pandas" }, + { name = "pillow" }, + { name = "polars" }, { name = "pydantic", extra = ["email"] }, { name = "pynamodb" }, { name = "python-dateutil" }, @@ -1124,6 +1236,10 @@ requires-dist = [ { name = "lambda-decorators", specifier = "==0.6.0" }, { name = "lambda-warmer-py", specifier = "==0.6.0" }, { name = "mangum", specifier = "==0.15.0" }, + { name = "openpyxl", specifier = ">=3.1.5" }, + { name = "pandas", specifier = ">=2.3.2" }, + { name = "pillow", specifier = ">=11.3.0" }, + { name = "polars", specifier = ">=1.33.1" }, { name = "pydantic", extras = ["email"], specifier = ">=1.10.0,<2.0.0" }, { name = "pynamodb", specifier = "==6.1.0" }, { name = "python-dateutil", specifier = "==2.9.0.post0" }, @@ -1200,6 +1316,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/21/7d397a4b7934ff4028987914ac1044d3b7d52712f30e2ac7a2ae5bc86dd0/typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", size = 31584 }, ] +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839 }, +] + [[package]] name = "ulid" version = "1.1" From aafda241b0ff06a76df25001404705806c9e40dd Mon Sep 17 00:00:00 2001 From: ASPactores Date: Mon, 22 Sep 2025 21:39:16 +0800 Subject: [PATCH 2/2] refactor(export_registration): refactor export registration data usecase --- .isort.cfg | 2 +- .../pycon_registrations/pycon_registration.py | 2 +- .../scripts/export_registrations_to_excel.py | 9 +- backend/usecase/export_data_usecase.py | 223 +++++++++++------- 4 files changed, 146 insertions(+), 90 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 19526fd3..8e21eb0b 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -known_third_party = PIL,aws,boto3,botocore,constants,controller,dotenv,external_gateway,fastapi,fastapi_cloudauth,lambda_decorators,lambdawarmer,mangum,model,openpyxl,pandas,pydantic,pynamodb,pytz,repository,requests,starlette,typing_extensions,ulid,usecase,utils +known_third_party = PIL,aws,boto3,botocore,constants,controller,dotenv,external_gateway,fastapi,fastapi_cloudauth,httpx,lambda_decorators,lambdawarmer,mangum,model,openpyxl,pandas,pydantic,pynamodb,pytz,repository,requests,starlette,typing_extensions,ulid,usecase,utils diff --git a/backend/model/pycon_registrations/pycon_registration.py b/backend/model/pycon_registrations/pycon_registration.py index 789594fa..eec4cf44 100644 --- a/backend/model/pycon_registrations/pycon_registration.py +++ b/backend/model/pycon_registrations/pycon_registration.py @@ -139,4 +139,4 @@ class PyconExportData(BaseModel): contactNumber: str = Field(..., title='Contact Number') organization: str = Field(..., title='Affiliated Company or Organization') ticketType: TicketTypes = Field(title='Ticket Type') - idURL: Optional[HttpUrl] = Field(None, title='ID URL') + imageIdUrl: Optional[HttpUrl] = Field(None, title='ID URL') diff --git a/backend/scripts/export_registrations_to_excel.py b/backend/scripts/export_registrations_to_excel.py index 76000506..cf408b91 100644 --- a/backend/scripts/export_registrations_to_excel.py +++ b/backend/scripts/export_registrations_to_excel.py @@ -13,9 +13,14 @@ args = parser.parse_args() load_dotenv(dotenv_path=args.env_file) +import asyncio + if __name__ == '__main__': from usecase.export_data_usecase import ExportDataUsecase - usecase = ExportDataUsecase() + async def main(): + usecase = ExportDataUsecase() + response = await usecase.export_registrations_to_excel(event_id=args.event_id, file_name=args.file_name) + print(response) - response = usecase.export_registrations_to_excel(event_id=args.event_id, file_name=args.file_name) + asyncio.run(main()) diff --git a/backend/usecase/export_data_usecase.py b/backend/usecase/export_data_usecase.py index ad1dd4fd..b61ae80e 100644 --- a/backend/usecase/export_data_usecase.py +++ b/backend/usecase/export_data_usecase.py @@ -1,14 +1,13 @@ +import asyncio import os from http import HTTPStatus from io import BytesIO from pathlib import Path +import httpx import pandas as pd -import requests from fastapi.responses import JSONResponse from model.pycon_registrations.pycon_registration import PyconExportData -from model.registrations.registration import Registration -from openpyxl import load_workbook from openpyxl.drawing.image import Image from PIL import Image as PilImage from repository.registrations_repository import RegistrationsRepository @@ -20,105 +19,157 @@ class ExportDataUsecase: def __init__(self): self.__registrations_repository = RegistrationsRepository() self.__pycon_registration_usecase = PyconRegistrationUsecase() + self.__FIXED_IMAGE_WIDTH_PX = 400 + self.__EXCEL_COLUMN_WIDTH_FACTOR = 0.15 + self.__EXCEL_ROW_HEIGHT_FACTOR = 0.75 + + async def export_registrations_to_excel(self, event_id: str, file_name: str): + """ + Exports an event's registration list to an Excel file, embedding ID images where available. + :param event_id: The ID of the event to export registrations for. + :param file_name: The desired name for the output Excel file (without extension). + :return: JSONResponse indicating success or failure, with the file path if successful. + """ + try: + registrations_data = self._fetch_and_prepare_data(event_id) + if not registrations_data: + logger.info('No registrations found to export.') + return JSONResponse(status_code=HTTPStatus.OK, content={'message': 'No registrations to export.'}) - def export_registrations_to_excel(self, event_id: str, file_name: str): - reg_status, registration, reg_message = self.__registrations_repository.query_registrations(event_id=event_id) + df, column_mapping = self._create_dataframe(registrations_data) + output_path = await self._write_excel_with_images_async(df, file_name, column_mapping) - if reg_status != HTTPStatus.OK: - return JSONResponse(status_code=reg_status, content={'message': reg_message}) + logger.info(f'Successfully exported data to {output_path}') + return JSONResponse( + status_code=HTTPStatus.OK, content={'message': f'Data exported to {Path(output_path).name}'} + ) - registration_with_presigned_url = [ - self.__pycon_registration_usecase.collect_pre_signed_url_pycon(registration=reg) for reg in registration - ] + except ValueError as e: + return JSONResponse(status_code=HTTPStatus.BAD_REQUEST, content={'message': str(e)}) + except Exception as e: + logger.error(f'An unexpected error occurred during Excel export: {e}', exc_info=True) + return JSONResponse( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + content={'message': f'An error occurred during Excel export: {e}'}, + ) + + def _fetch_and_prepare_data(self, event_id: str) -> list[PyconExportData]: + status, registrations, message = self.__registrations_repository.query_registrations(event_id=event_id) + if status != HTTPStatus.OK: + raise ValueError(f'Failed to query registrations: {message}') - export_data_dicts = [ - { - 'firstName': reg.firstName, - 'lastName': reg.lastName, - 'nickname': reg.nickname, - 'jobTitle': reg.jobTitle, - 'email': reg.email, - 'contactNumber': reg.contactNumber, - 'organization': reg.organization, - 'ticketType': str(reg.ticketType), - 'idURL': reg.imageIdUrl, - } - for reg in registration_with_presigned_url + registrations_with_url = [ + self.__pycon_registration_usecase.collect_pre_signed_url_pycon(registration=reg) for reg in registrations ] - column_mapping = {} - for field_name, field_info in PyconExportData.__fields__.items(): - if field_name != 'idURL': - column_mapping[field_name] = field_info.field_info.title + export_data = [ + PyconExportData( + firstName=reg.firstName, + lastName=reg.lastName, + nickname=reg.nickname, + jobTitle=reg.jobTitle, + email=reg.email, + contactNumber=reg.contactNumber, + organization=reg.organization, + ticketType=reg.ticketType, + imageIdUrl=getattr(reg, 'imageIdUrl', None), + ) + for reg in registrations_with_url + ] - df = pd.DataFrame(export_data_dicts) + return export_data - df_to_excel = df.drop('idURL', axis=1) + def _create_dataframe(self, data: list[PyconExportData]) -> tuple[pd.DataFrame, dict]: + column_mapping = { + field.name: field.field_info.title + for field in PyconExportData.__fields__.values() + if field.name != 'imageIdUrl' + } - df_to_excel['ID Image'] = '' - column_mapping['ID Image'] = 'ID Image' + processed_records = [] + for item in data: + record = item.dict() + if 'ticketType' in record and hasattr(record['ticketType'], 'value'): + record['ticketType'] = record['ticketType'].value + processed_records.append(record) - df_to_excel.rename(columns=column_mapping, inplace=True) + df = pd.DataFrame(processed_records) + return df, column_mapping + async def _write_excel_with_images_async(self, df: pd.DataFrame, file_name: str, column_mapping: dict) -> str: output_file_name = Path(file_name).with_suffix('.xlsx').name output_path = os.path.join(os.getcwd(), output_file_name) - try: - with pd.ExcelWriter(output_path, engine='openpyxl') as writer: - df_to_excel.to_excel(writer, sheet_name='Registrations', index=False) - - workbook = writer.book - worksheet = writer.sheets['Registrations'] - - FIXED_IMAGE_WIDTH = 400 - - image_column_idx = df_to_excel.columns.get_loc('ID Image') + 1 - image_column_letter = chr(65 + image_column_idx - 1) - worksheet.column_dimensions[image_column_letter].width = FIXED_IMAGE_WIDTH * 0.15 - - for index, row in df.iterrows(): - if row['idURL']: - try: - response = requests.get(row['idURL']) - - if response.status_code == HTTPStatus.OK: - image_content = response.content - - with PilImage.open(BytesIO(image_content)) as pil_img: - original_width, original_height = pil_img.size - - new_height = int((FIXED_IMAGE_WIDTH / original_width) * original_height) - - image_stream_for_openpyxl = BytesIO(image_content) - - img = Image(image_stream_for_openpyxl) - img.width = FIXED_IMAGE_WIDTH - img.height = new_height - - row_height_in_points = new_height * 0.75 - worksheet.row_dimensions[index + 2].height = row_height_in_points - - cell = f'{image_column_letter}{index + 2}' - worksheet.add_image(img, cell) + df_to_excel = df.drop(columns=['imageIdUrl'], errors='ignore') + df_to_excel['ID Image'] = '' + df_to_excel.rename(columns=column_mapping, inplace=True) - else: - logger.error( - f"Failed to download image from {row['idURL']}. Status code: {response.status_code}" - ) - worksheet.cell( - row=index + 2, column=image_column_idx - ).value = f'Error: {response.status_code}' + with pd.ExcelWriter(output_path, engine='openpyxl') as writer: + df_to_excel.to_excel(writer, sheet_name='Registrations', index=False) + worksheet = writer.sheets['Registrations'] + await self._embed_images_async(worksheet, df, df_to_excel.columns) - except Exception as e: - logger.error(f"An error occurred while processing image from {row['idURL']}: {e}") - worksheet.cell(row=index + 2, column=image_column_idx).value = f'Error: {str(e)}' + return output_path - logger.info(f'Successfully exported data to {output_path}') - return JSONResponse(status_code=HTTPStatus.OK, content={'message': f'Data exported to {output_file_name}'}) + async def _download_and_process_image_async(self, client: httpx.AsyncClient, url: str) -> Image | str | None: + if not url or not isinstance(url, str) or not url.strip(): + return None + try: + response = await client.get(url, timeout=30) + response.raise_for_status() + + input_stream = BytesIO(response.content) + + with PilImage.open(input_stream) as pil_img: + output_stream = BytesIO() + pil_img.save(output_stream, format='PNG') + + original_width, original_height = pil_img.size + if original_width == 0: + return 'Error: Invalid image width' + aspect_ratio = original_height / original_width + new_height = int(self.__FIXED_IMAGE_WIDTH_PX * aspect_ratio) + + output_stream.seek(0) + + img = Image(output_stream) + img.width = self.__FIXED_IMAGE_WIDTH_PX + img.height = new_height + return img + + except httpx.HTTPStatusError as e: + logger.error(f'HTTP error for {url}: {e.response.status_code}') + return f'Error: {e.response.status_code}' + except httpx.RequestError as e: + logger.error(f'Network error for {url}: {e}') + return 'Error: Network issue' except Exception as e: - logger.error(f'An error occurred during Excel export: {e}') - return JSONResponse( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - content={'message': f'An error occurred during Excel export: {e}'}, - ) + logger.error(f'Processing error for {url}: {e}') + return 'Error: Corrupt image' + + async def _embed_images_async(self, worksheet, source_df: pd.DataFrame, final_columns: pd.Index): + image_column_idx = final_columns.get_loc('ID Image') + 1 + image_column_letter = chr(64 + image_column_idx) + worksheet.column_dimensions[image_column_letter].width = ( + self.__FIXED_IMAGE_WIDTH_PX * self.__EXCEL_COLUMN_WIDTH_FACTOR + ) + + async with httpx.AsyncClient() as client: + tasks = [ + self._download_and_process_image_async(client, row.get('imageIdUrl')) for _, row in source_df.iterrows() + ] + results = await asyncio.gather(*tasks) + + for idx, result in enumerate(results): + row_idx = idx + 2 + + if result is None: + continue + + if isinstance(result, Image): + img = result + worksheet.row_dimensions[row_idx].height = img.height * self.__EXCEL_ROW_HEIGHT_FACTOR + worksheet.add_image(img, f'{image_column_letter}{row_idx}') + else: + worksheet.cell(row=row_idx, column=image_column_idx, value=str(result))