diff --git a/.github/workflows/keyfactor-starter-workflow.yml b/.github/workflows/keyfactor-starter-workflow.yml index 07fe0be..56756c6 100644 --- a/.github/workflows/keyfactor-starter-workflow.yml +++ b/.github/workflows/keyfactor-starter-workflow.yml @@ -11,10 +11,19 @@ on: jobs: call-starter-workflow: - uses: keyfactor/actions/.github/workflows/starter.yml@v3 + uses: keyfactor/actions/.github/workflows/starter.yml@v4 + permissions: + contents: write # Explicitly grant write permission + with: + command_token_url: ${{ vars.COMMAND_TOKEN_URL }} + command_hostname: ${{ vars.COMMAND_HOSTNAME }} + command_base_api_path: ${{ vars.COMMAND_API_PATH }} secrets: token: ${{ secrets.V2BUILDTOKEN}} - APPROVE_README_PUSH: ${{ secrets.APPROVE_README_PUSH}} gpg_key: ${{ secrets.KF_GPG_PRIVATE_KEY }} gpg_pass: ${{ secrets.KF_GPG_PASSPHRASE }} scan_token: ${{ secrets.SAST_TOKEN }} + entra_username: ${{ secrets.DOCTOOL_ENTRA_USERNAME }} + entra_password: ${{ secrets.DOCTOOL_ENTRA_PASSWD }} + command_client_id: ${{ secrets.COMMAND_CLIENT_ID }} + command_client_secret: ${{ secrets.COMMAND_CLIENT_SECRET }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index b70c75f..cc87242 100644 --- a/.gitignore +++ b/.gitignore @@ -353,3 +353,7 @@ MigrationBackup/ /sectigo-metadata-sync/config.json /sectigo-metadata-sync/config/config.json /sectigo-metadata-sync/config/config-az.json +/sectigo-metadata-sync/config/fields - Copy (2).json +/sectigo-metadata-sync/config/fields - Copy.json +/sectigo-metadata-sync/config/config-ses.json +/sectigo-metadata-sync/config/config-az.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 3edd275..0507b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,2 +1,23 @@ -# 1.0.0 -Initial release +## 1.1.0 - October 15th 2025 + +### ⚠️ Important Notice +- Added a value truncation setting to config based on Keyfactor and Sectigo field character length limits. +- Setting must be specified for the tool to run - it will exit unless the setting is specified. Please review documentation for the `enableTruncation` setting before using this version! + +### Fixed +- Fixed configuration loading and verification. + +### Changed +- Improved logging and improved memory consumption. +- Updated nuget packages. +- Improved clients for Sectigo and Keyfactor APIs. +- Improved documentation, added diagrams with clarifications. + +### Added +- Added OAuth login support for Keyfactor API. +- Added the ability to limit sync to certificates imported into Keyfactor after a given date via config file. +- **Breaking** Added value truncation setting to config. + +## 1.0.0 - June 23rd 2025 + +- Initial release diff --git a/README.md b/README.md index eeb2b98..f27b9b0 100644 --- a/README.md +++ b/README.md @@ -33,31 +33,29 @@ The Sectigo Metadata Sync is open source and there is **no SLA**. Keyfactor will ## Overview -This tool automates the synchronization of metadata fields between Sectigo and Keyfactor. It performs two primary operations: +This tool performs the synchronization of metadata fields and their contents between Sectigo and Keyfactor. It has two modes of operation: 1. **SCtoKF** – Synchronizes custom fields and contained data, and additional requested data from Sectigo into Keyfactor. 2. **KFtoSC** – Synchronizes custom field data from Keyfactor back to Sectigo. -Fields listed in `fields.json` that do not already exist in Keyfactor will be created automatically. -> **Note:** Certificates must already be imported into Keyfactor for metadata synchronization to work. The tool does *not* import certificates themselves. - ---- +Fields listed in `fields.json` that do not already exist in Keyfactor will be created automatically if they do not already exist, and updated otherwise. +> **Note:** Certificates must already be imported into Keyfactor for metadata synchronization to work. The tool does *not* import the certificates themselves, and functions separately from the Sectigo Gateway. The tool does not need to be installed on the same server as Keyfactor Command or the Sectigo Gateway, but needs API access to both Command and the Sectigo SCM API. ## Installation and Usage 1. **Prerequisites** - * .NET 9 or newer runtime. + * .NET 9 runtime, installed through the hosting package. * A valid Sectigo account with API access credentials. - * A Keyfactor account with API access credentials. - * The following config files filled in within the config sub-directory: + * A Keyfactor account with API access credentials for use with basic authentication, or OAuth token retrieval data if OAuth authentication is used (see OAuth login section in this manual for specific details). + * The following config files set up within the `config` sub-directory: * `config.json` * `fields.json` * `bannedcharacters.json`, which will be generated during the first run if needed. - * The tool has been designed for Keyfactor 25.1, but was tested as compatible with older versions. + * The tool has been designed for Keyfactor 25.3, but has been tested as compatible with older versions. -2. **Running The Tool** +2. **Running the Tool** ```powershell SectigoMetadataSync.exe sctokf ``` @@ -67,35 +65,20 @@ Fields listed in `fields.json` that do not already exist in Keyfactor will be cr ```powershell SectigoMetadataSync.exe kftosc ``` - > **Note:** sctokf sync must be run at least once before kftosc can be. + > **Note:** SCtoKF sync must be run at least once before KFtoSC can be. Please carefully review all the available configuration options below before running the tool, as some settings may irreversibly impact existing data. ---- +## Settings -## Command Line Arguments +### The Philosophy of Manual Fields and Custom Fields -One of the following two modes must be specified as the **first (and only) argument** when launching the executable: +Manual Fields is a term used to represent fields containing data from “static” Sectigo certificate attributes (e.g., externalRequester, reasonCode) obtained when using the Sectigo API "Get SSL certificate details" endpoint for a given certificadte into Keyfactor. +This is useful in cases where certain data is not automatically loaded into Keyfactor when the certificate is imported via the Sectigo Gateway, but you wish to have it stored in Keyfactor regardless using a Keyfactor Metadata Field. -* `sctokf` - Synchronizes custom and manual fields **from Sectigo into Keyfactor**. +Custom Fields is a term used to represent Sectigo custom fields, which are user-defined fields, and exist in Sectigo the same way Metadata Fields exist in Keyfactor. - * Reads each entry in `fields.json`. - * For each “ManualField,” extracts the specified data from Sectigo’s certificate details JSON. - * For each “CustomField,” reads Sectigo’s custom-field value and writes it into Keyfactor. - * The required metadata fields are created in Keyfactor if they do not already exist. - -* `kftosc` - Synchronizes custom fields (NOT manual fields) **from Keyfactor back into Sectigo**. - - * Reads each `CustomField` entry in `fields.json`. - * For each field, retrieves the value from Keyfactor and updates Sectigo. +These terms are only used in the context of this tool, and do not represent any official terminology from either Sectigo or Keyfactor, but are useful to differentiate between the two different scenarios that occur during synchronization. -> **Note:** If no argument or an invalid argument is provided, the tool will log an error and exit. - ---- - -## Settings - -### 1. `config\config.json` Settings +### 1.`config\config.json` Settings * **sectigoLogin** Login name/email for Sectigo API (e.g., the account you use to log into Sectigo). @@ -104,19 +87,19 @@ One of the following two modes must be specified as the **first (and only) argum Password for the Sectigo account. * **sectigoCustomerUri** - This is a static value that determined the customer's account on the Certificate Platform. This can be found as part of the portal login URL `https://hard.cert-manager.com/customer/{CustomerUri}` + This is a static value that determines the customer's account on the Certificate Platform. This can be found as part of the portal login URL `https://hard.cert-manager.com/customer/{CustomerUri}` * **sectigoAPIUrl** Base URL for Sectigo’s API (e.g., `https://cert-manager.com`). * **keyfactorLogin** - Keyfactor domain and username, in the form `DOMAIN\\Username`. + Keyfactor domain and username, in the form `DOMAIN\\Username`. If using OAuth authentication, this can be set to `null`. The Keyfactor account must have API access and permissions to create/update metadata fields, and alter certificate data. * **keyfactorPassword** - Password for the Keyfactor account. + Password for the Keyfactor account. If using OAuth authentication, this can be set to `null`. * **keyfactorAPIUrl** - Full Keyfactor API endpoint (e.g., `https://your-keyfactor-server.com/keyfactorapi`). + Full Keyfactor API endpoint (e.g., `https://your-keyfactor-server.com/keyfactorapi`). This applies to both basic and OAuth authentication. * **keyfactorDateFormat** Date/time format to use when reading Date information from Keyfactor for SCtoKF mode. Varies based on your Keyfactor Command version. @@ -140,7 +123,7 @@ One of the following two modes must be specified as the **first (and only) argum * **issuerDNLookupTerm** A substring to match against the Issuer Distinguished Name, which is how the tool identifies Sectigo-issued certificates in Keyfactor. - * Only certificates whose Issuer DN contains this term (e.g., `"Sectigo"`) will be considered. + * Only certificates whose Issuer DN contains this term (e.g., `"Sectigo"`) will be considered. Certificates that are not present in your Sectigo account will be ignored, and their metadata will not be altered. * **enableDisabledFieldSync** String `"true"` or `"false"`. @@ -156,35 +139,62 @@ One of the following two modes must be specified as the **first (and only) argum * **sectigoPageSize** Maximum number of certificates to fetch per page from Sectigo (pagination). - Default in code is `25`. + Default is `25`. * **keyfactorPageSize** Maximum number of certificates to fetch per page from Keyfactor (pagination). - Default in code is `100`. + Default is `100`. ---- +* **keyfactorAddedSince** + Allows you to specify a date that will be used to limit the sync to certificates imported into Keyfactor after that date. + For example, if you specify `"2023-09-01"`, only certificates added to Keyfactor after September 1, 2023 will be considered for metadata sync. + If you wish to sync all certificates, use the value of `null`. -### 2. `config\fields.json` Settings -* **ManualFields and CustomFields** +* **enableTruncation** + This setting controls data truncation to comply with Sectigo and Keyfactor field length character limits. + Keyfactor specifies the Keyfactor limits in their documentation [here](https://software.keyfactor.com/Core-OnPrem/v25.3/Content/ReferenceGuide/CreatingaMetadataField.htm#_Ref498525440) and Sectigo specifies the limits in their API documentation [here](https://www.sectigo.com/knowledge-base/detail/Sectigo-Certificate-Manager-SCM-REST-API/kA01N000000XDkE). + The Keyfactor length limits are varied depending on the field data type, and Sectigo custom fields have a maximum length of 255 characters. - Manual Fields are used to import data from “static” Sectigo certificate attributes obtained using the Sectigo API "Get SSL certificate details" endpoint (e.g., serial number, common name) into Keyfactor. - To retrieve this data, you specify the path to the attribute you wish to retrieve in the sectigoFieldName, using `.` for separation. For example, to retrieve certificateDetails.issuer - from certificateDetails, list certificateDetails.issuer as the sectigoFieldName, as issuer is a part of certificateDetails. - Review the Sectigo API documentation for the SSL "Get SSL certificate details" endpoint to view the available attributes and subattributes. + You may run into these limits when attempting to sync `Custom Field` data from Keyfactor to Sectigo, but also when trying to sync a `Manual Field` from Sectigo to Keyfactor, as the data retrieved from the Sectigo "Get SSL certificate details" endpoint may technically exceed the limits for either system. - Custm Fields are used to import data from Sectigo custom fields, which are user-defined fields. If importAllCustomFields is set to true, the tool will match the field types and other information contained in Sectigo when it creates the fields within Keyfactor. - Otherwise, information listed in the CustomFields array within `fields.json` will be used to create the fields within Keyfactor. - > **Note:** The Keyfactor fields listed in ManualFields and KeyfactorFields are compatible with Keyfactor Command 25.1, - but the tool will work with older versions of Keyfactor and the unused fields and contained data will be ignored. + If this setting is set to `true`, the tool will automatically truncate data to fit within the limits when writing to either system. + + ⚠️ WARNING: *If you sync the data to Keyfactor (in SCtoKF mode) and it ends up getting truncated, and then sync the data back to Sectigo (in KFtoSC), the portion that was truncated will be permanently lost. The same applies in reverse. To avoid this scenario, be mindful of running sync both ways when you have data exceeding character length limits for either Keyfactor or Sectigo. Review the logs following sync to identify any truncation related issues early.* - -* **ManualFields** + If this setting is set to `false`, the tool will ignore any data that exceeds the limits and will log a warning. The original value will be used and will likely result in an error, as well as the value not getting synced. + +If you are using OAuth authentication for Keyfactor, the following section must be present (and must be absent if you are using basic authentication, see `stock-config-oauth.json` for OAuth and `stock-config.json` for basic auth): + +* **keyfactorOAuth** + * **tokenUrl** + The URL to retrieve the token from. This tool has been tested for use with Keyfactor using both Keycloak and Auth0 to obtain tokens. + + * **clientId** + Token client id. + + * **clientSecret** + Token client secret. + * **scopesCsv** + Comma separated list of scopes to request the token for, can be left blank. + + * **audience** + Token audience. + + * **requestedWith** + String to specify the requested with header value for all OAuth related requests. + + * **refreshSkewSeconds** + Default is 60 seconds. This is the amount of time before the token expiration that the tool will attempt to refresh the token. If a refresh time is returned with the token, and that value exceeds the value given here, the returned value will be used instead. + +### 2. `config\fields.json` Settings +* **ManualFields** + This is a json list of objects defining Sectigo *static* fields to be synchronized. Each object must include: 1. `sectigoFieldName` - * The exact JSON path of the field in Sectigo’s “certificate details” response. + * The exact JSON path of the field in Sectigo’s “Get SSL certificate details” response. * Use dot notation if nested (e.g., `'certificate.subject.organization'`). 2. `keyfactorMetadataFieldName` @@ -231,7 +241,23 @@ One of the following two modes must be specified as the **first (and only) argum * **CustomFields** Uses the same fields as above. An array of objects defining Sectigo *custom* fields to be synchronized. If `importAllCustomFields = true` (in `stock-config.json`), you may omit individual entries here and let the tool import all custom fields automatically. ---- +### Use Guidelines for `config.json` + +To retrieve data via a Manual Field, you specify the path to the attribute from the Sectigo "Get SSL certificate details" in the sectigoFieldName in the `ManualFields` section in `fields.json`, using `.` to address into additional layers of objects. For example, to have `certificateDetails.issuer` +imported into Keyfactor as a metadata field, list `certificateDetails.issuer` as the `sectigoFieldName` for a given field. +For an object retrieved from the Sectigo API, you will only be able to store one leaf (lowest level) attribute per Keyfactor metadata field. If an object is a list, the Metadata Sync Tool will attempt to store the entire list as a comma-separated string. +Review the [Sectigo API](https://www.sectigo.com/knowledge-base/detail/Sectigo-Certificate-Manager-SCM-REST-API/kA01N000000XDkE) documentation for the SSL "Get SSL certificate details" endpoint to view all of the available attributes and objects. + +If `importAllCustomFields` is set to true, the tool will attempt to match the field types between Keyfactor and Sectigo when it creates the fields within Keyfactor. +Otherwise, information listed for each field specified in the CustomFields array within `fields.json` will be used to create the fields within Keyfactor. +It is recommended to only use `importAllCustomFields` in cases where you have a very large number of custom fields in Sectigo and do not wish to manually list them all in `fields.json`. +Using `fields.json` is the preferred method, as it provides granular control over metadata field settings. + +Additional information on what attributes like `keyfactorDataType` and `keyfactorValidation` represent, as well as details of other configuration options can be found in the [Keyfactor Metadata Field API Endpoint documentation](https://software.keyfactor.com/Core-OnPrem/v25.2/Content/WebAPI/KeyfactorAPI/MetadataFieldsPost.htm). +> **Note:** The Keyfactor fields supported by this tool are compatible with Keyfactor Command 25.1, +but the tool will work with older versions of Keyfactor and the unused fields and contained data will be ignored. + +> **Note:** Set the value of all parameters you are not using to `null`. ### 3. config\bannedcharacters.json @@ -261,7 +287,28 @@ On the very first run, the tool inspects all Sectigo custom field names and comp If any `"replacementCharacter"` remains `null`, the tool will exit with an error on the next run. Once you populate all replacements, fields will be created in Keyfactor with names free of banned characters. ---- +If you are using `importAllCustomFields = false`, the tool will still check for banned characters in the `CustomFields` listed in `fields.json`, and will generate an error if any are found, and will also create `bannedCharacters.json`. +However, in this case, you should manually alter the field names in `fields.json`, then remove the `bannedCharacters.json` file, and rerun the tool. + +## Command Line Arguments + +One of the following two modes must be specified as the **first (and only) argument** when launching the executable: + +* `sctokf` + Synchronizes custom and manual fields **from Sectigo into Keyfactor**. + + * Reads each entry in `fields.json`. + * For each “ManualField,” extracts the specified data from Sectigo’s certificate details JSON. + * For each “CustomField,” reads Sectigo’s custom-field value and writes it into Keyfactor. + * The required metadata fields are created in Keyfactor if they do not already exist. + +* `kftosc` + Synchronizes custom fields (NOT manual fields) **from Keyfactor back into Sectigo**. + + * Reads each `CustomField` entry in `fields.json`. + * For each field, retrieves the value from Keyfactor and updates Sectigo. + +> **Note:** If no argument or an invalid argument is provided, the tool will log an error and exit. ## Logging @@ -286,8 +333,6 @@ Logging is managed via NLog and is configured in the accompanying `config\NLog.c * **ErrorLogFile (`MetadataSync-Errors.log`)** Captures only `Error` and `Fatal` entries. Use this file to quickly locate failed operations without sifting through lower‐level debug or info messages. ---- - #### Example Log Entry (MainLogFile) ``` @@ -311,66 +356,82 @@ Logging is managed via NLog and is configured in the accompanying `config\NLog.c Always restart the tool after modifying `NLog.config` to ensure changes take effect. ---- - ## Example Workflow 1. **Initial Setup** - - * Populate `config\config.json` with your Sectigo and Keyfactor API credentials. +Extract the zip and have the tool files in a folder (e.g., `C:\Tools\SectigoSync\`). If you want to run the tool on a schedule, consider creating a scheduled task in Windows Task Scheduler. +Make sure that the account used to run the tool has read/write permissions to the folder and subfolders. + * Populate `config\config.json` with your Sectigo and Keyfactor API credentials, as well as other settings. The tool ships with the `stock` config files, so you should copy one of those as `config.json` and edit it. * Populate `config\fields.json` with the manual and/or custom fields you wish to sync. -2. **First Run (Detect Banned Characters)** +2. **First Run (Detection of Banned Characters)** ```powershell cd C:\Tools\SectigoSync\ .\SectigoMetadataSync.exe SCtoKF ``` - * If Sectigo custom field names contain banned characters, you will see warnings in the log and the tool will exit. - * A file named `bannedcharacters.json` will be created listing each banned character with `"replacementCharacter": null`. + * If a Keyfactor field name contains banned characters, you will see warnings in the log and the tool will exit. + * A file named `bannedcharacters.json` will be created listing each banned character with `"replacementCharacter": null`. + If `importAllCustomFields` is set to ` true`, populate the `bannedcharacters.json` file as detailed below. If `importAllCustomFields` is set to `false`, manually edit the field names in `fields.json`, delete `bannedcharacters.json`, and rerun the tool. + + **Populate Banned Characters** + + * Open `config\bannedcharacters.json`. + * For each entry where `"replacementCharacter": null`, supply a valid replacement (alphanumeric, `-`, or `_`). + + ```jsonc + [ + { + "id": 1, + "character": " ", + "replacementCharacter": "_" + }, + { + "id": 2, + "character": "/", + "replacementCharacter": "-" + } + ] + ``` + * Save the file. + +3. **Second Run (Create Fields & Sync Data)** -3. **Populate Banned Characters** + ```powershell + .\SectigoMetadataSync.exe SCtoKF + ``` - * Open `config\bannedcharacters.json`. - * For each entry where `"replacementCharacter": null`, supply a valid replacement (alphanumeric, `-`, or `_`). + * The tool will now convert Sectigo custom‐field names (using your replacements) if `importAllCustomFields` is enabled, create new Keyfactor metadata fields if needed (and update them otherwise), and write all custom/manual field data into Keyfactor. - ```jsonc - [ - { - "id": 1, - "character": " ", - "replacementCharacter": "_" - }, - { - "id": 2, - "character": "/", - "replacementCharacter": "-" - } - ] - ``` - * Save the file. +## Data Flow +![title](docsource/dataflow.png) -4. **Second Run (Create Fields & Sync Data)** +The Metadata Sync Tool operates independently of the Sectigo Gateway and does not require installation on the same server as Keyfactor Command or the Sectigo Gateway. It requires API access to both systems. - ```powershell - .\SectigoMetadataSync.exe SCtoKF - ``` +## Tool Process Diagram +![title](docsource/logicdiagram.png) - * The tool will now convert Sectigo custom‐field names (using your replacements), create new Keyfactor metadata fields if needed, and write all custom/manual data into Keyfactor. +## Value Coercion +When syncing data between Keyfactor and Sectigo, the tool attempts to coerce values to match the expected data types in the target system. ---- +For example, if a Keyfactor metadata field is of type `Date`, the tool will try to parse the string value from Keyfactor into a date format that Sectigo accepts (typically `yyyy-MM-dd`). + +If you set up a field in Keyfactor as email, and the data exists as a string in Sectigo, Keyfactor will attempt to parse the values from Sectigo, validate them as email addresses, and submit them to Keyfactor as a properly formatted email list. + +The truncation process occurs at this stage of the process as well, if enabled in `config.json`. -### Troubleshooting +## Troubleshooting * **Missing or Invalid JSON** - * If `stock-config.json` or `fields.json` is malformed or missing required sections, the tool logs an error and exits. + * If `config.json` or `fields.json` is malformed or missing required sections, the tool logs an error and exits. * Ensure both files exist, are valid JSON, and contain the required properties (see “Settings” above). * **Authentication Failures** * Double‐check `sectigoLogin`/`sectigoPassword` and `keyfactorLogin`/`keyfactorPassword`. + * If you are using OAuth, consider using Postman to test your OAuth credentials and ensure token retrieval works as expected. * Ensure your API user has adequate permissions to create/update metadata fields. * **Field Creation Errors** @@ -378,8 +439,23 @@ Always restart the tool after modifying `NLog.config` to ensure changes take eff * If Keyfactor rejects a field name (e.g., still contains a banned character), verify that `bannedCharacters.json` is up to date. * If Sectigo rejects a custom‐field update, ensure you are using the correct custom‐field “name” as visible in Sectigo’s administration UI. * Consider deleting 'bannedCharacters.json' and re-running the tool to regenerate it from scratch. + +* **Keyfactor metadata field type update failure** ---- + * Keyfactor will not allow updating the data type of an existing metadata field. If you need to change the data type, you must delete the existing field in Keyfactor and let the tool recreate it. + +* **Field content truncation** + + * Both Keyfactor and Sectigo impose maximum lengths on metadata field names and values. If you see warnings about truncation in the logs, consider truncating the data manually prior to running sync. These limits can be found in the respective product documentation for [Keyfactor](https://software.keyfactor.com/Core-OnPrem/v25.3/Content/ReferenceGuide/CreatingaMetadataField.htm#_Ref498525440) and [Sectigo](https://www.sectigo.com/knowledge-base/detail/Sectigo-Certificate-Manager-SCM-REST-API/kA01N000000XDkE). + +* **Account permission issues** + * Ensure the API user accounts have permissions to edit certificate details and edit metadata fields. + * If you are running into these issues, you can try running the tool using an admin account and see if you can match the required permissions. + +* **API Access issues** + * Ensure that the API endpoints specified in `config.json` are correct and reachable from the machine running the tool. Sometimes a missing slash or typo can cause connection issues. + * Check for network issues, firewalls, or proxies that might be blocking access. + * Consider using Postman or a similar tool to manually test the API endpoint connectivity. diff --git a/sectigo-metadata-sync/SectigoMetadataSync.sln b/SectigoMetadataSync.sln similarity index 53% rename from sectigo-metadata-sync/SectigoMetadataSync.sln rename to SectigoMetadataSync.sln index de85f9b..7069c50 100644 --- a/sectigo-metadata-sync/SectigoMetadataSync.sln +++ b/SectigoMetadataSync.sln @@ -1,14 +1,17 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 -VisualStudioVersion = 17.13.35919.96 +VisualStudioVersion = 17.14.36511.14 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SectigoMetadataSync", "SectigoMetadataSync.csproj", "{12C10854-E2CF-4244-A0D0-6FF1C2102058}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SectigoMetadataSync", "sectigo-metadata-sync\SectigoMetadataSync.csproj", "{8B00BC70-F403-0D2D-2792-3951211C6665}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8EC462FD-D22E-90A8-E5CE-7E832BA40C5D}" ProjectSection(SolutionItems) = preProject - ..\docsource\content.md = ..\docsource\content.md - ..\integration-manifest.json = ..\integration-manifest.json + CHANGELOG.md = CHANGELOG.md + docsource\content.md = docsource\content.md + docsource\dataflow.png = docsource\dataflow.png + integration-manifest.json = integration-manifest.json + .github\workflows\keyfactor-starter-workflow.yml = .github\workflows\keyfactor-starter-workflow.yml EndProjectSection EndProject Global @@ -17,15 +20,15 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {12C10854-E2CF-4244-A0D0-6FF1C2102058}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {12C10854-E2CF-4244-A0D0-6FF1C2102058}.Debug|Any CPU.Build.0 = Debug|Any CPU - {12C10854-E2CF-4244-A0D0-6FF1C2102058}.Release|Any CPU.ActiveCfg = Release|Any CPU - {12C10854-E2CF-4244-A0D0-6FF1C2102058}.Release|Any CPU.Build.0 = Release|Any CPU + {8B00BC70-F403-0D2D-2792-3951211C6665}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B00BC70-F403-0D2D-2792-3951211C6665}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B00BC70-F403-0D2D-2792-3951211C6665}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B00BC70-F403-0D2D-2792-3951211C6665}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {94154D4D-3D14-471A-A082-CF4F6D69DC90} + SolutionGuid = {25C852B6-8FAF-4218-B453-6DBB1A896754} EndGlobalSection EndGlobal diff --git a/docsource/content.md b/docsource/content.md index a776cea..8f6c53f 100644 --- a/docsource/content.md +++ b/docsource/content.md @@ -2,31 +2,30 @@ ## Overview -This tool automates the synchronization of metadata fields between Sectigo and Keyfactor. It performs two primary operations: +This tool performs the synchronization of metadata fields and their contents between Sectigo and Keyfactor. It has two modes of operation: 1. **SCtoKF** – Synchronizes custom fields and contained data, and additional requested data from Sectigo into Keyfactor. 2. **KFtoSC** – Synchronizes custom field data from Keyfactor back to Sectigo. -Fields listed in `fields.json` that do not already exist in Keyfactor will be created automatically. -> **Note:** Certificates must already be imported into Keyfactor for metadata synchronization to work. The tool does *not* import certificates themselves. +Fields listed in `fields.json` that do not already exist in Keyfactor will be created automatically if they do not already exist, and updated otherwise. +> **Note:** Certificates must already be imported into Keyfactor for metadata synchronization to work. The tool does *not* import the certificates themselves, and functions separately from the Sectigo Gateway. The tool does not need to be installed on the same server as Keyfactor Command or the Sectigo Gateway, but needs API access to both Command and the Sectigo SCM API. ---- ## Installation and Usage 1. **Prerequisites** - * .NET 9 or newer runtime. + * .NET 9 runtime, installed through the hosting package. * A valid Sectigo account with API access credentials. - * A Keyfactor account with API access credentials. - * The following config files filled in within the config sub-directory: + * A Keyfactor account with API access credentials for use with basic authentication, or OAuth token retrieval data if OAuth authentication is used (see OAuth login section in this manual for specific details). + * The following config files set up within the `config` sub-directory: * `config.json` * `fields.json` * `bannedcharacters.json`, which will be generated during the first run if needed. - * The tool has been designed for Keyfactor 25.1, but was tested as compatible with older versions. + * The tool has been designed for Keyfactor 25.3, but has been tested as compatible with older versions. -2. **Running The Tool** +2. **Running the Tool** ```powershell SectigoMetadataSync.exe sctokf ``` @@ -36,35 +35,20 @@ Fields listed in `fields.json` that do not already exist in Keyfactor will be cr ```powershell SectigoMetadataSync.exe kftosc ``` - > **Note:** sctokf sync must be run at least once before kftosc can be. - ---- + > **Note:** SCtoKF sync must be run at least once before KFtoSC can be. Please carefully review all the available configuration options below before running the tool, as some settings may irreversibly impact existing data. -## Command Line Arguments - -One of the following two modes must be specified as the **first (and only) argument** when launching the executable: -* `sctokf` - Synchronizes custom and manual fields **from Sectigo into Keyfactor**. - - * Reads each entry in `fields.json`. - * For each “ManualField,” extracts the specified data from Sectigo’s certificate details JSON. - * For each “CustomField,” reads Sectigo’s custom-field value and writes it into Keyfactor. - * The required metadata fields are created in Keyfactor if they do not already exist. +## Settings +### The Philosophy of Manual Fields and Custom Fields -* `kftosc` - Synchronizes custom fields (NOT manual fields) **from Keyfactor back into Sectigo**. +Manual Fields is a term used to represent fields containing data from “static” Sectigo certificate attributes (e.g., externalRequester, reasonCode) obtained when using the Sectigo API "Get SSL certificate details" endpoint for a given certificadte into Keyfactor. +This is useful in cases where certain data is not automatically loaded into Keyfactor when the certificate is imported via the Sectigo Gateway, but you wish to have it stored in Keyfactor regardless using a Keyfactor Metadata Field. - * Reads each `CustomField` entry in `fields.json`. - * For each field, retrieves the value from Keyfactor and updates Sectigo. +Custom Fields is a term used to represent Sectigo custom fields, which are user-defined fields, and exist in Sectigo the same way Metadata Fields exist in Keyfactor. -> **Note:** If no argument or an invalid argument is provided, the tool will log an error and exit. +These terms are only used in the context of this tool, and do not represent any official terminology from either Sectigo or Keyfactor, but are useful to differentiate between the two different scenarios that occur during synchronization. ---- - -## Settings - -### 1. `config\config.json` Settings +### 1.`config\config.json` Settings * **sectigoLogin** Login name/email for Sectigo API (e.g., the account you use to log into Sectigo). @@ -73,19 +57,19 @@ One of the following two modes must be specified as the **first (and only) argum Password for the Sectigo account. * **sectigoCustomerUri** - This is a static value that determined the customer's account on the Certificate Platform. This can be found as part of the portal login URL `https://hard.cert-manager.com/customer/{CustomerUri}` + This is a static value that determines the customer's account on the Certificate Platform. This can be found as part of the portal login URL `https://hard.cert-manager.com/customer/{CustomerUri}` * **sectigoAPIUrl** Base URL for Sectigo’s API (e.g., `https://cert-manager.com`). * **keyfactorLogin** - Keyfactor domain and username, in the form `DOMAIN\\Username`. + Keyfactor domain and username, in the form `DOMAIN\\Username`. If using OAuth authentication, this can be set to `null`. The Keyfactor account must have API access and permissions to create/update metadata fields, and alter certificate data. * **keyfactorPassword** - Password for the Keyfactor account. + Password for the Keyfactor account. If using OAuth authentication, this can be set to `null`. * **keyfactorAPIUrl** - Full Keyfactor API endpoint (e.g., `https://your-keyfactor-server.com/keyfactorapi`). + Full Keyfactor API endpoint (e.g., `https://your-keyfactor-server.com/keyfactorapi`). This applies to both basic and OAuth authentication. * **keyfactorDateFormat** Date/time format to use when reading Date information from Keyfactor for SCtoKF mode. Varies based on your Keyfactor Command version. @@ -109,7 +93,7 @@ One of the following two modes must be specified as the **first (and only) argum * **issuerDNLookupTerm** A substring to match against the Issuer Distinguished Name, which is how the tool identifies Sectigo-issued certificates in Keyfactor. - * Only certificates whose Issuer DN contains this term (e.g., `"Sectigo"`) will be considered. + * Only certificates whose Issuer DN contains this term (e.g., `"Sectigo"`) will be considered. Certificates that are not present in your Sectigo account will be ignored, and their metadata will not be altered. * **enableDisabledFieldSync** String `"true"` or `"false"`. @@ -125,35 +109,62 @@ One of the following two modes must be specified as the **first (and only) argum * **sectigoPageSize** Maximum number of certificates to fetch per page from Sectigo (pagination). - Default in code is `25`. + Default is `25`. * **keyfactorPageSize** Maximum number of certificates to fetch per page from Keyfactor (pagination). - Default in code is `100`. + Default is `100`. ---- +* **keyfactorAddedSince** + Allows you to specify a date that will be used to limit the sync to certificates imported into Keyfactor after that date. + For example, if you specify `"2023-09-01"`, only certificates added to Keyfactor after September 1, 2023 will be considered for metadata sync. + If you wish to sync all certificates, use the value of `null`. -### 2. `config\fields.json` Settings -* **ManualFields and CustomFields** +* **enableTruncation** + This setting controls data truncation to comply with Sectigo and Keyfactor field length character limits. + Keyfactor specifies the Keyfactor limits in their documentation [here](https://software.keyfactor.com/Core-OnPrem/v25.3/Content/ReferenceGuide/CreatingaMetadataField.htm#_Ref498525440) and Sectigo specifies the limits in their API documentation [here](https://www.sectigo.com/knowledge-base/detail/Sectigo-Certificate-Manager-SCM-REST-API/kA01N000000XDkE). + The Keyfactor length limits are varied depending on the field data type, and Sectigo custom fields have a maximum length of 255 characters. - Manual Fields are used to import data from “static” Sectigo certificate attributes obtained using the Sectigo API "Get SSL certificate details" endpoint (e.g., serial number, common name) into Keyfactor. - To retrieve this data, you specify the path to the attribute you wish to retrieve in the sectigoFieldName, using `.` for separation. For example, to retrieve certificateDetails.issuer - from certificateDetails, list certificateDetails.issuer as the sectigoFieldName, as issuer is a part of certificateDetails. - Review the Sectigo API documentation for the SSL "Get SSL certificate details" endpoint to view the available attributes and subattributes. + You may run into these limits when attempting to sync `Custom Field` data from Keyfactor to Sectigo, but also when trying to sync a `Manual Field` from Sectigo to Keyfactor, as the data retrieved from the Sectigo "Get SSL certificate details" endpoint may technically exceed the limits for either system. - Custm Fields are used to import data from Sectigo custom fields, which are user-defined fields. If importAllCustomFields is set to true, the tool will match the field types and other information contained in Sectigo when it creates the fields within Keyfactor. - Otherwise, information listed in the CustomFields array within `fields.json` will be used to create the fields within Keyfactor. - > **Note:** The Keyfactor fields listed in ManualFields and KeyfactorFields are compatible with Keyfactor Command 25.1, - but the tool will work with older versions of Keyfactor and the unused fields and contained data will be ignored. + If this setting is set to `true`, the tool will automatically truncate data to fit within the limits when writing to either system. + + ⚠️ WARNING: *If you sync the data to Keyfactor (in SCtoKF mode) and it ends up getting truncated, and then sync the data back to Sectigo (in KFtoSC), the portion that was truncated will be permanently lost. The same applies in reverse. To avoid this scenario, be mindful of running sync both ways when you have data exceeding character length limits for either Keyfactor or Sectigo. Review the logs following sync to identify any truncation related issues early.* - -* **ManualFields** + If this setting is set to `false`, the tool will ignore any data that exceeds the limits and will log a warning. The original value will be used and will likely result in an error, as well as the value not getting synced. + +If you are using OAuth authentication for Keyfactor, the following section must be present (and must be absent if you are using basic authentication, see `stock-config-oauth.json` for OAuth and `stock-config.json` for basic auth): + +* **keyfactorOAuth** + * **tokenUrl** + The URL to retrieve the token from. This tool has been tested for use with Keyfactor using both Keycloak and Auth0 to obtain tokens. + + * **clientId** + Token client id. + + * **clientSecret** + Token client secret. + + * **scopesCsv** + Comma separated list of scopes to request the token for, can be left blank. + * **audience** + Token audience. + + * **requestedWith** + String to specify the requested with header value for all OAuth related requests. + + * **refreshSkewSeconds** + Default is 60 seconds. This is the amount of time before the token expiration that the tool will attempt to refresh the token. If a refresh time is returned with the token, and that value exceeds the value given here, the returned value will be used instead. + +### 2. `config\fields.json` Settings +* **ManualFields** + This is a json list of objects defining Sectigo *static* fields to be synchronized. Each object must include: 1. `sectigoFieldName` - * The exact JSON path of the field in Sectigo’s “certificate details” response. + * The exact JSON path of the field in Sectigo’s “Get SSL certificate details” response. * Use dot notation if nested (e.g., `'certificate.subject.organization'`). 2. `keyfactorMetadataFieldName` @@ -200,7 +211,23 @@ One of the following two modes must be specified as the **first (and only) argum * **CustomFields** Uses the same fields as above. An array of objects defining Sectigo *custom* fields to be synchronized. If `importAllCustomFields = true` (in `stock-config.json`), you may omit individual entries here and let the tool import all custom fields automatically. ---- +### Use Guidelines for `config.json` + +To retrieve data via a Manual Field, you specify the path to the attribute from the Sectigo "Get SSL certificate details" in the sectigoFieldName in the `ManualFields` section in `fields.json`, using `.` to address into additional layers of objects. For example, to have `certificateDetails.issuer` +imported into Keyfactor as a metadata field, list `certificateDetails.issuer` as the `sectigoFieldName` for a given field. +For an object retrieved from the Sectigo API, you will only be able to store one leaf (lowest level) attribute per Keyfactor metadata field. If an object is a list, the Metadata Sync Tool will attempt to store the entire list as a comma-separated string. +Review the [Sectigo API](https://www.sectigo.com/knowledge-base/detail/Sectigo-Certificate-Manager-SCM-REST-API/kA01N000000XDkE) documentation for the SSL "Get SSL certificate details" endpoint to view all of the available attributes and objects. + +If `importAllCustomFields` is set to true, the tool will attempt to match the field types between Keyfactor and Sectigo when it creates the fields within Keyfactor. +Otherwise, information listed for each field specified in the CustomFields array within `fields.json` will be used to create the fields within Keyfactor. +It is recommended to only use `importAllCustomFields` in cases where you have a very large number of custom fields in Sectigo and do not wish to manually list them all in `fields.json`. +Using `fields.json` is the preferred method, as it provides granular control over metadata field settings. + +Additional information on what attributes like `keyfactorDataType` and `keyfactorValidation` represent, as well as details of other configuration options can be found in the [Keyfactor Metadata Field API Endpoint documentation](https://software.keyfactor.com/Core-OnPrem/v25.2/Content/WebAPI/KeyfactorAPI/MetadataFieldsPost.htm). +> **Note:** The Keyfactor fields supported by this tool are compatible with Keyfactor Command 25.1, +but the tool will work with older versions of Keyfactor and the unused fields and contained data will be ignored. + +> **Note:** Set the value of all parameters you are not using to `null`. ### 3. config\bannedcharacters.json @@ -230,7 +257,28 @@ On the very first run, the tool inspects all Sectigo custom field names and comp If any `"replacementCharacter"` remains `null`, the tool will exit with an error on the next run. Once you populate all replacements, fields will be created in Keyfactor with names free of banned characters. ---- +If you are using `importAllCustomFields = false`, the tool will still check for banned characters in the `CustomFields` listed in `fields.json`, and will generate an error if any are found, and will also create `bannedCharacters.json`. +However, in this case, you should manually alter the field names in `fields.json`, then remove the `bannedCharacters.json` file, and rerun the tool. + +## Command Line Arguments + +One of the following two modes must be specified as the **first (and only) argument** when launching the executable: + +* `sctokf` + Synchronizes custom and manual fields **from Sectigo into Keyfactor**. + + * Reads each entry in `fields.json`. + * For each “ManualField,” extracts the specified data from Sectigo’s certificate details JSON. + * For each “CustomField,” reads Sectigo’s custom-field value and writes it into Keyfactor. + * The required metadata fields are created in Keyfactor if they do not already exist. + +* `kftosc` + Synchronizes custom fields (NOT manual fields) **from Keyfactor back into Sectigo**. + + * Reads each `CustomField` entry in `fields.json`. + * For each field, retrieves the value from Keyfactor and updates Sectigo. + +> **Note:** If no argument or an invalid argument is provided, the tool will log an error and exit. ## Logging @@ -255,8 +303,6 @@ Logging is managed via NLog and is configured in the accompanying `config\NLog.c * **ErrorLogFile (`MetadataSync-Errors.log`)** Captures only `Error` and `Fatal` entries. Use this file to quickly locate failed operations without sifting through lower‐level debug or info messages. ---- - #### Example Log Entry (MainLogFile) ``` @@ -280,66 +326,82 @@ Logging is managed via NLog and is configured in the accompanying `config\NLog.c Always restart the tool after modifying `NLog.config` to ensure changes take effect. ---- - ## Example Workflow 1. **Initial Setup** - - * Populate `config\config.json` with your Sectigo and Keyfactor API credentials. +Extract the zip and have the tool files in a folder (e.g., `C:\Tools\SectigoSync\`). If you want to run the tool on a schedule, consider creating a scheduled task in Windows Task Scheduler. +Make sure that the account used to run the tool has read/write permissions to the folder and subfolders. + * Populate `config\config.json` with your Sectigo and Keyfactor API credentials, as well as other settings. The tool ships with the `stock` config files, so you should copy one of those as `config.json` and edit it. * Populate `config\fields.json` with the manual and/or custom fields you wish to sync. -2. **First Run (Detect Banned Characters)** +2. **First Run (Detection of Banned Characters)** ```powershell cd C:\Tools\SectigoSync\ .\SectigoMetadataSync.exe SCtoKF ``` - * If Sectigo custom field names contain banned characters, you will see warnings in the log and the tool will exit. - * A file named `bannedcharacters.json` will be created listing each banned character with `"replacementCharacter": null`. + * If a Keyfactor field name contains banned characters, you will see warnings in the log and the tool will exit. + * A file named `bannedcharacters.json` will be created listing each banned character with `"replacementCharacter": null`. + If `importAllCustomFields` is set to ` true`, populate the `bannedcharacters.json` file as detailed below. If `importAllCustomFields` is set to `false`, manually edit the field names in `fields.json`, delete `bannedcharacters.json`, and rerun the tool. + + **Populate Banned Characters** + + * Open `config\bannedcharacters.json`. + * For each entry where `"replacementCharacter": null`, supply a valid replacement (alphanumeric, `-`, or `_`). + + ```jsonc + [ + { + "id": 1, + "character": " ", + "replacementCharacter": "_" + }, + { + "id": 2, + "character": "/", + "replacementCharacter": "-" + } + ] + ``` + * Save the file. + +3. **Second Run (Create Fields & Sync Data)** -3. **Populate Banned Characters** + ```powershell + .\SectigoMetadataSync.exe SCtoKF + ``` - * Open `config\bannedcharacters.json`. - * For each entry where `"replacementCharacter": null`, supply a valid replacement (alphanumeric, `-`, or `_`). + * The tool will now convert Sectigo custom‐field names (using your replacements) if `importAllCustomFields` is enabled, create new Keyfactor metadata fields if needed (and update them otherwise), and write all custom/manual field data into Keyfactor. - ```jsonc - [ - { - "id": 1, - "character": " ", - "replacementCharacter": "_" - }, - { - "id": 2, - "character": "/", - "replacementCharacter": "-" - } - ] - ``` - * Save the file. +## Data Flow +![title](docsource/dataflow.png) -4. **Second Run (Create Fields & Sync Data)** +The Metadata Sync Tool operates independently of the Sectigo Gateway and does not require installation on the same server as Keyfactor Command or the Sectigo Gateway. It requires API access to both systems. - ```powershell - .\SectigoMetadataSync.exe SCtoKF - ``` +## Tool Process Diagram +![title](docsource/logicdiagram.png) - * The tool will now convert Sectigo custom‐field names (using your replacements), create new Keyfactor metadata fields if needed, and write all custom/manual data into Keyfactor. +## Value Coercion +When syncing data between Keyfactor and Sectigo, the tool attempts to coerce values to match the expected data types in the target system. ---- +For example, if a Keyfactor metadata field is of type `Date`, the tool will try to parse the string value from Keyfactor into a date format that Sectigo accepts (typically `yyyy-MM-dd`). + +If you set up a field in Keyfactor as email, and the data exists as a string in Sectigo, Keyfactor will attempt to parse the values from Sectigo, validate them as email addresses, and submit them to Keyfactor as a properly formatted email list. + +The truncation process occurs at this stage of the process as well, if enabled in `config.json`. -### Troubleshooting +## Troubleshooting * **Missing or Invalid JSON** - * If `stock-config.json` or `fields.json` is malformed or missing required sections, the tool logs an error and exits. + * If `config.json` or `fields.json` is malformed or missing required sections, the tool logs an error and exits. * Ensure both files exist, are valid JSON, and contain the required properties (see “Settings” above). * **Authentication Failures** * Double‐check `sectigoLogin`/`sectigoPassword` and `keyfactorLogin`/`keyfactorPassword`. + * If you are using OAuth, consider using Postman to test your OAuth credentials and ensure token retrieval works as expected. * Ensure your API user has adequate permissions to create/update metadata fields. * **Field Creation Errors** @@ -347,5 +409,20 @@ Always restart the tool after modifying `NLog.config` to ensure changes take eff * If Keyfactor rejects a field name (e.g., still contains a banned character), verify that `bannedCharacters.json` is up to date. * If Sectigo rejects a custom‐field update, ensure you are using the correct custom‐field “name” as visible in Sectigo’s administration UI. * Consider deleting 'bannedCharacters.json' and re-running the tool to regenerate it from scratch. + +* **Keyfactor metadata field type update failure** ---- + * Keyfactor will not allow updating the data type of an existing metadata field. If you need to change the data type, you must delete the existing field in Keyfactor and let the tool recreate it. + +* **Field content truncation** + + * Both Keyfactor and Sectigo impose maximum lengths on metadata field names and values. If you see warnings about truncation in the logs, consider truncating the data manually prior to running sync. These limits can be found in the respective product documentation for [Keyfactor](https://software.keyfactor.com/Core-OnPrem/v25.3/Content/ReferenceGuide/CreatingaMetadataField.htm#_Ref498525440) and [Sectigo](https://www.sectigo.com/knowledge-base/detail/Sectigo-Certificate-Manager-SCM-REST-API/kA01N000000XDkE). + +* **Account permission issues** + * Ensure the API user accounts have permissions to edit certificate details and edit metadata fields. + * If you are running into these issues, you can try running the tool using an admin account and see if you can match the required permissions. + +* **API Access issues** + * Ensure that the API endpoints specified in `config.json` are correct and reachable from the machine running the tool. Sometimes a missing slash or typo can cause connection issues. + * Check for network issues, firewalls, or proxies that might be blocking access. + * Consider using Postman or a similar tool to manually test the API endpoint connectivity. \ No newline at end of file diff --git a/docsource/dataflow.png b/docsource/dataflow.png new file mode 100644 index 0000000..59dba0c Binary files /dev/null and b/docsource/dataflow.png differ diff --git a/docsource/logicdiagram.png b/docsource/logicdiagram.png new file mode 100644 index 0000000..a049919 Binary files /dev/null and b/docsource/logicdiagram.png differ diff --git a/readme_source.md b/readme_source.md deleted file mode 100644 index 8c3650b..0000000 --- a/readme_source.md +++ /dev/null @@ -1,461 +0,0 @@ -# Sectigo Metadata Sync Tool - -## Overview - -This tool automates the synchronization of metadata fields between Sectigo and Keyfactor. It performs two primary operations: - -1. **SCtoKF** – Synchronizes both custom and manual (non-custom) fields from Sectigo into Keyfactor. Any fields listed in `fields.json` that do not already exist in Keyfactor will be created automatically. -2. **KFtoSC** – Synchronizes custom fields from Keyfactor back to Sectigo. Fields are created in Sectigo on demand if they do not exist. - -This utility requires a working Sectigo API account, a Keyfactor API endpoint, and a correctly configured set of JSON files (`stock-config.json`, `fields.json`, and optionally a banned-characters file). Each time the executable runs, it will scan for new or modified fields and update the target system accordingly. - -> **Note:** Certificates must already be imported into Keyfactor for metadata synchronization to work. The tool does *not* import certificates themselves. - ---- - -## Installation and Usage - -1. **Prerequisites** - - * Windows environment (supports scheduling via Task Scheduler). - * .NET runtime that matches the build target of the executable. - * A valid Sectigo account with API access credentials. - * A Keyfactor account with API access credentials. - * Network access from the machine running this tool to both the Sectigo API endpoint and the Keyfactor API endpoint. - * The following files placed in a single directory alongside the executable: - - * `SectigoMetadataSync.exe` (the compiled tool) - * `stock-config.json` - * `fields.json` - * *(Optional)* A banned-characters JSON file, if the first run reports unsupported characters. - -2. **Directory Layout** - - ``` - C:\SomeFolder\SectigoMetadataSync\ - ├─ SectigoMetadataSync.exe - ├─ stock-config.json - ├─ fields.json - └─ (bannedCharacters.json) ← Optional; see “Banned Characters” below - ``` - -3. **Running Manually** - Open a Command Prompt, navigate to the directory above, and invoke: - - ```powershell - SectigoMetadataSync.exe SCtoKF - ``` - - or - - ```powershell - SectigoMetadataSync.exe KFtoSC - ``` - - The tool will: - - * Read `stock-config.json` and `fields.json`. - * Connect to Sectigo and Keyfactor APIs. - * Create any missing metadata fields. - * Populate metadata fields on certificates according to the selected mode. - -4. **Scheduling (Windows Task Scheduler)** - To run this tool automatically (recommended interval: once per week): - - 1. Open Task Scheduler. - 2. Create a new task with “Run whether user is logged on or not.” - 3. In **Actions**, point to `SectigoMetadataSync.exe` and add an argument (`SCtoKF` or `KFtoSC`). - 4. In **Triggers**, set the desired recurrence (e.g., weekly on Monday at 2:00 AM). - 5. Ensure the **Start in** field is set to the folder containing the executable and JSON files. - 6. Save with the appropriate credentials. - ---- - -## Command Line Arguments - -One of the following two modes must be specified as the **first (and only) argument** when launching the executable: - -* `SCtoKF` - Synchronizes custom and manual fields **from Sectigo into Keyfactor**. - - * Reads each entry in `fields.json`. - * For each “ManualField,” extracts the specified data from Sectigo’s certificate JSON and writes it into Keyfactor. - * For each “CustomField,” reads Sectigo’s custom-field value and writes it into Keyfactor. - * If a field does not exist in Keyfactor, it is created on the fly. - -* `KFtoSC` - Synchronizes custom fields **from Keyfactor back into Sectigo**. - - * Reads each `CustomField` entry in `fields.json`. - * For each field, retrieves the value from Keyfactor and updates Sectigo. - * If the custom field does not exist in Sectigo, it is created first (using Sectigo’s API). - -> If no argument or an invalid argument is provided, the tool will log an error and exit. - ---- - -## Settings - -### 1. `stock-config.json` Settings - -```jsonc -{ - "config": { - "sectigoLogin": "user@example.com", - "sectigoPassword": "*********", - "sectigoCustomerUri": "sectigo-customer", - "sectigoAPIUrl": "https://api.sectigo.com", - "keyfactorLogin": "DOMAIN\\Username", - "keyfactorPassword": "********", - "keyfactorAPIUrl": "https://your-keyfactor-server.com/keyfactorapi", - "keyfactorDateFormat": "M/d/yyyy h:mm:ss tt", - "importAllCustomFields": "false", - "syncRevokedAndExpiredCerts": "true", - "issuerDNLookupTerm": "Sectigo", - "enableDisabledFieldSync": "false", - "sslTypeIds": [ 111111, 222222 ], - "sectigoPageSize": 25, - "keyfactorPageSize": 100 - } -} -``` - -* **sectigoLogin** - Login name/email for Sectigo API (e.g., the account you use to log into Sectigo). - -* **sectigoPassword** - Password for the Sectigo account. - -* **sectigoCustomerUri** - The “Customer URI” or customer identifier used in Sectigo API URIs. - Example: if your certificates endpoint is `https://api.sectigo.com/cert-manager/v1/certificates/sectigo-customer`, then `sectigoCustomerUri` = `"sectigo-customer"`. - -* **sectigoAPIUrl** - Base URL for Sectigo’s API (e.g., `https://api.sectigo.com/cert-manager/v1`). - -* **keyfactorLogin** - Keyfactor domain and username, in the form `DOMAIN\Username`. - -* **keyfactorPassword** - Password for the Keyfactor account. - -* **keyfactorAPIUrl** - Full Keyfactor API endpoint (e.g., `https://your-keyfactor-server.com/keyfactorapi`). - -* **keyfactorDateFormat** - Date/time format to use when parsing or writing datetime fields in Keyfactor. - Example: `"M/d/yyyy h:mm:ss tt"` (for `6/30/2025 11:45:00 AM`). - -* **importAllCustomFields** - String `"true"` or `"false"`. - - * If `"true"`, on `SCtoKF` mode the tool will import *all* custom metadata fields that exist in Sectigo. - * If `"false"`, it will only import the ones explicitly listed under `"CustomFields"` in `fields.json`. - -* **syncRevokedAndExpiredCerts** - String `"true"` or `"false"`. - - * If `"true"`, certificates with status “revoked” or “expired” in Sectigo will also be included for metadata sync. - * If `"false"`, those certificates are skipped. - -* **issuerDNLookupTerm** - A substring to match against the Issuer Distinguished Name. - - * Only certificates whose Issuer DN contains this term (e.g., `"Sectigo"`) will be considered. - -* **enableDisabledFieldSync** - String `"true"` or `"false"`. - - * If `"true"`, fields in Sectigo that are marked “disabled” will still be synchronized (both their schema and data). - * If `"false"`, disabled fields are ignored. - -* **sslTypeIds** - Array of integers specifying which Sectigo SSL certificate Type IDs should be queried. - Example: `[ 111111, 222222 ]` might correspond to “Domain Validated”, “Organization Validated”, etc. - - * Only certificates whose `sslType` matches one of these IDs are processed. - -* **sectigoPageSize** - Maximum number of certificates to fetch per page from Sectigo (pagination). - Default in code is `25`; you may increase if you expect larger result sets. - -* **keyfactorPageSize** - Maximum number of certificates to fetch per page from Keyfactor (pagination). - Default in code is `100`; you may increase if you expect larger result sets. - ---- - -### 2. `fields.json` Settings - -```jsonc -{ - "_comments": { - "ManualFields": "List of fields used for loading static information from Sectigo as certificate metadata in Keyfactor.", - "CustomFields": "List of custom metadata fields to be synced between Keyfactor and Sectigo.", - "sectigoFieldName": "The name of the field in Sectigo. If nested in the certificate JSON, use dot notation, e.g. \"certificate.subject.organization\".", - "keyfactorMetadataFieldName": "The name of the field in Keyfactor. Use only alphanumeric, '-' or '_' characters (no spaces).", - "keyfactorDescription": "A description of the metadata field for display in Keyfactor.", - "keyfactorDataType": "The data type of the field (1 = String, 2 = Integer, 3 = Date, 4 = Boolean, 5 = MultipleChoice, 6 = BigText, 7 = Email).", - "keyfactorHint": "A short hint to guide users on what to enter in the field (Keyfactor UI).", - "keyfactorValidation": "A regular expression to validate the field’s input (only for string fields).", - "keyfactorEnrollment": "How the field is handled during certificate enrollment in Keyfactor (0 = Optional, 1 = Required, etc.).", - "keyfactorMessage": "Validation failure message to display if the input does not match `keyfactorValidation`.", - "keyfactorOptions": "Array of values for a multiple-choice field (ignored if not a MultipleChoice type).", - "keyfactorDefaultValue": "Default value for the field (used if none is provided).", - "keyfactorDisplayOrder": "Numeric order in which the field appears in Keyfactor’s UI.", - "keyfactorCaseSensitive": "Boolean indicating if validation is case‐sensitive (only for string fields)." - }, - "ManualFields": [ - { - "sectigoFieldName": "certificate.serialNumber", - "keyfactorMetadataFieldName": "SerialNumber", - "keyfactorDescription": "Certificate serial number from Sectigo", - "keyfactorDataType": 1, - "keyfactorHint": "Automatically populated from Sectigo", - "keyfactorValidation": null, - "keyfactorEnrollment": 0, - "keyfactorMessage": null, - "keyfactorOptions": null, - "keyfactorDefaultValue": null, - "keyfactorDisplayOrder": 1, - "keyfactorCaseSensitive": false - }, - { - "sectigoFieldName": "certificate.commonName", - "keyfactorMetadataFieldName": "CommonName", - "keyfactorDescription": "Certificate Common Name (CN)", - "keyfactorDataType": 1, - "keyfactorHint": "Automatically populated from Sectigo", - "keyfactorValidation": null, - "keyfactorEnrollment": 0, - "keyfactorMessage": null, - "keyfactorOptions": null, - "keyfactorDefaultValue": null, - "keyfactorDisplayOrder": 2, - "keyfactorCaseSensitive": false - } - // … add additional ManualFields as needed - ], - "CustomFields": [ - { - "sectigoFieldName": "DeviceLocation", - "keyfactorMetadataFieldName": "DeviceLocation", - "keyfactorDescription": "Location of the device (custom field)", - "keyfactorDataType": 1, - "keyfactorHint": "Enter where the device is located", - "keyfactorValidation": null, - "keyfactorEnrollment": 0, - "keyfactorMessage": null, - "keyfactorOptions": null, - "keyfactorDefaultValue": null, - "keyfactorDisplayOrder": 3, - "keyfactorCaseSensitive": false - }, - { - "sectigoFieldName": "WarrantyExpiry", - "keyfactorMetadataFieldName": "WarrantyExpiry", - "keyfactorDescription": "Warranty expiration date for the device", - "keyfactorDataType": 3, - "keyfactorHint": "Format: MM/dd/yyyy", - "keyfactorValidation": "\\d{2}/\\d{2}/\\d{4}", - "keyfactorEnrollment": 0, - "keyfactorMessage": "Date must be in MM/dd/yyyy format.", - "keyfactorOptions": null, - "keyfactorDefaultValue": null, - "keyfactorDisplayOrder": 4, - "keyfactorCaseSensitive": false - } - // … add additional CustomFields as needed - ] -} -``` - -* **ManualFields** - An array of objects defining “static” Sectigo certificate attributes (e.g., serial number, common name) to be imported into Keyfactor. Each object must include: - - 1. `sectigoFieldName` - - * The exact JSON path of the field in Sectigo’s “certificate details” response. - * Use dot notation if nested (e.g., `'certificate.subject.organization'`). - 2. `keyfactorMetadataFieldName` - - * The desired metadata field name in Keyfactor. Must contain only `[A–Za–z0–9-_]`. - 3. `keyfactorDescription` - - * A short description for display in Keyfactor’s UI. - 4. `keyfactorDataType` - - * Integer code for Keyfactor’s data type: - - * `1` = String - * `2` = Integer - * `3` = Date - * `4` = Boolean - * `5` = MultipleChoice - * `6` = BigText - * `7` = Email - 5. `keyfactorHint` - - * Hint text shown in Keyfactor when entering or viewing data. - 6. `keyfactorValidation` (nullable) - - * A regular expression that the input must match (only for string fields). - 7. `keyfactorEnrollment` - - * How the field behaves during enrollment. Valid values depend on your Keyfactor setup (typically `0` = optional, `1` = required). - 8. `keyfactorMessage` (nullable) - - * Message shown if validation fails. - 9. `keyfactorOptions` (nullable) - - * Array of strings for multiple‐choice fields. Ignored if `keyfactorDataType` ≠ `5`. - 10. `keyfactorDefaultValue` (nullable) - - * Default value for the metadata field. - 11. `keyfactorDisplayOrder` - - * Integer specifying the display order in the Keyfactor administration view. - 12. `keyfactorCaseSensitive` - - * `true` or `false`. Only relevant if `keyfactorValidation` is provided for a string field. - -* **CustomFields** - An array of objects defining Sectigo *custom* metadata fields to be synchronized. The property names are identical to those in `ManualFields`, but the tool will read/write against Sectigo’s “customFields” APIs rather than the static certificate JSON. If `importAllCustomFields = true` (in `stock-config.json`), you may omit individual entries here and let the tool import all custom fields automatically. - ---- - -### 3. Banned Characters (Optional) - -On the very first run, the tool inspects all Sectigo custom field names and compares them against Keyfactor’s allowed metadata‐field naming rules (alphanumeric, `-`, and `_` only). If it finds unsupported characters, it will produce a file called `bannedCharacters.json` in the same directory, with entries like: - -```jsonc -[ - { - "id": 1, - "character": " ", - "replacementCharacter": null - }, - { - "id": 2, - "character": "/", - "replacementCharacter": null - } -] -``` - -* **character** - The unsupported character detected in the Sectigo field name. - -* **replacementCharacter** (initially `null`) - A value you must supply before rerunning. It should be any alphanumeric, `-`, or `_` string that the tool can use to replace the banned character. - For example, to replace spaces (`" "`) with underscores (`"_"`), set `"replacementCharacter": "_"`. - -If any `"replacementCharacter"` remains `null`, the tool will exit with an error on the next run. Once you populate all replacements, fields will be created in Keyfactor with names free of banned characters. - ---- - -## Logging - -Logging is handled via NLog. By default, `NLog.config` is included alongside the executable. In this file you can set: - -```xml - -``` - -Change `value="Info"` to one of: - -* `"Debug"` — Verbose logs (development/troubleshooting). -* `"Trace"` — All internal steps (maximum verbosity). -* `"Info"` — High‐level progress messages (default). -* `"Warn"` — Warnings only. -* `"Error"` — Errors only. - -Logs (by default) are written to: - -* A console window. -* A rolling file named `SectigoMetadataSync.log` in the same directory. - -You can adjust targets, layouts, and file paths in `NLog.config` as desired. - ---- - -## Example Workflow - -1. **Initial Setup** - - * Place `SectigoMetadataSync.exe`, `stock-config.json`, and `fields.json` in `C:\Tools\SectigoSync\`. - * Populate `stock-config.json` with your Sectigo and Keyfactor API credentials. - * Populate `fields.json` with the manual and/or custom fields you wish to sync. - -2. **First Run (Detect Banned Characters)** - - ```powershell - cd C:\Tools\SectigoSync\ - .\SectigoMetadataSync.exe SCtoKF - ``` - - * If Sectigo custom field names contain banned characters, you will see warnings in the log. - * A file named `bannedCharacters.json` will be created listing each banned character with `"replacementCharacter": null`. - -3. **Populate Banned Characters** - - * Open `bannedCharacters.json`. - * For each entry where `"replacementCharacter": null`, supply a valid replacement (alphanumeric, `-`, or `_`). - - ```jsonc - [ - { - "id": 1, - "character": " ", - "replacementCharacter": "_" - }, - { - "id": 2, - "character": "/", - "replacementCharacter": "-" - } - ] - ``` - * Save the file. - -4. **Second Run (Create Fields & Sync Data)** - - ```powershell - .\SectigoMetadataSync.exe SCtoKF - ``` - - * The tool will now convert Sectigo custom‐field names (using your replacements), create new Keyfactor metadata fields if needed, and write all custom/manual data into Keyfactor. - -5. **Ongoing Use** - - * Schedule the executable with the argument of your choice (`SCtoKF` or `KFtoSC`) once per week via Task Scheduler. - * Each run will create any newly added fields and keep all values in sync. - ---- - -### Troubleshooting - -* **Missing or Invalid JSON** - - * If `stock-config.json` or `fields.json` is malformed or missing required sections, the tool logs an error and exits. - * Ensure both files exist, are valid JSON, and contain the required properties (see “Settings” above). - -* **Authentication Failures** - - * Double‐check `sectigoLogin`/`sectigoPassword` and `keyfactorLogin`/`keyfactorPassword`. - * Ensure your API user has adequate permissions to create/update metadata fields. - -* **Rate Limits or Timeouts** - - * Sectigo and Keyfactor may throttle large requests. - * If you repeatedly see HTTP 429 or timeout errors, consider reducing `sectigoPageSize`/`keyfactorPageSize` in `stock-config.json`. - -* **Field Creation Errors** - - * If Keyfactor rejects a field name (e.g., still contains a banned character), verify that `bannedCharacters.json` is up to date. - * If Sectigo rejects a custom‐field update, ensure you are using the correct custom‐field “name” as visible in Sectigo’s administration UI. - ---- - -**Copyright & License** -This tool and its source code are provided “as‐is.” Modify, redistribute, or use at your own risk. Check with your organization’s policies before deploying in a production environment. diff --git a/sectigo-metadata-sync/Client/KeyfactorClient.cs b/sectigo-metadata-sync/Client/KeyfactorClient.cs index 22d2095..78fec42 100644 --- a/sectigo-metadata-sync/Client/KeyfactorClient.cs +++ b/sectigo-metadata-sync/Client/KeyfactorClient.cs @@ -1,4 +1,4 @@ -// Copyright 2021 Keyfactor +// Copyright 2024 Keyfactor // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. // You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 // Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -7,12 +7,15 @@ using System; using System.Collections.Generic; +using System.Globalization; +using System.IO; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using Microsoft.Extensions.DependencyInjection; using NLog; using SectigoMetadataSync.Client; @@ -21,8 +24,7 @@ namespace SectigoMetadataSync.Client { /// - /// Synchronous client for retrieving Keyfactor metadata fields. - /// Designed for use as a typed HttpClient via IHttpClientFactory. + /// Fully synchronous Keyfactor client implemented around HttpClient.Send (NET 8+). /// public class KeyfactorMetadataClient { @@ -32,187 +34,288 @@ public class KeyfactorMetadataClient private readonly JsonSerializerOptions _jsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; - /// - /// Initializes a new instance of the KeyfactorMetadataClient. - /// HttpClient is injected/configured via IHttpClientFactory; its BaseAddress should be set externally. - /// + private DateTimeOffset _bearerExpiresUtc; + private string? _bearerToken; + public KeyfactorMetadataClient(HttpClient httpClient) { _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); } + // OAuth support + public void AuthenticateBearer(string accessToken, DateTimeOffset? expiresUtc = null, + string requestedWith = "APIClient") + { + if (string.IsNullOrWhiteSpace(accessToken)) + throw new ArgumentException("Access token must be provided.", nameof(accessToken)); + + _bearerToken = accessToken; + _bearerExpiresUtc = + expiresUtc ?? DateTimeOffset.UtcNow.AddMinutes(5); // conservative default if none provided + + var headers = _httpClient.DefaultRequestHeaders; + headers.Authorization = new AuthenticationHeaderValue("Bearer", _bearerToken); + headers.Remove("x-keyfactor-requested-with"); + headers.Add("x-keyfactor-requested-with", requestedWith); + + // Remove any stale Basic header if caller switched modes + if (headers.TryGetValues("Authorization", out _)) + { + // Nothing else to do; new Bearer replaces any previous Authorization. + } + } + + // Ensure token validity before API calls + private void EnsureBearer(Func<(string token, DateTimeOffset expiresUtc)> refresh) + { + if (string.IsNullOrEmpty(_bearerToken) || DateTimeOffset.UtcNow >= _bearerExpiresUtc.AddMinutes(-2)) + { + var (tkn, exp) = refresh(); + AuthenticateBearer(tkn, exp); + } + } + + /// - /// Configures Basic authentication and required headers for Keyfactor API. - /// Must be called before invoking any list/get methods. + /// Configure basic auth and required headers. /// - /// Keyfactor API username - /// Keyfactor API password - /// Value for the x-keyfactor-requested-with header (default: APIClient) public void Authenticate(string username, string password, string requestedWith = "APIClient") { - var credentials = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}")); - var headers = _httpClient.DefaultRequestHeaders; + if (username is null) throw new ArgumentNullException(nameof(username)); + if (password is null) throw new ArgumentNullException(nameof(password)); - headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); + var creds = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}")); + var headers = _httpClient.DefaultRequestHeaders; + headers.Authorization = new AuthenticationHeaderValue("Basic", creds); headers.Remove("x-keyfactor-requested-with"); headers.Add("x-keyfactor-requested-with", requestedWith); } + // ---- Helpers ------------------------------------------------------ + private Uri BuildUri(string relativePath, string? query = null) + { + var baseUri = _httpClient.BaseAddress ?? + throw new InvalidOperationException("HttpClient.BaseAddress must be set."); + var path = relativePath.TrimStart('/') + (string.IsNullOrEmpty(query) + ? string.Empty + : (relativePath.Contains('?') ? "&" : "?") + query); + return new Uri(baseUri, path); + } + + private HttpResponseMessage Send(HttpMethod method, string relativePath, HttpContent? content = null) + { + if (method is null) throw new ArgumentNullException(nameof(method)); + if (string.IsNullOrWhiteSpace(relativePath)) + throw new ArgumentException("relativePath cannot be null or whitespace.", nameof(relativePath)); + + var uri = BuildUri(relativePath); + _logger.Trace("-> Send: {0} {1}", method, uri); + + using var req = new HttpRequestMessage(method, uri); + if (content != null) req.Content = content; + + HttpResponseMessage resp = null!; + try + { + // Prefer headers-first so we control when/how we buffer the body. + resp = _httpClient.Send(req, HttpCompletionOption.ResponseHeadersRead); + + if (resp.IsSuccessStatusCode) + { + _logger.Trace("<- Send OK: {0} {1} [{2}]", method, uri, (int)resp.StatusCode); + return resp; // caller disposes + } + + // Non-success: read body synchronously, log, then throw with full message + var status = resp.StatusCode; + var reason = resp.ReasonPhrase ?? string.Empty; + var body = ReadString(resp); // uses Content.ReadAsStream() (sync) + + var snippet = body.Length > 2000 ? body.Substring(0, 2000) + "…[truncated]" : body; + _logger.Error("HTTP failure {0} {1}: {2} {3}. Body={4}", + method, uri, (int)status, reason, snippet); + + resp.Dispose(); // not returning it + throw new HttpRequestException( + $"HTTP {(int)status} {reason} for {uri}. Body: {body}", + null, + status); + } + finally + { + _logger.Trace("<- Send exit: {0} {1}", method, uri); + } + } + + + private T? ReadJson(HttpResponseMessage resp) + { + using var s = resp.Content.ReadAsStream(); // synchronous + return JsonSerializer.Deserialize(s, _jsonOptions); + } + + private string ReadString(HttpResponseMessage resp) + { + using var s = resp.Content.ReadAsStream(); // synchronous + using var sr = new StreamReader(s, Encoding.UTF8, true, 8192, false); + return sr.ReadToEnd(); + } + + private StringContent JsonBody(T value) + { + return new StringContent(JsonSerializer.Serialize(value, _jsonOptions), Encoding.UTF8, "application/json"); + } + + // ---- API methods -------------------------------------------------- + /// - /// Lists metadata field definitions. Maps to GET /KeyfactorAPI/MetadataFields with optional filtering, paging, and - /// sorting. + /// Lists metadata field definitions. /// public List ListMetadataFields() { - var sb = new StringBuilder(_httpClient.BaseAddress + "/MetadataFields"); - var response = _httpClient.GetAsync(sb.ToString()).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); - var json = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - return JsonSerializer.Deserialize>(json, _jsonOptions) - ?? new List(); + var resp = Send(HttpMethod.Get, "MetadataFields"); + return ReadJson>(resp) ?? new List(); } - /// - /// Sends a list of unified metadata fields to Keyfactor. - /// If a field already exists, it is updated using an HTTP PUT request. - /// Otherwise, it is created using an HTTP POST request. + /// Sends/Upserts unified metadata fields to Keyfactor. /// - /// List of unified metadata fields to send. - /// List of existing metadata fields in Keyfactor. public void SendUnifiedMetadataFields(List unifiedFields, List existingFields) { - if (unifiedFields == null || unifiedFields.Count == 0) - throw new ArgumentException("The list of unified metadata fields cannot be null or empty."); + if (unifiedFields is null || unifiedFields.Count == 0) + throw new ArgumentException("The list of unified metadata fields cannot be null or empty.", + nameof(unifiedFields)); - var createdCount = 0; - var updatedCount = 0; + existingFields ??= new List(); + + var created = 0; + var updated = 0; foreach (var field in unifiedFields) try { - // Check if the field already exists in Keyfactor - var existingField = existingFields.FirstOrDefault(f => + var existing = existingFields.FirstOrDefault(f => f.Name.Equals(field.KeyfactorMetadataFieldName, StringComparison.OrdinalIgnoreCase)); - - // Convert UnifiedFormatField to KeyfactorMetadataField - var keyfactorField = new KeyfactorMetadataField + KeyfactorMetadataField payload; + HttpResponseMessage resp; + if (existing is not null) { - Id = existingField?.Id ?? 0, // Use existing ID if available, otherwise 0 for new field - Name = field.KeyfactorMetadataFieldName, - Description = field.KeyfactorDescription, - DataType = field.KeyfactorDataType, - Hint = field.KeyfactorHint, - Validation = field.KeyfactorValidation, - Enrollment = field.KeyfactorEnrollment, - Message = field.KeyfactorMessage, - Options = field.KeyfactorOptions != null - ? string.Join(",", field.KeyfactorOptions) - : null, // Convert array to string - DefaultValue = field.KeyfactorDefaultValue, - DisplayOrder = field.KeyfactorDisplayOrder, - CaseSensitive = field.KeyfactorCaseSensitive - }; - - // Serialize the field to JSON - var jsonContent = JsonSerializer.Serialize(keyfactorField, _jsonOptions); - var content = new StringContent(jsonContent, Encoding.UTF8, "application/json"); - - // Log the JSON payload being sent - _logger.Trace($"Sending JSON Payload: {jsonContent}"); - - HttpResponseMessage response; - if (existingField != null) - { - // Field exists, update it using PUT - var endpoint = $"{_httpClient.BaseAddress}/MetadataFields"; - response = _httpClient.PutAsync(endpoint, content).GetAwaiter().GetResult(); - updatedCount++; + // PUT: include Id + payload = new KeyfactorMetadataField + { + Id = existing.Id, + Name = field.KeyfactorMetadataFieldName, + Description = field.KeyfactorDescription, + DataType = field.KeyfactorDataType, + Hint = field.KeyfactorHint, + Validation = field.KeyfactorValidation, + Enrollment = field.KeyfactorEnrollment, + Message = field.KeyfactorMessage, + Options = field.KeyfactorOptions != null ? string.Join(",", field.KeyfactorOptions) : null, + DefaultValue = field.KeyfactorDefaultValue, + DisplayOrder = field.KeyfactorDisplayOrder, + CaseSensitive = field.KeyfactorCaseSensitive + }; + var json = JsonSerializer.Serialize(payload, _jsonOptions); + _logger.Trace($"Sending JSON Payload: {json}"); + resp = Send(HttpMethod.Put, "MetadataFields", JsonBody(payload)); + updated++; } else { - // Field does not exist, create it using POST - var endpoint = $"{_httpClient.BaseAddress}/MetadataFields"; - response = _httpClient.PostAsync(endpoint, content).GetAwaiter().GetResult(); - createdCount++; + // POST: do not include Id + payload = new KeyfactorMetadataField + { + Id = 0, + Name = field.KeyfactorMetadataFieldName, + Description = field.KeyfactorDescription, + DataType = field.KeyfactorDataType, + Hint = field.KeyfactorHint, + Validation = field.KeyfactorValidation, + Enrollment = field.KeyfactorEnrollment, + Message = field.KeyfactorMessage, + Options = field.KeyfactorOptions != null ? string.Join(",", field.KeyfactorOptions) : null, + DefaultValue = field.KeyfactorDefaultValue, + DisplayOrder = field.KeyfactorDisplayOrder, + CaseSensitive = field.KeyfactorCaseSensitive + }; + var json = JsonSerializer.Serialize(payload, _jsonOptions); + _logger.Trace($"Sending JSON Payload: {json}"); + resp = Send(HttpMethod.Post, "MetadataFields", JsonBody(payload)); + created++; } - response.EnsureSuccessStatusCode(); - - // Deserialize the response to get the KeyfactorMetadataFieldId - var responseJson = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - var responseField = JsonSerializer.Deserialize(responseJson, _jsonOptions); - if (responseField != null) + var returned = ReadJson(resp); + if (returned is not null) { - field.KeyfactorMetadataFieldId = - responseField.Id; // Update the UnifiedFormatField with the returned ID + field.KeyfactorMetadataFieldId = returned.Id; _logger.Trace( $"Field '{field.KeyfactorMetadataFieldName}' updated with KeyfactorMetadataFieldId: {field.KeyfactorMetadataFieldId}"); } } catch (Exception ex) { - // Log errors into error logs _logger.Error(ex, $"Error processing metadata field: {field.KeyfactorMetadataFieldName}"); } - // Log counts of created and updated fields into info logs - _logger.Info($"Metadata fields processed: {createdCount} created, {updatedCount} updated."); + _logger.Info($"Metadata fields processed: {created} created, {updated} updated."); } + /// - /// Retrieves a list of certificates from Keyfactor where the issuer DN contains the specified substring. - /// Optionally includes revoked and expired certificates, and includes metadata. + /// Get certificates by issuer (paged). + /// Optionally restrict to certs that were added (imported) on/after . /// - /// The substring to search for in the issuer DN (default: "Sectigo"). - /// Whether to include revoked and expired certificates (default: false). - /// A list of certificates matching the criteria, including metadata. - public List GetCertificatesByIssuer(string issuerSubstring = "Sectigo", - bool includeRevokedAndExpired = false) + public List GetCertificatesByIssuer( + string issuerSubstring, + bool includeRevokedAndExpired, + int pageNumber, + int pageSize, + string? addedSince = null) { - if (string.IsNullOrEmpty(issuerSubstring)) + if (string.IsNullOrWhiteSpace(issuerSubstring)) throw new ArgumentException("Issuer substring cannot be null or empty.", nameof(issuerSubstring)); - // Construct the query string in a readable format - var queryString = $"IssuerDN -contains \"{issuerSubstring}\""; + if (pageNumber <= 0) pageNumber = 1; + if (pageSize <= 0) pageSize = 100; + + // Base query + var q = $"IssuerDN -contains \"{issuerSubstring}\""; - // Encode the query string for safe transmission - var encodedQueryString = Uri.EscapeDataString(queryString); - // Build the full query parameters - var queryParameters = new StringBuilder($"QueryString={encodedQueryString}&includeMetadata=true"); - if (includeRevokedAndExpired) queryParameters.Append("&IncludeRevoked=true&IncludeExpired=true"); + var encoded = Uri.EscapeDataString(q); - // Construct the endpoint URL - var endpoint = $"{_httpClient.BaseAddress}/Certificates?{queryParameters}"; + var query = new StringBuilder() + .Append("QueryString=").Append(encoded) + .Append("&includeMetadata=true") + .Append("&PageReturned=").Append(pageNumber) + .Append("&ReturnLimit=").Append(pageSize) + .Append("&SortField=NotBefore") + .Append("&SortAscending=1"); // keep existing behavior - // Send the GET request - var response = _httpClient.GetAsync(endpoint).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); + if (!string.IsNullOrEmpty(addedSince)) query.Append($"&DateImported -ge \"{addedSince}\""); - // Deserialize the response JSON - var json = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + if (includeRevokedAndExpired) + query.Append("&IncludeRevoked=true&IncludeExpired=true"); - // Log raw JSON response into trace logs + var resp = Send(HttpMethod.Get, $"Certificates?{query}"); + + var json = ReadString(resp); _logger.Trace($"Raw JSON Response from GetCertificatesByIssuer: {json}"); try { - return JsonSerializer.Deserialize>(json, new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }) ?? new List(); + return JsonSerializer.Deserialize>(json, _jsonOptions) + ?? new List(); } catch (JsonException ex) { - // Log errors into error logs _logger.Error(ex, "Failed to deserialize the certificate list from Keyfactor API."); throw new InvalidOperationException("Failed to deserialize the certificate list from Keyfactor API.", ex); @@ -220,140 +323,216 @@ public List GetCertificatesByIssuer(string issuerSubstring } /// - /// Retrieves a paginated list of certificates from Keyfactor where the issuer DN contains the specified substring. - /// Optionally includes revoked and expired certificates, and includes metadata. + /// Update certificate metadata (typed payload). Prefer this overload. + /// Sends integers and booleans as JSON numbers/bools; other types as strings. /// - /// The substring to search for in the issuer DN (default: "Sectigo"). - /// Whether to include revoked and expired certificates (default: false). - /// The page number to retrieve (default: 1). - /// The number of certificates per page (default: 100). - /// A list of certificates matching the criteria, including metadata. - public List GetCertificatesByIssuer(string issuerSubstring = "Sectigo", - bool includeRevokedAndExpired = false, int pageNumber = 1, int pageSize = 100) + public bool UpdateCertificateMetadata(int certificateId, IReadOnlyDictionary metadata) { - if (string.IsNullOrEmpty(issuerSubstring)) - throw new ArgumentException("Issuer substring cannot be null or empty.", nameof(issuerSubstring)); - - // Construct the query string in a readable format - var queryString = $"IssuerDN -contains \"{issuerSubstring}\""; - - // Encode the query string for safe transmission - var encodedQueryString = Uri.EscapeDataString(queryString); + if (certificateId <= 0) + throw new ArgumentException("Certificate ID must be greater than zero.", nameof(certificateId)); + if (metadata is null || metadata.Count == 0) + throw new ArgumentException("Metadata cannot be null or empty.", nameof(metadata)); - // Build the full query parameters - var queryParameters = new StringBuilder($"QueryString={encodedQueryString}&includeMetadata=true"); - queryParameters.Append($"&PageReturned={pageNumber}&ReturnLimit={pageSize}"); - if (includeRevokedAndExpired) queryParameters.Append("&IncludeRevoked=true&IncludeExpired=true"); + // Final pass: sanitize strings, normalize numeric/bool types, drop null/blank values. + var normalized = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var kvp in metadata) + { + if (string.IsNullOrWhiteSpace(kvp.Key)) + continue; - // Construct the endpoint URL - var endpoint = $"{_httpClient.BaseAddress}/Certificates?{queryParameters}"; + var v = NormalizeForWire(kvp.Value, out var keep); + if (keep) + normalized[kvp.Key] = v; + } - // Send the GET request - var response = _httpClient.GetAsync(endpoint).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); + if (normalized.Count == 0) + throw new ArgumentException("No non-empty metadata values after normalization.", nameof(metadata)); - // Deserialize the response JSON - var json = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + var body = new { Id = certificateId, Metadata = normalized }; - // Log raw JSON response into trace logs - _logger.Trace($"Raw JSON Response from GetCertificatesByIssuer: {json}"); + // Log the exact JSON we're about to send (useful for troubleshooting type issues). + var json = JsonSerializer.Serialize(body, _jsonOptions); + _logger.Trace($"Sending JSON Payload to update metadata: {json}"); try { - return JsonSerializer.Deserialize>(json, new JsonSerializerOptions + var resp = Send(HttpMethod.Put, "Certificates/Metadata", JsonBody(body)); + using (resp) { - PropertyNameCaseInsensitive = true, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }) ?? new List(); + /* disposing response */ + } + + return true; } - catch (JsonException ex) + catch (Exception ex) { - // Log errors into error logs - _logger.Error(ex, "Failed to deserialize the certificate list from Keyfactor API."); - throw new InvalidOperationException("Failed to deserialize the certificate list from Keyfactor API.", - ex); + _logger.Error(ex, $"Failed to update metadata for certificate ID: {certificateId}"); + return false; } - } - /// - /// Updates metadata for a given certificate in Keyfactor. - /// - /// The ID of the certificate to update. - /// A dictionary containing the metadata key-value pairs to update. - /// True if the update was successful, otherwise false. - public bool UpdateCertificateMetadata(int certificateId, Dictionary metadata) - { - if (certificateId <= 0) - throw new ArgumentException("Certificate ID must be greater than zero.", nameof(certificateId)); + // ---- local helpers ---- - if (metadata == null || metadata.Count == 0) - throw new ArgumentException("Metadata cannot be null or empty.", nameof(metadata)); + static object? NormalizeForWire(object? value, out bool keep) + { + keep = true; + if (value is null) + { + keep = false; + return null; + } - // Construct the endpoint URL - var endpoint = $"{_httpClient.BaseAddress}/Certificates/Metadata"; + switch (value) + { + // Already-typed primitives pass through + case bool b: + return b; - // Create the request body - var requestBody = new - { - Id = certificateId, - Metadata = metadata - }; + case sbyte or byte or short or ushort or int or uint or long: + // cast all signed/unsigned integrals to long for uniform JSON emission + return Convert.ToInt64(value, CultureInfo.InvariantCulture); - // Serialize the request body to JSON - var jsonContent = JsonSerializer.Serialize(requestBody, _jsonOptions); - var content = new StringContent(jsonContent, Encoding.UTF8, "application/json"); + case ulong ul: + // STJ writes ulong; Keyfactor integer fields are signed. Clamp if needed. + if (ul > long.MaxValue) ul = long.MaxValue; + return (long)ul; - // Log the JSON payload being sent - _logger.Trace($"Sending JSON Payload to update metadata: {jsonContent}"); + case float f: + // If it’s integral, keep as integer; else round toward zero + return (long)f; - try - { - // Send the PUT request - var response = _httpClient.PutAsync(endpoint, content).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); + case double d: + return (long)d; + + case decimal m: + return (long)m; + + case DateTime dt: + // Use ISO-8601 UTC string + return dt.ToUniversalTime().ToString("o", CultureInfo.InvariantCulture); + + case DateTimeOffset dto: + return dto.ToUniversalTime().ToString("o", CultureInfo.InvariantCulture); - return true; // Indicate success + case string s: + { + var cleaned = SanitizeString(s); + if (string.IsNullOrWhiteSpace(cleaned)) + { + keep = false; + return null; + } + + return cleaned; + } + + // If a JsonElement slipped through, preserve its JSON scalar type where possible. + case JsonElement el: + { + switch (el.ValueKind) + { + case JsonValueKind.Null: + case JsonValueKind.Undefined: + keep = false; + return null; + case JsonValueKind.True: return true; + case JsonValueKind.False: return false; + case JsonValueKind.Number: + if (el.TryGetInt64(out var n)) return n; + if (el.TryGetDouble(out var dbl)) return (long)dbl; + return SanitizeString(el.GetRawText()); + case JsonValueKind.String: + var s = el.GetString(); + var cleaned = SanitizeString(s); + if (string.IsNullOrWhiteSpace(cleaned)) + { + keep = false; + return null; + } + + return cleaned; + default: + // objects/arrays should have been flattened earlier; keep raw JSON as string + var raw = el.GetRawText(); + var cleanedJson = SanitizeString(raw, false); + if (string.IsNullOrWhiteSpace(cleanedJson)) + { + keep = false; + return null; + } + + return cleanedJson; + } + } + + default: + // Fallback: ToString() + sanitize (avoids exceptions on unexpected types) + var txt = SanitizeString(value.ToString()); + if (string.IsNullOrWhiteSpace(txt)) + { + keep = false; + return null; + } + + return txt; + } } - catch (Exception ex) + + static string SanitizeString(string? s, bool collapseInnerWhitespace = true) { - // Log errors into error logs - _logger.Error(ex, $"Failed to update metadata for certificate ID: {certificateId}"); - return false; // Indicate failure + if (string.IsNullOrEmpty(s)) return string.Empty; + + // Normalize Unicode, drop hidden/zero-width/bidi, replace NBSP, remove control chars (except CR/LF/TAB) + Span hidden = stackalloc char[] + { + '\u200B', '\u200C', '\u200D', '\uFEFF', '\u200E', '\u200F', '\u202A', '\u202B', '\u202C', '\u202D', + '\u202E' + }; + var norm = s.Normalize(NormalizationForm.FormKC); + var sb = new StringBuilder(norm.Length); + foreach (var ch in norm) + { + var isHidden = false; + for (var i = 0; i < hidden.Length; i++) + if (ch == hidden[i]) + { + isHidden = true; + break; + } + + if (isHidden) continue; + + if (ch == '\u00A0') + { + sb.Append(' '); + continue; + } // NBSP -> space + + if (char.IsControl(ch) && ch != '\r' && ch != '\n' && ch != '\t') continue; + sb.Append(ch); + } + + var cleaned = sb.ToString().Trim(); + + if (!collapseInnerWhitespace) return cleaned; + + // Collapse all whitespace to single spaces + return Regex.Replace(cleaned, @"\s+", " ").Trim(); } } } } /// -/// Extension methods for registering Keyfactor clients in the DI container. +/// DI registration helpers. /// -public static class ServiceCollectionExtensions +public static class KeyfactorServiceCollectionExtensions { - /// - /// Registers a typed KeyfactorMetadataClient with IHttpClientFactory. - /// - public static IServiceCollection AddKeyfactorMetadataClient( - this IServiceCollection services, - string baseAddress) + public static IServiceCollection AddKeyfactorMetadataClient(this IServiceCollection services, string baseAddress) { services.AddHttpClient(client => { client.BaseAddress = new Uri(baseAddress); - client.DefaultRequestHeaders.Accept.Add( - new MediaTypeWithQualityHeaderValue("application/json")); - }); - return services; - } - - public static IServiceCollection AddSectigoCustomFieldsClient( - this IServiceCollection services, - string baseAddress) - { - services.AddHttpClient(client => - { - client.BaseAddress = new Uri(baseAddress); - // other defaults like Accept headers can be configured here + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); }); return services; } diff --git a/sectigo-metadata-sync/Client/OAuthTokenClient.cs b/sectigo-metadata-sync/Client/OAuthTokenClient.cs new file mode 100644 index 0000000..9bd25b4 --- /dev/null +++ b/sectigo-metadata-sync/Client/OAuthTokenClient.cs @@ -0,0 +1,439 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using System.Threading; +using Microsoft.Extensions.DependencyInjection; +using NLog; + +namespace SectigoMetadataSync.Client; + +public sealed class OAuthTokenClient +{ + private static readonly JsonSerializerOptions _json = new() { PropertyNameCaseInsensitive = true }; + private static readonly Logger _logger = LogManager.GetCurrentClassLogger(); + + // Aggressive redaction for common secret-bearing fields / tokens + private static readonly Regex _redactRegex = new( + @"(?ix) + (access[_-]?token) \s*[:=]\s*[""']? [^""'\s}]+ + |(refresh[_-]?token) \s*[:=]\s*[""']? [^""'\s}]+ + |(client[_-]?secret) \s*[:=]\s*[""']? [^""'\s}]+ + |(authorization)\s*:\s*bearer\s+[A-Za-z0-9\-\._~\+/=]+ + ", RegexOptions.Compiled); + + private readonly string? _audience; // optional + private readonly string _clientId; + private readonly string _clientSecret; + private readonly TimeSpan _defaultLifetime; // used when expires_in is absent + private readonly HttpClient _http; + private readonly string? _scopeCsv; // optional + private readonly string _tokenUrl; + + public OAuthTokenClient( + HttpClient http, + string tokenUrl, + string clientId, + string clientSecret, + string? scopesCsv = null, + string? audience = null, + TimeSpan? defaultLifetime = null) + { + _logger.Trace( + "Entering OAuthTokenClient::.ctor (tokenUrl={tokenUrl}, hasScope={hasScope}, hasAudience={hasAudience})", + Safe(tokenUrl), !string.IsNullOrWhiteSpace(scopesCsv), !string.IsNullOrWhiteSpace(audience)); + + _http = http ?? throw new ArgumentNullException(nameof(http)); + _tokenUrl = tokenUrl ?? throw new ArgumentNullException(nameof(tokenUrl)); + _clientId = clientId ?? throw new ArgumentNullException(nameof(clientId)); + _clientSecret = clientSecret ?? throw new ArgumentNullException(nameof(clientSecret)); + _scopeCsv = string.IsNullOrWhiteSpace(scopesCsv) ? null : scopesCsv; + _audience = string.IsNullOrWhiteSpace(audience) ? null : audience; + _defaultLifetime = defaultLifetime ?? TimeSpan.FromMinutes(5); + + // Be explicit about Accept; Content-Type is set by FormUrlEncodedContent. + if (!_http.DefaultRequestHeaders.Accept.Any(h => h.MediaType == "application/json")) + _http.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + _logger.Trace("Exiting OAuthTokenClient::.ctor (defaultLifetime={lifetime})", _defaultLifetime); + } + + /// + /// Mirrors: + /// curl/Invoke-RestMethod -X POST tokenUrl + /// Content-Type: application/x-www-form-urlencoded + /// grant_type=client_credentials&client_id=...&client_secret=...(&scope=...|audience=...) + /// Returns (accessToken, expiresUtc). If the IdP does not return expires_in, a conservative + /// synthetic expiry is applied (default 5 minutes). + /// + public (string accessToken, DateTimeOffset expiresUtc) GetTokenWithClientCredentials() + { + _logger.Trace("Entering GetTokenWithClientCredentials (url={url}, scopeSet={scopeSet}, audienceSet={audSet})", + Safe(_tokenUrl), _scopeCsv is not null, _audience is not null); + + var form = new List> + { + new("grant_type", "client_credentials"), + new("client_id", _clientId), + new("client_secret", _clientSecret) + }; + + // Optional extras: many IdPs accept these; ignored if null/empty. + if (!string.IsNullOrWhiteSpace(_scopeCsv)) + form.Add(new KeyValuePair("scope", _scopeCsv!.Replace(",", " ").Trim())); + if (!string.IsNullOrWhiteSpace(_audience)) + form.Add(new KeyValuePair("audience", _audience!)); + + using var req = new HttpRequestMessage(HttpMethod.Post, _tokenUrl) + { + Content = new FormUrlEncodedContent(form) // sets Content-Type: application/x-www-form-urlencoded + }; + + const int maxAttempts = 5; + var attempt = 0; + var rand = new Random(); + + while (true) + { + attempt++; + _logger.Debug("Token request attempt {attempt} -> {method} {url}", attempt, req.Method, Safe(_tokenUrl)); + + using var resp = _http.Send(req); + var sc = (int)resp.StatusCode; + _logger.Debug("Token response attempt {attempt}: HTTP {status} ({reason})", attempt, sc, resp.ReasonPhrase); + + if ((int)resp.StatusCode is >= 200 and < 300) + { + using var s = resp.Content.ReadAsStream(); + var dto = JsonSerializer.Deserialize(s, _json) + ?? throw new InvalidOperationException("Token JSON was empty."); + if (string.IsNullOrWhiteSpace(dto.AccessToken)) + { + _logger.Error("Token response missing access_token (attempt {attempt})", attempt); + throw new InvalidOperationException("Token response missing 'access_token'."); + } + + var now = DateTimeOffset.UtcNow; + // If expires_in is present, use it; else synthesize defaultLifetime. + var expires = dto.ExpiresIn.HasValue && dto.ExpiresIn.Value > 0 + ? now.AddSeconds(dto.ExpiresIn.Value) + : now.Add(_defaultLifetime); + _logger.Info("Obtained OAuth token (type={type}).", + string.IsNullOrWhiteSpace(dto.TokenType) ? "bearer?" : dto.TokenType); + _logger.Trace("Exiting GetTokenWithClientCredentials (expiresUtc={expires:o})", expires); + return (dto.AccessToken!, expires); + } + + // Transient handling (rate limit / server errors) + if ((resp.StatusCode == (HttpStatusCode)429 || (int)resp.StatusCode >= 500) && attempt < maxAttempts) + { + var delayMs = GetRetryAfterMs(resp) ?? + (int)Math.Min(30000, Math.Pow(2, attempt) * 250 + rand.Next(0, 250)); + _logger.Warn( + "Transient token failure (HTTP {status}). Retrying attempt {nextAttempt}/{max} after {delay} ms.", + sc, attempt + 1, maxAttempts, delayMs); + Thread.Sleep(delayMs); + continue; + } + + var body = ReadBody(resp); + _logger.Error("Token request failed (HTTP {status} {reason}). Body: {body}", + sc, resp.ReasonPhrase, body); + throw new InvalidOperationException( + $"Token request failed ({(int)resp.StatusCode} {resp.ReasonPhrase}). Body: {body}"); + } + + static int? GetRetryAfterMs(HttpResponseMessage resp) + { + if (resp.Headers.TryGetValues("Retry-After", out var values)) + { + var v = values.FirstOrDefault(); + if (int.TryParse(v, out var secs) && secs >= 0) return secs * 1000; + if (DateTimeOffset.TryParse(v, out var when)) + return (int)Math.Max(0, (when - DateTimeOffset.UtcNow).TotalMilliseconds); + } + + return null; + } + + static string ReadBody(HttpResponseMessage resp) + { + using var s = resp.Content.ReadAsStream(); + using var sr = new StreamReader(s, Encoding.UTF8, true, 8192, false); + return sr.ReadToEnd(); + } + } + + private static string Safe(string? s) + { + return string.IsNullOrWhiteSpace(s) ? "(null)" : s.Length > 2048 ? s[..2048] + "…" : s; + } + + private sealed class TokenResponseMinimal + { + // Exact match for {"access_token":"..."} + [JsonPropertyName("access_token")] public string? AccessToken { get; set; } + + // Optional fields that some IdPs include + [JsonPropertyName("expires_in")] public int? ExpiresIn { get; set; } + + [JsonPropertyName("token_type")] public string? TokenType { get; set; } + + // Capture any other fields without failing deserialization + [JsonExtensionData] public Dictionary? Extra { get; set; } + + /// + /// Best-effort normalization for alternate casings (e.g., "accessToken"). + /// Call after deserialization if you need to be defensive. + /// + public void Normalize() + { + if (!string.IsNullOrWhiteSpace(AccessToken) || Extra is null) return; + + if (Extra.TryGetValue("accessToken", out var alt) && alt.ValueKind == JsonValueKind.String) + AccessToken = alt.GetString(); + + if (!ExpiresIn.HasValue && + Extra.TryGetValue("expiresIn", out var exp) && exp.ValueKind == JsonValueKind.Number && + exp.TryGetInt32(out var secs)) + ExpiresIn = secs; + + if (string.IsNullOrWhiteSpace(TokenType) && + Extra.TryGetValue("tokenType", out var tt) && tt.ValueKind == JsonValueKind.String) + TokenType = tt.GetString(); + } + } +} + +public static class OAuthServiceCollectionExtensions +{ + // inside: public static class OAuthServiceCollectionExtensions + public static IServiceCollection AddKeyfactorMetadataClientOAuth( + this IServiceCollection services, + string baseAddress, + OAuthOptions oauthOptions, + string requestedWith = "APIClient") + { + if (oauthOptions is null) throw new ArgumentNullException(nameof(oauthOptions)); + if (string.IsNullOrWhiteSpace(baseAddress)) throw new ArgumentNullException(nameof(baseAddress)); + + // 1) Ensure the OAuth token client is registered with its ctor args + services.AddHttpClient(c => + { + c.Timeout = TimeSpan.FromSeconds(30); + c.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + }) + .AddTypedClient(http => new OAuthTokenClient( + http, + oauthOptions.TokenUrl, + oauthOptions.ClientId, + oauthOptions.ClientSecret, + oauthOptions.ScopesCsv, + oauthOptions.Audience)); + + // 2) Register the handler itself (so DI can resolve it) + services.AddTransient(sp => + new OAuthTokenHandler( + sp.GetRequiredService(), + oauthOptions, + requestedWith)); + + // 3) Register the Keyfactor client and plug the handler into the pipeline + services.AddHttpClient(client => + { + client.BaseAddress = new Uri(baseAddress); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + }) + .AddHttpMessageHandler(); + + return services; + } + + public sealed class OAuthOptions + { + public string TokenUrl { get; init; } = string.Empty; + public string ClientId { get; init; } = string.Empty; + public string ClientSecret { get; init; } = string.Empty; + public string? ScopesCsv { get; init; } // e.g., "openid,profile" + public string? Audience { get; init; } + public int refreshSkewSeconds { get; init; } = 120; + } +} + +internal sealed class OAuthTokenHandler : DelegatingHandler +{ + private static readonly Logger _logger = LogManager.GetCurrentClassLogger(); + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly OAuthServiceCollectionExtensions.OAuthOptions _opt; + private readonly string _requestedWith; + private readonly OAuthTokenClient _tokenClient; + private string? _accessToken; + private DateTimeOffset _expiresUtc; + + public OAuthTokenHandler( + OAuthTokenClient tokenClient, + OAuthServiceCollectionExtensions.OAuthOptions opt, + string requestedWith) + { + _tokenClient = tokenClient ?? throw new ArgumentNullException(nameof(tokenClient)); + _opt = opt ?? throw new ArgumentNullException(nameof(opt)); + _requestedWith = string.IsNullOrWhiteSpace(requestedWith) ? "APIClient" : requestedWith; + } + + // -------- SYNC PIPELINE -------- + protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken ct) + { + if (request is null) throw new ArgumentNullException(nameof(request)); + _logger.Trace("-> OAuthTokenHandler.Send: {method} {uri}", request.Method, request.RequestUri); + + EnsureToken(ct); + ApplyHeaders(request); + + var resp = base.Send(request, ct); + _logger.Debug("Downstream response: HTTP {status}", (int)resp.StatusCode); + + if (resp.StatusCode == HttpStatusCode.Unauthorized) + { + _logger.Warn("401 Unauthorized detected. Forcing token refresh and retrying once."); + resp.Dispose(); + ForceRefresh(ct); + ApplyHeaders(request, true); + resp = base.Send(request, ct); + _logger.Debug("Retry response: HTTP {status}", (int)resp.StatusCode); + } + + if (resp.IsSuccessStatusCode) + { + _logger.Trace("<- OAuthTokenHandler.Send OK [{status}]", (int)resp.StatusCode); + return resp; // caller disposes + } + + // Non-success: read body synchronously, log, then throw with full body in exception + var status = resp.StatusCode; + var reason = resp.ReasonPhrase ?? string.Empty; + + var body = string.Empty; + try + { + if (resp.Content != null) + { + using var s = resp.Content.ReadAsStream(); // sync path on .NET 8 + using var sr = new StreamReader(s, Encoding.UTF8, true, 8192, false); + body = sr.ReadToEnd(); + } + } + catch + { + // Swallow body-read failures; keep body empty for logging/exception. + } + + var snippet = body.Length > 2000 ? body[..2000] + "…[truncated]" : body; + string? reqId = null; + try + { + if (!resp.Headers.TryGetValues("x-request-id", out var v) || (reqId = v.FirstOrDefault()) is null) + if (resp.Headers.TryGetValues("request-id", out var v2)) + reqId = v2.FirstOrDefault(); + } + catch + { + /* ignore header parsing issues */ + } + + _logger.Error( + "OAuth downstream failure {method} {uri}: {code} {reason}. RequestId={reqId}. Body={body}", + request.Method, request.RequestUri, (int)status, reason, reqId ?? "n/a", snippet); + + resp.Dispose(); + _logger.Trace("<- OAuthTokenHandler.Send throwing for {method} {uri}", request.Method, request.RequestUri); + + throw new HttpRequestException( + $"HTTP {(int)status} {reason} for {request.RequestUri}. RequestId={reqId}. Body: {body}", + null, + status); + } + + + private void ApplyHeaders(HttpRequestMessage req, bool replaceAuth = false) + { + if (replaceAuth || req.Headers.Authorization is null) + req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _accessToken); + + if (!req.Headers.Contains("x-keyfactor-requested-with")) + req.Headers.Add("x-keyfactor-requested-with", _requestedWith); + _logger.Trace("Applied auth + headers (replaceAuth={replaceAuth})", replaceAuth); + } + + private void EnsureToken(CancellationToken ct) + { + if (!IsExpiringSoon()) + { + _logger.Trace("Token considered fresh (exp={exp:o})", _expiresUtc); + return; + } + + _logger.Debug("Token missing/expiring soon. Acquiring under lock."); + _gate.Wait(ct); + try + { + if (!IsExpiringSoon()) + { + _logger.Trace("Token became fresh while waiting (exp={exp:o})", _expiresUtc); + return; + } + + var (tkn, expires) = _tokenClient.GetTokenWithClientCredentials(); + _accessToken = tkn; + _expiresUtc = expires; + _logger.Info("Token refreshed (exp={exp:o}, ttlSec≈{ttl})", + _expiresUtc, (int)(_expiresUtc - DateTimeOffset.UtcNow).TotalSeconds); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to acquire OAuth token."); + throw; + } + finally + { + _gate.Release(); + } + } + + private void ForceRefresh(CancellationToken ct) + { + _logger.Debug("Forcing token refresh."); + _gate.Wait(ct); + try + { + var (tkn, expires) = _tokenClient.GetTokenWithClientCredentials(); + _accessToken = tkn; + _expiresUtc = expires; + _logger.Info("Token forced refresh complete (exp={exp:o})", _expiresUtc); + } + catch (Exception ex) + { + _logger.Error(ex, "Forced token refresh failed."); + throw; + } + finally + { + _gate.Release(); + } + } + + private bool IsExpiringSoon() + { + var soon = string.IsNullOrWhiteSpace(_accessToken) + || DateTimeOffset.UtcNow >= + _expiresUtc - TimeSpan.FromSeconds(Math.Max(30, _opt.refreshSkewSeconds)); + _logger.Trace("IsExpiringSoon? {soon} (now={now:o}, exp={exp:o})", soon, DateTimeOffset.UtcNow, _expiresUtc); + return soon; + } +} \ No newline at end of file diff --git a/sectigo-metadata-sync/Client/SectigoClient.cs b/sectigo-metadata-sync/Client/SectigoClient.cs index 9428ee6..0c81b32 100644 --- a/sectigo-metadata-sync/Client/SectigoClient.cs +++ b/sectigo-metadata-sync/Client/SectigoClient.cs @@ -1,295 +1,374 @@ -// Copyright 2021 Keyfactor -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -using System; +using System; using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; using System.Net.Http; +using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Threading; +using Microsoft.Extensions.DependencyInjection; using NLog; using SectigoMetadataSync.Models; namespace SectigoMetadataSync.Client; /// -/// Synchronous client for retrieving Sectigo custom fields (metadata) and SSL certificates. -/// Designed for use as a typed HttpClient via IHttpClientFactory. +/// Fully synchronous Sectigo API client using HttpClient.Send with automatic retry + backoff. +/// Safe for use via IHttpClientFactory (typed client) or manual construction. /// -public class SectigoCustomFieldsClient +public sealed class SectigoClient { - private static readonly Logger _logger = LogManager.GetCurrentClassLogger(); // NLog Logger + private static readonly Logger Log = LogManager.GetCurrentClassLogger(); - private readonly HttpClient _httpClient; - - private readonly JsonSerializerOptions _jsonOptions = new() + // ---- JSON options + private static readonly JsonSerializerOptions JsonOpts = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; - /// - /// Initializes a new instance of the SectigoCustomFieldsClient. - /// HttpClient is injected/configured via IHttpClientFactory; its BaseAddress should be set externally. - /// - public SectigoCustomFieldsClient(HttpClient httpClient) + private readonly TimeSpan _baseDelay; + private readonly HttpClient _http; + private readonly TimeSpan _maxDelay; + + // ---- Retry/backoff policy (tune as desired) + private readonly int _maxRetries; + private readonly Random _rng = new(); + + /// + /// HttpClient with BaseAddress set to your Sectigo endpoint (e.g., https://cert-manager.com/api/). + /// + /// Total attempts = maxRetries + 1 initial. + /// Initial backoff delay when Retry-After is absent. + /// Ceiling for backoff delay. + public SectigoClient(HttpClient http, int maxRetries = 6, TimeSpan? baseDelay = null, TimeSpan? maxDelay = null) { - _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _http = http ?? throw new ArgumentNullException(nameof(http)); + _maxRetries = Math.Max(0, maxRetries); + _baseDelay = baseDelay ?? TimeSpan.FromMilliseconds(500); + _maxDelay = maxDelay ?? TimeSpan.FromSeconds(20); + + // Recommended: JSON headers default + if (!_http.DefaultRequestHeaders.Accept.Contains(new MediaTypeWithQualityHeaderValue("application/json"))) + _http.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); } /// - /// Configures API credentials for subsequent requests. - /// Must be called before invoking any list/get methods. - /// Uses header-based authentication: login, password, customerUri. + /// Configure per-request authentication headers (Sectigo SCM style). /// public void Authenticate(string login, string password, string customerUri) { - var headers = _httpClient.DefaultRequestHeaders; - headers.Remove("login"); - headers.Remove("password"); - headers.Remove("customerUri"); + var h = _http.DefaultRequestHeaders; + h.Remove("login"); + h.Add("login", login ?? string.Empty); + h.Remove("password"); + h.Add("password", password ?? string.Empty); + h.Remove("customerUri"); + h.Add("customerUri", customerUri ?? string.Empty); + + Log.Info("Sectigo auth headers configured (login/customerUri set)."); + } - headers.Add("login", login); - headers.Add("password", password); - headers.Add("customerUri", customerUri); + // ---------- Public API surface (mirror of common operations) ---------- - _logger.Info("Sectigo API authentication headers configured."); + public List ListCustomFields() + { + return SendJson>(HttpMethod.Get, "api/customField/v2") + ?? new List(); } - /// - /// Lists all custom fields (full details). Maps to GET /api/customField/v2 - /// - public List ListCustomFields() + public List GetCertificatesByProfileId(List profileIds, + bool includeRevokedAndExpired = false, int pageSize = 25) { - _logger.Debug("Fetching all custom fields from Sectigo API."); - var response = _httpClient.GetAsync("api/customField/v2").GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); - var json = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - return JsonSerializer.Deserialize>(json, _jsonOptions) ?? - new List(); + if (profileIds == null || profileIds.Count == 0) + throw new ArgumentException("profileIds required", nameof(profileIds)); + var acc = new List(); + foreach (var pid in profileIds) + // If includeRevokedAndExpired=false, Sectigo typically returns current/issued with no explicit status filter. + acc.AddRange( + GetCertificatesByProfileIdAndStatus(pid, includeRevokedAndExpired ? null : "Issued", pageSize)); + return acc; } - /// - /// Lists custom fields filtered by certificate type. Maps to GET /api/customField/v2?certType={type} - /// - public List ListCustomFieldsByCertificateType(string certType) + public SectigoCertificateDetails GetCertificateDetails(int sectigoCertId) { - _logger.Debug($"Fetching custom fields for certificate type: {certType}."); - var uri = $"api/customField/v2?certType={Uri.EscapeDataString(certType)}"; - var response = _httpClient.GetAsync(uri).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); - var json = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - return JsonSerializer.Deserialize>(json, _jsonOptions) ?? - new List(); + return SendJson(HttpMethod.Get, $"api/ssl/v1/{sectigoCertId}") + ?? throw new InvalidOperationException($"Certificate {sectigoCertId} not found."); } - /// - /// Retrieves detailed information for a specific custom field by ID. Maps to GET /api/customField/v2/{id} - /// - public SectigoCustomField GetCustomFieldDetails(int id) + public SectigoCertificateDetails UpdateCertificateMetadata( + int sslId, + List? customFields = null, + string? comments = null) { - _logger.Debug($"Fetching details for custom field ID: {id}."); - var uri = $"api/customField/v2/{id}"; - var response = _httpClient.GetAsync(uri).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); - var json = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - return JsonSerializer.Deserialize(json, _jsonOptions) - ?? throw new InvalidOperationException( - $"Custom field with id {id} not found or failed to deserialize."); + var payload = new { sslId, customFields, comments }; + return SendJson(HttpMethod.Put, "api/ssl/v1", payload) + ?? throw new InvalidOperationException("Null response when updating certificate metadata."); } - /// - /// Retrieves a list of SSL certificates by sending individual requests for each profile ID. - /// If syncRevokedAndExpired is enabled, additional lookups are performed for Revoked and Expired statuses. - /// - /// List of profile IDs to filter by. - /// Whether to include revoked and expired certificates. - /// The size of each page for pagination. - /// A combined list of SSL certificates matching the profile IDs. - public List GetCertificatesByProfileId(List profileIds, bool syncRevokedAndExpired = false, - int sectigoPageSize = 25) + // ---------- Internals ---------- + + private List GetCertificatesByProfileIdAndStatus(int profileId, string? status, int pageSize) { - if (profileIds == null || profileIds.Count == 0) + var position = 0; + var acc = new List(); + while (true) { - _logger.Debug("GetCertificatesByProfileId called with an empty or null profileIds list."); - throw new ArgumentException("Profile IDs cannot be null or empty.", nameof(profileIds)); + var qs = $"sslTypeId={profileId}&position={position}&size={pageSize}"; + if (!string.IsNullOrWhiteSpace(status)) qs += $"&status={Uri.EscapeDataString(status)}"; + + var page = SendJson>(HttpMethod.Get, $"api/ssl/v1?{qs}") ?? + new List(); + acc.AddRange(page); + if (page.Count < pageSize) break; + position += pageSize; } - var combinedCertificates = new List(); + return acc; + } - foreach (var profileId in profileIds) + /// + /// Core JSON sender (request body optional). Fully synchronous. + /// + private TOut? SendJson(HttpMethod method, string relativeUrl, object? body = null) + { + using var req = new HttpRequestMessage(method, relativeUrl); + + if (body != null) { - _logger.Trace($"Processing profile ID: {profileId}"); + var json = JsonSerializer.Serialize(body, JsonOpts); + req.Content = new StringContent(json, Encoding.UTF8, "application/json"); + } - try - { - if (syncRevokedAndExpired) - { - _logger.Trace( - $"SyncRevokedAndExpired is enabled. Fetching revoked and expired certificates for profile ID: {profileId}."); + using var res = SendWithRetry(req); + var text = res.Content is null ? null : ReadString(res.Content); - // Fetch revoked certificates - _logger.Trace($"Fetching ALL certificates for profile ID: {profileId}."); - combinedCertificates.AddRange( - GetCertificatesByProfileIdAndStatus(profileId, null, sectigoPageSize)); - } - else - { - _logger.Trace($"Fetching active/Issued certificates for profile ID: {profileId}."); - combinedCertificates.AddRange( - GetCertificatesByProfileIdAndStatus(profileId, "Issued", sectigoPageSize)); - } - } - catch (Exception ex) - { - _logger.Error(ex, $"Error processing profile ID: {profileId}"); - } - } + if (string.IsNullOrWhiteSpace(text)) + return default; - return combinedCertificates; + try + { + return JsonSerializer.Deserialize(text!, JsonOpts); + } + catch (Exception ex) + { + Log.Error(ex, "JSON deserialization error for {Url}. Payload (truncated): {Snippet}", + relativeUrl, text!.Length > 512 ? text.Substring(0, 512) + "…" : text); + throw; + } } - /// - /// Helper method to retrieve certificates for a specific profile ID and status with pagination. - /// - /// The profile ID to filter by. - /// The status to filter by (e.g., "Revoked", "Expired"). Pass null for active certificates. - /// The size of each page for pagination. - /// A list of SSL certificates matching the profile ID and status. - private List GetCertificatesByProfileIdAndStatus(int profileId, string? status, - int sectigoPageSize = 25) + private HttpResponseMessage SendWithRetry(HttpRequestMessage request) { - var pageSize = sectigoPageSize; // Define the size of each page - var position = 0; // Start at the first entry - var combinedCertificates = new List(); + _ = request ?? throw new ArgumentNullException(nameof(request)); + + var attempt = 0; + var nextDelay = _baseDelay; + + bool IsIdempotent(HttpMethod m) + { + return m == HttpMethod.Get || m == HttpMethod.Head || m == HttpMethod.Put || m == HttpMethod.Delete; + } while (true) { - // Build the query string for the current profile ID, status, and pagination - var queryString = $"sslTypeId={profileId}&position={position}&size={pageSize}"; - if (!string.IsNullOrEmpty(status)) queryString += $"&status={Uri.EscapeDataString(status)}"; + attempt++; + var start = DateTimeOffset.UtcNow; - // Construct the endpoint URL - var endpoint = $"api/ssl/v1?{queryString}"; + HttpResponseMessage? res = null; + Exception? sendEx = null; try { - // Log the pagination details at debug level - _logger.Trace( - $"Fetching certificates for profile ID {profileId} with status '{status ?? "Active"}'. Position: {position}, Page Size: {pageSize}"); + res = _http.Send(request, HttpCompletionOption.ResponseHeadersRead); + } + catch (Exception ex) + { + sendEx = ex; + } + + // Network failure -> retry as transient (respect attempt budget) + if (sendEx != null) + { + if (attempt > _maxRetries) + { + Log.Error(sendEx, "Too many retries ({Attempt}/{Max}) for {Method} {Uri}.", attempt, _maxRetries, + request.Method, request.RequestUri); + throw new HttpRequestException( + $"Network send failed after {attempt} attempts for {request.Method} {request.RequestUri}.", + sendEx); + } + + SleepWithJitter(ref nextDelay); + Log.Warn(sendEx, "Transient send error on attempt {Attempt} for {Method} {Uri}. Retrying.", + attempt, request.Method, request.RequestUri); + continue; + } - // Send the GET request - var response = _httpClient.GetAsync(endpoint).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); + // Success path + if (res!.IsSuccessStatusCode) + { + Log.Trace("HTTP {Status} in {Ms}ms for {Method} {Uri}", + (int)res.StatusCode, (DateTimeOffset.UtcNow - start).TotalMilliseconds, + request.Method, request.RequestUri); + return res; // caller disposes + } - // Deserialize the response JSON - var json = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - var certificates = JsonSerializer.Deserialize>(json, _jsonOptions) - ?? new List(); + // Decide whether to retry based on status + var sc = (int)res.StatusCode; + var retryable = + sc == 429 || // too many requests (rate limited) + sc == 503 || // service unavailable + sc == 408 || sc == 500 || sc == 502 || sc == 504; // classic transient errors - // Log the number of certificates retrieved in the current page - _logger.Trace( - $"Retrieved {certificates.Count} certificates for profile ID {profileId} with status '{status ?? "Active"}'."); + // For non-idempotent methods, only retry if explicitly rate-limited with Retry-After + if (!IsIdempotent(request.Method) && retryable && sc != 429) + retryable = false; - // Add the retrieved certificates to the combined list - combinedCertificates.AddRange(certificates); + // Honor Retry-After (seconds or HTTP-date) if present + var retryAfter = ParseRetryAfter(res); + if (retryAfter > TimeSpan.Zero) retryable = true; + + if (!retryable || attempt > _maxRetries) + { + // Read body fully for exception message; log truncated snippet + var reason = res.ReasonPhrase ?? string.Empty; + var fullBody = res.Content is null ? string.Empty : ReadString(res.Content); + var snippet = fullBody.Length > 2000 ? fullBody.Substring(0, 2000) + "…[truncated]" : fullBody; - // If the number of certificates returned is less than the page size, we have reached the end - if (certificates.Count < pageSize) + string? reqId = null; + try { - _logger.Trace( - $"No more certificates to fetch for profile ID {profileId} with status '{status ?? "Active"}'."); - break; + if (!res.Headers.TryGetValues("x-request-id", out var v) || (reqId = v.FirstOrDefault()) is null) + if (res.Headers.TryGetValues("request-id", out var v2)) + reqId = v2.FirstOrDefault(); } + catch + { + /* ignore header parsing issues */ + } + + Log.Error( + "HTTP {Status} not retryable or attempts exhausted ({Attempt}/{Max}). {Method} {Uri}. RequestId={ReqId}. Body={Body}", + sc, attempt, _maxRetries, request.Method, request.RequestUri, reqId ?? "n/a", snippet); - // Increment the position for the next page - position += pageSize; + // Dispose before throwing since we won't return it + res.Dispose(); + + throw new HttpRequestException( + $"HTTP {sc} {reason} for {request.Method} {request.RequestUri}. RequestId={reqId}. Body: {fullBody}", + null, + (HttpStatusCode)sc); } - catch (Exception ex) + + // Sleep based on server guidance first + if (retryAfter > TimeSpan.Zero) { - // Log the error and return the certificates retrieved so far - _logger.Error(ex, - $"Error retrieving certificates for profile ID {profileId} with status '{status ?? "Active"}'."); - break; + Log.Warn("Rate limited ({Status}). Honoring Retry-After={RetryAfter}. Attempt {Attempt}/{Max}.", + sc, retryAfter, attempt, _maxRetries); + Thread.Sleep(retryAfter); + } + else + { + // Alternatively, respect X-RateLimit-Reset if available (epoch seconds) + var resetAt = ParseRateLimitReset(res); + if (resetAt > DateTimeOffset.UtcNow) + { + var toWait = resetAt - DateTimeOffset.UtcNow; + Log.Warn("Rate hint via X-RateLimit-Reset. Sleeping {Wait}. Attempt {Attempt}/{Max}.", + toWait, attempt, _maxRetries); + Thread.Sleep(toWait); + } + else + { + // Exponential backoff + Full-Jitter + SleepWithJitter(ref nextDelay); + } } - } - return combinedCertificates; + // Dispose the non-success response before the next attempt + res.Dispose(); + } } - /// - /// Retrieves detailed information for a specific SSL certificate by its ID. - /// - /// The ID of the SSL certificate. - /// The detailed information of the SSL certificate. - public SectigoCertificateDetails GetCertificateDetails(int sectigoCertId) + private void SleepWithJitter(ref TimeSpan nextDelay) { - if (sectigoCertId <= 0) - throw new ArgumentException("Certificate ID must be greater than zero.", nameof(sectigoCertId)); - - // Construct the endpoint URL - var endpoint = $"api/ssl/v1/{sectigoCertId}"; - - // Send the GET request - var response = _httpClient.GetAsync(endpoint).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); - _logger.Trace("GET request successful. Status code: " + response.StatusCode); - - // Deserialize the response JSON - var json = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - return JsonSerializer.Deserialize(json, _jsonOptions) - ?? throw new InvalidOperationException( - $"Certificate with ID {sectigoCertId} not found or failed to deserialize."); + // Full-Jitter: sleep = random(0, min(cap, base * 2^attempt)) + var cap = _maxDelay; + var max = nextDelay < cap ? nextDelay : cap; + var millis = _rng.Next(0, (int)Math.Max(1, max.TotalMilliseconds)); + Thread.Sleep(TimeSpan.FromMilliseconds(millis)); + + // Increase for next time + var doubled = TimeSpan.FromMilliseconds(Math.Min(cap.TotalMilliseconds, nextDelay.TotalMilliseconds * 2.0)); + nextDelay = doubled; } - /// - /// Updates metadata for a given SSL certificate by its ID. - /// Maps to PUT /api/ssl/v1 - /// - /// The ID of the SSL certificate to update. - /// Custom fields to update (optional). - /// Comments to update (optional). - /// The updated SSL certificate details. - public SectigoCertificateDetails UpdateCertificateMetadata( - int sslId, - List? customFields = null, - string? comments = null) + private static TimeSpan ParseRetryAfter(HttpResponseMessage res) { - if (sslId <= 0) - throw new ArgumentException("Certificate ID must be greater than zero.", nameof(sslId)); - - // Construct the request payload - var payload = new + if (res.Headers.TryGetValues("Retry-After", out var values)) { - sslId, - customFields, - comments - }; - - // Serialize the payload to JSON - var jsonPayload = JsonSerializer.Serialize(payload, _jsonOptions); - _logger.Trace($"Constructed JSON payload for updating certificate metadata: {jsonPayload}"); + var ra = values.FirstOrDefault(); + if (string.IsNullOrWhiteSpace(ra)) return TimeSpan.Zero; + + // Seconds? + if (int.TryParse(ra.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var secs) && secs >= 0) + return TimeSpan.FromSeconds(secs); + + // HTTP-date? + if (DateTimeOffset.TryParseExact( + ra.Trim(), + new[] { "r", "ddd, dd MMM yyyy HH':'mm':'ss 'GMT'" }, // RFC1123 + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var when) && when > DateTimeOffset.UtcNow) + return when - DateTimeOffset.UtcNow; + } - // Construct the endpoint URL - var endpoint = "api/ssl/v1"; + return TimeSpan.Zero; + } - // Send the PUT request - var content = new StringContent(jsonPayload, Encoding.UTF8, "application/json"); - var response = _httpClient.PutAsync(endpoint, content).GetAwaiter().GetResult(); + private static DateTimeOffset ParseRateLimitReset(HttpResponseMessage res) + { + // Optional: X-RateLimit-Reset (epoch seconds) + if (res.Headers.TryGetValues("X-RateLimit-Reset", out var vals)) + { + var v = vals.FirstOrDefault(); + if (long.TryParse(v, NumberStyles.Integer, CultureInfo.InvariantCulture, out var epoch) && epoch > 0) + try + { + return DateTimeOffset.FromUnixTimeSeconds(epoch); + } + catch + { + /* ignore */ + } + } - // Log the response status - _logger.Trace($"Received response with status code: {response.StatusCode}"); - response.EnsureSuccessStatusCode(); + return DateTimeOffset.MinValue; + } - // Deserialize the response JSON - var jsonResponse = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - _logger.Trace($"Response JSON: {jsonResponse}"); + private static string ReadString(HttpContent content) + { + return content.ReadAsStringAsync().GetAwaiter().GetResult() ?? string.Empty; + } +} - return JsonSerializer.Deserialize(jsonResponse, _jsonOptions) - ?? throw new InvalidOperationException("Failed to deserialize the updated certificate details."); +/// +/// DI registration helpers. +/// +public static class SectigoServiceCollectionExtensions +{ + public static IServiceCollection AddSectigoClient(this IServiceCollection services, string baseAddress) + { + services.AddHttpClient(client => + { + client.BaseAddress = new Uri(baseAddress); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + }); + return services; } } \ No newline at end of file diff --git a/sectigo-metadata-sync/Logic/DateParser.cs b/sectigo-metadata-sync/Logic/DateParser.cs new file mode 100644 index 0000000..66ba7fd --- /dev/null +++ b/sectigo-metadata-sync/Logic/DateParser.cs @@ -0,0 +1,66 @@ +using System; +using System.Globalization; + +public static class DateParser +{ + // Good coverage for ISO + your configured format + US date-only + RFC1123 + private static readonly string[] Formats = + { + "o", // ISO 8601 round-trip + "yyyy-MM-dd'T'HH:mmK", + "yyyy-MM-dd'T'HH:mm:ssK", + "yyyy-MM-dd'T'HH:mm", + "yyyy-MM-dd'T'HH:mm:ss", + "yyyy-MM-dd", + "M/d/yyyy h:mm tt", + "M/d/yyyy h:mm:ss tt", + "M/d/yyyy", + "r" // RFC1123 + }; + + public static DateTimeOffset? ParseToUtcOrNull( + string? raw, + TimeZoneInfo assumeZone, + string? preferredFormat = null) + { + if (string.IsNullOrWhiteSpace(raw)) return null; + + // 1) If a preferred format is supplied (from config), try it first. + if (!string.IsNullOrWhiteSpace(preferredFormat) + && DateTime.TryParseExact(raw, preferredFormat, CultureInfo.InvariantCulture, + DateTimeStyles.AllowWhiteSpaces, out var dtPref)) + return AssumeZoneToUtc(dtPref, assumeZone); + + // 2) Try exact with known formats into DateTimeOffset (captures explicit offsets/UTC) + if (DateTimeOffset.TryParseExact(raw, Formats, CultureInfo.InvariantCulture, + DateTimeStyles.AllowWhiteSpaces | DateTimeStyles.AssumeUniversal, + out var dtoExact)) + return dtoExact.ToUniversalTime(); + + // 3) Try broad DateTimeOffset parse + if (DateTimeOffset.TryParse(raw, CultureInfo.InvariantCulture, + DateTimeStyles.AllowWhiteSpaces | DateTimeStyles.AssumeUniversal, + out var dtoAny)) + return dtoAny.ToUniversalTime(); + + // 4) If date-only, map to midnight in assumeZone + if (DateOnly.TryParse(raw, CultureInfo.InvariantCulture, DateTimeStyles.AllowWhiteSpaces, out var dOnly)) + { + var unspecified = new DateTime(dOnly.Year, dOnly.Month, dOnly.Day, 0, 0, 0, DateTimeKind.Unspecified); + return AssumeZoneToUtc(unspecified, assumeZone); + } + + return null; + } + + private static DateTimeOffset AssumeZoneToUtc(DateTime dt, TimeZoneInfo zone) + { + // Treat Unspecified as ‘zone’; Local stays local; Utc stays Utc + return dt.Kind switch + { + DateTimeKind.Utc => new DateTimeOffset(dt, TimeSpan.Zero), + DateTimeKind.Local => new DateTimeOffset(dt).ToUniversalTime(), + _ => new DateTimeOffset(dt, zone.GetUtcOffset(dt)).ToUniversalTime() + }; + } +} \ No newline at end of file diff --git a/sectigo-metadata-sync/Logic/Helpers.cs b/sectigo-metadata-sync/Logic/Helpers.cs index e4d608a..e72dc29 100644 --- a/sectigo-metadata-sync/Logic/Helpers.cs +++ b/sectigo-metadata-sync/Logic/Helpers.cs @@ -6,69 +6,179 @@ // and limitations under the License. using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using Microsoft.Extensions.Configuration; using SectigoMetadataSync.Models; namespace SectigoMetadataSync.Logic; public class Helpers { - /// - /// Verifies the provided mode. - /// - public static bool CheckMode(string mode) + public static IConfigurationSection GetRootConfigSection(IConfiguration cfg) + { + // Support either "Config" or "config" in JSON + var sec = cfg.GetSection("Config"); + if (!sec.Exists()) sec = cfg.GetSection("config"); + return sec; + } + + public static bool IsOAuthBlockUsable(KeyfactorOAuthOptions? opt) { - if (mode == "kftosc" || mode == "sctokf") return true; - return false; + if (opt is null) return false; + // Minimal requirements to consider OAuth “present” + if (string.IsNullOrWhiteSpace(opt.TokenUrl)) return false; + if (string.IsNullOrWhiteSpace(opt.ClientId)) return false; + if (string.IsNullOrWhiteSpace(opt.ClientSecret)) return false; + return true; } - public static MetadataDataType ToKeyfactorDataType(CustomFieldInputType inputType) + public static KeyfactorMetadataDataType ToKeyfactorDataType(CustomFieldInputType inputType) { return inputType switch { - CustomFieldInputType.TEXT_SINGLE_LINE => MetadataDataType.String, - CustomFieldInputType.TEXT_MULTI_LINE => MetadataDataType.BigText, - CustomFieldInputType.EMAIL => MetadataDataType.Email, - CustomFieldInputType.NUMBER => MetadataDataType.Integer, - CustomFieldInputType.TEXT_OPTION => MetadataDataType.MultipleChoice, - CustomFieldInputType.DATE => MetadataDataType.Date, - _ => MetadataDataType.String + CustomFieldInputType.TEXT_SINGLE_LINE => KeyfactorMetadataDataType.String, + CustomFieldInputType.TEXT_MULTI_LINE => KeyfactorMetadataDataType.BigText, + CustomFieldInputType.EMAIL => KeyfactorMetadataDataType.Email, + CustomFieldInputType.NUMBER => KeyfactorMetadataDataType.Integer, + CustomFieldInputType.TEXT_OPTION => KeyfactorMetadataDataType.MultipleChoice, + CustomFieldInputType.DATE => KeyfactorMetadataDataType.Date, + _ => KeyfactorMetadataDataType.String }; } + // Mirrors: public static JObject Flatten(JObject jObject, string parentName = "") + public static JsonObject Flatten(JsonObject obj, string parentName = "") + { + if (obj is null) throw new ArgumentNullException(nameof(obj)); + + var result = new JsonObject(); + + void Recurse(JsonNode? node, string prefix) + { + switch (node) + { + case JsonObject o: + foreach (var kvp in o) + { + var name = string.IsNullOrEmpty(prefix) ? kvp.Key : $"{prefix}.{kvp.Key}"; + Recurse(kvp.Value, name); + } + + break; + + case JsonArray arr: + for (var i = 0; i < arr.Count; i++) + { + var name = string.IsNullOrEmpty(prefix) ? $"[{i}]" : $"{prefix}[{i}]"; + Recurse(arr[i], name); + } + + break; + + default: + // Leaf (string/number/bool/null). DeepClone to detach from source graph. + result[prefix] = node?.DeepClone(); + break; + } + } + + Recurse(obj, parentName ?? string.Empty); + return result; + } + /// - /// Retrieves the value of a property from a SectigoCertificateDetails instance based on a string path. - /// The path can include nested properties separated by dots (e.g., "certificateDetails.sha1Hash"). + /// Resolves a dot-path on an object graph using reflection and [JsonPropertyName] matches. + /// If stringifyLeaf=true, collections/JsonElement/etc. are converted to a scalar string. /// - /// The instance of SectigoCertificateDetails to retrieve the value from. - /// The string path pointing to the desired property (e.g., "renewed" or "certificateDetails.sha1Hash"). - /// The value of the property, or null if the path is invalid or the property does not exist. - public static object? GetPropertyValue(SectigoCertificateDetails certDetails, string path) + public static object? GetPropertyValue(object root, string path, bool stringifyLeaf = true) { - if (certDetails == null) throw new ArgumentNullException(nameof(certDetails)); - if (string.IsNullOrEmpty(path)) throw new ArgumentException("Path cannot be null or empty.", nameof(path)); + if (root is null) throw new ArgumentNullException(nameof(root)); + if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Path cannot be null or empty.", nameof(path)); - object? currentObject = certDetails; - foreach (var propertyName in path.Split('.')) + var current = root; + + foreach (var propertyName in path.Split('.', + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) { - if (currentObject == null) return null; + if (current is null) return null; - // Get all public instance properties - var properties = currentObject.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance); + // Support JsonElement containers too + if (current is JsonElement je && je.ValueKind == JsonValueKind.Object) + { + if (!je.TryGetProperty(propertyName, out je)) return null; + current = je; + continue; + } - // Find the property by name (case-insensitive) or by JsonPropertyName attribute - var propertyInfo = properties.FirstOrDefault(p => + var props = current.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance); + var pi = props.FirstOrDefault(p => string.Equals(p.Name, propertyName, StringComparison.OrdinalIgnoreCase) || p.GetCustomAttribute()?.Name == propertyName); - if (propertyInfo == null) return null; // Property not found + if (pi is null) return null; + current = pi.GetValue(current); + } + + return stringifyLeaf ? CoerceToScalarString(current) : current; + } + + private static string? CoerceToScalarString(object? value) + { + if (value is null) return null; + + // Already string + if (value is string s) return s; + + // JsonElement -> scalar/CSV/JSON + if (value is JsonElement je) return JsonElementToString(je); + + // Date/Time + if (value is DateTime dt) return dt.ToString("O", CultureInfo.InvariantCulture); + if (value is DateTimeOffset dto) return dto.ToString("O", CultureInfo.InvariantCulture); - // Get the value of the property - currentObject = propertyInfo.GetValue(currentObject); + // Numbers/booleans/etc. + if (value is IFormattable f) return f.ToString(null, CultureInfo.InvariantCulture); + + // IEnumerable -> CSV + if (value is IEnumerable enumerable && value is not IEnumerable) + { + var parts = new List(); + foreach (var item in enumerable) + { + var text = CoerceToScalarString(item); + if (!string.IsNullOrWhiteSpace(text)) + parts.Add(text); + } + + return parts.Count == 0 ? null : string.Join(", ", parts); } - return currentObject; + // Complex object -> stable JSON snapshot + return JsonSerializer.Serialize(value); + } + + private static string? JsonElementToString(JsonElement el) + { + return el.ValueKind switch + { + JsonValueKind.String => el.GetString(), + JsonValueKind.Number => el.GetRawText(), + JsonValueKind.True => "true", + JsonValueKind.False => "false", + JsonValueKind.Null => null, + JsonValueKind.Array => string.Join(", ", + el.EnumerateArray() + .Select(JsonElementToString) + .Where(x => !string.IsNullOrWhiteSpace(x))), + JsonValueKind.Object => el.GetRawText(), + _ => el.GetRawText() + }; } } \ No newline at end of file diff --git a/sectigo-metadata-sync/Logic/ListFlushExtensions.cs b/sectigo-metadata-sync/Logic/ListFlushExtensions.cs new file mode 100644 index 0000000..6722ad8 --- /dev/null +++ b/sectigo-metadata-sync/Logic/ListFlushExtensions.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; +using NLog; + +namespace SectigoMetadataSync.Logic; + +public static class ListFlushExtensions +{ + /// + /// Flush any remaining items after a page/loop ends. + /// + public static void FlushRemainder(this List buffer, + Logger log, + string label, + ref int totalCount) + { + FlushToTrace(buffer, log, label, ref totalCount); + } + + private static void FlushToTrace(List buffer, + Logger log, + string label, + ref int totalCount) + { + if (buffer.Count == 0) return; + + // One line per item keeps logs searchable and prevents jumbo lines. + foreach (var s in buffer) + log.Trace("{Label}: {Item}", label, s); + + totalCount += buffer.Count; + buffer.Clear(); // release memory + buffer.TrimExcess(); // optional: shrink backing array + } +} \ No newline at end of file diff --git a/sectigo-metadata-sync/Logic/MappingLogic.cs b/sectigo-metadata-sync/Logic/MappingLogic.cs deleted file mode 100644 index 7bf43a7..0000000 --- a/sectigo-metadata-sync/Logic/MappingLogic.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2021 Keyfactor -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -using SectigoMetadataSync.Models; - -namespace SectigoMetadataSync.Logic; - -public class MappingLogic -{ - /// - /// Maps a Sectigo custom field to a unified format field. - /// - /// The Sectigo custom field to map. - /// A unified format field with mapped properties. - public static UnifiedFormatField MapSectigoToUnified(SectigoCustomField sectigoField) - { - return new UnifiedFormatField - { - SectigoFieldName = sectigoField.Name, - KeyfactorMetadataFieldName = sectigoField.Name, // Example mapping - KeyfactorDescription = sectigoField.CertType, // Example mapping - KeyfactorDataType = MapDataType(sectigoField.Input.Type), - KeyfactorHint = null, // Add hint if needed - KeyfactorValidation = null, // Add validation if needed - KeyfactorEnrollment = 0, // Default to Optional - KeyfactorMessage = null, // Add message if needed - KeyfactorOptions = sectigoField.Input.Type == CustomFieldInputType.TEXT_OPTION - ? sectigoField.Input.Options?.ToArray() - : null, // Map options only for TEXT_OPTION - KeyfactorDefaultValue = null, // Add default value if needed - KeyfactorDisplayOrder = 0, // Add display order if needed - KeyfactorCaseSensitive = false // Default to false - }; - } - - private static int MapDataType(CustomFieldInputType inputType) - { - return inputType switch - { - CustomFieldInputType.TEXT_SINGLE_LINE => 1, // String - CustomFieldInputType.TEXT_MULTI_LINE => 1, // String - CustomFieldInputType.EMAIL => 1, // String - CustomFieldInputType.NUMBER => 2, // Integer - CustomFieldInputType.TEXT_OPTION => 3, // Multiple Choice - CustomFieldInputType.DATE => 4, // Date - _ => 1 // Default to String - }; - } -} \ No newline at end of file diff --git a/sectigo-metadata-sync/Logic/ValueCoercion.cs b/sectigo-metadata-sync/Logic/ValueCoercion.cs new file mode 100644 index 0000000..561f8d8 --- /dev/null +++ b/sectigo-metadata-sync/Logic/ValueCoercion.cs @@ -0,0 +1,367 @@ +// Copyright 2024 Keyfactor +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions +// and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using NLog; +using SectigoMetadataSync.Models; + +public static class ValueCoercion +{ + // --- ADD: configurable Keyfactor date format (default keeps current behavior) --- + private const string DefaultKfDateFormat = "yyyy-MM-dd"; + + + // --- LENGTH LIMITS (per Keyfactor docs) --- + private const int MaxStringLen = 400; // String fields limited to 400 chars + private const int MaxBigTextLen = 4000; // Big Text fields limited to 4000 chars + private const int MaxEmailPerAddr = 100; // Email data type: 100 chars per address + private static string _kfDateFormat = DefaultKfDateFormat; + private static readonly Logger _logger = LogManager.GetCurrentClassLogger(); + + private static readonly Regex EmailRx = new(@"^[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}$", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + /// + /// Output format for Keyfactor Date metadata (e.g., "M/d/yyyy h:mm:ss tt"). + /// Set this once at startup from config. Null/empty resets to default "yyyy-MM-dd". + /// + public static string KeyfactorDateFormat + { + get => _kfDateFormat; + set => _kfDateFormat = string.IsNullOrWhiteSpace(value) ? DefaultKfDateFormat : value!; + } + + private static bool _enableTruncation = false; + public static bool EnableTruncation + { + get => _enableTruncation; + set => _enableTruncation = value; + } + // ---- public API ---- + + // ---- public API ---- + public static object? Coerce(JsonElement value, + KeyfactorMetadataDataType type, + string[]? options) + { + var coerced = type switch + { + KeyfactorMetadataDataType.Integer => CoerceInt(value), + KeyfactorMetadataDataType.Boolean => CoerceBool(value), + KeyfactorMetadataDataType.Date => CoerceDateYmd(value), + KeyfactorMetadataDataType.Email => CoerceEmailCsv(value), // per-address truncation inside + KeyfactorMetadataDataType.MultipleChoice => CoerceChoice(value, options), + KeyfactorMetadataDataType.BigText => CoerceString(value, true), + KeyfactorMetadataDataType.String => CoerceString(value), + _ => CoerceString(value) + }; + + // Apply centralized length caps for string-like outputs. + return ApplyLengthLimits(coerced, type); + } + + // Back-compat overload unchanged... + public static object? Coerce(JsonElement value, int keyfactorTypeCode, string[]? options) + { + var type = Enum.IsDefined(typeof(KeyfactorMetadataDataType), keyfactorTypeCode) + ? (KeyfactorMetadataDataType)keyfactorTypeCode + : KeyfactorMetadataDataType.String; + + var coerced = Coerce(value, type, options); + return coerced; + } + + // ---- scalars ---- + + private static int? CoerceInt(JsonElement v) + { + switch (v.ValueKind) + { + case JsonValueKind.Number: + if (v.TryGetInt32(out var n)) return n; + if (v.TryGetInt64(out var l)) + { + if (l > int.MaxValue || l < int.MinValue) return null; + return (int)l; + } + + return null; + + case JsonValueKind.String: + var s = v.GetString(); + if (string.IsNullOrWhiteSpace(s)) return null; + if (int.TryParse(s.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var i)) + return i; + return null; + + case JsonValueKind.True: return 1; + case JsonValueKind.False: return 0; + default: return null; + } + } + + private static bool? CoerceBool(JsonElement v) + { + return v.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Number => v.TryGetInt32(out var n) ? n != 0 : null, + JsonValueKind.String => ParseBoolLoose(v.GetString()), + _ => null + }; + } + + private static string? CoerceDateYmd(JsonElement v) + { + // Accept ISO 8601 or y/M/d etc., emit in configurable Keyfactor format + if (TryExtractString(v, out var s) && !string.IsNullOrWhiteSpace(s)) + if (DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var dto)) + return dto.ToString(_kfDateFormat, CultureInfo.InvariantCulture); + + // If it's already yyyy-MM-dd but config requests a different format, reformat + if (v.ValueKind == JsonValueKind.String) + { + var raw = v.GetString(); + if (!string.IsNullOrWhiteSpace(raw) && Regex.IsMatch(raw!, @"^\d{4}-\d{2}-\d{2}$")) + { + if (_kfDateFormat == DefaultKfDateFormat) + return raw; + + if (DateTime.TryParseExact(raw, DefaultKfDateFormat, CultureInfo.InvariantCulture, + DateTimeStyles.None, out var dt)) + return dt.ToString(_kfDateFormat, CultureInfo.InvariantCulture); + } + } + + return null; + } + + private static string? CoerceString(JsonElement v, bool multiline = false) + { + switch (v.ValueKind) + { + case JsonValueKind.String: + var s = v.GetString(); + return string.IsNullOrWhiteSpace(s) ? null : s; + + case JsonValueKind.Number: + case JsonValueKind.True: + case JsonValueKind.False: + return v.GetRawText(); // culture-invariant + + case JsonValueKind.Array: + var parts = v.EnumerateArray() + .Select(e => CoerceString(e)) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .ToArray(); + return parts.Length == 0 ? null : string.Join(", ", parts); + + case JsonValueKind.Object: + return multiline ? v.GetRawText() : null; + + default: + return null; + } + } + + // replace CoerceEmailCsv with this version + private static string? CoerceEmailCsv(JsonElement v) + { + var all = ExtractEmails(v); + if (all.Count == 0) return null; + + var normalized = all.Select(x => x.Trim()) + .Where(x => x.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (normalized.Length == 0) return null; + + // Join and truncate at whole email boundaries to 400 chars total + return TruncateJoinedList(normalized, MaxStringLen, "email.list"); + } + + private static object? CoerceChoice(JsonElement v, string[]? options) + { + var val = CoerceString(v); + if (string.IsNullOrWhiteSpace(val)) + return null; + + if (options is null || options.Length == 0) + return Truncate(val, MaxStringLen); // store-as-text cap + + var norm = NormalizeChoice(val); + var match = options.FirstOrDefault(o => NormalizeChoice(o) == norm); + return match is null ? null : Truncate(match, MaxStringLen); + } + + // ---- helpers ---- + + private static bool? ParseBoolLoose(string? s) + { + if (string.IsNullOrWhiteSpace(s)) return null; + s = s.Trim().ToLowerInvariant(); + return s switch + { + "true" or "t" or "yes" or "y" or "1" => true, + "false" or "f" or "no" or "n" or "0" => false, + _ => null + }; + } + + // Keep centralized caps as-is; Email uses MaxStringLen (400) at the end. + private static object? ApplyLengthLimits(object? value, KeyfactorMetadataDataType type) + { + if (value is not string s) return value; + + return type switch + { + KeyfactorMetadataDataType.BigText => Truncate(s, MaxBigTextLen), + KeyfactorMetadataDataType.String => Truncate(s, MaxStringLen), + KeyfactorMetadataDataType.MultipleChoice => Truncate(s, MaxStringLen), + KeyfactorMetadataDataType.Email => Truncate(s, MaxStringLen), // total list capped at 400 + _ => s + }; + } + + private static string Truncate(string s, int max) + { + if (string.IsNullOrEmpty(s)) return s; + if (s.Length <= max) return s; + + if (!EnableTruncation) + { + _logger.Warn( + $"Field value exceeds a Keyfactor maximum length of {max} characters for Metadata Field content and truncation is disabled in config. Original value will be returned and will result in the value not getting synced."); + return s; + } + _logger.Warn( + "Truncating value to fit {MaxLength} characters. OriginalLength={OriginalLength}.", + max, s.Length); + + return s.Substring(0, max); + } + + // add this new helper (keeps only whole items that fit; logs one warning if truncated) + private static string TruncateJoinedList(IReadOnlyList items, int max, string context, + string separator = ", ") + { + if (items is null || items.Count == 0) return string.Empty; + + if (!EnableTruncation) + { + _logger.Warn( + $"Field value exceeds a Keyfactor maximum length of {max} characters for Metadata Field contents and truncation is disabled in config. Original value will be returned and will result in the value not getting synced."); + return String.Join(", ", items.ToArray()); ; + } + var sb = new StringBuilder(max); + var total = 0; + var kept = 0; + + for (var i = 0; i < items.Count; i++) + { + var part = items[i]; + if (string.IsNullOrWhiteSpace(part)) continue; + + var addLen = (kept == 0 ? 0 : separator.Length) + part.Length; + if (total + addLen > max) break; + + if (kept > 0) sb.Append(separator); + sb.Append(part); + total += addLen; + kept++; + } + + if (kept < items.Count) + // One warning; metadata-only (no values); says we cut at a whole item + _logger.Warn( + "Truncated {Context} at whole-item boundary: kept {Kept}/{Total} items to stay under {Max} chars (final length {Len}).", + context, kept, items.Count, max, total); + + return sb.ToString(); + } + + + private static bool TryExtractString(JsonElement v, out string? s) + { + switch (v.ValueKind) + { + case JsonValueKind.String: + s = v.GetString(); + return true; + case JsonValueKind.Number: + case JsonValueKind.True: + case JsonValueKind.False: + s = v.GetRawText(); + return true; + default: + s = null; + return false; + } + } + + private static string NormalizeChoice(string s) + { + return Regex.Replace(s.Trim(), @"\s+", " ").ToLowerInvariant(); + } + + private static List ExtractEmails(JsonElement v) + { + var outList = new List(8); + + void FromText(string? text) + { + if (string.IsNullOrWhiteSpace(text)) return; + + var pieces = text.Split(new[] { ',', ';', ' ', '\t', '\r', '\n' }, + StringSplitOptions.RemoveEmptyEntries); + foreach (var raw in pieces) + { + var candidate = raw.Trim().Trim('"', '\'', '<', '>', '(', ')', '[', ']'); + if (EmailRx.IsMatch(candidate)) + outList.Add(candidate); + } + } + + switch (v.ValueKind) + { + case JsonValueKind.String: + FromText(v.GetString()); + break; + + case JsonValueKind.Array: + foreach (var item in v.EnumerateArray()) + if (item.ValueKind == JsonValueKind.String) FromText(item.GetString()); + else if (item.ValueKind == JsonValueKind.Object && item.TryGetProperty("email", out var one) && + one.ValueKind == JsonValueKind.String) + FromText(one.GetString()); + break; + + case JsonValueKind.Object: + if (v.TryGetProperty("email", out var e) && e.ValueKind == JsonValueKind.String) + FromText(e.GetString()); + else if (v.TryGetProperty("emails", out var es)) outList.AddRange(ExtractEmails(es)); + else FromText(v.GetRawText()); + break; + + default: + FromText(v.GetRawText()); + break; + } + + return outList; + } +} \ No newline at end of file diff --git a/sectigo-metadata-sync/Logic/ValueCoercionSC.cs b/sectigo-metadata-sync/Logic/ValueCoercionSC.cs new file mode 100644 index 0000000..d39d35b --- /dev/null +++ b/sectigo-metadata-sync/Logic/ValueCoercionSC.cs @@ -0,0 +1,199 @@ +// Copyright 2024 Keyfactor +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions +// and limitations under the License. + +using System; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using NLog; +using SectigoMetadataSync.Models; + +namespace SectigoMetadataSync.Logic; + +/// +/// Coercion helpers for pushing Keyfactor metadata values into Sectigo custom fields. +/// Mirrors the approach used by the DigiCert implementation, but targets Sectigo's input types. +/// +public static class ValueCoercionSC +{ + private const int MaxCustomFieldValue = 256; + + // Keep the same email validator pattern used elsewhere for consistency. + private static readonly Regex EmailRx = new(@"^[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}$", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static bool _enableTruncation = false; + public static bool EnableTruncation + { + get => _enableTruncation; + set => _enableTruncation = value; + } + private static readonly Logger _logger = LogManager.GetCurrentClassLogger(); + + /// + /// Coerces an input value (usually from Keyfactor) into a Sectigo-compatible string value + /// based on the Sectigo field's input type. + /// + /// Raw value, typically a Keyfactor metadata string. + /// Sectigo custom field input type (TEXT_SINGLE_LINE, NUMBER, DATE, etc.). + /// + /// Optional Keyfactor-side options for Multiple Choice style fields; if provided for TEXT_OPTION, we try to + /// normalize against these options so we push a canonical value. + /// + /// + /// The date format Keyfactor uses for Date metadata (e.g., "yyyy-MM-dd" or "M/d/yyyy h:mm:ss tt"). + /// When provided for DATE fields, this is used to parse the incoming value and emit "yyyy-MM-dd". + /// + /// String value suitable for Sectigo's API, or null when the value is invalid/empty for the target type. + public static string? CoerceForSectigo(string? value, + CustomFieldInputType inputType, + string[]? kfOptions = null, + string? kfDateFormat = "yyyy-MM-dd") + { + if (value is null) return null; + + var s = value.Trim(); + if (s.Length == 0) return null; + + switch (inputType) + { + case CustomFieldInputType.TEXT_SINGLE_LINE: + case CustomFieldInputType.TEXT_MULTI_LINE: + return Truncate(s, MaxCustomFieldValue); // cap to 256 + + case CustomFieldInputType.EMAIL: + return EmailRx.IsMatch(s) ? Truncate(s, MaxCustomFieldValue) : null; + + case CustomFieldInputType.NUMBER: + return int.TryParse(s, NumberStyles.Integer, CultureInfo.InvariantCulture, out var n) + ? Truncate(n.ToString(CultureInfo.InvariantCulture), MaxCustomFieldValue) + : null; + + case CustomFieldInputType.TEXT_OPTION: + { + if (kfOptions is { Length: > 0 }) + { + var norm = NormalizeChoice(s); + var match = kfOptions.FirstOrDefault(o => NormalizeChoice(o) == norm); + return Truncate(match ?? s, MaxCustomFieldValue); + } + + return Truncate(s, MaxCustomFieldValue); + } + + case CustomFieldInputType.DATE: + { + if (!string.IsNullOrWhiteSpace(kfDateFormat) && + DateTime.TryParseExact(s, kfDateFormat, CultureInfo.InvariantCulture, + DateTimeStyles.None, out var dtExact)) + return Truncate(dtExact.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), MaxCustomFieldValue); + + if (DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var dto)) + return Truncate(dto.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), MaxCustomFieldValue); + + if (DateTime.TryParseExact(s, "yyyy-MM-dd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var dtYmd)) + return Truncate(dtYmd.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), MaxCustomFieldValue); + + return null; + } + + default: + // Treat unknown types as text + return Truncate(s, MaxCustomFieldValue); + } + } + + /// + /// Looser variant that attempts to coerce values based solely on Keyfactor-type hints. + /// Useful if you do not have a Sectigo input type handy. + /// + public static string? CoerceFromKeyfactorType(string? value, int keyfactorDataTypeCode, string[]? kfOptions = null, + string? kfDateFormat = "yyyy-MM-dd") + { + if (value is null) return null; + var s = value.Trim(); + if (s.Length == 0) return null; + + switch (keyfactorDataTypeCode) + { + case 2: // Integer + return int.TryParse(s, NumberStyles.Integer, CultureInfo.InvariantCulture, out var n) + ? Truncate(n.ToString(CultureInfo.InvariantCulture), MaxCustomFieldValue) + : null; + + case 4: // Date + if (!string.IsNullOrWhiteSpace(kfDateFormat) && + DateTime.TryParseExact(s, kfDateFormat, CultureInfo.InvariantCulture, + DateTimeStyles.None, out var dtExact)) + return Truncate(dtExact.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), MaxCustomFieldValue); + + if (DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var dto)) + return Truncate(dto.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), MaxCustomFieldValue); + + if (DateTime.TryParseExact(s, "yyyy-MM-dd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var dtYmd)) + return Truncate(dtYmd.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), MaxCustomFieldValue); + + return null; + + case 3: // MultipleChoice + if (kfOptions is { Length: > 0 }) + { + var norm = NormalizeChoice(s); + var match = kfOptions.FirstOrDefault(o => NormalizeChoice(o) == norm); + return Truncate(match ?? s, MaxCustomFieldValue); + } + + return Truncate(s, MaxCustomFieldValue); + + // String / BigText / Email -> best-effort email sanity then cap + default: + if (s.Contains('@')) + return EmailRx.IsMatch(s) ? Truncate(s, MaxCustomFieldValue) : null; + + return Truncate(s, MaxCustomFieldValue); + } + } + + private static string? Truncate(string? x, int max, string? context = null) + { + if (string.IsNullOrEmpty(x)) return x; + if (x.Length <= max) return x; + + if (!EnableTruncation) + { + _logger.Warn( + $"Input value exceeds Sectigo limits on character length for Custom Field contents ({x.Length} > {max}) and truncation is disabled in config. Original value will be returned and will result in the value not getting synced."); + return x; + } + // Keep logs metadata-only; do not echo the actual value. + // Example contexts: "sectigo.custom_field.name", "sectigo.custom_field.value", "sectigo.comments" + try + { + _logger.Warn( + "WARNING: Keyfactor metadata field contents exceed Sectigo limits on character length and truncation is enabled. Truncating {Context} from {OriginalLength} to {MaxLength} characters.", + context ?? "value", x.Length, max); + } + catch + { + // Never throw from logging + } + + return x.Substring(0, max); + } + + private static string NormalizeChoice(string x) + { + return Regex.Replace(x.Trim(), @"\s+", " ").ToLowerInvariant(); + } +} \ No newline at end of file diff --git a/sectigo-metadata-sync/MetadataSync.cs b/sectigo-metadata-sync/MetadataSync.cs index c1a71d3..73708ba 100644 --- a/sectigo-metadata-sync/MetadataSync.cs +++ b/sectigo-metadata-sync/MetadataSync.cs @@ -1,4 +1,4 @@ -// Copyright 2021 Keyfactor +// Copyright 2025 Keyfactor // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. // You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 // Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -55,13 +55,13 @@ private static void Main(string[] args) try { if (args.Length == 0) - throw new ArgumentException("No configuration mode provided. Please specify KFtoSC or SCtoKF."); + throw new ArgumentException("No sync mode provided. Please specify either KFtoSC or SCtoKF using a command line argument."); // Parse the config mode from the command-line arguments if (!Enum.TryParse(args[0], true, out configMode)) { - _logger.Error("Invalid configuration mode. Please specify KFtoSC or SCtoKF."); - throw new ArgumentException("Invalid configuration mode. Please specify KFtoSC or SCtoKF."); + _logger.Error("Invalid sync mode. Please specify KFtoSC or SCtoKF using a command line argument."); + throw new ArgumentException("Invalid sync mode. Please specify KFtoSC or SCtoKF using a command line argument."); } } catch (Exception ex) @@ -70,7 +70,7 @@ private static void Main(string[] args) throw; // Use 'throw;' to preserve the original stack trace } - _logger.Info($"Configuration mode set to: {configMode}"); + _logger.Info($"Tool sync mode set to: {configMode}"); // Build the config @@ -87,51 +87,103 @@ private static void Main(string[] args) catch (Exception ex) { _logger.Error($"Unable to load config file: {ex.Message}"); - throw; // Use 'throw;' to preserve the original stack trace + throw; // preserve stack } Config settings = new(); - var manualFields = new List(); - var customFields = new List(); List bannedCharList = new(); try { - // Bind config to Config class - settings = config.GetSection("Config") - .Get() - ?? throw new InvalidOperationException("Missing config section in the config json file."); + // Bind config to Config class (supports "Config" or "config" root) + var rootSection = Helpers.GetRootConfigSection(config); + settings = rootSection.Get() + ?? throw new InvalidOperationException("Missing 'config' section in config.json."); + + // Compute the bool *without* throwing if OAuth block is missing/empty + settings.UseKeyfactorOAuth = Helpers.IsOAuthBlockUsable(settings.KeyfactorOAuth); + + // Bind other sections (keep your strictness) _ = config.GetSection("ManualFields") - .Get>() - ?? throw new InvalidOperationException( - "Missing manual fields section in the fields json file."); + .Get>(o => o.ErrorOnUnknownConfiguration = true) + ?? new List(); + _ = config.GetSection("CustomFields") - .Get>() - ?? throw new InvalidOperationException( - "Missing custom fields section in the fields json file."); + .Get>(o => o.ErrorOnUnknownConfiguration = true) + ?? new List(); + bannedCharList = config.GetSection("BannedCharacters") - .Get>() ?? new List() - ?? throw new InvalidOperationException("Missing banned characters section in the config json file."); + .Get>(o => o.ErrorOnUnknownConfiguration = true) + ?? new List(); } catch (Exception ex) { _logger.Error($"Unable to process config file: {ex.Message}"); - throw; // Use 'throw;' to preserve the original stack trace + throw; } + // Parsing date from loaded config + var tz = TimeZoneInfo.Local; + settings.KeyfactorAddedSinceUtc = DateParser.ParseToUtcOrNull( + settings.keyfactorAddedSince, + tz, + settings.keyfactorDateFormat); + // warn if user supplied a value that couldn't be parsed + if (!string.IsNullOrWhiteSpace(settings.keyfactorAddedSince) && settings.KeyfactorAddedSinceUtc is null) + _logger.Warn($"Could not parse keyfactorAddedSince='{settings.keyfactorAddedSince}'. " + + $"Acceptable examples: '2025-09-01', '2025-09-01T13:45', '2025-09-01T13:45:00Z', " + + $"or matching keyfactorDateFormat='{settings.keyfactorDateFormat}'."); + if (settings.KeyfactorAddedSinceUtc != null) + _logger.Info($"Only syncing data for certs imported after {settings.KeyfactorAddedSinceUtc.ToString()}"); _logger.Info("Configuration loaded successfully. Testing connection to Sectigo API and Keyfactor API."); + ValueCoercion.KeyfactorDateFormat = settings.keyfactorDateFormat; + if (settings.enableTruncation) + { + _logger.Info("IMPORTANT: Value truncation is enabled for this sync. Data will be truncated to fit Keyfactor/Sectigo character length limits."); + } + else + { + _logger.Info("IMPORTANT: Value truncation is disabled for this sync. Fields containing data that exceeds Keyfactor character length limits will not be synced."); + } + ValueCoercion.EnableTruncation = settings.enableTruncation; + ValueCoercionSC.EnableTruncation = settings.enableTruncation; // Setup the service var services = new ServiceCollection(); - services.AddSectigoCustomFieldsClient(settings.sectigoAPIUrl); - services.AddKeyfactorMetadataClient(settings.keyfactorAPIUrl); + services.AddSectigoClient(settings.sectigoAPIUrl); + + if (settings.UseKeyfactorOAuth) + { + _logger.Info("Utilizing OAuth for Authentication to Keyfactor API."); + services.AddKeyfactorMetadataClientOAuth( + settings.keyfactorAPIUrl, + new OAuthServiceCollectionExtensions.OAuthOptions + { + TokenUrl = settings.KeyfactorOAuth.TokenUrl, + ClientId = settings.KeyfactorOAuth.ClientId, + ClientSecret = settings.KeyfactorOAuth.ClientSecret, + ScopesCsv = settings.KeyfactorOAuth.ScopesCsv, + Audience = settings.KeyfactorOAuth.Audience, + refreshSkewSeconds = settings.KeyfactorOAuth!.RefreshSkewSeconds + }, + string.IsNullOrWhiteSpace(settings.KeyfactorOAuth.RequestedWith) + ? "APIClient" + : settings.KeyfactorOAuth.RequestedWith + ); + } + else + { + _logger.Info("Utilizing Basic Auth for Authentication to Keyfactor API."); + // Fall back to Basic/Windows (your existing flow) + services.AddKeyfactorMetadataClient(settings.keyfactorAPIUrl); + } // Build the service provider var provider = services.BuildServiceProvider(); // Test Sectigo connection - var scClient = provider.GetRequiredService(); + var scClient = provider.GetRequiredService(); scClient.Authenticate( settings.sectigoLogin, settings.sectigoPassword, @@ -154,11 +206,13 @@ private static void Main(string[] args) // Test Keyfactor connection var kfClient = provider.GetRequiredService(); - // Authenticate - kfClient.Authenticate( - settings.keyfactorLogin, - settings.keyfactorPassword - ); + // Authenticate if not using oauth + if (!settings.UseKeyfactorOAuth) + kfClient.Authenticate( + settings.keyfactorLogin, + settings.keyfactorPassword + ); + var kfFields = new List(); try { @@ -212,6 +266,7 @@ private static void Main(string[] args) KeyfactorDefaultValue = null, // Default to null KeyfactorDisplayOrder = 0, // Default to 0 KeyfactorCaseSensitive = false, // Default to false + KeyfactorMetadataFieldId = 0, ToolFieldType = UnifiedFieldType.Custom }) .ToList(); @@ -223,8 +278,9 @@ private static void Main(string[] args) _logger.Info("importAllCustomFields is disabled. Using field mapping."); // This loads custom metadata using the manualfields config. // Converts blank fields etc and preps the data. - unifiedFieldList = config.GetSection("CustomFields").Get>() ?? - new List(); + unifiedFieldList = config + .GetSection("CustomFields") + .Get>(o => o.ErrorOnUnknownConfiguration = true); foreach (var item in unifiedFieldList) item.ToolFieldType = UnifiedFieldType.Custom; } } @@ -313,13 +369,32 @@ private static void Main(string[] args) // Initialize cumulative lists for unmatched and successfully updated certificates var cumulativeUnmatchedCerts = new List(); + var unmatchedCount = 0; var cumulativePartiallyProcessedCerts = new List(); + var partiallyProcessedCount = 0; var cumulativeSuccessfullyUpdatedCerts = new List(); + var successfullyUpdatedCount = 0; // Initialize a list to collect certificates with missing custom fields var cumulativeMissingCustomFields = new List(); + var missingCustomFields = 0; + + + // Loading Sectigo metadata field IDs into the unified list (for custom fields only) + foreach (var unifiedField in unifiedFieldList) + { + var matchingScField = scFields + .FirstOrDefault(sc => + string.Equals(sc.Name, unifiedField.SectigoFieldName, + StringComparison.OrdinalIgnoreCase)); + if (matchingScField != null) + { + unifiedField.SectigoMetadataFieldID = matchingScField.Id; + unifiedField.SectigoCustomFieldType = matchingScField.Input.Type; + } + } - _logger.Info("Starting paginated retrieval of certificates from Keyfactor."); + _logger.Info("Retrieving base database of Sectigo Certs."); // This list only contains a Sectigo Cert Serial and a Sectigo ID to get extra details. var sectigoCertsDB = scClient.GetCertificatesByProfileId(settings.sslTypeIds, settings.syncRevokedAndExpiredCerts, settings.sectigoPageSize); @@ -330,12 +405,13 @@ private static void Main(string[] args) { // Get the current page of certificates var certsPage = kfClient.GetCertificatesByIssuer(settings.issuerDNLookupTerm, - settings.syncRevokedAndExpiredCerts, pageNumber, pageSize); + settings.syncRevokedAndExpiredCerts, pageNumber, pageSize, + settings.KeyfactorAddedSinceUtc?.ToString("MM-dd-yyyy", CultureInfo.InvariantCulture)); if (certsPage.Count > 0) { - _logger.Debug( - $"[PAGE INFO] Retrieved {certsPage.Count} certificates on page {pageNumber}. Processing batch."); + _logger.Info( + $"[PAGE INFO] Retrieved {certsPage.Count} certificates from Keyfactor on page {pageNumber}. Processing batch."); pageNumber++; // Process the current page of certificates @@ -363,7 +439,7 @@ private static void Main(string[] args) var hasPartialProcessing = false; // Now we process and prep the data for Keyfactor - first load manual fields. - var keyfactorMetadataPayload = new Dictionary(); + var keyfactorMetadataPayload = new Dictionary(); // Process manual fields foreach (var field in unifiedFieldList.Where(f => f.ToolFieldType == UnifiedFieldType.Manual)) @@ -392,9 +468,17 @@ private static void Main(string[] args) .FirstOrDefault(cf => cf.Name.Equals(field.SectigoFieldName, StringComparison.OrdinalIgnoreCase)); - if (localCustomField != null) - keyfactorMetadataPayload[field.KeyfactorMetadataFieldName] = - localCustomField.Value; + // Example when mapping a Sectigo custom field into a KF metadata field: + var raw = localCustomField?.Value; // string + using var doc = + JsonDocument.Parse($"\"{raw?.Replace("\\", "\\\\").Replace("\"", "\\\"")}\""); + var coerced = ValueCoercion.Coerce( + doc.RootElement, + field.KeyfactorDataType, + field.KeyfactorOptions + ); + if (coerced is not null && !(coerced is string s && string.IsNullOrWhiteSpace(s))) + keyfactorMetadataPayload[field.KeyfactorMetadataFieldName] = coerced; } catch (Exception ex) { @@ -408,8 +492,17 @@ private static void Main(string[] args) // Update metadata in Keyfactor try { - kfClient.UpdateCertificateMetadata(localKfCert.Id, keyfactorMetadataPayload); - cumulativeSuccessfullyUpdatedCerts.Add(localScCert.SerialNumber); + if (keyfactorMetadataPayload.Count > 0) + { + kfClient.UpdateCertificateMetadata(localKfCert.Id, + keyfactorMetadataPayload); + cumulativeSuccessfullyUpdatedCerts.Add(localScCert.SerialNumber); + } + else + { + _logger.Trace( + $"Empty metadata payload for cert {localKfCert.SerialNumber}. Skipping upload."); + } } catch (Exception ex) { @@ -430,103 +523,116 @@ private static void Main(string[] args) { // Strip leading zeros from the Keyfactor serial number var strippedSerialNumber = localKfCert.SerialNumber.TrimStart('0'); - - // Find the matching Sectigo cert by serial number - var localScCert = sectigoCertsDB.FirstOrDefault(cert => - cert.SerialNumber.Equals(strippedSerialNumber, StringComparison.OrdinalIgnoreCase)); - - if (localScCert == null) + var hasPartialProcessing = false; + if (localKfCert.Metadata != null && localKfCert.Metadata.Count != 0) { - cumulativeUnmatchedCerts.Add(strippedSerialNumber); - continue; // Skip to the next Keyfactor cert - } + // Find the matching Sectigo cert by serial number + var localScCert = sectigoCertsDB.FirstOrDefault(cert => + cert.SerialNumber.Equals(strippedSerialNumber, StringComparison.OrdinalIgnoreCase)); - // As we have the matched Sectigo ID, we now download the full Sectigo cert details. - var sectigoCertDetails = scClient.GetCertificateDetails(localScCert.SslId); + if (localScCert == null) + { + cumulativeUnmatchedCerts.Add(strippedSerialNumber); + continue; // Skip to the next Keyfactor cert + } - var hasPartialProcessing = false; + // As we have the matched Sectigo ID, we now download the full Sectigo cert details. + var sectigoCertDetails = scClient.GetCertificateDetails(localScCert.SslId); - // Update the Sectigo certificate metadata - var sectigoDataPayload = new List(); - // Retrieve each existing Keyfactor metadata field - if (localKfCert.Metadata != null && localKfCert.Metadata.Count != 0) - foreach (var field in unifiedFieldList.Where(f => - f.ToolFieldType == UnifiedFieldType.Custom)) - try - { - // Find the custom field in SectigoCertificateDetails by SectigoFieldName - var localCustomField = localKfCert.Metadata - .FirstOrDefault(cf => cf.Key.Equals(field.KeyfactorMetadataFieldName, - StringComparison.OrdinalIgnoreCase)); + // Update the Sectigo certificate metadata + var sectigoDataPayload = new List(); - if (!localCustomField.Equals(default(KeyValuePair))) + // Retrieve each existing Keyfactor metadata field + if (localKfCert.Metadata != null && localKfCert.Metadata.Count != 0) + foreach (var field in unifiedFieldList.Where(f => + f.ToolFieldType == UnifiedFieldType.Custom)) + try { - if (field.KeyfactorDataType.Equals((int)MetadataDataType.Date)) + // Find the custom field in SectigoCertificateDetails by SectigoFieldName + var localCustomField = localKfCert.Metadata + .FirstOrDefault(cf => cf.Key.Equals(field.KeyfactorMetadataFieldName, + StringComparison.OrdinalIgnoreCase)); + if (!localCustomField.Equals(default(KeyValuePair))) { - // Define the expected date format - var dateFormat = settings.keyfactorDateFormat; - - if (DateTime.TryParseExact(localCustomField.Value, dateFormat, null, - DateTimeStyles.None, out var parsedDate)) - { - var formattedDate = parsedDate.ToString("yyyy-MM-dd"); + var coerced = ValueCoercionSC.CoerceForSectigo( + localCustomField.Value, field.SectigoCustomFieldType, + field.KeyfactorOptions, + settings.keyfactorDateFormat /* e.g., "M/d/yyyy h:mm:ss tt" */); + if (!string.IsNullOrWhiteSpace(coerced)) sectigoDataPayload.Add(new CustomFieldDetails { Name = field.SectigoFieldName, - Value = formattedDate + Value = coerced }); - } - else - { - _logger.Warn( - $"Invalid date format for field {field.KeyfactorMetadataFieldName}. Expected format: {dateFormat}. Date received: {localCustomField.Value}. Date parsed: {parsedDate}."); - } - } - else - { - sectigoDataPayload.Add(new CustomFieldDetails - { - Name = field.SectigoFieldName, - Value = localCustomField.Value - }); } } + catch (Exception ex) + { + _logger.Warn( + $"[PAGE ERROR] Error processing custom field '{field.KeyfactorMetadataFieldName}' for cert {localScCert.SerialNumber}: {ex.Message}"); + hasPartialProcessing = true; + } + + if (sectigoDataPayload.Count == 0) + { + certsWithoutCustomFields++; + cumulativeMissingCustomFields.Add(localScCert.SerialNumber); + } + else + { + // Update metadata in Sectigo + try + { + scClient.UpdateCertificateMetadata(sectigoCertDetails.SslId, sectigoDataPayload, + "update"); + totalCertsProcessed++; // Increment total processed count + cumulativeSuccessfullyUpdatedCerts.Add(localScCert.SerialNumber); } catch (Exception ex) { _logger.Warn( - $"[PAGE ERROR] Error processing custom field '{field.KeyfactorMetadataFieldName}' for cert {localScCert.SerialNumber}: {ex.Message}"); + $"[PAGE ERROR] Error updating metadata for cert {localScCert.SerialNumber}: {ex.Message}"); hasPartialProcessing = true; } - - if (sectigoDataPayload.Count == 0) - { - certsWithoutCustomFields++; - cumulativeMissingCustomFields.Add(localScCert.SerialNumber); + } } else { - // Update metadata in Sectigo - try - { - scClient.UpdateCertificateMetadata(sectigoCertDetails.SslId, sectigoDataPayload, - "update"); - totalCertsProcessed++; // Increment total processed count - cumulativeSuccessfullyUpdatedCerts.Add(localScCert.SerialNumber); - } - catch (Exception ex) - { - _logger.Warn( - $"[PAGE ERROR] Error updating metadata for cert {localScCert.SerialNumber}: {ex.Message}"); - hasPartialProcessing = true; - } + _logger.Trace($"No custom fields data contained for certificate {localKfCert.SerialNumber}"); + certsWithoutCustomFields++; + cumulativeMissingCustomFields.Add(localKfCert.SerialNumber); } + // Update counters + if (hasPartialProcessing) + cumulativePartiallyProcessedCerts.Add(strippedSerialNumber); + else + totalCertsProcessed++; } else throw new ArgumentException("Invalid configuration mode. Please specify KFtoSC or SCtoKF."); - _logger.Info($"[PAGE PROCESSING] Processed page {pageNumber - 1}."); + // Flushing lists to avoid memory issues on large syncs + cumulativeSuccessfullyUpdatedCerts.FlushRemainder( + _logger, + "SuccessfullyUpdated", + ref successfullyUpdatedCount + ); + cumulativePartiallyProcessedCerts.FlushRemainder( + _logger, + "PartiallyProcessed", + ref partiallyProcessedCount + ); + cumulativeUnmatchedCerts.FlushRemainder( + _logger, + "UnmatchedBetweenKfAndDc", + ref unmatchedCount + ); + cumulativeMissingCustomFields.FlushRemainder( + _logger, + "MissingCustomFields", + ref missingCustomFields + ); } else { @@ -536,28 +642,17 @@ private static void Main(string[] args) // Log cumulative results before the application finishes _logger.Info( - $"[SUMMARY] Completed retrieval and processing of certificates. Total certificates processed successfully: {totalCertsProcessed}. Certs without Custom Fields: {certsWithoutCustomFields}"); - if (cumulativePartiallyProcessedCerts.Count + cumulativeUnmatchedCerts.Count > 0) - _logger.Warn( - $"[SUMMARY] Total certificates with partial processing or errors: {cumulativePartiallyProcessedCerts.Count + cumulativeUnmatchedCerts.Count}"); - if (cumulativeUnmatchedCerts.Any()) + $"[SUMMARY] Completed retrieval and processing of certificates. Total certificates processed successfully: {totalCertsProcessed}. Certs without Custom Fields data: {certsWithoutCustomFields}."); + if (partiallyProcessedCount + unmatchedCount > 0) _logger.Warn( - $"[SUMMARY] No matching Sectigo certificates found for the following Keyfactor certs: {string.Join(", ", cumulativeUnmatchedCerts)}"); - if (cumulativePartiallyProcessedCerts.Any()) + $"[SUMMARY] Total certificates with partial processing or errors: {partiallyProcessedCount + unmatchedCount}."); + if (unmatchedCount > 0) _logger.Warn( - $"[SUMMARY] Following certificates were only partially processed: {string.Join(", ", cumulativePartiallyProcessedCerts)}"); - if (cumulativeSuccessfullyUpdatedCerts.Any()) - _logger.Debug( - $"[SUMMARY] Successfully updated metadata for the following certificates: {string.Join(", ", cumulativeSuccessfullyUpdatedCerts)}"); + $"[SUMMARY] No matching DigiCert certificates found for {unmatchedCount} Keyfactor certs."); // Log aggregated warnings for missing custom fields during SCtoKF sync - if (cumulativeMissingCustomFields.Any()) - { + if (missingCustomFields > 0) _logger.Info( - $"[SUMMARY] No Metadata found for {cumulativeMissingCustomFields.Count} Sectigo certificates in Keyfactor)"); - _logger.Debug( - $"[SUMMARY] No Metadata found for the following Sectigo certificates in Keyfactor: {string.Join(", ", cumulativeMissingCustomFields)}"); - } - + $"[SUMMARY] No Metadata found for {missingCustomFields} DigiCert certificates in Keyfactor."); // End of the run _logger.Info("============================================================"); _logger.Info($"[END] Sectigo Metadata Sync - Run completed at {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); diff --git a/sectigo-metadata-sync/Models/Config.cs b/sectigo-metadata-sync/Models/Config.cs index 528d9ca..981c350 100644 --- a/sectigo-metadata-sync/Models/Config.cs +++ b/sectigo-metadata-sync/Models/Config.cs @@ -5,7 +5,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions // and limitations under the License. +using System; using System.Collections.Generic; +using System.Text.Json.Serialization; namespace SectigoMetadataSync.Models; @@ -26,6 +28,22 @@ public class Config public bool enableDisabledFieldSync { get; set; } = false; public int sectigoPageSize { get; set; } = 25; public int keyfactorPageSize { get; set; } = 100; + public KeyfactorOAuthOptions? KeyfactorOAuth { get; init; } + public bool UseKeyfactorOAuth { get; set; } + public string? keyfactorAddedSince { get; set; } + [JsonIgnore] public DateTimeOffset? KeyfactorAddedSinceUtc { get; set; } + public bool enableTruncation { get; set; } +} + +public sealed class KeyfactorOAuthOptions +{ + public string TokenUrl { get; init; } = string.Empty; + public string ClientId { get; init; } = string.Empty; + public string ClientSecret { get; init; } = string.Empty; + public string? ScopesCsv { get; init; } + public string? Audience { get; init; } + public string RequestedWith { get; init; } = "APIClient"; + public int RefreshSkewSeconds { get; init; } = 120; } public class ManualField diff --git a/sectigo-metadata-sync/Models/Internal.cs b/sectigo-metadata-sync/Models/Internal.cs index f24c29b..308e947 100644 --- a/sectigo-metadata-sync/Models/Internal.cs +++ b/sectigo-metadata-sync/Models/Internal.cs @@ -15,6 +15,8 @@ public class UnifiedFormatField public string SectigoFieldName { get; set; } = string.Empty; public string KeyfactorMetadataFieldName { get; set; } = string.Empty; public string KeyfactorDescription { get; set; } = string.Empty; + public int SectigoMetadataFieldID { get; set; } = 0; + public CustomFieldInputType SectigoCustomFieldType { get; set; } public int KeyfactorDataType { get; set; } public string? KeyfactorHint { get; set; } public string? KeyfactorValidation { get; set; } @@ -24,7 +26,7 @@ public class UnifiedFormatField public string? KeyfactorDefaultValue { get; set; } public int KeyfactorDisplayOrder { get; set; } public bool KeyfactorCaseSensitive { get; set; } = false; // Default to false - public int KeyfactorMetadataFieldId { get; set; } = 0; // Default to 0 (not set) + public int KeyfactorMetadataFieldId { get; set; } // Default to 0 (not set) public UnifiedFieldType ToolFieldType { get; set; } = UnifiedFieldType.Custom; // Default to Custom } diff --git a/sectigo-metadata-sync/Models/KeyfactorCertificate.cs b/sectigo-metadata-sync/Models/KeyfactorCertificate.cs new file mode 100644 index 0000000..e69de29 diff --git a/sectigo-metadata-sync/Models/KeyfactorModels.cs b/sectigo-metadata-sync/Models/KeyfactorModels.cs index fe27ce8..5655a94 100644 --- a/sectigo-metadata-sync/Models/KeyfactorModels.cs +++ b/sectigo-metadata-sync/Models/KeyfactorModels.cs @@ -30,7 +30,7 @@ public class KeyfactorMetadataField /// /// Represents the data types for Keyfactor metadata fields. /// -public enum MetadataDataType +public enum KeyfactorMetadataDataType { String = 1, Integer = 2, diff --git a/sectigo-metadata-sync/SectigoMetadataSync.csproj b/sectigo-metadata-sync/SectigoMetadataSync.csproj index e07c98a..c159e88 100644 --- a/sectigo-metadata-sync/SectigoMetadataSync.csproj +++ b/sectigo-metadata-sync/SectigoMetadataSync.csproj @@ -9,13 +9,12 @@ - - - - - - - + + + + + + @@ -23,17 +22,26 @@ PreserveNewest - Always + Always Always + + Always + + + Always + Always Always + + Always + Always diff --git a/sectigo-metadata-sync/config/bannedcharacters.json b/sectigo-metadata-sync/config/bannedcharacters.json index 3bee5c6..f435a94 100644 --- a/sectigo-metadata-sync/config/bannedcharacters.json +++ b/sectigo-metadata-sync/config/bannedcharacters.json @@ -1,3 +1,4 @@ { - "BannedCharacters": [] -} + "BannedCharacters": [ + ] +} \ No newline at end of file diff --git a/sectigo-metadata-sync/config/fields.json b/sectigo-metadata-sync/config/fields.json index 324b0be..332efa9 100644 --- a/sectigo-metadata-sync/config/fields.json +++ b/sectigo-metadata-sync/config/fields.json @@ -5,12 +5,12 @@ "sectigoFieldName": "The name of the field in Sectigo. If manual field, an address into the Sectigo Certificate Details json response.", "keyfactorMetadataFieldName": "The name of the field in Keyfactor. This must not contain spaces, or a variety of other characters. Only [a-zA-Z0-9-_] fitting characters are accepted.", "keyfactorDescription": "A description of the metadata field for use with Keyfactor.", - "keyfactorDataType": "The data type of the field. (1 = String, 2 = Integer, etc.)", + "keyfactorDataType": "The data type of the field for Keyfactor. String = 1, Integer = 2, Date = 3, Boolean = 4, MultipleChoice = 5, BigText = 6, Email = 7.", "keyfactorHint": "A short hint to guide users on what to enter in the field, displayed in Keyfactor.", "keyfactorValidation": "A RegEx expression to validate the field's input. Only applicable for string fields.", - "keyfactorEnrollment": "How the metadata field is handled in Keyfactor during certificate enrollment. String = 1, Integer = 2, Date = 3, Boolean = 4, MultipleChoice = 5, BigText = 6, Email = 7. ", + "keyfactorEnrollment": "How the metadata field is handled in Keyfactor during certificate enrollment. 0=Optional, 1=Required, 2=Hidden", "keyfactorMessage": "A message to be displayed when validation fails.", - "keyfactorOptions": "An array of values for multiple-choice fields. Ignored for other data types.", + "keyfactorOptions": "An array of values for multiple-choice fields. Ignored for other data types. (e.g. ['option1','option2'])", "keyfactorDefaultValue": "A default value for the field. Only applicable for certain data types.", "keyfactorDisplayOrder": "The order in which the field is displayed.", "keyfactorCaseSensitive": "Whether validation is case-sensitive. Only applicable for string fields with validation." @@ -27,7 +27,7 @@ "keyfactorMessage": null, "keyfactorOptions": null, "keyfactorDefaultValue": null, - "keyfactorDisplayOrder": 1, + "keyfactorDisplayOrder": 0, "keyfactorCaseSensitive": false }, { @@ -41,7 +41,7 @@ "keyfactorMessage": null, "keyfactorOptions": null, "keyfactorDefaultValue": null, - "keyfactorDisplayOrder": 2, + "keyfactorDisplayOrder": 0, "keyfactorCaseSensitive": false } ], @@ -50,14 +50,14 @@ "sectigoFieldName": "Special String", "keyfactorMetadataFieldName": "TestField", "keyfactorDescription": "Just an empty test field.", - "keyfactorDataType": 1, + "keyfactorDataType": 7, "keyfactorHint": "Enter test data", "keyfactorValidation": null, "keyfactorEnrollment": 0, "keyfactorMessage": null, "keyfactorOptions": null, "keyfactorDefaultValue": null, - "keyfactorDisplayOrder": 3, + "keyfactorDisplayOrder": 0, "keyfactorCaseSensitive": false }, { @@ -71,7 +71,7 @@ "keyfactorMessage": null, "keyfactorOptions": null, "keyfactorDefaultValue": null, - "keyfactorDisplayOrder": 4, + "keyfactorDisplayOrder": 0, "keyfactorCaseSensitive": false } ] diff --git a/sectigo-metadata-sync/config/stock-config-oauth.json b/sectigo-metadata-sync/config/stock-config-oauth.json new file mode 100644 index 0000000..f885c58 --- /dev/null +++ b/sectigo-metadata-sync/config/stock-config-oauth.json @@ -0,0 +1,30 @@ +{ + "config": { + "sectigoLogin": "test@localtest.com", + "sectigoPassword": "*****", + "sectigoCustomerUri": "test-partner", + "sectigoAPIUrl": "https://cert-manager.com", + "keyfactorLogin": null, + "keyfactorPassword": null, + "keyfactorAPIUrl": "https://test.local/Keyfactor/API/", + "keyfactorDateFormat": "M/d/yyyy h:mm:ss tt", + "importAllCustomFields": "false", + "syncRevokedAndExpiredCerts": "true", + "issuerDNLookupTerm": "Sectigo", + "enableDisabledFieldSync": "false", + "sslTypeIds": [ 11111, 11112 ], + "sectigoPageSize": 25, + "keyfactorPageSize": 100, + "keyfactorAddedSince": null, + "enableTruncation": null, + "keyfactorOAuth": { + "tokenUrl": "https://test.us.auth0.com/oauth/token", + "clientId": "******", + "clientSecret": "*******", + "scopesCsv": "", + "audience": "https://test.us.auth0.com/api/v2/", + "requestedWith": "APIClient", + "refreshSkewSeconds": 60 + } + } +} \ No newline at end of file diff --git a/sectigo-metadata-sync/config/stock-config.json b/sectigo-metadata-sync/config/stock-config.json index 593c889..1f12fc3 100644 --- a/sectigo-metadata-sync/config/stock-config.json +++ b/sectigo-metadata-sync/config/stock-config.json @@ -1,19 +1,21 @@ { "config": { - "sectigoLogin": "test@example.com", - "sectigoPassword": "*********", - "sectigoCustomerUri": "sectigo-customer", + "sectigoLogin": "test@localtest.com", + "sectigoPassword": "*****", + "sectigoCustomerUri": "test-partner", "sectigoAPIUrl": "https://cert-manager.com", - "keyfactorLogin": "DOMAIN\\Username", - "keyfactorPassword": "********", - "keyfactorAPIUrl": "https://keyfactorexample.com/keyfactorapi", + "keyfactorLogin": "DOMAIN\\username", + "keyfactorPassword": "******", + "keyfactorAPIUrl": "https://test.local/Keyfactor/API/", "keyfactorDateFormat": "M/d/yyyy h:mm:ss tt", "importAllCustomFields": "false", "syncRevokedAndExpiredCerts": "true", "issuerDNLookupTerm": "Sectigo", "enableDisabledFieldSync": "false", - "sslTypeIds": [ 111111, 222222 ], + "sslTypeIds": [ 11111, 11112 ], "sectigoPageSize": 25, - "keyfactorPageSize": 100 + "keyfactorPageSize": 100, + "keyfactorAddedSince": null, + "enableTruncation": null } } \ No newline at end of file diff --git a/sectigo-metadata-sync/config/stock-fields.json b/sectigo-metadata-sync/config/stock-fields.json new file mode 100644 index 0000000..ccaaaf5 --- /dev/null +++ b/sectigo-metadata-sync/config/stock-fields.json @@ -0,0 +1,78 @@ +{ + "_comments": { + "ManualFields": "List of fields used for loading static information from Sectigo as certificate Metadata in Keyfactor.", + "CustomFields": "List of Custom Fields/Metadata Fields to be synced between Keyfactor and Sectigo.", + "sectigoFieldName": "The name of the field in Sectigo. If manual field, an address into the Sectigo Certificate Details json response.", + "keyfactorMetadataFieldName": "The name of the field in Keyfactor. This must not contain spaces, or a variety of other characters. Only [a-zA-Z0-9-_] fitting characters are accepted.", + "keyfactorDescription": "A description of the metadata field for use with Keyfactor.", + "keyfactorDataType": "The data type of the field for Keyfactor. String = 1, Integer = 2, Date = 3, Boolean = 4, MultipleChoice = 5, BigText = 6, Email = 7.", + "keyfactorHint": "A short hint to guide users on what to enter in the field, displayed in Keyfactor.", + "keyfactorValidation": "A RegEx expression to validate the field's input. Only applicable for string fields.", + "keyfactorEnrollment": "How the metadata field is handled in Keyfactor during certificate enrollment. 0=Optional, 1=Required, 2=Hidden", + "keyfactorMessage": "A message to be displayed when validation fails.", + "keyfactorOptions": "An array of values for multiple-choice fields. Ignored for other data types. (e.g. ['option1','option2'])", + "keyfactorDefaultValue": "A default value for the field. Only applicable for certain data types.", + "keyfactorDisplayOrder": "The order in which the field is displayed.", + "keyfactorCaseSensitive": "Whether validation is case-sensitive. Only applicable for string fields with validation." + }, + "ManualFields": [ + { + "sectigoFieldName": "id", + "keyfactorMetadataFieldName": "SectigoID", + "keyfactorDescription": "Sectigo Assigned Cert ID", + "keyfactorDataType": 2, + "keyfactorHint": "", + "keyfactorValidation": null, + "keyfactorEnrollment": 0, + "keyfactorMessage": null, + "keyfactorOptions": null, + "keyfactorDefaultValue": null, + "keyfactorDisplayOrder": 1, + "keyfactorCaseSensitive": false + }, + { + "sectigoFieldName": "certType.id", + "keyfactorMetadataFieldName": "SectigoCertTypeID", + "keyfactorDescription": "Sectigo Cert Type ID For This Cert", + "keyfactorDataType": 2, + "keyfactorHint": "", + "keyfactorValidation": null, + "keyfactorEnrollment": 0, + "keyfactorMessage": null, + "keyfactorOptions": null, + "keyfactorDefaultValue": null, + "keyfactorDisplayOrder": 2, + "keyfactorCaseSensitive": false + } + ], + "CustomFields": [ + { + "sectigoFieldName": "Special String", + "keyfactorMetadataFieldName": "TestField", + "keyfactorDescription": "Just an empty test field.", + "keyfactorDataType": 1, + "keyfactorHint": "Enter test data", + "keyfactorValidation": null, + "keyfactorEnrollment": 0, + "keyfactorMessage": null, + "keyfactorOptions": null, + "keyfactorDefaultValue": null, + "keyfactorDisplayOrder": 3, + "keyfactorCaseSensitive": false + }, + { + "sectigoFieldName": "Special Number", + "keyfactorMetadataFieldName": "IntegerTestField", + "keyfactorDescription": "Another test field but with an integer.", + "keyfactorDataType": 2, + "keyfactorHint": "Enter an integer value", + "keyfactorValidation": null, + "keyfactorEnrollment": 0, + "keyfactorMessage": null, + "keyfactorOptions": null, + "keyfactorDefaultValue": null, + "keyfactorDisplayOrder": 4, + "keyfactorCaseSensitive": false + } + ] +} \ No newline at end of file