fix(fetch): fixup generators and add error handling (#8395)

* fix(fetch): fixup generators and add error handling

* fixup!

* fixup!

* fixup!

* fixup!

Author: avivkeller

apps/site/next-data/generators/supportersData.mjs

@@ -1,13 +1,14 @@
+import { OPENCOLLECTIVE_MEMBERS_URL } from '#site/next.constants.mjs';
+import { fetchWithRetry } from '#site/util/fetch';
+
 /**
  * Fetches supporters data from Open Collective API, filters active backers,
  * and maps it to the Supporters type.
  *
- * @returns {Promise<Array<import('#site/types/supporters')>>} Array of supporters
+ * @returns {Promise<Array<import('#site/types/supporters').OpenCollectiveSupporter>>} Array of supporters
  */
 async function fetchOpenCollectiveData() {
-  const endpoint = 'https://opencollective.com/nodejs/members/all.json';
-
-  const response = await fetch(endpoint);
+  const response = await fetchWithRetry(OPENCOLLECTIVE_MEMBERS_URL);
 
   const payload = await response.json();
 

apps/site/next-data/generators/vulnerabilities.mjs

@@ -1,6 +1,9 @@
 import { VULNERABILITIES_URL } from '#site/next.constants.mjs';
+import { fetchWithRetry } from '#site/util/fetch';
 
 const RANGE_REGEX = /([<>]=?)\s*(\d+)(?:\.(\d+))?/;
+const V0_REGEX = /^0\.\d+(\.x)?$/;
+const VER_REGEX = /^\d+\.x$/;
 
 /**
  * Fetches vulnerability data from the Node.js Security Working Group repository,
@@ -9,7 +12,7 @@ const RANGE_REGEX = /([<>]=?)\s*(\d+)(?:\.(\d+))?/;
  * @returns {Promise<import('#site/types/vulnerabilities').GroupedVulnerabilities>} Grouped vulnerabilities
  */
 export default async function generateVulnerabilityData() {
-  const response = await fetch(VULNERABILITIES_URL);
+  const response = await fetchWithRetry(VULNERABILITIES_URL);
 
   /** @type {Array<import('#site/types/vulnerabilities').RawVulnerability>} */
   const data = Object.values(await response.json());
@@ -26,14 +29,14 @@ export default async function generateVulnerabilityData() {
   // Helper function to process version patterns
   const processVersion = (version, vulnerability) => {
     // Handle 0.X versions (pre-semver)
-    if (/^0\.\d+(\.x)?$/.test(version)) {
+    if (V0_REGEX.test(version)) {
       addToGroup('0', vulnerability);
 
       return;
     }
 
     // Handle simple major.x patterns (e.g., 12.x)
-    if (/^\d+\.x$/.test(version)) {
+    if (VER_REGEX.test(version)) {
       const majorVersion = version.split('.')[0];
 
       addToGroup(majorVersion, vulnerability);
@@ -67,25 +70,14 @@ export default async function generateVulnerabilityData() {
     }
   };
 
-  for (const vulnerability of Object.values(data)) {
-    const parsedVulnerability = {
-      cve: vulnerability.cve,
-      url: vulnerability.ref,
-      vulnerable: vulnerability.vulnerable,
-      patched: vulnerability.patched,
-      description: vulnerability.description,
-      overview: vulnerability.overview,
-      affectedEnvironments: vulnerability.affectedEnvironments,
-      severity: vulnerability.severity,
-    };
+  for (const { ref, ...vulnerability } of Object.values(data)) {
+    vulnerability.url = ref;
 
     // Process all potential versions from the vulnerable field
-    const versions = parsedVulnerability.vulnerable
-      .split(' || ')
-      .filter(Boolean);
+    const versions = vulnerability.vulnerable.split(' || ').filter(Boolean);
 
     for (const version of versions) {
-      processVersion(version, parsedVulnerability);
+      processVersion(version, vulnerability);
     }
   }
 

apps/site/next.calendar.mjs

@@ -4,6 +4,7 @@ import {
   BASE_CALENDAR_URL,
   SHARED_CALENDAR_KEY,
 } from './next.calendar.constants.mjs';
+import { fetchWithRetry } from './util/fetch';
 
 /**
  *
@@ -33,7 +34,7 @@ export const getCalendarEvents = async (calendarId = '', maxResults = 20) => {
     calendarQueryUrl.searchParams.append(key, value)
   );
 
-  return fetch(calendarQueryUrl)
+  return fetchWithRetry(calendarQueryUrl)
     .then(response => response.json())
     .then(calendar => calendar.items ?? []);
 };

apps/site/next.constants.mjs

@@ -213,3 +213,9 @@ export const EOL_VERSION_IDENTIFIER = 'End-of-life';
  */
 export const VULNERABILITIES_URL =
   'https://raw.githubusercontent.com/nodejs/security-wg/main/vuln/core/index.json';
+
+/**
+ * The location of the OpenCollective data
+ */
+export const OPENCOLLECTIVE_MEMBERS_URL =
+  'https://opencollective.com/nodejs/members/all.json';

apps/site/types/index.ts

@@ -16,3 +16,4 @@ export * from './download';
 export * from './userAgent';
 export * from './vulnerabilities';
 export * from './page';
+export * from './supporters';

apps/site/types/supporters.ts

@@ -0,0 +1,9 @@
+export type Supporter<T extends string> = {
+  name: string;
+  image: string;
+  url: string;
+  profile: string;
+  source: T;
+};
+
+export type OpenCollectiveSupporter = Supporter<'opencollective'>;

apps/site/util/fetch.ts

@@ -0,0 +1,34 @@
+import { setTimeout } from 'node:timers/promises';
+
+type RetryOptions = RequestInit & {
+  maxRetry?: number;
+  delay?: number;
+};
+
+type FetchError = {
+  cause: {
+    code: string;
+  };
+};
+
+export const fetchWithRetry = async (
+  url: string,
+  { maxRetry = 3, delay = 100, ...options }: RetryOptions = {}
+) => {
+  for (let i = 1; i <= maxRetry; i++) {
+    try {
+      return fetch(url, options);
+    } catch (e) {
+      console.debug(
+        `fetch of ${url} failed at ${Date.now()}, attempt ${i}/${maxRetry}`,
+        e
+      );
+
+      if (i === maxRetry || (e as FetchError).cause.code !== 'ETIMEDOUT') {
+        throw e;
+      }
+
+      await setTimeout(delay * i);
+    }
+  }
+};