Skip to content

Commit 8bbdf32

Browse files
joshjohanningjoshjohanning
committed
fix: implement retry logic for discussion creation to handle rate limits
1 parent 95187df commit 8bbdf32

File tree

1 file changed

+144
-84
lines changed

1 file changed

+144
-84
lines changed

‎scripts/copy-discussions.js‎

Lines changed: 144 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
const INCLUDE_POLL_MERMAID_CHART = true; // Set to false to disable Mermaid pie chart for polls
3030
const RATE_LIMIT_SLEEP_SECONDS = 0.5; // Default sleep duration between API calls to avoid rate limiting
3131
const DISCUSSION_PROCESSING_DELAY_SECONDS = 5; // Delay between processing discussions
32+
const RATE_LIMIT_RETRY_DELAY_SECONDS = 60; // Delay when hitting rate limits before retrying
33+
const MAX_RETRIES = 3; // Maximum number of retries for failed operations
3234

3335
const { Octokit } = require("octokit");
3436

@@ -169,6 +171,39 @@ async function rateLimitSleep(seconds = RATE_LIMIT_SLEEP_SECONDS) {
169171
await sleep(seconds);
170172
}
171173

174+
function isRateLimitError(err) {
175+
const message = err.message?.toLowerCase() || '';
176+
const status = err.status || 0;
177+
178+
// Check for primary rate limit (403 with rate limit message)
179+
if (status === 403 && (message.includes('rate limit') || message.includes('api rate limit'))) {
180+
return true;
181+
}
182+
183+
// Check for secondary rate limit (403 with abuse/secondary message)
184+
if (status === 403 && (message.includes('secondary') || message.includes('abuse'))) {
185+
return true;
186+
}
187+
188+
// Check for retry-after header indication
189+
if (message.includes('retry after') || message.includes('try again later')) {
190+
return true;
191+
}
192+
193+
return false;
194+
}
195+
196+
async function handleRateLimitError(err, attemptNumber) {
197+
if (isRateLimitError(err)) {
198+
const waitTime = RATE_LIMIT_RETRY_DELAY_SECONDS * attemptNumber; // Exponential-ish backoff
199+
warn(`Rate limit detected (attempt ${attemptNumber}). Waiting ${waitTime}s before retry...`);
200+
warn(`Error details: ${err.message}`);
201+
await sleep(waitTime);
202+
return true;
203+
}
204+
return false;
205+
}
206+
172207
function formatPollData(poll) {
173208
if (!poll || !poll.options || poll.options.nodes.length === 0) {
174209
return '';
@@ -997,99 +1032,124 @@ async function processDiscussionsPage(sourceOctokit, targetOctokit, owner, repo,
9971032
// Check if discussion is pinned
9981033
const isPinned = pinnedDiscussionIds.has(discussion.id);
9991034

1000-
// Create discussion
1001-
try {
1002-
const newDiscussion = await createDiscussion(
1003-
targetOctokit,
1004-
targetRepoId,
1005-
targetCategoryId,
1006-
discussion.title,
1007-
discussion.body || "",
1008-
discussion.url,
1009-
discussion.author?.login || "unknown",
1010-
discussion.createdAt,
1011-
discussion.poll || null,
1012-
discussion.locked || false,
1013-
isPinned,
1014-
discussion.reactionGroups || []
1015-
);
1016-
1017-
createdDiscussions++;
1018-
log(`✓ Created discussion #${discussion.number}: '${discussion.title}'`);
1019-
1020-
// Log additional metadata info
1021-
if (discussion.poll && discussion.poll.options?.nodes?.length > 0) {
1022-
log(` ℹ️ Poll included with ${discussion.poll.options.nodes.length} options (${discussion.poll.totalVoteCount} total votes)`);
1023-
}
1024-
if (discussion.locked) {
1025-
log(` 🔒 Discussion was locked in source and has been locked in target`);
1026-
}
1027-
if (isPinned) {
1028-
log(` 📌 Discussion was pinned in source (indicator added to body)`);
1029-
}
1030-
const totalReactions = discussion.reactionGroups?.reduce((sum, group) => sum + (group.users.totalCount || 0), 0) || 0;
1031-
if (totalReactions > 0) {
1032-
log(` ❤️ ${totalReactions} reaction${totalReactions !== 1 ? 's' : ''} copied`);
1033-
}
1034-
1035-
// Process labels
1036-
if (discussion.labels.nodes.length > 0) {
1037-
const labelIds = [];
1035+
// Create discussion with retry logic
1036+
let newDiscussion = null;
1037+
let createSuccess = false;
1038+
1039+
for (let attempt = 1; attempt <= MAX_RETRIES && !createSuccess; attempt++) {
1040+
try {
1041+
newDiscussion = await createDiscussion(
1042+
targetOctokit,
1043+
targetRepoId,
1044+
targetCategoryId,
1045+
discussion.title,
1046+
discussion.body || "",
1047+
discussion.url,
1048+
discussion.author?.login || "unknown",
1049+
discussion.createdAt,
1050+
discussion.poll || null,
1051+
discussion.locked || false,
1052+
isPinned,
1053+
discussion.reactionGroups || []
1054+
);
10381055

1039-
for (const label of discussion.labels.nodes) {
1040-
log(`Processing label: '${label.name}' (color: ${label.color})`);
1041-
1042-
const labelId = await getOrCreateLabelId(
1043-
targetOctokit,
1044-
targetRepoId,
1045-
label.name,
1046-
label.color,
1047-
label.description || "",
1048-
targetLabels
1049-
);
1050-
1051-
if (labelId) {
1052-
labelIds.push(labelId);
1053-
}
1054-
}
1056+
createSuccess = true;
1057+
createdDiscussions++;
1058+
log(`✓ Created discussion #${discussion.number}: '${discussion.title}'`);
1059+
1060+
} catch (err) {
1061+
// Handle rate limit errors with retry
1062+
const shouldRetry = await handleRateLimitError(err, attempt);
10551063

1056-
if (labelIds.length > 0) {
1057-
await addLabelsToDiscussion(targetOctokit, newDiscussion.id, labelIds);
1064+
if (!shouldRetry) {
1065+
// Not a rate limit error, or unrecoverable error
1066+
error(`Failed to create discussion #${discussion.number}: '${discussion.title}' - ${err.message}`);
1067+
if (attempt < MAX_RETRIES) {
1068+
warn(`Retrying (attempt ${attempt + 1}/${MAX_RETRIES})...`);
1069+
await sleep(5); // Brief pause before retry
1070+
} else {
1071+
error(`Max retries (${MAX_RETRIES}) reached. Skipping discussion #${discussion.number}.`);
1072+
skippedDiscussions++;
1073+
break;
1074+
}
10581075
}
1076+
// If shouldRetry is true, loop will continue to next attempt
10591077
}
1078+
}
1079+
1080+
// If we exhausted retries without success, skip this discussion
1081+
if (!createSuccess) {
1082+
continue;
1083+
}
1084+
1085+
// Log additional metadata info
1086+
if (discussion.poll && discussion.poll.options?.nodes?.length > 0) {
1087+
log(` ℹ️ Poll included with ${discussion.poll.options.nodes.length} options (${discussion.poll.totalVoteCount} total votes)`);
1088+
}
1089+
if (discussion.locked) {
1090+
log(` 🔒 Discussion was locked in source and has been locked in target`);
1091+
}
1092+
if (isPinned) {
1093+
log(` 📌 Discussion was pinned in source (indicator added to body)`);
1094+
}
1095+
const totalReactions = discussion.reactionGroups?.reduce((sum, group) => sum + (group.users.totalCount || 0), 0) || 0;
1096+
if (totalReactions > 0) {
1097+
log(` ❤️ ${totalReactions} reaction${totalReactions !== 1 ? 's' : ''} copied`);
1098+
}
1099+
1100+
// Process labels
1101+
if (discussion.labels.nodes.length > 0) {
1102+
const labelIds = [];
10601103

1061-
// Copy comments
1062-
log("Processing comments for discussion...");
1063-
const comments = await fetchDiscussionComments(sourceOctokit, discussion.id);
1064-
const answerCommentId = discussion.answer?.id || null;
1065-
const newAnswerCommentId = await copyDiscussionComments(
1066-
targetOctokit,
1067-
newDiscussion.id,
1068-
comments,
1069-
answerCommentId
1070-
);
1071-
1072-
// Mark answer if applicable
1073-
if (newAnswerCommentId) {
1074-
log("Source discussion has an answer comment, marking it in target...");
1075-
await markCommentAsAnswer(targetOctokit, newAnswerCommentId);
1104+
for (const label of discussion.labels.nodes) {
1105+
log(`Processing label: '${label.name}' (color: ${label.color})`);
1106+
1107+
const labelId = await getOrCreateLabelId(
1108+
targetOctokit,
1109+
targetRepoId,
1110+
label.name,
1111+
label.color,
1112+
label.description || "",
1113+
targetLabels
1114+
);
1115+
1116+
if (labelId) {
1117+
labelIds.push(labelId);
1118+
}
10761119
}
10771120

1078-
// Close discussion if it was closed in source
1079-
if (discussion.closed) {
1080-
log("Source discussion is closed, closing target discussion...");
1081-
await closeDiscussion(targetOctokit, newDiscussion.id);
1121+
if (labelIds.length > 0) {
1122+
await addLabelsToDiscussion(targetOctokit, newDiscussion.id, labelIds);
10821123
}
1083-
1084-
log(`✅ Finished processing discussion #${discussion.number}: '${discussion.title}'`);
1085-
1086-
// Delay between discussions
1087-
await sleep(DISCUSSION_PROCESSING_DELAY_SECONDS);
1088-
1089-
} catch (err) {
1090-
error(`Failed to create discussion #${discussion.number}: '${discussion.title}' - ${err.message}`);
1091-
skippedDiscussions++;
10921124
}
1125+
1126+
// Copy comments
1127+
log("Processing comments for discussion...");
1128+
const comments = await fetchDiscussionComments(sourceOctokit, discussion.id);
1129+
const answerCommentId = discussion.answer?.id || null;
1130+
const newAnswerCommentId = await copyDiscussionComments(
1131+
targetOctokit,
1132+
newDiscussion.id,
1133+
comments,
1134+
answerCommentId
1135+
);
1136+
1137+
// Mark answer if applicable
1138+
if (newAnswerCommentId) {
1139+
log("Source discussion has an answer comment, marking it in target...");
1140+
await markCommentAsAnswer(targetOctokit, newAnswerCommentId);
1141+
}
1142+
1143+
// Close discussion if it was closed in source
1144+
if (discussion.closed) {
1145+
log("Source discussion is closed, closing target discussion...");
1146+
await closeDiscussion(targetOctokit, newDiscussion.id);
1147+
}
1148+
1149+
log(`✅ Finished processing discussion #${discussion.number}: '${discussion.title}'`);
1150+
1151+
// Delay between discussions
1152+
await sleep(DISCUSSION_PROCESSING_DELAY_SECONDS);
10931153
}
10941154

10951155
// Process next page if exists

0 commit comments

Comments
 (0)