Updated podcasts

Author: ofriw

scripts/README.md

@@ -89,7 +89,7 @@ node generate-podcast.js
 
 ### Speakers
 - **Alex**: "Kore" voice (firm, professional)
-- **Sam**: "Puck" voice (upbeat, curious)
+- **Sam**: "Charon" voice (neutral, professional)
 
 ### Rate Limiting
 - 2-second delay between files to avoid API rate limits

scripts/generate-podcast.js

@@ -100,10 +100,26 @@ function findMarkdownFiles(dir) {
   return files.sort();
 }
 
+/**
+ * Count tokens in dialogue text using Gemini API
+ */
+async function countDialogueTokens(dialogue) {
+  const model = genAI.getGenerativeModel({ model: TTS_MODEL });
+
+  const result = await model.countTokens({
+    contents: [{
+      role: 'user',
+      parts: [{ text: dialogue }]
+    }]
+  });
+
+  return result.totalTokens;
+}
+
 /**
  * Generate conversational dialogue from technical content
  */
-async function generateDialogue(content, fileName) {
+async function generateDialogue(content, fileName, maxTokens = null) {
   console.log(`  🎭 Generating dialogue script...`);
 
   const model = genAI.getGenerativeModel({ model: DIALOGUE_MODEL });
@@ -115,21 +131,30 @@ async function generateDialogue(content, fileName) {
 SPECIAL INSTRUCTIONS FOR THIS EPISODE:
 This is the course introduction. When discussing the section about how the course was developed using AI (including the podcast itself), have Alex and Sam briefly acknowledge in a natural, professional way that they themselves are AI-generated voices. This should feel like a thoughtful meta-moment, not gimmicky. Keep it concise - one or two exchanges where they note the self-referential nature of AI-generated hosts discussing AI-generated content. Make it feel authentic to how senior engineers would react to this realization.` : '';
 
+  const lengthConstraint = maxTokens ? `
+
+CRITICAL LENGTH REQUIREMENT:
+The dialogue MUST be under ${maxTokens} tokens (approximately ${Math.floor(maxTokens * 4)} characters or ${Math.floor(maxTokens * 0.75)} words). This is a hard constraint due to TTS API limitations. Prioritize the most important concepts and keep explanations concise while maintaining quality.` : `
+
+LENGTH GUIDELINE:
+Aim for a concise dialogue under 7,000 tokens to ensure it fits within TTS API limits while maintaining quality.`;
+
   const prompt = `You are converting technical course content into a natural, engaging two-person podcast conversation.
 
 Speakers:
-- Alex: The instructor - knowledgeable, clear, engaging teacher
-- Sam: Senior software engineer - curious, asks clarifying questions, relates concepts to real-world scenarios
+- Alex: The instructor - knowledgeable, clear, measured teaching style
+- Sam: Senior software engineer - thoughtful, asks clarifying questions, relates concepts to real-world scenarios
 
 Guidelines:
-- Keep the conversation natural and flowing
-- Sam should ask relevant questions that a senior engineer would ask
+- Keep the conversation natural and flowing, but maintain professional composure
+- Conversational yet measured - avoid excessive enthusiasm or exclamations
+- Sam should ask relevant, thoughtful questions that a senior engineer would ask
 - Alex should explain concepts clearly but not patronizingly
-- Include brief moments of insight or "aha" moments
+- Include brief moments of insight or understanding
 - Keep technical accuracy - don't dumb down the content
-- Make it engaging but professional
+- Make it engaging but professional - prioritize clarity over energy
 - Break down complex concepts through dialogue
-- Reference real-world scenarios and examples${metaCommentary}
+- Reference real-world scenarios and examples${metaCommentary}${lengthConstraint}
 
 Technical Content Title: ${fileName}
 
@@ -180,46 +205,107 @@ function createWavHeader(pcmDataLength) {
   return header;
 }
 
+/**
+ * Retry a function with exponential backoff for transient errors
+ * @param {Function} fn - Async function to retry
+ * @param {number} maxAttempts - Maximum retry attempts (default: 4)
+ * @returns {Promise} - Result of successful function call
+ */
+async function retryWithBackoff(fn, maxAttempts = 4) {
+  let lastError;
+
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+
+      // Check if error is retryable (transient network/API errors)
+      const isRetryable =
+        error.message?.includes('fetch failed') ||
+        error.message?.includes('ECONNRESET') ||
+        error.message?.includes('ETIMEDOUT') ||
+        error.message?.includes('ENOTFOUND') ||
+        error.status === 429 ||  // Rate limit
+        error.status === 500 ||  // Internal server error
+        error.status === 503 ||  // Service unavailable
+        error.status === 504;    // Gateway timeout
+
+      // Don't retry permanent errors (auth, permission, not found)
+      const isPermanent =
+        error.status === 400 ||  // Bad request
+        error.status === 401 ||  // Unauthorized
+        error.status === 403 ||  // Forbidden
+        error.status === 404;    // Not found
+
+      if (isPermanent) {
+        throw error;  // Fail fast on permanent errors
+      }
+
+      if (!isRetryable || attempt === maxAttempts - 1) {
+        throw lastError;  // Last attempt or non-retryable error
+      }
+
+      // Exponential backoff: 1s, 2s, 4s, 8s
+      const delay = Math.pow(2, attempt) * 1000;
+      console.log(`  ⏳ Retry ${attempt + 1}/${maxAttempts} after ${delay}ms (${error.message})`);
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+
+  throw lastError;
+}
+
 /**
  * Convert dialogue text to audio using multi-speaker TTS
  */
 async function generateAudio(dialogue, outputPath) {
   console.log(`  🎙️  Synthesizing audio...`);
 
-  const model = genAI.getGenerativeModel({
-    model: TTS_MODEL,
-  });
-
-  const result = await model.generateContent({
-    contents: [{
-      role: 'user',
-      parts: [{ text: dialogue }]
-    }],
-    generationConfig: {
-      responseModalities: ['AUDIO'],
-      speechConfig: {
-        multiSpeakerVoiceConfig: {
-          speakerVoiceConfigs: [
-            {
-              speaker: 'Alex',
-              voiceConfig: {
-                prebuiltVoiceConfig: {
-                  voiceName: 'Kore' // Firm, professional voice
+  // Wrap TTS API call with retry logic
+  const result = await retryWithBackoff(async () => {
+    const model = genAI.getGenerativeModel({
+      model: TTS_MODEL,
+    });
+
+    const response = await model.generateContent({
+      contents: [{
+        role: 'user',
+        parts: [{ text: dialogue }]
+      }],
+      generationConfig: {
+        responseModalities: ['AUDIO'],
+        speechConfig: {
+          multiSpeakerVoiceConfig: {
+            speakerVoiceConfigs: [
+              {
+                speaker: 'Alex',
+                voiceConfig: {
+                  prebuiltVoiceConfig: {
+                    voiceName: 'Kore' // Firm, professional voice
+                  }
                 }
-              }
-            },
-            {
-              speaker: 'Sam',
-              voiceConfig: {
-                prebuiltVoiceConfig: {
-                  voiceName: 'Puck' // Upbeat, curious voice
+              },
+              {
+                speaker: 'Sam',
+                voiceConfig: {
+                  prebuiltVoiceConfig: {
+                    voiceName: 'Charon' // Neutral, professional voice
+                  }
                 }
               }
-            }
-          ]
+            ]
+          }
         }
       }
+    });
+
+    // Guarded response parsing - validate structure before accessing
+    if (!response?.response?.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data) {
+      throw new Error('TTS API returned malformed response - missing inlineData.data');
     }
+
+    return response;
   });
 
   const audioData = result.response.candidates[0].content.parts[0].inlineData;
@@ -238,6 +324,8 @@ async function generateAudio(dialogue, outputPath) {
 
   writeFileSync(outputPath, wavBuffer);
 
+  console.log(`  ✅ Audio synthesized successfully`);
+
   return {
     size: wavBuffer.length,
     format: 'audio/wav'
@@ -263,8 +351,46 @@ async function processFile(filePath, manifest) {
       return;
     }
 
-    // Generate dialogue
-    const dialogue = await generateDialogue(content, fileName);
+    // Generate dialogue with retry logic for token limit
+    const TOKEN_LIMIT = 8192;
+    const TOKEN_SAFETY_MARGIN = 500;
+    const MAX_TOKENS = TOKEN_LIMIT - TOKEN_SAFETY_MARGIN;
+
+    const retryLimits = [
+      { maxTokens: null, attempt: 0 },        // First try: soft guideline (7,000 tokens)
+      { maxTokens: 7000, attempt: 1 },        // Retry 1: 7,000 tokens
+      { maxTokens: 6000, attempt: 2 },        // Retry 2: 6,000 tokens
+      { maxTokens: 5500, attempt: 3 },        // Retry 3: 5,500 tokens (last resort)
+    ];
+
+    let dialogue;
+    let tokenCount;
+    let attemptSucceeded = false;
+
+    for (const { maxTokens, attempt } of retryLimits) {
+      if (attempt > 0) {
+        console.log(`  🔄 Retry ${attempt}: Regenerating with ${maxTokens} token limit...`);
+      }
+
+      dialogue = await generateDialogue(content, fileName, maxTokens);
+      tokenCount = await countDialogueTokens(dialogue);
+
+      console.log(`  📊 Token count: ${tokenCount} / ${MAX_TOKENS} (${((tokenCount / MAX_TOKENS) * 100).toFixed(1)}%)`);
+
+      if (tokenCount <= MAX_TOKENS) {
+        attemptSucceeded = true;
+        if (attempt > 0) {
+          console.log(`  ✅ Success on attempt ${attempt + 1}`);
+        }
+        break;
+      } else {
+        console.log(`  ⚠️  Exceeds limit by ${tokenCount - MAX_TOKENS} tokens`);
+      }
+    }
+
+    if (!attemptSucceeded) {
+      throw new Error(`Failed to generate dialogue within token limit after ${retryLimits.length} attempts. Final count: ${tokenCount} tokens`);
+    }
 
     // Determine output path
     const outputFileName = `${fileName}.wav`;
@@ -279,18 +405,53 @@ async function processFile(filePath, manifest) {
       audioUrl,
       size: audioInfo.size,
       format: audioInfo.format,
+      tokenCount: tokenCount,
       generatedAt: new Date().toISOString()
     };
 
     console.log(`  ✅ Generated: ${audioUrl}`);
-    console.log(`  📊 Size: ${(audioInfo.size / 1024 / 1024).toFixed(2)} MB`);
+    console.log(`  📊 Audio size: ${(audioInfo.size / 1024 / 1024).toFixed(2)} MB`);
+    console.log(`  📊 Tokens: ${tokenCount}`);
 
   } catch (error) {
     console.error(`  ❌ Error: ${error.message}`);
     console.error(`  Skipping this file and continuing...`);
   }
 }
 
+/**
+ * Process files with concurrency limit
+ * @param {string[]} files - Array of file paths to process
+ * @param {Object} manifest - Manifest object to update
+ * @param {number} concurrency - Max concurrent operations (default: 3)
+ * @returns {Promise<{processed: number, failed: number}>}
+ */
+async function processFilesWithConcurrency(files, manifest, concurrency = 3) {
+  const results = { processed: 0, failed: 0 };
+
+  // Process files in batches of `concurrency`
+  for (let i = 0; i < files.length; i += concurrency) {
+    const batch = files.slice(i, i + concurrency);
+
+    console.log(`\n🔄 Processing batch ${Math.floor(i / concurrency) + 1}/${Math.ceil(files.length / concurrency)} (${batch.length} files concurrently)...`);
+
+    // Process batch concurrently
+    await Promise.all(
+      batch.map(async (file) => {
+        try {
+          await processFile(file, manifest);
+          results.processed++;
+        } catch (error) {
+          console.error(`\n❌ Failed to process ${file}:`, error.message);
+          results.failed++;
+        }
+      })
+    );
+  }
+
+  return results;
+}
+
 /**
  * Main execution
  */
@@ -312,25 +473,8 @@ async function main() {
     console.log(`📋 Loaded existing manifest with ${Object.keys(manifest).length} entries\n`);
   }
 
-  // Process files sequentially (to avoid rate limits)
-  let processed = 0;
-  let skipped = 0;
-  let failed = 0;
-
-  for (const file of files) {
-    try {
-      await processFile(file, manifest);
-      processed++;
-
-      // Rate limiting - wait 2 seconds between files
-      if (processed < files.length) {
-        await new Promise(resolve => setTimeout(resolve, 2000));
-      }
-    } catch (error) {
-      console.error(`Failed to process ${file}:`, error.message);
-      failed++;
-    }
-  }
+  // Process files with concurrency limit of 3
+  const results = await processFilesWithConcurrency(files, manifest, 3);
 
   // Save manifest
   mkdirSync(dirname(MANIFEST_PATH), { recursive: true });
@@ -339,9 +483,9 @@ async function main() {
   console.log('\n' + '='.repeat(60));
   console.log('✨ Podcast generation complete!\n');
   console.log(`📊 Summary:`);
-  console.log(`   ✅ Processed: ${processed}`);
-  console.log(`   ⚠️  Skipped: ${skipped}`);
-  console.log(`   ❌ Failed: ${failed}`);
+  console.log(`   ✅ Processed: ${results.processed}`);
+  console.log(`   ❌ Failed: ${results.failed}`);
+  console.log(`   📁 Total files: ${files.length}`);
   console.log(`\n📋 Manifest saved to: ${MANIFEST_PATH}`);
   console.log('='.repeat(60));
 }