Add downloads dir to cache external dependency within the crawl (#921)

Fixes #920 
- Downloads profile, custom behavior, and seed list to `/downloads`
directory in the crawl
- Seed File: Downloaded into downloads. Never refetched if already
exists on subsequent crawl restarts.
- Custom Behaviors: Git: Downloaded into dir, then moved to
/downloads/behaviors/<dir name>. if already exist, failure to downloaded
will reuse existing directory
- Custom Behaviors: File: Downloaded into temp file, then moved to
/downloads/behaviors/<name.js>. if already exists, failure to download
will reuse existing file.
- Profile: using `/profile` directory to contain the browser profile
- Profile: downloaded to temp file, then placed into
/downloads/profile.tar.gz. If failed to download, but already exists,
existing /profile directory is used
- Also fixes #897

Author: ikreymer

src/crawler.ts

@@ -156,6 +156,8 @@ export class Crawler {
   warcCdxDir: string;
   indexesDir: string;
 
+  downloadsDir: string;
+
   screenshotWriter: WARCWriter | null;
   textWriter: WARCWriter | null;
 
@@ -289,6 +291,9 @@ export class Crawler {
     this.warcCdxDir = path.join(this.collDir, "warc-cdx");
     this.indexesDir = path.join(this.collDir, "indexes");
 
+    // download dirs
+    this.downloadsDir = path.join(this.collDir, "downloads");
+
     this.screenshotWriter = null;
     this.textWriter = null;
 
@@ -307,7 +312,7 @@ export class Crawler {
 
     this.customBehaviors = "";
 
-    this.browser = new Browser();
+    this.browser = new Browser(this.collDir);
   }
 
   protected parseArgs() {
@@ -503,6 +508,8 @@ export class Crawler {
       await fsp.mkdir(this.warcCdxDir, { recursive: true });
     }
 
+    await fsp.mkdir(this.downloadsDir, { recursive: true });
+
     this.logFH = fs.createWriteStream(this.logFilename, { flags: "a" });
     logger.setExternalLogStream(this.logFH);
 
@@ -514,7 +521,7 @@ export class Crawler {
     this.proxyServer = res.proxyServer;
     this.proxyPacUrl = res.proxyPacUrl;
 
-    this.seeds = await parseSeeds(this.params);
+    this.seeds = await parseSeeds(this.downloadsDir, this.params);
     this.numOriginalSeeds = this.seeds.length;
 
     logger.info("Seeds", this.seeds);
@@ -1015,7 +1022,10 @@ self.__bx_behaviors.selectMainBehavior();
   async loadCustomBehaviors(sources: string[]) {
     let str = "";
 
-    for (const { contents } of await collectCustomBehaviors(sources)) {
+    for (const { contents } of await collectCustomBehaviors(
+      this.downloadsDir,
+      sources,
+    )) {
       str += `self.__bx_behaviors.load(${contents});\n`;
     }
 
@@ -1029,7 +1039,10 @@ self.__bx_behaviors.selectMainBehavior();
       return;
     }
 
-    for (const { path, contents } of await collectCustomBehaviors(sources)) {
+    for (const { path, contents } of await collectCustomBehaviors(
+      this.downloadsDir,
+      sources,
+    )) {
       await this.browser.checkScript(cdp, path, contents);
     }
   }

src/create-login-profile.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 
 import fs from "fs";
+import os from "os";
 import http, { IncomingMessage, ServerResponse } from "http";
 
 import readline from "readline";
@@ -203,7 +204,7 @@ async function main() {
     ]);
   }
 
-  const browser = new Browser();
+  const browser = new Browser(os.tmpdir());
 
   await browser.launch({
     profileUrl: params.profile,

src/util/browser.ts

@@ -1,9 +1,10 @@
 import * as child_process from "child_process";
 import fs from "fs";
+import fsp from "node:fs/promises";
 import { pipeline } from "node:stream/promises";
 import { Readable } from "node:stream";
+import crypto from "crypto";
 
-import os from "os";
 import path from "path";
 
 import { formatErr, LogContext, logger } from "./logger.js";
@@ -31,6 +32,7 @@ import puppeteer, {
 import { Recorder } from "./recorder.js";
 import { timedRun } from "./timing.js";
 import assert from "node:assert";
+import { replaceDir } from "./file_reader.js";
 
 type BtrixChromeOpts = {
   proxyServer?: string;
@@ -61,6 +63,7 @@ const BROWSER_HEIGHT_OFFSET = 81;
 
 // ==================================================================
 export class Browser {
+  downloadsDir: string;
   profileDir: string;
   customProfile = false;
   // TODO: Fix this the next time the file is edited.
@@ -81,12 +84,9 @@ export class Browser {
   screenHeight: number;
   screenWHRatio: number;
 
-  constructor() {
-    this.profileDir = path.join(os.tmpdir(), "btrixProfile");
-    if (fs.existsSync(this.profileDir)) {
-      fs.rmSync(this.profileDir, { recursive: true, force: true });
-    }
-    fs.mkdirSync(this.profileDir);
+  constructor(rootDir: string) {
+    this.downloadsDir = path.join(rootDir, "downloads");
+    this.profileDir = path.join(rootDir, "profile");
 
     // must be provided, part of Dockerfile
     assert(process.env.GEOMETRY);
@@ -112,9 +112,7 @@ export class Browser {
       return;
     }
 
-    if (profileUrl) {
-      this.customProfile = await this.loadProfile(profileUrl);
-    }
+    await this.installProfile(profileUrl);
 
     this.swOpt = swOpt;
 
@@ -190,61 +188,97 @@ export class Browser {
     }
   }
 
-  async loadProfile(profileFilename: string): Promise<boolean> {
-    const targetFilename = path.join(os.tmpdir(), "profile.tar.gz");
+  async installProfile(profileUrl: string) {
+    await fsp.mkdir(this.profileDir, { recursive: true });
+
+    if (!profileUrl) {
+      return;
+    }
+
+    const profileTarGz = path.join(this.downloadsDir, "profile.tar.gz");
+
+    const exists = fs.existsSync(profileTarGz);
+
+    const suffix = crypto.randomBytes(4).toString("hex");
+
+    const tmpProfileDest = path.join(
+      this.downloadsDir,
+      `profile-${suffix}.tar.gz`,
+    );
+    const tmpProfileDir = path.join(this.downloadsDir, `profile-${suffix}`);
+
+    await fsp.mkdir(tmpProfileDir, { recursive: true });
+
+    try {
+      await this.loadProfile(profileUrl, tmpProfileDest, tmpProfileDir);
+
+      // replace old profile dir with new profile dir
+      await replaceDir(tmpProfileDir, this.profileDir, exists);
+
+      // replace old tarball with new tarball
+      await fsp.rename(tmpProfileDest, profileTarGz);
+    } catch (e) {
+      if (exists) {
+        logger.warn(
+          "Error updating profile, using existing profile",
+          formatErr(e),
+          "browser",
+        );
+      } else {
+        // remove the temp profile dir, likely empty
+        await fsp.rm(tmpProfileDir, { recursive: true });
+        logger.fatal("Profile setup failed", formatErr(e), "browser");
+      }
+    }
+    this.customProfile = true;
+  }
 
+  async loadProfile(
+    profileRemoteSrc: string,
+    profileLocalSrc: string,
+    profileDir: string,
+  ) {
     if (
-      profileFilename &&
-      (profileFilename.startsWith("http:") ||
-        profileFilename.startsWith("https:"))
+      profileRemoteSrc &&
+      (profileRemoteSrc.startsWith("http:") ||
+        profileRemoteSrc.startsWith("https:"))
     ) {
       logger.info(
-        `Downloading ${profileFilename} to ${targetFilename}`,
+        `Downloading ${profileRemoteSrc} to ${profileLocalSrc}`,
         {},
         "browser",
       );
 
-      const resp = await fetch(profileFilename);
+      const resp = await fetch(profileRemoteSrc);
       await pipeline(
         // TODO: Fix this the next time the file is edited.
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         Readable.fromWeb(resp.body as any),
-        fs.createWriteStream(targetFilename),
+        fs.createWriteStream(profileLocalSrc),
       );
-
-      profileFilename = targetFilename;
-    } else if (profileFilename && profileFilename.startsWith("@")) {
+    } else if (profileRemoteSrc && profileRemoteSrc.startsWith("@")) {
       const storage = initStorage();
 
       if (!storage) {
-        logger.fatal(
+        throw new Error(
           "Profile specified relative to s3 storage, but no S3 storage defined",
         );
-        return false;
       }
 
-      await storage.downloadFile(profileFilename.slice(1), targetFilename);
-
-      profileFilename = targetFilename;
+      await storage.downloadFile(profileRemoteSrc.slice(1), profileLocalSrc);
+    } else {
+      await fsp.copyFile(profileRemoteSrc, profileLocalSrc);
     }
 
-    if (profileFilename) {
-      try {
-        child_process.execSync("tar xvfz " + profileFilename, {
-          cwd: this.profileDir,
-        });
-        this.removeSingletons();
-        return true;
-      } catch (e) {
-        logger.fatal(
-          `Profile filename ${profileFilename} not a valid tar.gz, can not load profile, exiting`,
-          {},
-          "browser",
-        );
-      }
+    try {
+      child_process.execSync("tar xvfz " + profileLocalSrc, {
+        cwd: profileDir,
+        stdio: "ignore",
+      });
+      this.removeSingletons();
+    } catch (e) {
+      throw new Error(`Profile ${profileLocalSrc} not a valid tar.gz`);
     }
-
-    return false;
   }
 
   removeSingletons() {