feat: integrity check jobs (missing files, orphaned files, checksums)

e2e/src/api/specs/maintenance.e2e-spec.ts

@@ -1,9 +1,12 @@
-import { LoginResponseDto } from '@immich/sdk';
+import { IntegrityReportType, LoginResponseDto, ManualJobName, QueueName } from '@immich/sdk';
+import { readFile } from 'node:fs/promises';
 import { createUserDto } from 'src/fixtures';
 import { errorDto } from 'src/responses';
-import { app, utils } from 'src/utils';
+import { app, testAssetDir, utils } from 'src/utils';
 import request from 'supertest';
-import { beforeAll, describe, expect, it } from 'vitest';
+import { afterEach, beforeAll, describe, expect, it } from 'vitest';
+
+const assetFilepath = `${testAssetDir}/metadata/gps-position/thompson-springs.jpg`;
 
 describe('/admin/maintenance', () => {
   let cookie: string | undefined;
@@ -34,6 +37,188 @@
     });
   });
 
+  describe('POST /integrity/summary (& jobs)', async () => {
+    let baseline: Record<IntegrityReportType, number>;
+
+    beforeAll(async () => {
+      await utils.createAsset(admin.accessToken, {
+        assetData: {
+          filename: 'asset.jpg',
+          bytes: await readFile(assetFilepath),
+        },
+      });
+
+      await utils.copyFolder(`/data/upload/${admin.userId}`, `/data/upload/${admin.userId}-bak`);
+    });
+
+    afterEach(async () => {
+      await utils.deleteFolder(`/data/upload/${admin.userId}`);
+      await utils.copyFolder(`/data/upload/${admin.userId}-bak`, `/data/upload/${admin.userId}`);
+    });
+
+    it.sequential('may report issues', async () => {
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityOrphanFiles,
+      });
+
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityMissingFiles,
+      });
+
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityChecksumMismatch,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+
+      const { status, body } = await request(app)
+        .get('/admin/maintenance/integrity/summary')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send();
+
+      expect(status).toBe(200);
+      expect(body).toEqual({
+        missing_file: 0,
+        orphan_file: expect.any(Number),
+        checksum_mismatch: 0,
+      });
+
+      baseline = body;
+    });
+
+    it.sequential('should detect an orphan file (job: check orphan files)', async () => {
+      await utils.putTextFile('orphan', `/data/upload/${admin.userId}/orphan1.png`);
+
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityOrphanFiles,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+
+      const { status, body } = await request(app)
+        .get('/admin/maintenance/integrity/summary')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send();
+
+      expect(status).toBe(200);
+      expect(body).toEqual(
+        expect.objectContaining({
+          orphan_file: baseline.orphan_file + 1,
+        }),
+      );
+    });
+
+    it.sequential('should detect outdated orphan file reports (job: refresh orphan files)', async () => {
+      // these should not be detected:
+      await utils.putTextFile('orphan', `/data/upload/${admin.userId}/orphan2.png`);
+      await utils.putTextFile('orphan', `/data/upload/${admin.userId}/orphan3.png`);
+
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityOrphanFilesRefresh,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+
+      const { status, body } = await request(app)
+        .get('/admin/maintenance/integrity/summary')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send();
+
+      expect(status).toBe(200);
+      expect(body).toEqual(
+        expect.objectContaining({
+          orphan_file: baseline.orphan_file,
+        }),
+      );
+    });
+
+    it.sequential('should detect a missing file and not a checksum mismatch (job: check missing files)', async () => {
+      await utils.deleteFolder(`/data/upload/${admin.userId}`);
+
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityMissingFiles,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+
+      const { status, body } = await request(app)
+        .get('/admin/maintenance/integrity/summary')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send();
+
+      expect(status).toBe(200);
+      expect(body).toEqual(
+        expect.objectContaining({
+          missing_file: 1,
+          checksum_mismatch: 0,
+        }),
+      );
+    });
+
+    it.sequential('should detect outdated missing file reports (job: refresh missing files)', async () => {
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityMissingFilesRefresh,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+
+      const { status, body } = await request(app)
+        .get('/admin/maintenance/integrity/summary')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send();
+
+      expect(status).toBe(200);
+      expect(body).toEqual(
+        expect.objectContaining({
+          missing_file: 0,
+          checksum_mismatch: 0,
+        }),
+      );
+    });
+
+    it.sequential('should detect a checksum mismatch (job: check file checksums)', async () => {
+      await utils.truncateFolder(`/data/upload/${admin.userId}`);
+
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityChecksumMismatch,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+
+      const { status, body } = await request(app)
+        .get('/admin/maintenance/integrity/summary')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send();
+
+      expect(status).toBe(200);
+      expect(body).toEqual(
+        expect.objectContaining({
+          checksum_mismatch: 1,
+        }),
+      );
+    });
+
+    it.sequential('should detect outdated checksum mismatch reports (job: refresh file checksums)', async () => {
+      await utils.createJob(admin.accessToken, {
+        name: ManualJobName.IntegrityChecksumMismatchRefresh,
+      });
+
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+
+      const { status, body } = await request(app)
+        .get('/admin/maintenance/integrity/summary')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send();
+
+      expect(status).toBe(200);
+      expect(body).toEqual(
+        expect.objectContaining({
+          checksum_mismatch: 0,
+        }),
+      );
+    });
+  });
+
   // => enter maintenance mode
 
   describe.sequential('POST /', () => {
@@ -83,8 +268,8 @@
             return body.maintenanceMode;
           },
           {
-            interval: 5e2,
-            timeout: 1e4,
+            interval: 500,
+            timeout: 60_000,
           },
         )
         .toBeTruthy();
@@ -148,7 +333,7 @@
   // => exit maintenance mode
 
   describe.sequential('POST /', () => {
     it('should exit maintenance mode', async () => {
       const { status } = await request(app).post('/admin/maintenance').set('cookie', cookie!).send({
         action: 'end',
       });
@@ -162,8 +347,8 @@
             return body.maintenanceMode;
           },
           {
-            interval: 5e2,
-            timeout: 1e4,
+            interval: 500,
+            timeout: 60_000,
           },
         )
         .toBeFalsy();

e2e/src/utils.ts

@@ -6,6 +6,7 @@ import {
   CheckExistingAssetsDto,
   CreateAlbumDto,
   CreateLibraryDto,
+  JobCreateDto,
   MaintenanceAction,
   MetadataSearchDto,
   Permission,
@@ -21,6 +22,7 @@ import {
   checkExistingAssets,
   createAlbum,
   createApiKey,
+  createJob,
   createLibrary,
   createPartner,
   createPerson,
@@ -52,9 +54,12 @@ import {
 import { BrowserContext } from '@playwright/test';
 import { exec, spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { createWriteStream, existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdtemp } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
-import { dirname, resolve } from 'node:path';
+import { dirname, join, resolve } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { setTimeout as setAsyncTimeout } from 'node:timers/promises';
 import { promisify } from 'node:util';
 import pg from 'pg';
@@ -171,6 +176,7 @@ export const utils = {
         'user',
         'system_metadata',
         'tag',
+        'integrity_report',
       ];
 
       const sql: string[] = [];
@@ -481,6 +487,9 @@ export const utils = {
   tagAssets: (accessToken: string, tagId: string, assetIds: string[]) =>
     tagAssets({ id: tagId, bulkIdsDto: { ids: assetIds } }, { headers: asBearerAuth(accessToken) }),
 
+  createJob: async (accessToken: string, jobCreateDto: JobCreateDto) =>
+    createJob({ jobCreateDto }, { headers: asBearerAuth(accessToken) }),
+
   queueCommand: async (accessToken: string, name: QueueName, queueCommandDto: QueueCommandDto) =>
     runQueueCommandLegacy({ name, queueCommandDto }, { headers: asBearerAuth(accessToken) }),
 
@@ -559,6 +568,50 @@ export const utils = {
     mkdirSync(`${testAssetDir}/temp`, { recursive: true });
   },
 
+  putFile(source: string, dest: string) {
+    return executeCommand('docker', ['cp', source, `immich-e2e-server:${dest}`]).promise;
+  },
+
+  async putTextFile(contents: string, dest: string) {
+    const dir = await mkdtemp(join(tmpdir(), 'test-'));
+    const fn = join(dir, 'file');
+    await pipeline(Readable.from(contents), createWriteStream(fn));
+    return executeCommand('docker', ['cp', fn, `immich-e2e-server:${dest}`]).promise;
+  },
+
+  async move(source: string, dest: string) {
+    return executeCommand('docker', ['exec', 'immich-e2e-server', 'mv', source, dest]).promise;
+  },
+
+  async copyFolder(source: string, dest: string) {
+    return executeCommand('docker', ['exec', 'immich-e2e-server', 'cp', '-r', source, dest]).promise;
+  },
+
+  async deleteFile(path: string) {
+    return executeCommand('docker', ['exec', 'immich-e2e-server', 'rm', path]).promise;
+  },
+
+  async deleteFolder(path: string) {
+    return executeCommand('docker', ['exec', 'immich-e2e-server', 'rm', '-r', path]).promise;
+  },
+
+  async truncateFolder(path: string) {
+    return executeCommand('docker', [
+      'exec',
+      'immich-e2e-server',
+      'find',
+      path,
+      '-type',
+      'f',
+      '-exec',
+      'truncate',
+      '-s',
+      '1',
+      '{}',
+      ';',
+    ]).promise;
+  },
+
   resetAdminConfig: async (accessToken: string) => {
     const defaultConfig = await getConfigDefaults({ headers: asBearerAuth(accessToken) });
     await updateConfig({ systemConfigDto: defaultConfig }, { headers: asBearerAuth(accessToken) });

i18n/en.json

@@ -177,6 +177,16 @@
     "machine_learning_smart_search_enabled": "Enable smart search",
     "machine_learning_smart_search_enabled_description": "If disabled, images will not be encoded for smart search.",
     "machine_learning_url_description": "The URL of the machine learning server. If more than one URL is provided, each server will be attempted one-at-a-time until one responds successfully, in order from first to last. Servers that don't respond will be temporarily ignored until they come back online.",
+    "maintenance_integrity_checksum_mismatch": "Checksum Mismatch",
+    "maintenance_integrity_checksum_mismatch_job": "Check for checksum mismatches",
+    "maintenance_integrity_checksum_mismatch_refresh_job": "Refresh checksum mismatch reports",
+    "maintenance_integrity_missing_file": "Missing Files",
+    "maintenance_integrity_missing_file_job": "Check for missing files",
+    "maintenance_integrity_missing_file_refresh_job": "Refresh missing file reports",
+    "maintenance_integrity_orphan_file": "Orphan Files",
+    "maintenance_integrity_orphan_file_job": "Check for orphaned files",
+    "maintenance_integrity_orphan_file_refresh_job": "Refresh orphan file reports",
+    "maintenance_integrity_report": "Integrity Report",
     "maintenance_settings": "Maintenance",
     "maintenance_settings_description": "Put Immich into maintenance mode.",
     "maintenance_start": "Start maintenance mode",