ignore perm errors

e2e/docker-compose.yml

@@ -32,6 +32,9 @@ services:
       - database
     ports:
       - 2285:2285
+    cap_drop:
+      # We need this to perform testing on permission errors
+      - DAC_OVERRIDE
 
   redis:
     image: redis:6.2-alpine@sha256:905c4ee67b8e0aa955331960d2aa745781e6bd89afc44a8584bfd13bc890f0ae

e2e/src/api/specs/library.e2e-spec.ts

@@ -1,5 +1,5 @@
 import { LibraryResponseDto, LoginResponseDto, getAllLibraries, scanLibrary } from '@immich/sdk';
-import { cpSync, existsSync, rmSync, unlinkSync } from 'node:fs';
+import { chmodSync, cpSync, existsSync, promises, rmSync, unlinkSync } from 'node:fs';
 import { Socket } from 'socket.io-client';
 import { userDto, uuidDto } from 'src/fixtures';
 import { errorDto } from 'src/responses';
@@ -492,6 +492,34 @@ describe('/libraries', () => {
       utils.removeImageFile(`${testAssetDir}/temp/folder${char}2/asset2.png`);
     });
 
+    it('should handle permission errors on import paths without error', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        ownerId: admin.userId,
+        importPaths: [`${testAssetDirInternal}/temp/`],
+      });
+
+      const stat = await promises.stat(`${testAssetDir}/temp/directoryA`);
+      const mode = stat.mode;
+
+      chmodSync(`${testAssetDir}/temp/directoryB`, 0o000);
+
+      const { status } = await request(app)
+        .post(`/libraries/${library.id}/scan`)
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send();
+      expect(status).toBe(204);
+
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      chmodSync(`${testAssetDir}/temp/directoryB`, mode);
+
+      const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
+
+      expect(assets.count).toBe(1);
+      expect(assets.items.find((asset) => asset.originalPath.includes('directoryA'))).toBeDefined();
+      expect(assets.items.find((asset) => asset.originalPath.includes('directoryB'))).not.toBeDefined();
+    });
+
     it('should reimport a modified file', async () => {
       const library = await utils.createLibrary(admin.accessToken, {
         ownerId: admin.userId,

server/package-lock.json

@@ -34,7 +34,7 @@
         "class-validator": "^0.14.0",
         "cookie-parser": "^1.4.6",
         "exiftool-vendored": "^28.3.1",
-        "fast-glob": "^3.3.2",
+        "fast-glob": "github:etnoy/fast-glob#built",
         "fluent-ffmpeg": "^2.1.2",
         "geo-tz": "^8.0.0",
         "handlebars": "^4.7.8",
@@ -2466,38 +2466,38 @@
       }
     },
     "node_modules/@nodelib/fs.scandir": {
-      "version": "2.1.5",
+      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-4.0.1.tgz",
-      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "integrity": "sha512-vAkI715yhnmiPupY+dq+xenu5Tdf2TBQ66jLvBIcCddtz+5Q8LbMKaf9CIJJreez8fQ8fgaY+RaywQx8RJIWpw==",
       "license": "MIT",
       "dependencies": {
-        "@nodelib/fs.stat": "2.0.5",
+        "@nodelib/fs.stat": "4.0.0",
-        "run-parallel": "^1.1.9"
+        "run-parallel": "^1.2.0"
       },
       "engines": {
-        "node": ">= 8"
+        "node": ">=18.18.0"
       }
     },
     "node_modules/@nodelib/fs.stat": {
-      "version": "2.0.5",
+      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-4.0.0.tgz",
-      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "integrity": "sha512-ctr6bByzksKRCV0bavi8WoQevU6plSp2IkllIsEqaiKe2mwNNnaluhnRhcsgGZHrrHk57B3lf95MkLMO3STYcg==",
       "license": "MIT",
       "engines": {
-        "node": ">= 8"
+        "node": ">=18.18.0"
       }
     },
     "node_modules/@nodelib/fs.walk": {
-      "version": "1.2.8",
+      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-3.0.1.tgz",
-      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "integrity": "sha512-nIh/M6Kh3ZtOmlY00DaUYB4xeeV6F3/ts1l29iwl3/cfyY/OuCfUx+v08zgx8TKPTifXRcjjqVQ4KB2zOYSbyw==",
       "license": "MIT",
       "dependencies": {
-        "@nodelib/fs.scandir": "2.1.5",
+        "@nodelib/fs.scandir": "4.0.1",
-        "fastq": "^1.6.0"
+        "fastq": "^1.15.0"
       },
       "engines": {
-        "node": ">= 8"
+        "node": ">=18.18.0"
       }
     },
     "node_modules/@nuxtjs/opencollective": {
@@ -5559,6 +5559,44 @@
         "typescript": ">=4.8.4 <5.8.0"
       }
     },
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.scandir": {
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.stat": "2.0.5",
+        "run-parallel": "^1.1.9"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.stat": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.walk": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.scandir": "2.1.5",
+        "fastq": "^1.6.0"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@@ -5569,6 +5607,23 @@
         "balanced-match": "^1.0.0"
       }
     },
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/fast-glob": {
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
+      "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.2",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.8"
+      },
+      "engines": {
+        "node": ">=8.6.0"
+      }
+    },
     "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {
       "version": "9.0.5",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
@@ -8701,19 +8756,30 @@
       "license": "MIT"
     },
     "node_modules/fast-glob": {
-      "version": "3.3.3",
+      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
+      "resolved": "git+ssh://git@github.com/etnoy/fast-glob.git#7d0d2f912eeadcbc3e4af55249cbbcabe8d0808b",
-      "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
       "license": "MIT",
       "dependencies": {
-        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.stat": "^4.0.0",
-        "@nodelib/fs.walk": "^1.2.3",
+        "@nodelib/fs.walk": "^3.0.0",
-        "glob-parent": "^5.1.2",
+        "glob-parent": "^6.0.2",
-        "merge2": "^1.3.0",
+        "merge2": "^1.4.1",
         "micromatch": "^4.0.8"
       },
       "engines": {
-        "node": ">=8.6.0"
+        "node": ">=18.18.0"
+      }
+    },
+    "node_modules/fast-glob/node_modules/glob-parent": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+      "license": "ISC",
+      "dependencies": {
+        "is-glob": "^4.0.3"
+      },
+      "engines": {
+        "node": ">=10.13.0"
       }
     },
     "node_modules/fast-json-stable-stringify": {
@@ -8744,9 +8810,9 @@
       "license": "BSD-3-Clause"
     },
     "node_modules/fastq": {
-      "version": "1.17.1",
+      "version": "1.19.0",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.0.tgz",
-      "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==",
+      "integrity": "sha512-7SFSRCNjBQIZH/xZR3iy5iQYR8aGBE0h3VG6/cwlbrpdciNYBMotQav8c1XI3HjHH+NikUpP53nPdlZSdWmFzA==",
       "license": "ISC",
       "dependencies": {
         "reusify": "^1.0.4"
@@ -14287,6 +14353,74 @@
         "tailwindcss": ">=3.4.17"
       }
     },
+    "node_modules/tailwindcss/node_modules/@nodelib/fs.scandir": {
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "license": "MIT",
+      "peer": true,
+      "dependencies": {
+        "@nodelib/fs.stat": "2.0.5",
+        "run-parallel": "^1.1.9"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/tailwindcss/node_modules/@nodelib/fs.stat": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "license": "MIT",
+      "peer": true,
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/tailwindcss/node_modules/@nodelib/fs.walk": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "license": "MIT",
+      "peer": true,
+      "dependencies": {
+        "@nodelib/fs.scandir": "2.1.5",
+        "fastq": "^1.6.0"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/tailwindcss/node_modules/fast-glob": {
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
+      "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
+      "license": "MIT",
+      "peer": true,
+      "dependencies": {
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.2",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.8"
+      },
+      "engines": {
+        "node": ">=8.6.0"
+      }
+    },
+    "node_modules/tailwindcss/node_modules/fast-glob/node_modules/glob-parent": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+      "license": "ISC",
+      "peer": true,
+      "dependencies": {
+        "is-glob": "^4.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/tailwindcss/node_modules/glob-parent": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",

server/package.json

@@ -60,7 +60,7 @@
     "class-validator": "^0.14.0",
     "cookie-parser": "^1.4.6",
     "exiftool-vendored": "^28.3.1",
-    "fast-glob": "^3.3.2",
+    "fast-glob": "github:etnoy/fast-glob#built",
     "fluent-ffmpeg": "^2.1.2",
     "geo-tz": "^8.0.0",
     "handlebars": "^4.7.8",

server/src/repositories/storage.repository.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
 import archiver from 'archiver';
 import chokidar, { WatchOptions } from 'chokidar';
 import { escapePath, glob, globStream } from 'fast-glob';
+import { ErrnoException } from 'fast-glob/out/types';
 import { constants, createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
@@ -170,6 +171,19 @@ export class StorageRepository implements IStorageRepository {
     });
   }
 
+  private errorHandler = (error: ErrnoException) => {
+    if (error.code === 'ENOENT') {
+      this.logger.warn(`Path ${error.path} does not exist, ignoring.`);
+      return true;
+    } else if (error.code === 'EACCES') {
+      this.logger.warn(`Permission denied for path ${error.path}, ignoring.`);
+      return true;
+    }
+
+    this.logger.error(`Error while walking path ${error.path}: ${error.message}`);
+    return false;
+  };
+
   async *walk(walkOptions: WalkOptionsDto): AsyncGenerator<string[]> {
     const { pathsToCrawl, exclusionPatterns, includeHidden } = walkOptions;
     if (pathsToCrawl.length === 0) {
@@ -185,6 +199,7 @@ export class StorageRepository implements IStorageRepository {
       onlyFiles: true,
       dot: includeHidden,
       ignore: exclusionPatterns,
+      errorHandler: this.errorHandler,
     });
 
     let batch: string[] = [];