Infrastructure/immich
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-02-18 01:24:26 -05:00
Code Issues Activity

ignore perm errors

Browse source
This commit is contained in:
Jonathan Jogenfors 2025-02-06 14:58:18 +01:00
parent 48d421e28c
commit 43d66c7a36
5 changed files with 210 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
e2e/docker-compose.yml
View file

@ -32,6 +32,9 @@ services:
- database
ports:
- 2285:2285
cap_drop:
# We need this to perform testing on permission errors
- DAC_OVERRIDE
redis:
image: redis:6.2-alpine@sha256:905c4ee67b8e0aa955331960d2aa745781e6bd89afc44a8584bfd13bc890f0ae

30
e2e/src/api/specs/library.e2e-spec.ts
View file

@ -1,5 +1,5 @@
import { LibraryResponseDto, LoginResponseDto, getAllLibraries, scanLibrary } from '@immich/sdk';
import { cpSync, existsSync, rmSync, unlinkSync } from 'node:fs';
import { chmodSync, cpSync, existsSync, promises, rmSync, unlinkSync } from 'node:fs';
import { Socket } from 'socket.io-client';
import { userDto, uuidDto } from 'src/fixtures';
import { errorDto } from 'src/responses';
@ -492,6 +492,34 @@ describe('/libraries', () => {
utils.removeImageFile(`${testAssetDir}/temp/folder${char}2/asset2.png`);
});
it('should handle permission errors on import paths without error', async () => {
const library = await utils.createLibrary(admin.accessToken, {
ownerId: admin.userId,
importPaths: [`${testAssetDirInternal}/temp/`],
});
const stat = await promises.stat(`${testAssetDir}/temp/directoryA`);
const mode = stat.mode;
chmodSync(`${testAssetDir}/temp/directoryB`, 0o000);
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send();
expect(status).toBe(204);
await utils.waitForQueueFinish(admin.accessToken, 'library');
chmodSync(`${testAssetDir}/temp/directoryB`, mode);
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
expect(assets.count).toBe(1);
expect(assets.items.find((asset) => asset.originalPath.includes('directoryA'))).toBeDefined();
expect(assets.items.find((asset) => asset.originalPath.includes('directoryB'))).not.toBeDefined();
});
it('should reimport a modified file', async () => {
const library = await utils.createLibrary(admin.accessToken, {
ownerId: admin.userId,

190
server/package-lock.json generated
View file

@ -34,7 +34,7 @@
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.6",
"exiftool-vendored": "^28.3.1",
"fast-glob": "^3.3.2",
"fast-glob": "github:etnoy/fast-glob#built",
"fluent-ffmpeg": "^2.1.2",
"geo-tz": "^8.0.0",
"handlebars": "^4.7.8",
@ -2466,38 +2466,38 @@
}
},
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-4.0.1.tgz",
"integrity": "sha512-vAkI715yhnmiPupY+dq+xenu5Tdf2TBQ66jLvBIcCddtz+5Q8LbMKaf9CIJJreez8fQ8fgaY+RaywQx8RJIWpw==",
"license": "MIT",
"dependencies": {
"@nodelib/fs.stat": "2.0.5",
"run-parallel": "^1.1.9"
"@nodelib/fs.stat": "4.0.0",
"run-parallel": "^1.2.0"
},
"engines": {
"node": ">= 8"
"node": ">=18.18.0"
}
},
"node_modules/@nodelib/fs.stat": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-4.0.0.tgz",
"integrity": "sha512-ctr6bByzksKRCV0bavi8WoQevU6plSp2IkllIsEqaiKe2mwNNnaluhnRhcsgGZHrrHk57B3lf95MkLMO3STYcg==",
"license": "MIT",
"engines": {
"node": ">= 8"
"node": ">=18.18.0"
}
},
"node_modules/@nodelib/fs.walk": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-3.0.1.tgz",
"integrity": "sha512-nIh/M6Kh3ZtOmlY00DaUYB4xeeV6F3/ts1l29iwl3/cfyY/OuCfUx+v08zgx8TKPTifXRcjjqVQ4KB2zOYSbyw==",
"license": "MIT",
"dependencies": {
"@nodelib/fs.scandir": "2.1.5",
"fastq": "^1.6.0"
"@nodelib/fs.scandir": "4.0.1",
"fastq": "^1.15.0"
},
"engines": {
"node": ">= 8"
"node": ">=18.18.0"
}
},
"node_modules/@nuxtjs/opencollective": {
@ -5559,6 +5559,44 @@
"typescript": ">=4.8.4 <5.8.0"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
"dev": true,
"license": "MIT",
"dependencies": {
"@nodelib/fs.stat": "2.0.5",
"run-parallel": "^1.1.9"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.stat": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 8"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.walk": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@nodelib/fs.scandir": "2.1.5",
"fastq": "^1.6.0"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@ -5569,6 +5607,23 @@
"balanced-match": "^1.0.0"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/fast-glob": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
"integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@nodelib/fs.stat": "^2.0.2",
"@nodelib/fs.walk": "^1.2.3",
"glob-parent": "^5.1.2",
"merge2": "^1.3.0",
"micromatch": "^4.0.8"
},
"engines": {
"node": ">=8.6.0"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
@ -8701,19 +8756,30 @@
"license": "MIT"
},
"node_modules/fast-glob": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
"integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
"version": "4.0.0",
"resolved": "git+ssh://git@github.com/etnoy/fast-glob.git#7d0d2f912eeadcbc3e4af55249cbbcabe8d0808b",
"license": "MIT",
"dependencies": {
"@nodelib/fs.stat": "^2.0.2",
"@nodelib/fs.walk": "^1.2.3",
"glob-parent": "^5.1.2",
"merge2": "^1.3.0",
"@nodelib/fs.stat": "^4.0.0",
"@nodelib/fs.walk": "^3.0.0",
"glob-parent": "^6.0.2",
"merge2": "^1.4.1",
"micromatch": "^4.0.8"
},
"engines": {
"node": ">=8.6.0"
"node": ">=18.18.0"
}
},
"node_modules/fast-glob/node_modules/glob-parent": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
"integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
"license": "ISC",
"dependencies": {
"is-glob": "^4.0.3"
},
"engines": {
"node": ">=10.13.0"
}
},
"node_modules/fast-json-stable-stringify": {
@ -8744,9 +8810,9 @@
"license": "BSD-3-Clause"
},
"node_modules/fastq": {
"version": "1.17.1",
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz",
"integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==",
"version": "1.19.0",
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.0.tgz",
"integrity": "sha512-7SFSRCNjBQIZH/xZR3iy5iQYR8aGBE0h3VG6/cwlbrpdciNYBMotQav8c1XI3HjHH+NikUpP53nPdlZSdWmFzA==",
"license": "ISC",
"dependencies": {
"reusify": "^1.0.4"
@ -14287,6 +14353,74 @@
"tailwindcss": ">=3.4.17"
}
},
"node_modules/tailwindcss/node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
"license": "MIT",
"peer": true,
"dependencies": {
"@nodelib/fs.stat": "2.0.5",
"run-parallel": "^1.1.9"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/tailwindcss/node_modules/@nodelib/fs.stat": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 8"
}
},
"node_modules/tailwindcss/node_modules/@nodelib/fs.walk": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
"license": "MIT",
"peer": true,
"dependencies": {
"@nodelib/fs.scandir": "2.1.5",
"fastq": "^1.6.0"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/tailwindcss/node_modules/fast-glob": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
"integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
"license": "MIT",
"peer": true,
"dependencies": {
"@nodelib/fs.stat": "^2.0.2",
"@nodelib/fs.walk": "^1.2.3",
"glob-parent": "^5.1.2",
"merge2": "^1.3.0",
"micromatch": "^4.0.8"
},
"engines": {
"node": ">=8.6.0"
}
},
"node_modules/tailwindcss/node_modules/fast-glob/node_modules/glob-parent": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
"license": "ISC",
"peer": true,
"dependencies": {
"is-glob": "^4.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/tailwindcss/node_modules/glob-parent": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",

2
server/package.json
View file

@ -60,7 +60,7 @@
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.6",
"exiftool-vendored": "^28.3.1",
"fast-glob": "^3.3.2",
"fast-glob": "github:etnoy/fast-glob#built",
"fluent-ffmpeg": "^2.1.2",
"geo-tz": "^8.0.0",
"handlebars": "^4.7.8",

15
server/src/repositories/storage.repository.ts
View file

@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
import archiver from 'archiver';
import chokidar, { WatchOptions } from 'chokidar';
import { escapePath, glob, globStream } from 'fast-glob';
import { ErrnoException } from 'fast-glob/out/types';
import { constants, createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs';
import fs from 'node:fs/promises';
import path from 'node:path';
@ -170,6 +171,19 @@ export class StorageRepository implements IStorageRepository {
});
}
private errorHandler = (error: ErrnoException) => {
if (error.code === 'ENOENT') {
this.logger.warn(`Path ${error.path} does not exist, ignoring.`);
return true;
} else if (error.code === 'EACCES') {
this.logger.warn(`Permission denied for path ${error.path}, ignoring.`);
return true;
}
this.logger.error(`Error while walking path ${error.path}: ${error.message}`);
return false;
};
async *walk(walkOptions: WalkOptionsDto): AsyncGenerator<string[]> {
const { pathsToCrawl, exclusionPatterns, includeHidden } = walkOptions;
if (pathsToCrawl.length === 0) {
@ -185,6 +199,7 @@ export class StorageRepository implements IStorageRepository {
onlyFiles: true,
dot: includeHidden,
ignore: exclusionPatterns,
errorHandler: this.errorHandler,
});
let batch: string[] = [];