From 5a3a2c729310fe6346d8d72c580572dd9b322218 Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Tue, 19 Dec 2023 20:15:11 +0100 Subject: [PATCH] feat(cli): Allow uploading a single file (#5837) * Allow building and installing cli * feat: add format fix * docs: remove cli folder * feat: use immich scoped package * feat: rewrite cli readme * docs: add info on running without building * cleanup * chore: remove import functionality from cli * feat: add logout to cli * docs: add todo for file format from server * docs: add compilation step to cli * fix: success message spacing * feat: can create albums * fix: add check step to cli * fix: typos * feat: pull file formats from server * chore: use crawl service from server * chore: fix lint * docs: add cli documentation * chore: rename ignore pattern * chore: add version number to cli * feat: use sdk * fix: cleanup * feat: album name on windows * chore: remove skipped asset field * feat: add more info to server-info command * chore: cleanup * wip * chore: remove unneeded packages * e2e test can start * git ignore for geocode in cli * add cli e2e to github actions * can do e2e tests in the cli * simplify e2e test * cleanup * set matrix strategy in workflow * run npm ci in server * choose different working directory * check out submodules too * increase test timeout * set node version * cli docker e2e tests * fix cli docker file * run cli e2e in correct folder * set docker context * correct docker build * remove cli from dockerignore * chore: fix docs links * feat: add cli v2 milestone * fix: set correct cli date * remove submodule * chore: add npmignore * chore(cli): push to npm * fix: server e2e * run npm ci in server * remove state from e2e * run npm ci in server * reshuffle docker compose files * use new e2e composes in makefile * increase test timeout to 10 minutes * make github actions run makefile e2e tests * cleanup github test names * assert on server version * chore: split cli e2e tests into one file per command * chore: set cli release working dir * chore: add repo url to npmjs * chore: bump node setup to v4 * chore: normalize the github url * check e2e code in lint * fix lint * test key login flow * feat: allow configurable config dir * fix session service tests * create missing dir * cleanup * bump cli version to 2.0.4 * remove form-data * feat: allow single files as argument * add version option * bump dependencies * fix lint * wip use axios as upload * version bump * cApiTALiZaTiON * don't touch package lock * wip: don't use job queues * don't use make for cli e2e * fix server e2e * feat: can upload single file * fix upload options dto --------- Co-authored-by: Alex Co-authored-by: Jason Rasmussen --- cli/src/commands/upload.ts | 1 + cli/src/cores/dto/upload-options-dto.ts | 1 + cli/src/index.ts | 1 + cli/src/services/crawl.service.spec.ts | 23 ++++++++++++- cli/src/services/crawl.service.ts | 45 ++++++++++++++++++++++--- 5 files changed, 66 insertions(+), 5 deletions(-) diff --git a/cli/src/commands/upload.ts b/cli/src/commands/upload.ts index 58c5785583..b0f192dca2 100644 --- a/cli/src/commands/upload.ts +++ b/cli/src/commands/upload.ts @@ -22,6 +22,7 @@ export default class Upload extends BaseCommand { crawlOptions.pathsToCrawl = paths; crawlOptions.recursive = options.recursive; crawlOptions.exclusionPatterns = options.exclusionPatterns; + crawlOptions.includeHidden = options.includeHidden; const files: string[] = []; diff --git a/cli/src/cores/dto/upload-options-dto.ts b/cli/src/cores/dto/upload-options-dto.ts index 77bd7cd493..8b5fbdc4a7 100644 --- a/cli/src/cores/dto/upload-options-dto.ts +++ b/cli/src/cores/dto/upload-options-dto.ts @@ -5,4 +5,5 @@ export class UploadOptionsDto { skipHash? = false; delete? = false; album? = false; + includeHidden? = false; } diff --git a/cli/src/index.ts b/cli/src/index.ts index 8f538ead8b..3e5b74c47d 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -26,6 +26,7 @@ program .addOption(new Option('-r, --recursive', 'Recursive').env('IMMICH_RECURSIVE').default(false)) .addOption(new Option('-i, --ignore [paths...]', 'Paths to ignore').env('IMMICH_IGNORE_PATHS')) .addOption(new Option('-h, --skip-hash', "Don't hash files before upload").env('IMMICH_SKIP_HASH').default(false)) + .addOption(new Option('-i, --include-hidden', 'Include hidden folders').env('IMMICH_INCLUDE_HIDDEN').default(false)) .addOption( new Option('-a, --album', 'Automatically create albums based on folder name') .env('IMMICH_AUTO_CREATE_ALBUM') diff --git a/cli/src/services/crawl.service.spec.ts b/cli/src/services/crawl.service.spec.ts index 3957f193a8..4a30669d69 100644 --- a/cli/src/services/crawl.service.spec.ts +++ b/cli/src/services/crawl.service.spec.ts @@ -19,7 +19,7 @@ const tests: Test[] = [ files: {}, }, { - test: 'should crawl a single path', + test: 'should crawl a single folder', options: { pathsToCrawl: ['/photos/'], }, @@ -27,6 +27,25 @@ const tests: Test[] = [ '/photos/image.jpg': true, }, }, + { + test: 'should crawl a single file', + options: { + pathsToCrawl: ['/photos/image.jpg'], + }, + files: { + '/photos/image.jpg': true, + }, + }, + { + test: 'should crawl a single file and a folder', + options: { + pathsToCrawl: ['/photos/image.jpg', '/images/'], + }, + files: { + '/photos/image.jpg': true, + '/images/image2.jpg': true, + }, + }, { test: 'should exclude by file extension', options: { @@ -54,6 +73,7 @@ const tests: Test[] = [ options: { pathsToCrawl: ['/photos/'], exclusionPatterns: ['**/raw/**'], + recursive: true, }, files: { '/photos/image.jpg': true, @@ -98,6 +118,7 @@ const tests: Test[] = [ test: 'should crawl a single path', options: { pathsToCrawl: ['/photos/'], + recursive: true, }, files: { '/photos/image.jpg': true, diff --git a/cli/src/services/crawl.service.ts b/cli/src/services/crawl.service.ts index 28d7fb9126..ff0f5e3dfc 100644 --- a/cli/src/services/crawl.service.ts +++ b/cli/src/services/crawl.service.ts @@ -1,5 +1,6 @@ import { CrawlOptionsDto } from 'src/cores/dto/crawl-options-dto'; import { glob } from 'glob'; +import * as fs from 'fs'; export class CrawlService { private readonly extensions!: string[]; @@ -8,21 +9,57 @@ export class CrawlService { this.extensions = image.concat(video).map((extension) => extension.replace('.', '')); } - crawl(crawlOptions: CrawlOptionsDto): Promise { + async crawl(crawlOptions: CrawlOptionsDto): Promise { const { pathsToCrawl, exclusionPatterns, includeHidden } = crawlOptions; if (!pathsToCrawl) { return Promise.resolve([]); } - const base = pathsToCrawl.length === 1 ? pathsToCrawl[0] : `{${pathsToCrawl.join(',')}}`; - const extensions = `*{${this.extensions}}`; + const patterns: string[] = []; + const crawledFiles: string[] = []; - return glob(`${base}/**/${extensions}`, { + for await (const currentPath of pathsToCrawl) { + try { + const stats = await fs.promises.stat(currentPath); + if (stats.isFile() || stats.isSymbolicLink()) { + crawledFiles.push(currentPath); + } else { + patterns.push(currentPath); + } + } catch (error: any) { + if (error.code === 'ENOENT') { + patterns.push(currentPath); + } else { + throw error; + } + } + } + + let searchPattern: string; + if (patterns.length === 1) { + searchPattern = patterns[0]; + } else if (patterns.length === 0) { + return crawledFiles; + } else { + searchPattern = '{' + patterns.join(',') + '}'; + } + + if (crawlOptions.recursive) { + searchPattern = searchPattern + '/**/'; + } + + searchPattern = `${searchPattern}/*.{${this.extensions.join(',')}}`; + + const globbedFiles = await glob(searchPattern, { absolute: true, nocase: true, nodir: true, dot: includeHidden, ignore: exclusionPatterns, }); + + const returnedFiles = crawledFiles.concat(globbedFiles); + returnedFiles.sort(); + return returnedFiles; } }