ashipov/immich
1
0
Fork 1
forked from dark_thunder/immich
Code Pull Requests Activity

feat(cli): concurrent upload (#7192)

Browse Source 
* concurrent cli upload

* added concurrency flag, progress bar refinements

* no data property 🦀

* use lodash-es

* rebase

* linting

* typing

* album bug fixes

* dev dependency for lodash typing

* fixed not deleting assets if album isn't specified

* formatting

* fixed tests

* use `arrayContaining`

* add more checks

* assert updates existing assets
This commit is contained in:
Mert
2024-02-19 19:32:57 -05:00
committed by GitHub
parent 947bcf2d68
commit d5ef91b1ae
5 changed files with 425 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
cli/package-lock.json generated
View File

@ -8,6 +8,9 @@
"name": "@immich/cli",
"version": "2.0.8",
"license": "GNU Affero General Public License version 3",
"dependencies": {
"lodash-es": "^4.17.21"
},
"bin": {
"immich": "dist/index.js"
},
@ -16,6 +19,7 @@
"@testcontainers/postgresql": "^10.7.1",
"@types/byte-size": "^8.1.0",
"@types/cli-progress": "^3.11.0",
"@types/lodash-es": "^4.17.12",
"@types/mock-fs": "^4.13.1",
"@types/node": "^20.3.1",
"@typescript-eslint/eslint-plugin": "^7.0.0",
@ -1296,6 +1300,21 @@
"integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
"dev": true
},
"node_modules/@types/lodash": {
"version": "4.14.202",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.202.tgz",
"integrity": "sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ==",
"dev": true
},
"node_modules/@types/lodash-es": {
"version": "4.17.12",
"resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz",
"integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==",
"dev": true,
"dependencies": {
"@types/lodash": "*"
}
},
"node_modules/@types/mock-fs": {
"version": "4.13.4",
"resolved": "https://registry.npmjs.org/@types/mock-fs/-/mock-fs-4.13.4.tgz",
@ -3539,6 +3558,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/lodash-es": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz",
"integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw=="
},
"node_modules/lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
@ -6432,6 +6456,21 @@
"integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
"dev": true
},
"@types/lodash": {
"version": "4.14.202",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.202.tgz",
"integrity": "sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ==",
"dev": true
},
"@types/lodash-es": {
"version": "4.17.12",
"resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz",
"integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==",
"dev": true,
"requires": {
"@types/lodash": "*"
}
},
"@types/mock-fs": {
"version": "4.13.4",
"resolved": "https://registry.npmjs.org/@types/mock-fs/-/mock-fs-4.13.4.tgz",
@ -8080,6 +8119,11 @@
"p-locate": "^5.0.0"
}
},
"lodash-es": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz",
"integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw=="
},
"lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",

4
cli/package.json
View File

@ -17,6 +17,7 @@
"@testcontainers/postgresql": "^10.7.1",
"@types/byte-size": "^8.1.0",
"@types/cli-progress": "^3.11.0",
"@types/lodash-es": "^4.17.12",
"@types/mock-fs": "^4.13.1",
"@types/node": "^20.3.1",
"@typescript-eslint/eslint-plugin": "^7.0.0",
@ -56,5 +57,8 @@
},
"engines": {
"node": ">=20.0.0"
},
"dependencies": {
"lodash-es": "^4.17.21"
}
}

438
cli/src/commands/upload.command.ts
View File

File diff suppressed because it is too large Load Diff

5
cli/src/index.ts
View File

@ -43,6 +43,11 @@ program
.env('IMMICH_DRY_RUN')
.default(false),
)
.addOption(
new Option('-c, --concurrency', 'Number of assets to upload at the same time')
.env('IMMICH_UPLOAD_CONCURRENCY')
.default(4),
)
.addOption(new Option('--delete', 'Delete local assets after upload').env('IMMICH_DELETE_ASSETS'))
.argument('[paths...]', 'One or more paths to assets to be uploaded')
.action(async (paths, options) => {

71
e2e/src/cli/specs/upload.e2e-spec.ts
View File

@ -8,6 +8,7 @@ import {
testAssetDir,
} from 'src/utils';
import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { mkdir, readdir, rm, symlink } from 'fs/promises';
describe(`immich upload`, () => {
let key: string;
@ -29,9 +30,11 @@ describe(`immich upload`, () => {
'--recursive',
]);
expect(stderr).toBe('');
expect(stdout.split('\n')).toEqual([
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining('Successfully uploaded 9 assets'),
]);
])
);
expect(exitCode).toBe(0);
const assets = await getAllAssets({}, { headers: asKeyAuth(key) });
@ -47,9 +50,13 @@ describe(`immich upload`, () => {
'--recursive',
'--album',
]);
expect(stdout.split('\n')).toEqual([
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining('Successfully uploaded 9 assets'),
]);
expect.stringContaining('Successfully created 1 new album'),
expect.stringContaining('Successfully updated 9 assets'),
])
);
expect(stderr).toBe('');
expect(exitCode).toBe(0);
@ -67,9 +74,11 @@ describe(`immich upload`, () => {
`${testAssetDir}/albums/nature/`,
'--recursive',
]);
expect(response1.stdout.split('\n')).toEqual([
expect(response1.stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining('Successfully uploaded 9 assets'),
]);
])
);
expect(response1.stderr).toBe('');
expect(response1.exitCode).toBe(0);
@ -85,11 +94,14 @@ describe(`immich upload`, () => {
'--recursive',
'--album',
]);
expect(response2.stdout.split('\n')).toEqual([
expect(response2.stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining(
'All assets were already uploaded, nothing to do.'
),
]);
expect.stringContaining('Successfully updated 9 assets'),
])
);
expect(response2.stderr).toBe('');
expect(response2.exitCode).toBe(0);
@ -110,9 +122,13 @@ describe(`immich upload`, () => {
'--recursive',
'--album-name=e2e',
]);
expect(stdout.split('\n')).toEqual([
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining('Successfully uploaded 9 assets'),
]);
expect.stringContaining('Successfully created 1 new album'),
expect.stringContaining('Successfully updated 9 assets'),
])
);
expect(stderr).toBe('');
expect(exitCode).toBe(0);
@ -124,4 +140,39 @@ describe(`immich upload`, () => {
expect(albums[0].albumName).toBe('e2e');
});
});
describe('immich upload --delete', () => {
it('should delete local files if specified', async () => {
await mkdir(`/tmp/albums/nature`, { recursive: true });
const filesToLink = await readdir(`${testAssetDir}/albums/nature`);
for (const file of filesToLink) {
await symlink(
`${testAssetDir}/albums/nature/${file}`,
`/tmp/albums/nature/${file}`
);
}
const { stderr, stdout, exitCode } = await immichCli([
'upload',
`/tmp/albums/nature`,
'--delete',
]);
const files = await readdir(`/tmp/albums/nature`);
await rm(`/tmp/albums/nature`, { recursive: true });
expect(files).toEqual([]);
expect(stdout.split('\n')).toEqual(
expect.arrayContaining([
expect.stringContaining('Successfully uploaded 9 assets'),
expect.stringContaining('Deleting assets that have been uploaded'),
])
);
expect(stderr).toBe('');
expect(exitCode).toBe(0);
const assets = await getAllAssets({}, { headers: asKeyAuth(key) });
expect(assets.length).toBe(9);
});
});
});