Infrastructure/immich
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-01-07 00:50:23 -05:00
Code Issues Activity

chore: better observability for geodata import (#13931)

Browse source
This commit is contained in:
Zack Pollard 2024-11-05 16:20:56 +00:00 committed by GitHub
parent 5edbb93a4d
commit c383e115af
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
server/src/repositories/map.repository.ts
View file

@ -249,6 +249,7 @@ export class MapRepository implements IMapRepository {
const input = createReadStream(filePath); const input = createReadStream(filePath);
let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesEntity>[] = []; let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesEntity>[] = [];
const lineReader = readLine.createInterface({ input }); const lineReader = readLine.createInterface({ input });
let count = 0;
for await (const line of lineReader) { for await (const line of lineReader) {
const lineSplit = line.split('\t'); const lineSplit = line.split('\t');
@ -257,8 +258,12 @@ export class MapRepository implements IMapRepository {
} }
const geoData = lineToEntityMapper(lineSplit); const geoData = lineToEntityMapper(lineSplit);
bufferGeodata.push(geoData); bufferGeodata.push(geoData);
if (bufferGeodata.length > 1000) { if (bufferGeodata.length >= 1000) {
await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']); await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
count += bufferGeodata.length;
if (count % 10_000 === 0) {
this.logger.log(`${count} geodata records imported`);
}
bufferGeodata = []; bufferGeodata = [];
} }
} }