Infrastructure/verdaccio
0
Fork 0
mirror of https://github.com/verdaccio/verdaccio.git synced 2025-04-01 02:42:23 -05:00
Code Issues Activity

Migrate fs-memory library to memfs (#2149)

Browse source 
* Update package.json

* Update MemoryFileSystem dependency

* Update storage.ts
This commit is contained in:
Paola Morales 2021-03-30 18:43:57 +02:00 • committed by Juan Picado
parent dc05edfe60
commit 5ccb2bad16
5 changed files with 321 additions and 315 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/plugins/memory/package.json
View file

@ -34,7 +34,8 @@
"@verdaccio/commons-api": "workspace:10.0.0-alpha.3",
"@verdaccio/streams": "workspace:10.0.0-alpha.3",
"memory-fs": "^0.5.0",
"debug": "^4.3.1"
"debug": "^4.3.1",
"memfs": "3.2.0"
},
"devDependencies": {
"@verdaccio/types": "workspace:10.0.0-alpha.3"

10
packages/plugins/memory/src/memory-handler.ts
View file

@ -6,7 +6,7 @@ import {
getConflict,
getNotFound,
} from '@verdaccio/commons-api';
import MemoryFileSystem from 'memory-fs';
import { fs } from 'memfs';
import { UploadTarball, ReadTarball } from '@verdaccio/streams';
import {
Callback,
@ -25,7 +25,6 @@ import {
import { parsePackage, stringifyPackage } from './utils';
const debug = buildDebug('verdaccio:plugin:storage:memory-storage');
const fs = new MemoryFileSystem();
export type DataHandler = {
[key: string]: string;
@ -158,16 +157,15 @@ class MemoryHandler implements IPackageStorageManager {
const readTarballStream: IReadTarball = new ReadTarball({});
process.nextTick(function () {
fs.stat(pathName, function (fileError, stats) {
if (fileError && !stats) {
fs.stat(pathName, function (error, stats) {
if (error && !stats) {
return readTarballStream.emit('error', getNotFound());
}
try {
const readStream = fs.createReadStream(pathName);
const contentLength: number = fs?.data[name]?.length || 0;
readTarballStream.emit('content-length', contentLength);
readTarballStream.emit('content-length', stats?.size);
readTarballStream.emit('open');
readStream.pipe(readTarballStream);
readStream.on('error', (error: VerdaccioError) => {

608
packages/plugins/memory/test/memory.spec.ts
View file

@ -34,347 +34,339 @@ jest.mock('../src/utils.ts', () => ({
}));
describe('memory unit test .', () => {
describe('MemoryHandler', () => {
test('should create an MemoryHandler instance', () => {
const memoryHandler = new MemoryHandler(
'test',
{
['foo']: 'bar',
},
logger
);
test('should create an MemoryHandler instance', () => {
const memoryHandler = new MemoryHandler(
'test',
{
['foo']: 'bar',
},
logger
);
expect(memoryHandler).toBeDefined();
});
expect(memoryHandler).toBeDefined();
});
test('should save a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
test('should save a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.readPackage(pkgName, (err, data) => {
expect(err).toBeNull();
expect(data).toEqual(pkgExample);
done();
});
});
}
});
test('should fails on save a package', (done) => {
mockStringify.mockImplementationOnce(() => {
throw new Error('error on parse');
});
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler: IPackageStorage = localMemory.getPackageStorage(
pkgName
) as ILocalPackageManager;
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toEqual(getInternalError('error on parse'));
done();
});
});
test('should fails on read a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
if (handler) {
handler.readPackage(pkgName, (err) => {
expect(err).not.toBeNull();
expect(err.code).toBe(404);
expect(err).toBeNull();
handler.readPackage(pkgName, (err, data) => {
expect(err).toBeNull();
expect(data).toEqual(pkgExample);
done();
});
}
});
test('should update a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
const onEnd = jest.fn();
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.updatePackage(
pkgName,
(json, callback) => {
expect(json).toBeDefined();
expect(json.name).toBe(pkgExample.name);
expect(callback).toBeDefined();
callback(null);
},
(name, data, onEnd) => {
expect(name).toBe(pkgName);
expect(data.name).toBe(pkgExample.name);
onEnd();
expect(onEnd).toHaveBeenCalled();
done();
},
(data) => {
expect(data).toBeDefined();
return data;
},
onEnd
);
});
}
});
test('should parse fails on update a package', (done) => {
mockParsePackage.mockImplementationOnce(() => {
throw new Error('error on parse');
});
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
}
});
const pkgName = 'test';
test('should fails on save a package', (done) => {
mockStringify.mockImplementationOnce(() => {
throw new Error('error on parse');
});
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
const onEnd = jest.fn((err) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler: IPackageStorage = localMemory.getPackageStorage(pkgName) as ILocalPackageManager;
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toEqual(getInternalError('error on parse'));
done();
});
});
test('should fails on read a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
if (handler) {
handler.readPackage(pkgName, (err) => {
expect(err).not.toBeNull();
expect(err).toEqual(getInternalError('error on parse'));
expect(err.code).toBe(404);
done();
});
}
});
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.updatePackage(
pkgName,
() => {},
() => {},
// @ts-ignore
() => {},
onEnd
);
});
}
});
test('should update a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
test('should fail updateHandler update a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
const onEnd = jest.fn();
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
const onEnd = jest.fn((err) => {
expect(err).not.toBeNull();
expect(err).toEqual(getInternalError('some error'));
done();
});
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.updatePackage(
pkgName,
(json, callback) => {
expect(json).toBeDefined();
expect(json.name).toBe(pkgExample.name);
expect(callback).toBeDefined();
callback(getInternalError('some error'));
},
() => {},
// @ts-ignore
() => {},
onEnd
);
});
}
});
test('should onWrite update a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
const onEnd = jest.fn((err) => {
expect(err).not.toBeNull();
expect(err).toEqual(getInternalError('error on parse the metadata'));
done();
});
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.updatePackage(
pkgName,
(json, callback) => {
expect(json).toBeDefined();
expect(json.name).toBe(pkgExample.name);
expect(callback).toBeDefined();
callback(null);
},
(name, data, onEnd) => {
expect(name).toBe(pkgName);
expect(data.name).toBe(pkgExample.name);
onEnd();
expect(onEnd).toHaveBeenCalled();
done();
},
() => {
throw new Error('dadsads');
},
onEnd
);
});
}
});
describe('writing/reading files', () => {
test('should write a tarball', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const dataTarball = '12345';
const handler = localMemory.getPackageStorage(pkgName);
if (handler) {
const stream = handler.writeTarball(pkgName);
stream.on('data', (data) => {
expect(data.toString()).toBe(dataTarball);
});
stream.on('open', () => {
stream.done();
stream.end();
});
stream.on('success', () => {
handler.updatePackage(
pkgName,
(json, callback) => {
expect(json).toBeDefined();
expect(json.name).toBe(pkgExample.name);
expect(callback).toBeDefined();
callback(null);
},
(name, data, onEnd) => {
expect(name).toBe(pkgName);
expect(data.name).toBe(pkgExample.name);
onEnd();
expect(onEnd).toHaveBeenCalled();
done();
});
stream.write(dataTarball);
}
},
(data) => {
expect(data).toBeDefined();
return data;
},
onEnd
);
});
}
});
test('should read a tarball', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test.tar.gz';
const dataTarball = '12345';
test('should parse fails on update a package', (done) => {
mockParsePackage.mockImplementationOnce(() => {
throw new Error('error on parse');
});
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const handler = localMemory.getPackageStorage(pkgName);
const pkgName = 'test';
if (handler) {
const stream = handler.writeTarball(pkgName);
stream.on('open', () => {
stream.done();
stream.end();
});
stream.on('success', () => {
const readStream = handler.readTarball(pkgName);
readStream.on('data', (data) => {
expect(data.toString()).toBe(dataTarball);
done();
});
});
stream.write(dataTarball);
}
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
const onEnd = jest.fn((err) => {
expect(err).not.toBeNull();
expect(err).toEqual(getInternalError('error on parse'));
done();
});
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.updatePackage(
pkgName,
() => {},
() => {},
// @ts-ignore
() => {},
onEnd
);
});
}
});
test('should abort read a tarball', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test2.tar.gz';
const dataTarball = '12345';
test('should fail updateHandler update a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const handler = localMemory.getPackageStorage(pkgName);
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
const onEnd = jest.fn((err) => {
expect(err).not.toBeNull();
expect(err).toEqual(getInternalError('some error'));
done();
});
if (handler) {
const stream = handler.writeTarball(pkgName);
stream.on('open', () => {
stream.done();
stream.end();
});
stream.on('success', () => {
const readStream = handler.readTarball(pkgName);
readStream.on('data', () => {
readStream.abort();
});
readStream.on('error', (err) => {
expect(err).not.toBeNull();
expect(err.message).toMatch(/read has been aborted/);
done();
});
});
stream.write(dataTarball);
}
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.updatePackage(
pkgName,
(json, callback) => {
expect(json).toBeDefined();
expect(json.name).toBe(pkgExample.name);
expect(callback).toBeDefined();
callback(getInternalError('some error'));
},
() => {},
// @ts-ignore
() => {},
onEnd
);
});
}
});
test('should fails read a tarball not found', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test2.tar.gz';
const handler = localMemory.getPackageStorage(pkgName);
test('should onWrite update a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
if (handler) {
const readStream = handler.readTarball('not-found');
readStream.on('error', (err) => {
const handler = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
const onEnd = jest.fn((err) => {
expect(err).not.toBeNull();
expect(err).toEqual(getInternalError('error on parse the metadata'));
done();
});
if (handler) {
handler.savePackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.updatePackage(
pkgName,
(json, callback) => {
expect(json).toBeDefined();
expect(json.name).toBe(pkgExample.name);
expect(callback).toBeDefined();
callback(null);
},
(name, data, onEnd) => {
expect(name).toBe(pkgName);
expect(data.name).toBe(pkgExample.name);
onEnd();
expect(onEnd).toHaveBeenCalled();
done();
},
() => {
throw new Error('dadsads');
},
onEnd
);
});
}
});
test('should delete a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test2';
const handler: IPackageStorage = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
if (handler) {
handler.createPackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.deletePackage(pkgName, (err) => {
expect(err).toBeNull();
handler.readPackage(pkgName, (err) => {
expect(err).not.toBeNull();
expect(err.message).toMatch(/no such package/);
done();
});
}
});
});
test('should abort while write a tarball', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test-abort.tar.gz';
const dataTarball = '12345';
const handler = localMemory.getPackageStorage(pkgName);
if (handler) {
const stream = handler.writeTarball(pkgName);
stream.on('error', (err) => {
expect(err).not.toBeNull();
expect(err.message).toMatch(/transmision aborted/);
done();
});
stream.on('open', () => {
stream.abort();
});
stream.write(dataTarball);
}
});
test('should delete a package', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test2';
const handler: IPackageStorage = localMemory.getPackageStorage(pkgName);
expect(handler).toBeDefined();
if (handler) {
handler.createPackage(pkgName, pkgExample, (err) => {
expect(err).toBeNull();
handler.deletePackage(pkgName, (err) => {
expect(err).toBeNull();
handler.readPackage(pkgName, (err) => {
expect(err).not.toBeNull();
expect(err.message).toMatch(/no such package/);
done();
});
});
});
}
});
});
}
});
});
describe('writing files', () => {
test('should write a tarball', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test';
const dataTarball = '12345';
const handler = localMemory.getPackageStorage(pkgName);
if (handler) {
const stream = handler.writeTarball(pkgName);
stream.on('data', (data) => {
expect(data.toString()).toBe(dataTarball);
});
stream.on('open', () => {
stream.done();
stream.end();
});
stream.on('success', () => {
done();
});
stream.write(dataTarball);
}
});
test('should abort while write a tarball', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test-abort.tar.gz';
const handler = localMemory.getPackageStorage(pkgName);
if (handler) {
const stream = handler.writeTarball(pkgName);
stream.on('error', (err) => {
expect(err).not.toBeNull();
expect(err.message).toMatch(/transmision aborted/);
done();
});
stream.on('open', () => {
stream.abort();
});
}
});
});
describe('reading files', () => {
test('should read a tarball', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test.tar.gz';
const dataTarball = '12345';
const handler = localMemory.getPackageStorage(pkgName);
if (handler) {
const stream = handler.writeTarball(pkgName);
stream.on('open', () => {
stream.done();
stream.end();
});
stream.on('success', () => {
const readStream = handler.readTarball(pkgName);
readStream.on('data', (data) => {
expect(data.toString()).toBe(dataTarball);
done();
});
});
stream.write(dataTarball);
}
});
test('should abort read a tarball', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test2.tar.gz';
const dataTarball = '12345';
const handler = localMemory.getPackageStorage(pkgName);
if (handler) {
const stream = handler.writeTarball(pkgName);
stream.on('open', () => {
stream.done();
stream.end();
});
stream.on('success', () => {
const readStream = handler.readTarball(pkgName);
readStream.on('data', () => {
readStream.abort();
});
readStream.on('error', (err) => {
expect(err).not.toBeNull();
expect(err.message).toMatch(/read has been aborted/);
done();
});
});
stream.write(dataTarball);
}
});
test('should fails read a tarball not found', (done) => {
const localMemory: IPluginStorage<ConfigMemory> = new LocalMemory(config, defaultConfig);
const pkgName = 'test2.tar.gz';
const handler = localMemory.getPackageStorage(pkgName);
if (handler) {
const readStream = handler.readTarball('not-found');
readStream.on('error', (err) => {
expect(err).not.toBeNull();
expect(err.message).toMatch(/no such package/);
done();
});
}
});
});

1
packages/store/src/storage.ts
View file

@ -293,6 +293,7 @@ class Storage {
localStream.on('content-length', function (v): void {
readStream.emit('content-length', v);
});
localStream.on('open', function (): void {
isOpen = true;
localStream.pipe(readStream);

14
pnpm-lock.yaml generated
View file

@ -680,6 +680,7 @@ importers:
'@verdaccio/commons-api': link:../../core/commons-api
'@verdaccio/streams': link:../../core/streams
debug: 4.3.1
memfs: 3.2.0
memory-fs: 0.5.0
devDependencies:
'@verdaccio/types': link:../../core/types
@ -688,6 +689,7 @@ importers:
'@verdaccio/streams': workspace:10.0.0-alpha.3
'@verdaccio/types': workspace:10.0.0-alpha.3
debug: ^4.3.1
memfs: 3.2.0
memory-fs: ^0.5.0
packages/plugins/ui-theme:
devDependencies:
@ -14324,6 +14326,10 @@ packages:
node: '>= 8'
resolution:
integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==
/fs-monkey/1.0.1:
dev: false
resolution:
integrity: sha512-fcSa+wyTqZa46iWweI7/ZiUfegOZl0SG8+dltIwFXo7+zYU9J9kpS3NB6pZcSlJdhvIwp81Adx2XhZorncxiaA==
/fs-readdir-recursive/1.1.0:
dev: true
resolution:
@ -19658,6 +19664,14 @@ packages:
node: '>= 0.6'
resolution:
integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=
/memfs/3.2.0:
dependencies:
fs-monkey: 1.0.1
dev: false
engines:
node: '>= 4.0.0'
resolution:
integrity: sha512-f/xxz2TpdKv6uDn6GtHee8ivFyxwxmPuXatBb1FBwxYNuVpbM3k/Y1Z+vC0mH/dIXXrukYfe3qe5J32Dfjg93A==
/memory-fs/0.4.1:
dependencies:
errno: 0.1.7