feat: add tarball details for published packages (#4653)

* feat: add tarball details for published packages

* remove throw err

.changeset/eight-icons-heal.md

@@ -0,0 +1,6 @@
+---
+'@verdaccio/tarball': patch
+'@verdaccio/store': patch
+---
+
+feat: add tarball details for published packages

packages/core/tarball/package.json

@@ -37,7 +37,9 @@
     "@verdaccio/url": "workspace:12.0.0-next-7.15",
     "@verdaccio/utils": "workspace:7.0.0-next-7.15",
     "debug": "4.3.4",
-    "lodash": "4.17.21"
+    "gunzip-maybe": "^1.4.2",
+    "lodash": "4.17.21",
+    "tar-stream": "^3.1.7"
   },
   "devDependencies": {
     "@verdaccio/types": "workspace:12.0.0-next-7.3",

packages/core/tarball/src/getTarballDetails.ts

@@ -0,0 +1,34 @@
+import gunzipMaybe from 'gunzip-maybe';
+import { Readable } from 'stream';
+import * as tarStream from 'tar-stream';
+
+export type TarballDetails = {
+  fileCount: number;
+  unpackedSize: number; // in bytes
+};
+
+export async function getTarballDetails(buffer: Buffer): Promise<TarballDetails> {
+  let fileCount = 0;
+  let unpackedSize = 0;
+  const readable = Readable.from(buffer);
+  const unpack = tarStream.extract();
+
+  return new Promise((resolve, reject) => {
+    readable
+      .pipe(gunzipMaybe())
+      .pipe(unpack)
+      .on('entry', (header, stream, next) => {
+        fileCount++;
+        unpackedSize += Number(header.size);
+        stream.resume(); // important to ensure that "entry" events keep firing
+        next();
+      })
+      .on('finish', () => {
+        resolve({
+          fileCount,
+          unpackedSize,
+        });
+      })
+      .on('error', reject);
+  });
+}

packages/core/tarball/src/index.ts

@@ -5,5 +5,6 @@ export {
   convertDistVersionToLocalTarballsUrl,
 } from './convertDistRemoteToLocalTarballUrls';
 export { extractTarballFromUrl, getLocalRegistryTarballUri } from './getLocalRegistryTarballUri';
+export { getTarballDetails, TarballDetails } from './getTarballDetails';
 
 export { RequestOptions };

packages/core/tarball/tests/getTarballDetails.spec.ts

@@ -0,0 +1,33 @@
+import fs from 'fs';
+import path from 'path';
+
+import { getTarballDetails } from '../src/getTarballDetails.ts';
+
+const getFilePath = (filename: string): string => {
+  return path.resolve(__dirname, `assets/${filename}`);
+};
+
+const getFileBuffer = async (filename: string): Promise<Buffer> => {
+  return fs.promises.readFile(getFilePath(filename));
+};
+
+describe('getTarballDetails', () => {
+  test('should return stats of tarball (gzipped)', async () => {
+    const buffer = await getFileBuffer('tarball.tgz');
+    const details = await getTarballDetails(buffer);
+    expect(details.fileCount).toBe(2);
+    expect(details.unpackedSize).toBe(1280);
+  });
+
+  test('should return stats of tarball (uncompressed)', async () => {
+    const buffer = await getFileBuffer('tarball.tar');
+    const details = await getTarballDetails(buffer);
+    expect(details.fileCount).toBe(2);
+    expect(details.unpackedSize).toBe(1280);
+  });
+
+  test('should throw an error if the buffer is corrupt', async () => {
+    const corruptBuffer = Buffer.from('this is not a tarball');
+    await expect(getTarballDetails(corruptBuffer)).rejects.toThrow();
+  });
+});

packages/store/src/storage.ts

@@ -34,9 +34,11 @@ import {
 } from '@verdaccio/proxy';
 import Search from '@verdaccio/search';
 import {
+  TarballDetails,
   convertDistRemoteToLocalTarballUrls,
   convertDistVersionToLocalTarballsUrl,
   extractTarballFromUrl,
+  getTarballDetails,
 } from '@verdaccio/tarball';
 import {
   AbbreviatedManifest,
@@ -1076,13 +1078,14 @@ class Storage {
 
     // 1. after tarball has been successfully uploaded, we update the version
     try {
+      const tarballStats = await this.getTarballStats(versions[versionToPublish], buffer);
       // Older package managers like npm6 do not send readme content as part of version but include it on root level
       if (_.isEmpty(versions[versionToPublish].readme)) {
         versions[versionToPublish].readme =
           _.isNil(manifest.readme) === false ? String(manifest.readme) : '';
       }
       // addVersion will move the readme from the the published version to the root level
-      await this.addVersion(name, versionToPublish, versions[versionToPublish], null);
+      await this.addVersion(name, versionToPublish, versions[versionToPublish], null, tarballStats);
     } catch (err: any) {
       logger.error({ err: err.message }, 'updated version has failed: @{err}');
       debug('error on create a version for %o with error %o', name, err.message);
@@ -1283,7 +1286,8 @@ class Storage {
     name: string,
     version: string,
     metadata: Version,
-    tag: StringValue
+    tag: StringValue,
+    tarballStats: TarballDetails
   ): Promise<void> {
     debug(`add version %s package for %s`, version, name);
     await this.updatePackage(name, async (data: Manifest): Promise<Manifest> => {
@@ -1295,6 +1299,12 @@ class Storage {
       metadata.contributors = normalizeContributors(metadata.contributors as Author[]);
       debug('%s` contributors normalized', name);
 
+      // Update tarball stats
+      if (metadata.dist) {
+        metadata.dist.fileCount = tarballStats.fileCount;
+        metadata.dist.unpackedSize = tarballStats.unpackedSize;
+      }
+
       // if uploaded tarball has a different shasum, it's very likely that we
       // have some kind of error
       if (validatioUtils.isObject(metadata.dist) && _.isString(metadata.dist.tarball)) {
@@ -1905,6 +1915,20 @@ class Storage {
       return cacheManifest;
     }
   }
+
+  private async getTarballStats(version: Version, buffer: Buffer): Promise<TarballDetails> {
+    if (
+      version.dist == undefined ||
+      version.dist?.fileCount == undefined ||
+      version.dist?.unpackedSize == undefined
+    ) {
+      debug('tarball stats not found, calculating');
+      return await getTarballDetails(buffer);
+    } else {
+      debug('tarball stats found');
+      return { fileCount: version.dist.fileCount, unpackedSize: version.dist.unpackedSize };
+    }
+  }
 }
 
 export { Storage };

packages/store/test/storage.spec.ts

@@ -262,10 +262,12 @@ describe('storage', () => {
         expect(manifestVersion._id).toEqual(`${pkgName}@1.0.1`);
         expect(manifestVersion.description).toEqual('package generated');
         expect(manifestVersion.dist).toEqual({
+          fileCount: 4,
           integrity:
             'sha512-6gHiERpiDgtb3hjqpQH5/i7zRmvYi9pmCjQf2ZMy3QEa9wVk9RgdZaPWUt7ZOnWUPFjcr9cmE6dUBf+XoPoH4g==',
           shasum: '2c03764f651a9f016ca0b7620421457b619151b9',
           tarball: 'http://localhost:5555/upstream/-/upstream-1.0.1.tgz',
+          unpackedSize: 543,
         });
 
         expect(manifestVersion.contributors).toEqual([]);
@@ -658,6 +660,47 @@ describe('storage', () => {
   });
 
   describe('getTarball', () => {
+    test('should get a package from local storage', (done) => {
+      const pkgName = 'foo';
+      const config = new Config(
+        configExample({
+          ...getDefaultConfig(),
+          storage: generateRandomStorage(),
+        })
+      );
+      const storage = new Storage(config);
+      storage.init(config).then(() => {
+        const ac = new AbortController();
+        const bodyNewManifest = generatePackageMetadata(pkgName, '1.0.0');
+        storage
+          .updateManifest(bodyNewManifest, {
+            signal: ac.signal,
+            name: pkgName,
+            uplinksLook: false,
+            requestOptions: defaultRequestOptions,
+          })
+          .then(() => {
+            const abort = new AbortController();
+            storage
+              .getTarball(pkgName, `${pkgName}-1.0.0.tgz`, {
+                signal: abort.signal,
+              })
+              .then((stream) => {
+                stream.on('data', (dat) => {
+                  expect(dat).toBeDefined();
+                  expect(dat.length).toEqual(512);
+                });
+                stream.on('end', () => {
+                  done();
+                });
+                stream.on('error', () => {
+                  done('this should not happen');
+                });
+              });
+          });
+      });
+    });
+
     test('should not found a package anywhere', (done) => {
       const config = new Config(
         configExample({