Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-01-27 22:49:56 -05:00
Code Issues Activity

Added media file handling to Ghost importer

Browse source 
refs https://github.com/TryGhost/Toolbox/issues/523

- When a zip file is imported into Ghost we need to recognize and process media files with following extensions:
".mp4",".webm", ".ogv", ".mp3", ".wav", ".ogg", ".m4a"
- The media files can come from a "media" or "content/media" folder inside of zip file
This commit is contained in:
Naz 2023-03-01 15:50:03 +08:00
parent 9347e996f8
commit ce5eea5707
No known key found for this signature in database
4 changed files with 49 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
ghost/core/core/server/data/importer/import-manager.js
View file

@ -11,6 +11,7 @@ const debug = require('@tryghost/debug')('import-manager');
const logging = require('@tryghost/logging'); const logging = require('@tryghost/logging');
const errors = require('@tryghost/errors'); const errors = require('@tryghost/errors');
const ImageHandler = require('./handlers/image'); const ImageHandler = require('./handlers/image');
const MediaHandler = require('@tryghost/importer-handler-media');
const RevueHandler = require('./handlers/revue'); const RevueHandler = require('./handlers/revue');
const JSONHandler = require('./handlers/json'); const JSONHandler = require('./handlers/json');
const MarkdownHandler = require('./handlers/markdown'); const MarkdownHandler = require('./handlers/markdown');
@ -20,6 +21,7 @@ const DataImporter = require('./importers/data');
const urlUtils = require('../../../shared/url-utils'); const urlUtils = require('../../../shared/url-utils');
const {GhostMailer} = require('../../services/mail'); const {GhostMailer} = require('../../services/mail');
const jobManager = require('../../services/jobs'); const jobManager = require('../../services/jobs');
const mediaStorage = require('../../adapters/storage').getStorage('media');
const emailTemplate = require('./email-template'); const emailTemplate = require('./email-template');
const ghostMailer = new GhostMailer(); const ghostMailer = new GhostMailer();
@ -51,6 +53,12 @@ let defaults = {
class ImportManager { class ImportManager {
constructor() { constructor() {
const mediaHandler = new MediaHandler({
config: config,
urlUtils: urlUtils,
storage: mediaStorage
});
/** /**
* @type {Importer[]} importers * @type {Importer[]} importers
*/ */
@ -59,7 +67,7 @@ class ImportManager {
/** /**
* @type {Handler[]} * @type {Handler[]}
*/ */
this.handlers = [ImageHandler, RevueHandler, JSONHandler, MarkdownHandler]; this.handlers = [ImageHandler, mediaHandler, RevueHandler, JSONHandler, MarkdownHandler];
// Keep track of file to cleanup at the end // Keep track of file to cleanup at the end
/** /**

1
ghost/core/package.json
View file

@ -88,6 +88,7 @@
"@tryghost/html-to-plaintext": "0.0.0", "@tryghost/html-to-plaintext": "0.0.0",
"@tryghost/http-cache-utils": "0.1.6", "@tryghost/http-cache-utils": "0.1.6",
"@tryghost/image-transform": "1.2.3", "@tryghost/image-transform": "1.2.3",
"@tryghost/importer-handler-media": "0.0.0",
"@tryghost/importer-revue": "0.0.0", "@tryghost/importer-revue": "0.0.0",
"@tryghost/job-manager": "0.0.0", "@tryghost/job-manager": "0.0.0",
"@tryghost/kg-card-factory": "4.0.0", "@tryghost/kg-card-factory": "4.0.0",

54
ghost/core/test/unit/server/data/importer/index.test.js
View file

@ -30,7 +30,7 @@ describe('Importer', function () {
describe('ImportManager', function () { describe('ImportManager', function () {
it('has the correct interface', function () { it('has the correct interface', function () {
ImportManager.handlers.should.be.instanceof(Array).and.have.lengthOf(4); ImportManager.handlers.should.be.instanceof(Array).and.have.lengthOf(5);
ImportManager.importers.should.be.instanceof(Array).and.have.lengthOf(3); ImportManager.importers.should.be.instanceof(Array).and.have.lengthOf(3);
ImportManager.loadFile.should.be.instanceof(Function); ImportManager.loadFile.should.be.instanceof(Function);
ImportManager.preProcess.should.be.instanceof(Function); ImportManager.preProcess.should.be.instanceof(Function);
@ -39,17 +39,23 @@ describe('Importer', function () {
}); });
it('gets the correct extensions', function () { it('gets the correct extensions', function () {
ImportManager.getExtensions().should.be.instanceof(Array).and.have.lengthOf(13); ImportManager.getExtensions().should.be.instanceof(Array).and.have.lengthOf(20);
ImportManager.getExtensions().should.containEql('.csv'); ImportManager.getExtensions().should.containEql('.csv');
ImportManager.getExtensions().should.containEql('.json'); ImportManager.getExtensions().should.containEql('.json');
ImportManager.getExtensions().should.containEql('.zip'); ImportManager.getExtensions().should.containEql('.zip');
ImportManager.getExtensions().should.containEql('.jpg'); ImportManager.getExtensions().should.containEql('.jpg');
ImportManager.getExtensions().should.containEql('.md'); ImportManager.getExtensions().should.containEql('.md');
ImportManager.getExtensions().should.containEql('.webp'); ImportManager.getExtensions().should.containEql('.webp');
ImportManager.getExtensions().should.containEql('.mp4');
ImportManager.getExtensions().should.containEql('.ogv');
ImportManager.getExtensions().should.containEql('.mp3');
ImportManager.getExtensions().should.containEql('.wav');
ImportManager.getExtensions().should.containEql('.ogg');
ImportManager.getExtensions().should.containEql('.m4a');
}); });
it('gets the correct types', function () { it('gets the correct types', function () {
ImportManager.getContentTypes().should.be.instanceof(Array).and.have.lengthOf(12); ImportManager.getContentTypes().should.be.instanceof(Array).and.have.lengthOf(22);
ImportManager.getContentTypes().should.containEql('image/jpeg'); ImportManager.getContentTypes().should.containEql('image/jpeg');
ImportManager.getContentTypes().should.containEql('image/png'); ImportManager.getContentTypes().should.containEql('image/png');
ImportManager.getContentTypes().should.containEql('image/gif'); ImportManager.getContentTypes().should.containEql('image/gif');
@ -58,6 +64,17 @@ describe('Importer', function () {
ImportManager.getContentTypes().should.containEql('image/vnd.microsoft.icon'); ImportManager.getContentTypes().should.containEql('image/vnd.microsoft.icon');
ImportManager.getContentTypes().should.containEql('image/webp'); ImportManager.getContentTypes().should.containEql('image/webp');
ImportManager.getContentTypes().should.containEql('video/mp4');
ImportManager.getContentTypes().should.containEql('video/webm');
ImportManager.getContentTypes().should.containEql('video/ogg');
ImportManager.getContentTypes().should.containEql('audio/mpeg');
ImportManager.getContentTypes().should.containEql('audio/vnd.wav');
ImportManager.getContentTypes().should.containEql('audio/wave');
ImportManager.getContentTypes().should.containEql('audio/wav');
ImportManager.getContentTypes().should.containEql('audio/x-wav');
ImportManager.getContentTypes().should.containEql('audio/ogg');
ImportManager.getContentTypes().should.containEql('audio/x-m4a');
ImportManager.getContentTypes().should.containEql('application/octet-stream'); ImportManager.getContentTypes().should.containEql('application/octet-stream');
ImportManager.getContentTypes().should.containEql('application/json'); ImportManager.getContentTypes().should.containEql('application/json');
@ -68,36 +85,37 @@ describe('Importer', function () {
}); });
it('gets the correct directories', function () { it('gets the correct directories', function () {
ImportManager.getDirectories().should.be.instanceof(Array).and.have.lengthOf(2); ImportManager.getDirectories().should.be.instanceof(Array).and.have.lengthOf(3);
ImportManager.getDirectories().should.containEql('images'); ImportManager.getDirectories().should.containEql('images');
ImportManager.getDirectories().should.containEql('content'); ImportManager.getDirectories().should.containEql('content');
ImportManager.getDirectories().should.containEql('media');
}); });
it('globs extensions correctly', function () { it('globs extensions correctly', function () {
ImportManager.getGlobPattern(ImportManager.getExtensions()) ImportManager.getGlobPattern(ImportManager.getExtensions())
.should.equal('+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.csv|.json|.md|.markdown|.zip)'); .should.equal('+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.mp4|.webm|.ogv|.mp3|.wav|.ogg|.m4a|.csv|.json|.md|.markdown|.zip)');
ImportManager.getGlobPattern(ImportManager.getDirectories()) ImportManager.getGlobPattern(ImportManager.getDirectories())
.should.equal('+(images|content)'); .should.equal('+(images|content|media)');
ImportManager.getGlobPattern(JSONHandler.extensions) ImportManager.getGlobPattern(JSONHandler.extensions)
.should.equal('+(.json)'); .should.equal('+(.json)');
ImportManager.getGlobPattern(ImageHandler.extensions) ImportManager.getGlobPattern(ImageHandler.extensions)
.should.equal('+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp)'); .should.equal('+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp)');
ImportManager.getExtensionGlob(ImportManager.getExtensions()) ImportManager.getExtensionGlob(ImportManager.getExtensions())
.should.equal('*+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.csv|.json|.md|.markdown|.zip)'); .should.equal('*+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.mp4|.webm|.ogv|.mp3|.wav|.ogg|.m4a|.csv|.json|.md|.markdown|.zip)');
ImportManager.getDirectoryGlob(ImportManager.getDirectories()) ImportManager.getDirectoryGlob(ImportManager.getDirectories())
.should.equal('+(images|content)'); .should.equal('+(images|content|media)');
ImportManager.getExtensionGlob(ImportManager.getExtensions(), 0) ImportManager.getExtensionGlob(ImportManager.getExtensions(), 0)
.should.equal('*+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.csv|.json|.md|.markdown|.zip)'); .should.equal('*+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.mp4|.webm|.ogv|.mp3|.wav|.ogg|.m4a|.csv|.json|.md|.markdown|.zip)');
ImportManager.getDirectoryGlob(ImportManager.getDirectories(), 0) ImportManager.getDirectoryGlob(ImportManager.getDirectories(), 0)
.should.equal('+(images|content)'); .should.equal('+(images|content|media)');
ImportManager.getExtensionGlob(ImportManager.getExtensions(), 1) ImportManager.getExtensionGlob(ImportManager.getExtensions(), 1)
.should.equal('{*/*,*}+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.csv|.json|.md|.markdown|.zip)'); .should.equal('{*/*,*}+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.mp4|.webm|.ogv|.mp3|.wav|.ogg|.m4a|.csv|.json|.md|.markdown|.zip)');
ImportManager.getDirectoryGlob(ImportManager.getDirectories(), 1) ImportManager.getDirectoryGlob(ImportManager.getDirectories(), 1)
.should.equal('{*/,}+(images|content)'); .should.equal('{*/,}+(images|content|media)');
ImportManager.getExtensionGlob(ImportManager.getExtensions(), 2) ImportManager.getExtensionGlob(ImportManager.getExtensions(), 2)
.should.equal('**/*+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.csv|.json|.md|.markdown|.zip)'); .should.equal('**/*+(.jpg|.jpeg|.gif|.png|.svg|.svgz|.ico|.webp|.mp4|.webm|.ogv|.mp3|.wav|.ogg|.m4a|.csv|.json|.md|.markdown|.zip)');
ImportManager.getDirectoryGlob(ImportManager.getDirectories(), 2) ImportManager.getDirectoryGlob(ImportManager.getDirectories(), 2)
.should.equal('**/+(images|content)'); .should.equal('**/+(images|content|media)');
}); });
it('cleans up', async function () { it('cleans up', async function () {
@ -178,7 +196,7 @@ describe('Importer', function () {
extractSpy.calledOnce.should.be.true(); extractSpy.calledOnce.should.be.true();
validSpy.calledOnce.should.be.true(); validSpy.calledOnce.should.be.true();
baseDirSpy.calledOnce.should.be.true(); baseDirSpy.calledOnce.should.be.true();
getFileSpy.callCount.should.eql(4); getFileSpy.callCount.should.eql(5);
jsonSpy.calledOnce.should.be.true(); jsonSpy.calledOnce.should.be.true();
imageSpy.called.should.be.false(); imageSpy.called.should.be.false();
mdSpy.called.should.be.false(); mdSpy.called.should.be.false();
@ -227,6 +245,12 @@ describe('Importer', function () {
ImportManager.isValidZip(testDir).should.be.ok(); ImportManager.isValidZip(testDir).should.be.ok();
}); });
it('accepts a zip with a media directory', function () {
const testDir = path.resolve('test/utils/fixtures/import/zips/zip-media-dir');
ImportManager.isValidZip(testDir).should.be.ok();
});
it('accepts a zip with uppercase image extensions', function () { it('accepts a zip with uppercase image extensions', function () {
const testDir = path.resolve('test/utils/fixtures/import/zips/zip-uppercase-extensions'); const testDir = path.resolve('test/utils/fixtures/import/zips/zip-uppercase-extensions');

0
ghost/core/test/utils/fixtures/import/zips/zip-media-dir/media/video.mp4 Normal file
View file