🐛 Fixed error in sitemap with >50k posts (#13317)

closes: CORE-34
refs: https://github.com/TryGhost/Team/issues/1044

- this is a super basic fix, it adds a max nodes concept and limits the node in each sub-sitemap to 50k by default
- this will prevent the error in google console
- a better fix is in progress, but we want to at least solve the errors ASAP

core/frontend/services/sitemap/base-generator.js

@@ -19,30 +19,34 @@ class BaseSiteMapGenerator {
         this.nodeTimeLookup = {};
         this.siteMapContent = null;
         this.lastModified = 0;
+        this.maxNodes = 50000;
     }
 
     generateXmlFromNodes() {
-        const self = this;
-
         // Get a mapping of node to timestamp
-        const timedNodes = _.map(this.nodeLookup, function (node, id) {
+        let nodesToProcess = _.map(this.nodeLookup, (node, id) => {
             return {
                 id: id,
                 // Using negative here to sort newest to oldest
-                ts: -(self.nodeTimeLookup[id] || 0),
+                ts: -(this.nodeTimeLookup[id] || 0),
                 node: node
             };
         });
 
+        // Limit to 50k nodes - this is a quick fix to prevent errors in google console
+        if (this.maxNodes) {
+            nodesToProcess = nodesToProcess.slice(0, this.maxNodes);
+        }
+
         // Sort nodes by timestamp
-        const sortedNodes = _.sortBy(timedNodes, 'ts');
+        nodesToProcess = _.sortBy(nodesToProcess, 'ts');
 
         // Grab just the nodes
-        const urlElements = _.map(sortedNodes, 'node');
+        nodesToProcess = _.map(nodesToProcess, 'node');
 
         const data = {
             // Concat the elements to the _attr declaration
-            urlset: [XMLNS_DECLS].concat(urlElements)
+            urlset: [XMLNS_DECLS].concat(nodesToProcess)
         };
 
         // Generate full xml

test/unit/services/sitemap/generator.test.js

@@ -52,6 +52,32 @@ describe('Generators', function () {
         sinon.restore();
     });
 
+    it('max node setting results in the right number of nodes', function () {
+        generator = new PostGenerator({maxNodes: 5});
+
+        for (let i = 0; i < 10; i++) {
+            generator.addUrl(`http://my-ghost-blog.com/episode-${i}/`, testUtils.DataGenerator.forKnex.createPost({
+                created_at: (Date.UTC(2014, 11, 22, 12) - 360000) + 200,
+                updated_at: null,
+                published_at: null,
+                slug: `episode-${i}`
+            }));
+        }
+
+        generator.getXml();
+
+        // We end up with 10 nodes
+        Object.keys(generator.nodeLookup).should.be.Array().with.lengthOf(10);
+
+        // But only 5 are output in the xml
+        generator.siteMapContent.match(/<loc>/g).should.be.Array().with.lengthOf(5);
+    });
+
+    it('default is 50k', function () {
+        generator = new PostGenerator();
+        generator.maxNodes.should.eql(50000);
+    });
+
     describe('IndexGenerator', function () {
         beforeEach(function () {
             generator = new IndexGenerator({