diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js index 5f5ab8f18..9e3291ea2 100644 --- a/modules/tests-ui/models.js +++ b/modules/tests-ui/models.js @@ -92,6 +92,9 @@ return this.errors_list && this.errors_list.length > 0; }; + // Used in `loadPluginTests`, `loadTestSets`, `removeOldSets`. + TestUrlsSetSchema.index({plugin: 1, created_at: -1}); + var PageTestLogSchema = new Schema({ url: { @@ -151,6 +154,9 @@ return moment(this.created_at).format("DD-MM-YY HH:mm"); }; + // Used in `checkPageTestLogChangeNotification`. + PageTestLogSchema.index({plugin: 1, url: 1, created_at: -1}); + export const PluginTest = db.model('PluginTest', PluginTestSchema); export const PageTestLog = db.model('PageTestLog', PageTestLogSchema); export const TestUrlsSet = db.model('TestUrlsSet', TestUrlsSetSchema); diff --git a/modules/tests-ui/tester.js b/modules/tests-ui/tester.js index 537024663..d1f719872 100644 --- a/modules/tests-ui/tester.js +++ b/modules/tests-ui/tester.js @@ -50,6 +50,8 @@ var PageTestLog = models.PageTestLog; var TestUrlsSet = models.TestUrlsSet; var TestingProgress = models.TestingProgress; +var OLD_LOGS_MONTH_TTL = 30 * 24 * 60 * 60 * 1000; + function log() { if (CONFIG.DEBUG) { console.log.apply(console, arguments); @@ -134,8 +136,8 @@ function checkPageTestLogChangeNotification(logEntry) { PageTestLog .find({ - url: logEntry.url, plugin: logEntry.plugin, + url: logEntry.url, created_at: { $lt: logEntry.created_at } @@ -146,6 +148,19 @@ function checkPageTestLogChangeNotification(logEntry) { previousLogEntry = previousLogEntry && previousLogEntry.length && previousLogEntry[0]; + if (previousLogEntry) { + // Keep only current and previous log per plugin+url, older ones are unused. + PageTestLog.deleteMany({ + plugin: logEntry.plugin, + url: logEntry.url, + created_at: { + $lt: previousLogEntry.created_at + } + }).catch(function(error) { + cerror('Error removing old page test logs', error); + }); + } + /* Case 1. @@ -564,6 +579,24 @@ function processPluginTests(pluginTest, plugin, count, cb) { cb(null, data); }) .catch(cb); + }, + + function removeOldLogs(data, cb) { + // Logs of urls no longer in the current set. + PageTestLog.deleteMany({ + plugin: plugin.id, + url: { + $nin: testUrlsSet.urls + }, + created_at: { + // Remove only old. + $lt: new Date(new Date() - OLD_LOGS_MONTH_TTL) + } + }) + .then(data => { + cb(null, data); + }) + .catch(cb); } ], cb); @@ -604,7 +637,26 @@ function testAll(cb) { ], cb); }, - function loadPluginTests(data, cb) { + function cleanupObsoleteData(data, cb) { + PluginTest.find({obsolete: true}).distinct('_id') + .then(obsoleteIds => { + if (obsoleteIds.length === 0) { + return Promise.resolve(); + } + // Keep recent data in case plugin comes back soon. + var monthAgo = new Date(new Date() - OLD_LOGS_MONTH_TTL); + return Promise.all([ + TestUrlsSet.deleteMany({plugin: {$in: obsoleteIds}, created_at: {$lt: monthAgo}}), + PageTestLog.deleteMany({plugin: {$in: obsoleteIds}, created_at: {$lt: monthAgo}}) + ]); + }) + .then(() => { + cb(); + }) + .catch(cb); + }, + + function loadPluginTests(cb) { if (testOnePlugin) { PluginTest.find({_id: testOnePlugin})