Save YouTube playlists to library in batch to avoid connection pool starvation

This commit is contained in:
Calvin Montgomery 2018-03-05 22:19:51 -08:00
parent 54bf7f1c5b
commit fcfc45dd70
4 changed files with 63 additions and 1 deletions

View File

@ -2,7 +2,7 @@
"author": "Calvin Montgomery",
"name": "CyTube",
"description": "Online media synchronizer and chat",
"version": "3.55.4",
"version": "3.55.5",
"repository": {
"url": "http://github.com/calzoneman/sync"
},

View File

@ -4,6 +4,7 @@ var util = require("../utilities");
var InfoGetter = require("../get-info");
var db = require("../database");
var Media = require("../media");
const LOGGER = require('@calzoneman/jsli')('channel/library');
const TYPE_UNCACHE = {
id: "string"
@ -31,6 +32,19 @@ LibraryModule.prototype.cacheMedia = function (media) {
}
};
LibraryModule.prototype.cacheMediaList = function (list) {
if (this.channel.is(Flags.C_REGISTERED)) {
LOGGER.info(
'Saving %d items to library for %s',
list.length,
this.channel.name
);
db.channels.addListToLibrary(this.channel.name, list).catch(error => {
LOGGER.error('Failed to add list to library: %s', error.stack);
});
}
};
LibraryModule.prototype.getItem = function (id, cb) {
db.channels.getLibraryItem(this.channel.name, id, function (err, row) {
if (err) {

View File

@ -592,10 +592,17 @@ PlaylistModule.prototype.queueYouTubePlaylist = function (user, data) {
}
self.channel.refCounter.ref("PlaylistModule::queueYouTubePlaylist");
if (self.channel.modules.library && data.shouldAddToLibrary) {
self.channel.modules.library.cacheMediaList(vids);
data.shouldAddToLibrary = false;
}
vids.forEach(function (media) {
data.link = util.formatLink(media.id, media.type);
self._addItem(media, data, user);
});
self.channel.refCounter.unref("PlaylistModule::queueYouTubePlaylist");
lock.release();

View File

@ -5,6 +5,8 @@ var path = require("path");
var tables = require("./tables");
var Flags = require("../flags");
var util = require("../utilities");
import { createMySQLDuplicateKeyUpdate } from '../util/on-duplicate-key-update';
import Config from '../config';
const LOGGER = require('@calzoneman/jsli')('database/channels');
@ -441,6 +443,45 @@ module.exports = {
[media.id, media.title, media.seconds, media.type, meta, chan], callback);
},
/**
* Adds a list of media items to the library
*/
addListToLibrary: async function addListToLibrary(chan, list) {
if (!valid(chan)) {
throw new Error("Invalid channel name");
}
if (list.length > Config.get("playlist.max-items")) {
throw new Error("Cannot save list to library: exceeds max-items");
}
const items = list.map(item => ({
id: item.id,
title: item.title,
seconds: item.seconds,
type: item.type,
meta: JSON.stringify({
bitrate: item.meta.bitrate,
codec: item.meta.codec,
scuri: item.meta.scuri,
embed: item.meta.embed,
direct: item.meta.direct
}),
channel: chan
}));
await db.getDB().runTransaction(tx => {
const insert = tx.table('channel_libraries')
.insert(items);
const update = tx.raw(createMySQLDuplicateKeyUpdate(
['title', 'seconds', 'meta']
));
return tx.raw(insert.toString() + update.toString());
});
},
/**
* Retrieves a media item from the library by id
*/