Save YouTube playlists to library in batch to avoid connection pool starvation

package.json

@@ -2,7 +2,7 @@
   "author": "Calvin Montgomery",
   "name": "CyTube",
   "description": "Online media synchronizer and chat",
-  "version": "3.55.4",
+  "version": "3.55.5",
   "repository": {
     "url": "http://github.com/calzoneman/sync"
   },

src/channel/library.js

@@ -4,6 +4,7 @@ var util = require("../utilities");
 var InfoGetter = require("../get-info");
 var db = require("../database");
 var Media = require("../media");
+const LOGGER = require('@calzoneman/jsli')('channel/library');
 
 const TYPE_UNCACHE = {
     id: "string"
@@ -31,6 +32,19 @@ LibraryModule.prototype.cacheMedia = function (media) {
     }
 };
 
+LibraryModule.prototype.cacheMediaList = function (list) {
+    if (this.channel.is(Flags.C_REGISTERED)) {
+        LOGGER.info(
+            'Saving %d items to library for %s',
+            list.length,
+            this.channel.name
+        );
+        db.channels.addListToLibrary(this.channel.name, list).catch(error => {
+            LOGGER.error('Failed to add list to library: %s', error.stack);
+        });
+    }
+};
+
 LibraryModule.prototype.getItem = function (id, cb) {
     db.channels.getLibraryItem(this.channel.name, id, function (err, row) {
         if (err) {

src/channel/playlist.js

@@ -592,10 +592,17 @@ PlaylistModule.prototype.queueYouTubePlaylist = function (user, data) {
             }
 
             self.channel.refCounter.ref("PlaylistModule::queueYouTubePlaylist");
+
+            if (self.channel.modules.library && data.shouldAddToLibrary) {
+                self.channel.modules.library.cacheMediaList(vids);
+                data.shouldAddToLibrary = false;
+            }
+
             vids.forEach(function (media) {
                 data.link = util.formatLink(media.id, media.type);
                 self._addItem(media, data, user);
             });
+
             self.channel.refCounter.unref("PlaylistModule::queueYouTubePlaylist");
 
             lock.release();

src/database/channels.js

@@ -5,6 +5,8 @@ var path = require("path");
 var tables = require("./tables");
 var Flags = require("../flags");
 var util = require("../utilities");
+import { createMySQLDuplicateKeyUpdate } from '../util/on-duplicate-key-update';
+import Config from '../config';
 
 const LOGGER = require('@calzoneman/jsli')('database/channels');
 
@@ -441,6 +443,45 @@ module.exports = {
                  [media.id, media.title, media.seconds, media.type, meta, chan], callback);
     },
 
+    /**
+     * Adds a list of media items to the library
+     */
+    addListToLibrary: async function addListToLibrary(chan, list) {
+        if (!valid(chan)) {
+            throw new Error("Invalid channel name");
+        }
+
+        if (list.length > Config.get("playlist.max-items")) {
+            throw new Error("Cannot save list to library: exceeds max-items");
+        }
+
+        const items = list.map(item => ({
+            id: item.id,
+            title: item.title,
+            seconds: item.seconds,
+            type: item.type,
+            meta: JSON.stringify({
+                bitrate: item.meta.bitrate,
+                codec: item.meta.codec,
+                scuri: item.meta.scuri,
+                embed: item.meta.embed,
+                direct: item.meta.direct
+            }),
+            channel: chan
+        }));
+
+        await db.getDB().runTransaction(tx => {
+            const insert = tx.table('channel_libraries')
+                    .insert(items);
+
+            const update = tx.raw(createMySQLDuplicateKeyUpdate(
+                    ['title', 'seconds', 'meta']
+            ));
+
+            return tx.raw(insert.toString() + update.toString());
+        });
+    },
+
     /**
      * Retrieves a media item from the library by id
      */