Merge pull request #461 from calzoneman/ytv3

Add YouTube v3 API

config.template.yaml

@@ -118,8 +118,11 @@ mail:
   from-address: 'some.user@gmail.com'
   from-name: 'CyTube Services'
 
-# GData API v2 developer key (for non-anonymous youtube requests)
+# YouTube v3 API key
-youtube-v2-key: ''
+# See https://developers.google.com/youtube/registering_an_application
+# Google is closing the v2 API (which allowed anonymous requests) on
+# April 20, 2015 so you must register a v3 API key now.
+youtube-v3-key: ''
 # Minutes between saving channel state to disk
 channel-save-interval: 5
 # Limit for the number of channels a user can register

lib/channel/library.js

@@ -67,7 +67,7 @@ LibraryModule.prototype.handleUncache = function (user, data) {
 LibraryModule.prototype.handleSearchMedia = function (user, data) {
     var query = data.query.substring(0, 100);
     var searchYT = function () {
-        InfoGetter.Getters.ytSearch(query.split(" "), function (e, vids) {
+        InfoGetter.Getters.ytSearch(query, function (e, vids) {
             if (!e) {
                 user.socket.emit("searchResults", {
                     source: "yt",

lib/channel/playlist.js

@@ -509,6 +509,7 @@ PlaylistModule.prototype.queueYouTubePlaylist = function (user, data) {
 
             self.channel.activeLock.lock();
             vids.forEach(function (media) {
+                data.link = util.formatLink(media.id, media.type);
                 self._addItem(media, data, user);
             });
             self.channel.activeLock.release();

lib/config.js

@@ -59,7 +59,7 @@ var defaults = {
         "from-address": "some.user@gmail.com",
         "from-name": "CyTube Services"
     },
-    "youtube-v2-key": "",
+    "youtube-v3-key": "",
     "channel-save-interval": 5,
     "max-channels-per-user": 5,
     "max-accounts-per-ip": 5,
@@ -347,6 +347,17 @@ function preprocessConfig(cfg) {
         cfg["link-domain-blacklist-regex"] = new RegExp("$^", "gi");
     }
 
+    if (cfg["youtube-v3-key"]) {
+        require("cytube-mediaquery/lib/provider/youtube").setApiKey(
+                cfg["youtube-v3-key"]);
+    } else {
+        Logger.errlog.log("Warning: No YouTube v3 API key set.  YouTube lookups will " +
+            "fall back to the v2 API, which is scheduled for closure soon after " +
+            "April 20, 2015.  See " +
+            "https://developers.google.com/youtube/registering_an_application for " +
+            "information on registering an API key.");
+    }
+
     return cfg;
 }
 

lib/get-info.js

@@ -7,6 +7,8 @@ var CustomEmbedFilter = require("./customembed").filter;
 var Server = require("./server");
 var Config = require("./config");
 var ffmpeg = require("./ffmpeg");
+require("cytube-mediaquery"); // Initialize sourcemaps
+var YouTube = require("cytube-mediaquery/lib/provider/youtube");
 
 /*
  * Preference map of quality => youtube formats.
@@ -63,249 +65,68 @@ var urlRetrieve = function (transport, options, callback) {
 var Getters = {
     /* youtube.com */
     yt: function (id, callback) {
-        var sv = Server.getServer();
+        if (!Config.get("youtube-v3-key")) {
-
+            return Getters.yt2(id, callback);
-        var m = id.match(/([\w-]{11})/);
-        if (m) {
-            id = m[1];
-        } else {
-            callback("Invalid ID", null);
-            return;
         }
 
-        var options = {
-            host: "gdata.youtube.com",
-            port: 443,
-            path: "/feeds/api/videos/" + id + "?v=2&alt=json",
-            method: "GET",
-            dataType: "jsonp",
-            timeout: 1000
-        };
 
-        if (Config.get("youtube-v2-key")) {
+        YouTube.lookup(id).then(function (video) {
-            options.headers = {
+            var meta = {};
-                "X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
+            if (video.meta.blocked) {
-            };
+                meta.restricted = video.meta.blocked;
-        }
-
-        urlRetrieve(https, options, function (status, data) {
-            switch (status) {
-                case 200:
-                    break; /* Request is OK, skip to handling data */
-                case 400:
-                    return callback("Invalid request", null);
-                case 403:
-                    return callback("Private video", null);
-                case 404:
-                    return callback("Video not found", null);
-                case 500:
-                case 503:
-                    return callback("Service unavailable", null);
-                default:
-                    return callback("HTTP " + status, null);
             }
 
-            var buffer = data;
+            var media = new Media(video.id, video.title, video.duration, "yt", meta);
-            try {
+            callback(false, media);
-                data = JSON.parse(data);
+        }).catch(function (err) {
-                /* Check for embedding restrictions */
+            callback(err.message, null);
-                if (data.entry.yt$accessControl) {
-                    var ac = data.entry.yt$accessControl;
-                    for (var i = 0; i < ac.length; i++) {
-                        if (ac[i].action === "embed") {
-                            if (ac[i].permission === "denied") {
-                                callback("Embedding disabled", null);
-                                return;
-                            }
-                            break;
-                        }
-                    }
-                }
-
-                var seconds = data.entry.media$group.yt$duration.seconds;
-                var title = data.entry.title.$t;
-                var meta = {};
-                /* Check for country restrictions */
-                if (data.entry.media$group.media$restriction) {
-                    var rest = data.entry.media$group.media$restriction;
-                    if (rest.length > 0) {
-                        if (rest[0].relationship === "deny") {
-                            meta.restricted = rest[0].$t;
-                        }
-                    }
-                }
-                var media = new Media(id, title, seconds, "yt", meta);
-                callback(false, media);
-            } catch (e) {
-                // Gdata version 2 has the rather silly habit of
-                // returning error codes in XML when I explicitly asked
-                // for JSON
-                var m = buffer.match(/<internalReason>([^<]+)<\/internalReason>/);
-                if (m === null)
-                    m = buffer.match(/<code>([^<]+)<\/code>/);
-
-                var err = e;
-                if (m) {
-                    if(m[1] === "too_many_recent_calls") {
-                        err = "YouTube is throttling the server right "+
-                               "now for making too many requests.  "+
-                               "Please try again in a moment.";
-                    } else {
-                        err = m[1];
-                    }
-                }
-
-                callback(err, null);
-            }
         });
     },
 
     /* youtube.com playlists */
-    yp: function (id, callback, url) {
+    yp: function (id, callback) {
-        /**
+        if (!Config.get("youtube-v3-key")) {
-         * NOTE: callback may be called multiple times, once for each <= 25 video
+            return Getters.yp2(id, callback);
-         * batch of videos in the list.  It will be called in order.
-         */
-        var m = id.match(/([\w-]+)/);
-        if (m) {
-            id = m[1];
-        } else {
-            callback("Invalid ID", null);
-            return;
-        }
-        var path = "/feeds/api/playlists/" + id + "?v=2&alt=json";
-        /**
-         * NOTE: the third parameter, url, is used to chain this retriever
-         * multiple times to get all the videos from a playlist, as each
-         * request only returns 25 videos.
-         */
-        if (url !== undefined) {
-            path = "/" + url.split("gdata.youtube.com")[1];
         }
 
-        var options = {
+        YouTube.lookupPlaylist(id).then(function (videos) {
-            host: "gdata.youtube.com",
+            videos = videos.map(function (video) {
-            port: 443,
+                var meta = {};
-            path: path,
+                if (video.meta.blocked) {
-            method: "GET",
+                    meta.restricted = video.meta.blocked;
-            dataType: "jsonp",
-            timeout: 1000
-        };
-
-        if (Config.get("youtube-v2-key")) {
-            options.headers = {
-                "X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
-            };
-        }
-
-        urlRetrieve(https, options, function (status, data) {
-            switch (status) {
-                case 200:
-                    break; /* Request is OK, skip to handling data */
-                case 400:
-                    return callback("Invalid request", null);
-                case 403:
-                    return callback("Private playlist", null);
-                case 404:
-                    return callback("Playlist not found", null);
-                case 500:
-                case 503:
-                    return callback("Service unavailable", null);
-                default:
-                    return callback("HTTP " + status, null);
-            }
-
-            try {
-                data = JSON.parse(data);
-                var vids = [];
-                for(var i in data.feed.entry) {
-                    try {
-                        /**
-                         * FIXME: This should probably check for embed restrictions
-                         * and country restrictions on each video in the list
-                         */
-                        var item = data.feed.entry[i];
-                        var id = item.media$group.yt$videoid.$t;
-                        var title = item.title.$t;
-                        var seconds = item.media$group.yt$duration.seconds;
-                        var media = new Media(id, title, seconds, "yt");
-                        vids.push(media);
-                    } catch(e) {
-                    }
                 }
 
-                callback(false, vids);
+                return new Media(video.id, video.title, video.duration, "yt", meta);
-
+            });
-                var links = data.feed.link;
-                for (var i in links) {
-                    if (links[i].rel === "next") {
-                        /* Look up the next batch of videos from the list */
-                        Getters["yp"](id, callback, links[i].href);
-                    }
-                }
-            } catch (e) {
-                callback(e, null);
-            }
 
+            callback(null, videos);
+        }).catch(function (err) {
+            callback(err.message, null);
         });
     },
 
     /* youtube.com search */
-    ytSearch: function (terms, callback) {
+    ytSearch: function (query, callback) {
-        /**
+        if (!Config.get("youtube-v3-key")) {
-         * terms is a list of words from the search query.  Each word must be
+            return Getters.ytSearch2(query.split(" "), callback);
-         * encoded properly for use in the request URI
-         */
-        for (var i in terms) {
-            terms[i] = encodeURIComponent(terms[i]);
-        }
-        var query = terms.join("+");
-
-        var options = {
-            host: "gdata.youtube.com",
-            port: 443,
-            path: "/feeds/api/videos/?q=" + query + "&v=2&alt=json",
-            method: "GET",
-            dataType: "jsonp",
-            timeout: 1000
-        };
-
-        if (Config.get("youtube-v2-key")) {
-            options.headers = {
-                "X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
-            };
         }
 
-        urlRetrieve(https, options, function (status, data) {
+        YouTube.search(query).then(function (res) {
-            if (status !== 200) {
+            var videos = res.results;
-                callback("YouTube search: HTTP " + status, null);
+            videos = videos.map(function (video) {
-                return;
+                var meta = {};
-            }
+                if (video.meta.blocked) {
-
+                    meta.restricted = video.meta.blocked;
-            try {
-                data = JSON.parse(data);
-                var vids = [];
-                for(var i in data.feed.entry) {
-                    try {
-                        /**
-                         * FIXME: This should probably check for embed restrictions
-                         * and country restrictions on each video in the list
-                         */
-                        var item = data.feed.entry[i];
-                        var id = item.media$group.yt$videoid.$t;
-                        var title = item.title.$t;
-                        var seconds = item.media$group.yt$duration.seconds;
-                        var media = new Media(id, title, seconds, "yt");
-                        media.thumb = item.media$group.media$thumbnail[0];
-                        vids.push(media);
-                    } catch(e) {
-                    }
                 }
 
-                callback(false, vids);
+                var media = new Media(video.id, video.title, video.duration, "yt", meta);
-            } catch(e) {
+                media.thumb = { url: video.meta.thumbnail };
-                callback(e, null);
+                return media;
-            }
+            });
+
+            callback(null, videos);
+        }).catch(function (err) {
+            callback(err.message, null);
         });
     },
 
@@ -905,6 +726,254 @@ var Getters = {
         var media = new Media(id, title, "--:--", "hb");
         callback(false, media);
     },
+
+    /* youtube.com - old v2 API */
+    yt2: function (id, callback) {
+        var sv = Server.getServer();
+
+        var m = id.match(/([\w-]{11})/);
+        if (m) {
+            id = m[1];
+        } else {
+            callback("Invalid ID", null);
+            return;
+        }
+
+        var options = {
+            host: "gdata.youtube.com",
+            port: 443,
+            path: "/feeds/api/videos/" + id + "?v=2&alt=json",
+            method: "GET",
+            dataType: "jsonp",
+            timeout: 1000
+        };
+
+        if (Config.get("youtube-v2-key")) {
+            options.headers = {
+                "X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
+            };
+        }
+
+        urlRetrieve(https, options, function (status, data) {
+            switch (status) {
+                case 200:
+                    break; /* Request is OK, skip to handling data */
+                case 400:
+                    return callback("Invalid request", null);
+                case 403:
+                    return callback("Private video", null);
+                case 404:
+                    return callback("Video not found", null);
+                case 500:
+                case 503:
+                    return callback("Service unavailable", null);
+                default:
+                    return callback("HTTP " + status, null);
+            }
+
+            var buffer = data;
+            try {
+                data = JSON.parse(data);
+                /* Check for embedding restrictions */
+                if (data.entry.yt$accessControl) {
+                    var ac = data.entry.yt$accessControl;
+                    for (var i = 0; i < ac.length; i++) {
+                        if (ac[i].action === "embed") {
+                            if (ac[i].permission === "denied") {
+                                callback("Embedding disabled", null);
+                                return;
+                            }
+                            break;
+                        }
+                    }
+                }
+
+                var seconds = data.entry.media$group.yt$duration.seconds;
+                var title = data.entry.title.$t;
+                var meta = {};
+                /* Check for country restrictions */
+                if (data.entry.media$group.media$restriction) {
+                    var rest = data.entry.media$group.media$restriction;
+                    if (rest.length > 0) {
+                        if (rest[0].relationship === "deny") {
+                            meta.restricted = rest[0].$t;
+                        }
+                    }
+                }
+                var media = new Media(id, title, seconds, "yt", meta);
+                callback(false, media);
+            } catch (e) {
+                // Gdata version 2 has the rather silly habit of
+                // returning error codes in XML when I explicitly asked
+                // for JSON
+                var m = buffer.match(/<internalReason>([^<]+)<\/internalReason>/);
+                if (m === null)
+                    m = buffer.match(/<code>([^<]+)<\/code>/);
+
+                var err = e;
+                if (m) {
+                    if(m[1] === "too_many_recent_calls") {
+                        err = "YouTube is throttling the server right "+
+                               "now for making too many requests.  "+
+                               "Please try again in a moment.";
+                    } else {
+                        err = m[1];
+                    }
+                }
+
+                callback(err, null);
+            }
+        });
+    },
+
+    /* youtube.com playlists - old v2 api */
+    yp2: function (id, callback, url) {
+        /**
+         * NOTE: callback may be called multiple times, once for each <= 25 video
+         * batch of videos in the list.  It will be called in order.
+         */
+        var m = id.match(/([\w-]+)/);
+        if (m) {
+            id = m[1];
+        } else {
+            callback("Invalid ID", null);
+            return;
+        }
+        var path = "/feeds/api/playlists/" + id + "?v=2&alt=json";
+        /**
+         * NOTE: the third parameter, url, is used to chain this retriever
+         * multiple times to get all the videos from a playlist, as each
+         * request only returns 25 videos.
+         */
+        if (url !== undefined) {
+            path = "/" + url.split("gdata.youtube.com")[1];
+        }
+
+        var options = {
+            host: "gdata.youtube.com",
+            port: 443,
+            path: path,
+            method: "GET",
+            dataType: "jsonp",
+            timeout: 1000
+        };
+
+        if (Config.get("youtube-v2-key")) {
+            options.headers = {
+                "X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
+            };
+        }
+
+        urlRetrieve(https, options, function (status, data) {
+            switch (status) {
+                case 200:
+                    break; /* Request is OK, skip to handling data */
+                case 400:
+                    return callback("Invalid request", null);
+                case 403:
+                    return callback("Private playlist", null);
+                case 404:
+                    return callback("Playlist not found", null);
+                case 500:
+                case 503:
+                    return callback("Service unavailable", null);
+                default:
+                    return callback("HTTP " + status, null);
+            }
+
+            try {
+                data = JSON.parse(data);
+                var vids = [];
+                for(var i in data.feed.entry) {
+                    try {
+                        /**
+                         * FIXME: This should probably check for embed restrictions
+                         * and country restrictions on each video in the list
+                         */
+                        var item = data.feed.entry[i];
+                        var id = item.media$group.yt$videoid.$t;
+                        var title = item.title.$t;
+                        var seconds = item.media$group.yt$duration.seconds;
+                        var media = new Media(id, title, seconds, "yt");
+                        vids.push(media);
+                    } catch(e) {
+                    }
+                }
+
+                callback(false, vids);
+
+                var links = data.feed.link;
+                for (var i in links) {
+                    if (links[i].rel === "next") {
+                        /* Look up the next batch of videos from the list */
+                        Getters["yp2"](id, callback, links[i].href);
+                    }
+                }
+            } catch (e) {
+                callback(e, null);
+            }
+
+        });
+    },
+
+    /* youtube.com search - old v2 api */
+    ytSearch2: function (terms, callback) {
+        /**
+         * terms is a list of words from the search query.  Each word must be
+         * encoded properly for use in the request URI
+         */
+        for (var i in terms) {
+            terms[i] = encodeURIComponent(terms[i]);
+        }
+        var query = terms.join("+");
+
+        var options = {
+            host: "gdata.youtube.com",
+            port: 443,
+            path: "/feeds/api/videos/?q=" + query + "&v=2&alt=json",
+            method: "GET",
+            dataType: "jsonp",
+            timeout: 1000
+        };
+
+        if (Config.get("youtube-v2-key")) {
+            options.headers = {
+                "X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
+            };
+        }
+
+        urlRetrieve(https, options, function (status, data) {
+            if (status !== 200) {
+                callback("YouTube search: HTTP " + status, null);
+                return;
+            }
+
+            try {
+                data = JSON.parse(data);
+                var vids = [];
+                for(var i in data.feed.entry) {
+                    try {
+                        /**
+                         * FIXME: This should probably check for embed restrictions
+                         * and country restrictions on each video in the list
+                         */
+                        var item = data.feed.entry[i];
+                        var id = item.media$group.yt$videoid.$t;
+                        var title = item.title.$t;
+                        var seconds = item.media$group.yt$duration.seconds;
+                        var media = new Media(id, title, seconds, "yt");
+                        media.thumb = item.media$group.media$thumbnail[0];
+                        vids.push(media);
+                    } catch(e) {
+                    }
+                }
+
+                callback(false, vids);
+            } catch(e) {
+                callback(e, null);
+            }
+        });
+    },
 };
 
 /**

package.json

@@ -13,6 +13,7 @@
     "compression": "^1.3.0",
     "cookie-parser": "^1.3.3",
     "csrf": "^2.0.6",
+    "cytube-mediaquery": "git://github.com/CyTube/mediaquery",
     "cytubefilters": "git://github.com/calzoneman/cytubefilters#7663b3a9",
     "express": "^4.11.1",
     "express-minify": "^0.1.3",

www/js/util.js

@@ -2019,7 +2019,7 @@ function queueMessage(data, type) {
                 data.link + "</a>";
     }
     makeAlert(title, text, type)
-        .addClass("qfalert qf-" + type)
+        .addClass("linewrap qfalert qf-" + type)
         .appendTo($("#queuefail"));
 }