diff --git a/config.template.yaml b/config.template.yaml index d43eca20..71fb870f 100644 --- a/config.template.yaml +++ b/config.template.yaml @@ -118,8 +118,11 @@ mail: from-address: 'some.user@gmail.com' from-name: 'CyTube Services' -# GData API v2 developer key (for non-anonymous youtube requests) -youtube-v2-key: '' +# YouTube v3 API key +# See https://developers.google.com/youtube/registering_an_application +# Google is closing the v2 API (which allowed anonymous requests) on +# April 20, 2015 so you must register a v3 API key now. +youtube-v3-key: '' # Minutes between saving channel state to disk channel-save-interval: 5 # Limit for the number of channels a user can register diff --git a/lib/channel/library.js b/lib/channel/library.js index b255d667..2d8eeb78 100644 --- a/lib/channel/library.js +++ b/lib/channel/library.js @@ -67,7 +67,7 @@ LibraryModule.prototype.handleUncache = function (user, data) { LibraryModule.prototype.handleSearchMedia = function (user, data) { var query = data.query.substring(0, 100); var searchYT = function () { - InfoGetter.Getters.ytSearch(query.split(" "), function (e, vids) { + InfoGetter.Getters.ytSearch(query, function (e, vids) { if (!e) { user.socket.emit("searchResults", { source: "yt", diff --git a/lib/channel/playlist.js b/lib/channel/playlist.js index f414576a..327d1323 100644 --- a/lib/channel/playlist.js +++ b/lib/channel/playlist.js @@ -509,6 +509,7 @@ PlaylistModule.prototype.queueYouTubePlaylist = function (user, data) { self.channel.activeLock.lock(); vids.forEach(function (media) { + data.link = util.formatLink(media.id, media.type); self._addItem(media, data, user); }); self.channel.activeLock.release(); diff --git a/lib/config.js b/lib/config.js index 197849db..d549fe0a 100644 --- a/lib/config.js +++ b/lib/config.js @@ -59,7 +59,7 @@ var defaults = { "from-address": "some.user@gmail.com", "from-name": "CyTube Services" }, - "youtube-v2-key": "", + "youtube-v3-key": "", "channel-save-interval": 5, "max-channels-per-user": 5, "max-accounts-per-ip": 5, @@ -347,6 +347,17 @@ function preprocessConfig(cfg) { cfg["link-domain-blacklist-regex"] = new RegExp("$^", "gi"); } + if (cfg["youtube-v3-key"]) { + require("cytube-mediaquery/lib/provider/youtube").setApiKey( + cfg["youtube-v3-key"]); + } else { + Logger.errlog.log("Warning: No YouTube v3 API key set. YouTube lookups will " + + "fall back to the v2 API, which is scheduled for closure soon after " + + "April 20, 2015. See " + + "https://developers.google.com/youtube/registering_an_application for " + + "information on registering an API key."); + } + return cfg; } diff --git a/lib/get-info.js b/lib/get-info.js index ce2b30fa..bd017b8f 100644 --- a/lib/get-info.js +++ b/lib/get-info.js @@ -7,6 +7,8 @@ var CustomEmbedFilter = require("./customembed").filter; var Server = require("./server"); var Config = require("./config"); var ffmpeg = require("./ffmpeg"); +require("cytube-mediaquery"); // Initialize sourcemaps +var YouTube = require("cytube-mediaquery/lib/provider/youtube"); /* * Preference map of quality => youtube formats. @@ -63,249 +65,68 @@ var urlRetrieve = function (transport, options, callback) { var Getters = { /* youtube.com */ yt: function (id, callback) { - var sv = Server.getServer(); - - var m = id.match(/([\w-]{11})/); - if (m) { - id = m[1]; - } else { - callback("Invalid ID", null); - return; + if (!Config.get("youtube-v3-key")) { + return Getters.yt2(id, callback); } - var options = { - host: "gdata.youtube.com", - port: 443, - path: "/feeds/api/videos/" + id + "?v=2&alt=json", - method: "GET", - dataType: "jsonp", - timeout: 1000 - }; - if (Config.get("youtube-v2-key")) { - options.headers = { - "X-Gdata-Key": "key=" + Config.get("youtube-v2-key") - }; - } - - urlRetrieve(https, options, function (status, data) { - switch (status) { - case 200: - break; /* Request is OK, skip to handling data */ - case 400: - return callback("Invalid request", null); - case 403: - return callback("Private video", null); - case 404: - return callback("Video not found", null); - case 500: - case 503: - return callback("Service unavailable", null); - default: - return callback("HTTP " + status, null); + YouTube.lookup(id).then(function (video) { + var meta = {}; + if (video.meta.blocked) { + meta.restricted = video.meta.blocked; } - var buffer = data; - try { - data = JSON.parse(data); - /* Check for embedding restrictions */ - if (data.entry.yt$accessControl) { - var ac = data.entry.yt$accessControl; - for (var i = 0; i < ac.length; i++) { - if (ac[i].action === "embed") { - if (ac[i].permission === "denied") { - callback("Embedding disabled", null); - return; - } - break; - } - } - } - - var seconds = data.entry.media$group.yt$duration.seconds; - var title = data.entry.title.$t; - var meta = {}; - /* Check for country restrictions */ - if (data.entry.media$group.media$restriction) { - var rest = data.entry.media$group.media$restriction; - if (rest.length > 0) { - if (rest[0].relationship === "deny") { - meta.restricted = rest[0].$t; - } - } - } - var media = new Media(id, title, seconds, "yt", meta); - callback(false, media); - } catch (e) { - // Gdata version 2 has the rather silly habit of - // returning error codes in XML when I explicitly asked - // for JSON - var m = buffer.match(/([^<]+)<\/internalReason>/); - if (m === null) - m = buffer.match(/([^<]+)<\/code>/); - - var err = e; - if (m) { - if(m[1] === "too_many_recent_calls") { - err = "YouTube is throttling the server right "+ - "now for making too many requests. "+ - "Please try again in a moment."; - } else { - err = m[1]; - } - } - - callback(err, null); - } + var media = new Media(video.id, video.title, video.duration, "yt", meta); + callback(false, media); + }).catch(function (err) { + callback(err.message, null); }); }, /* youtube.com playlists */ - yp: function (id, callback, url) { - /** - * NOTE: callback may be called multiple times, once for each <= 25 video - * batch of videos in the list. It will be called in order. - */ - var m = id.match(/([\w-]+)/); - if (m) { - id = m[1]; - } else { - callback("Invalid ID", null); - return; - } - var path = "/feeds/api/playlists/" + id + "?v=2&alt=json"; - /** - * NOTE: the third parameter, url, is used to chain this retriever - * multiple times to get all the videos from a playlist, as each - * request only returns 25 videos. - */ - if (url !== undefined) { - path = "/" + url.split("gdata.youtube.com")[1]; + yp: function (id, callback) { + if (!Config.get("youtube-v3-key")) { + return Getters.yp2(id, callback); } - var options = { - host: "gdata.youtube.com", - port: 443, - path: path, - method: "GET", - dataType: "jsonp", - timeout: 1000 - }; - - if (Config.get("youtube-v2-key")) { - options.headers = { - "X-Gdata-Key": "key=" + Config.get("youtube-v2-key") - }; - } - - urlRetrieve(https, options, function (status, data) { - switch (status) { - case 200: - break; /* Request is OK, skip to handling data */ - case 400: - return callback("Invalid request", null); - case 403: - return callback("Private playlist", null); - case 404: - return callback("Playlist not found", null); - case 500: - case 503: - return callback("Service unavailable", null); - default: - return callback("HTTP " + status, null); - } - - try { - data = JSON.parse(data); - var vids = []; - for(var i in data.feed.entry) { - try { - /** - * FIXME: This should probably check for embed restrictions - * and country restrictions on each video in the list - */ - var item = data.feed.entry[i]; - var id = item.media$group.yt$videoid.$t; - var title = item.title.$t; - var seconds = item.media$group.yt$duration.seconds; - var media = new Media(id, title, seconds, "yt"); - vids.push(media); - } catch(e) { - } + YouTube.lookupPlaylist(id).then(function (videos) { + videos = videos.map(function (video) { + var meta = {}; + if (video.meta.blocked) { + meta.restricted = video.meta.blocked; } - callback(false, vids); - - var links = data.feed.link; - for (var i in links) { - if (links[i].rel === "next") { - /* Look up the next batch of videos from the list */ - Getters["yp"](id, callback, links[i].href); - } - } - } catch (e) { - callback(e, null); - } + return new Media(video.id, video.title, video.duration, "yt", meta); + }); + callback(null, videos); + }).catch(function (err) { + callback(err.message, null); }); }, /* youtube.com search */ - ytSearch: function (terms, callback) { - /** - * terms is a list of words from the search query. Each word must be - * encoded properly for use in the request URI - */ - for (var i in terms) { - terms[i] = encodeURIComponent(terms[i]); - } - var query = terms.join("+"); - - var options = { - host: "gdata.youtube.com", - port: 443, - path: "/feeds/api/videos/?q=" + query + "&v=2&alt=json", - method: "GET", - dataType: "jsonp", - timeout: 1000 - }; - - if (Config.get("youtube-v2-key")) { - options.headers = { - "X-Gdata-Key": "key=" + Config.get("youtube-v2-key") - }; + ytSearch: function (query, callback) { + if (!Config.get("youtube-v3-key")) { + return Getters.ytSearch2(query.split(" "), callback); } - urlRetrieve(https, options, function (status, data) { - if (status !== 200) { - callback("YouTube search: HTTP " + status, null); - return; - } - - try { - data = JSON.parse(data); - var vids = []; - for(var i in data.feed.entry) { - try { - /** - * FIXME: This should probably check for embed restrictions - * and country restrictions on each video in the list - */ - var item = data.feed.entry[i]; - var id = item.media$group.yt$videoid.$t; - var title = item.title.$t; - var seconds = item.media$group.yt$duration.seconds; - var media = new Media(id, title, seconds, "yt"); - media.thumb = item.media$group.media$thumbnail[0]; - vids.push(media); - } catch(e) { - } + YouTube.search(query).then(function (res) { + var videos = res.results; + videos = videos.map(function (video) { + var meta = {}; + if (video.meta.blocked) { + meta.restricted = video.meta.blocked; } - callback(false, vids); - } catch(e) { - callback(e, null); - } + var media = new Media(video.id, video.title, video.duration, "yt", meta); + media.thumb = { url: video.meta.thumbnail }; + return media; + }); + + callback(null, videos); + }).catch(function (err) { + callback(err.message, null); }); }, @@ -905,6 +726,254 @@ var Getters = { var media = new Media(id, title, "--:--", "hb"); callback(false, media); }, + + /* youtube.com - old v2 API */ + yt2: function (id, callback) { + var sv = Server.getServer(); + + var m = id.match(/([\w-]{11})/); + if (m) { + id = m[1]; + } else { + callback("Invalid ID", null); + return; + } + + var options = { + host: "gdata.youtube.com", + port: 443, + path: "/feeds/api/videos/" + id + "?v=2&alt=json", + method: "GET", + dataType: "jsonp", + timeout: 1000 + }; + + if (Config.get("youtube-v2-key")) { + options.headers = { + "X-Gdata-Key": "key=" + Config.get("youtube-v2-key") + }; + } + + urlRetrieve(https, options, function (status, data) { + switch (status) { + case 200: + break; /* Request is OK, skip to handling data */ + case 400: + return callback("Invalid request", null); + case 403: + return callback("Private video", null); + case 404: + return callback("Video not found", null); + case 500: + case 503: + return callback("Service unavailable", null); + default: + return callback("HTTP " + status, null); + } + + var buffer = data; + try { + data = JSON.parse(data); + /* Check for embedding restrictions */ + if (data.entry.yt$accessControl) { + var ac = data.entry.yt$accessControl; + for (var i = 0; i < ac.length; i++) { + if (ac[i].action === "embed") { + if (ac[i].permission === "denied") { + callback("Embedding disabled", null); + return; + } + break; + } + } + } + + var seconds = data.entry.media$group.yt$duration.seconds; + var title = data.entry.title.$t; + var meta = {}; + /* Check for country restrictions */ + if (data.entry.media$group.media$restriction) { + var rest = data.entry.media$group.media$restriction; + if (rest.length > 0) { + if (rest[0].relationship === "deny") { + meta.restricted = rest[0].$t; + } + } + } + var media = new Media(id, title, seconds, "yt", meta); + callback(false, media); + } catch (e) { + // Gdata version 2 has the rather silly habit of + // returning error codes in XML when I explicitly asked + // for JSON + var m = buffer.match(/([^<]+)<\/internalReason>/); + if (m === null) + m = buffer.match(/([^<]+)<\/code>/); + + var err = e; + if (m) { + if(m[1] === "too_many_recent_calls") { + err = "YouTube is throttling the server right "+ + "now for making too many requests. "+ + "Please try again in a moment."; + } else { + err = m[1]; + } + } + + callback(err, null); + } + }); + }, + + /* youtube.com playlists - old v2 api */ + yp2: function (id, callback, url) { + /** + * NOTE: callback may be called multiple times, once for each <= 25 video + * batch of videos in the list. It will be called in order. + */ + var m = id.match(/([\w-]+)/); + if (m) { + id = m[1]; + } else { + callback("Invalid ID", null); + return; + } + var path = "/feeds/api/playlists/" + id + "?v=2&alt=json"; + /** + * NOTE: the third parameter, url, is used to chain this retriever + * multiple times to get all the videos from a playlist, as each + * request only returns 25 videos. + */ + if (url !== undefined) { + path = "/" + url.split("gdata.youtube.com")[1]; + } + + var options = { + host: "gdata.youtube.com", + port: 443, + path: path, + method: "GET", + dataType: "jsonp", + timeout: 1000 + }; + + if (Config.get("youtube-v2-key")) { + options.headers = { + "X-Gdata-Key": "key=" + Config.get("youtube-v2-key") + }; + } + + urlRetrieve(https, options, function (status, data) { + switch (status) { + case 200: + break; /* Request is OK, skip to handling data */ + case 400: + return callback("Invalid request", null); + case 403: + return callback("Private playlist", null); + case 404: + return callback("Playlist not found", null); + case 500: + case 503: + return callback("Service unavailable", null); + default: + return callback("HTTP " + status, null); + } + + try { + data = JSON.parse(data); + var vids = []; + for(var i in data.feed.entry) { + try { + /** + * FIXME: This should probably check for embed restrictions + * and country restrictions on each video in the list + */ + var item = data.feed.entry[i]; + var id = item.media$group.yt$videoid.$t; + var title = item.title.$t; + var seconds = item.media$group.yt$duration.seconds; + var media = new Media(id, title, seconds, "yt"); + vids.push(media); + } catch(e) { + } + } + + callback(false, vids); + + var links = data.feed.link; + for (var i in links) { + if (links[i].rel === "next") { + /* Look up the next batch of videos from the list */ + Getters["yp2"](id, callback, links[i].href); + } + } + } catch (e) { + callback(e, null); + } + + }); + }, + + /* youtube.com search - old v2 api */ + ytSearch2: function (terms, callback) { + /** + * terms is a list of words from the search query. Each word must be + * encoded properly for use in the request URI + */ + for (var i in terms) { + terms[i] = encodeURIComponent(terms[i]); + } + var query = terms.join("+"); + + var options = { + host: "gdata.youtube.com", + port: 443, + path: "/feeds/api/videos/?q=" + query + "&v=2&alt=json", + method: "GET", + dataType: "jsonp", + timeout: 1000 + }; + + if (Config.get("youtube-v2-key")) { + options.headers = { + "X-Gdata-Key": "key=" + Config.get("youtube-v2-key") + }; + } + + urlRetrieve(https, options, function (status, data) { + if (status !== 200) { + callback("YouTube search: HTTP " + status, null); + return; + } + + try { + data = JSON.parse(data); + var vids = []; + for(var i in data.feed.entry) { + try { + /** + * FIXME: This should probably check for embed restrictions + * and country restrictions on each video in the list + */ + var item = data.feed.entry[i]; + var id = item.media$group.yt$videoid.$t; + var title = item.title.$t; + var seconds = item.media$group.yt$duration.seconds; + var media = new Media(id, title, seconds, "yt"); + media.thumb = item.media$group.media$thumbnail[0]; + vids.push(media); + } catch(e) { + } + } + + callback(false, vids); + } catch(e) { + callback(e, null); + } + }); + }, }; /** diff --git a/package.json b/package.json index 6ab09307..f273aa26 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "compression": "^1.3.0", "cookie-parser": "^1.3.3", "csrf": "^2.0.6", + "cytube-mediaquery": "git://github.com/CyTube/mediaquery", "cytubefilters": "git://github.com/calzoneman/cytubefilters#7663b3a9", "express": "^4.11.1", "express-minify": "^0.1.3", diff --git a/www/js/util.js b/www/js/util.js index e55a218b..b24387de 100644 --- a/www/js/util.js +++ b/www/js/util.js @@ -2019,7 +2019,7 @@ function queueMessage(data, type) { data.link + ""; } makeAlert(title, text, type) - .addClass("qfalert qf-" + type) + .addClass("linewrap qfalert qf-" + type) .appendTo($("#queuefail")); }