From 1b7e7c74f5a4a0e39b2932be64fe281b7d7f2ec4 Mon Sep 17 00:00:00 2001 From: Calvin Montgomery Date: Thu, 12 Aug 2021 22:11:10 -0700 Subject: [PATCH] Remove legacy counters --- NEWS.md | 9 ++ src/channel/chat.js | 3 - src/channel/playlist.js | 2 - src/counters.js | 55 ---------- src/database.js | 5 - src/io/ioserver.js | 3 - src/metrics/jsonfilemetricsreporter.js | 73 ------------- src/metrics/metrics.js | 136 ------------------------- src/web/webserver.js | 5 - 9 files changed, 9 insertions(+), 282 deletions(-) delete mode 100644 src/counters.js delete mode 100644 src/metrics/jsonfilemetricsreporter.js delete mode 100644 src/metrics/metrics.js diff --git a/NEWS.md b/NEWS.md index 3fa094cc..1567855d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,15 @@ 2021-08-12 ========== +The legacy metrics recorder (`counters.log` file) has been removed. For over 4 +years now, CyTube has integrated with [Prometheus](https://prometheus.io/), +which provides a superior way to monitor the application. Copy +`conf/example/prometheus.toml` to `conf/prometheus.toml` and edit it to +configure CyTube's Prometheus support. + +2021-08-12 +========== + Due to changes in Soundcloud's authorization scheme, support has been dropped from core due to requiring each server owner to register an API key (which is currently impossible as they have not accepted new API key registrations for diff --git a/src/channel/chat.js b/src/channel/chat.js index d726d3c0..a6d53ea2 100644 --- a/src/channel/chat.js +++ b/src/channel/chat.js @@ -3,7 +3,6 @@ var XSS = require("../xss"); var ChannelModule = require("./module"); var util = require("../utilities"); var Flags = require("../flags"); -var counters = require("../counters"); import { transformImgTags } from '../camo'; import { Counter } from 'prom-client'; @@ -157,7 +156,6 @@ const chatIncomingCount = new Counter({ }); ChatModule.prototype.handleChatMsg = function (user, data) { var self = this; - counters.add("chat:incoming"); chatIncomingCount.inc(1, new Date()); if (!this.channel || !this.channel.modules.permissions.canChat(user)) { @@ -358,7 +356,6 @@ ChatModule.prototype.processChatMsg = function (user, data) { return; } this.sendMessage(msgobj); - counters.add("chat:sent"); chatSentCount.inc(1, new Date()); }; diff --git a/src/channel/playlist.js b/src/channel/playlist.js index 786c439f..7cc0146c 100644 --- a/src/channel/playlist.js +++ b/src/channel/playlist.js @@ -8,7 +8,6 @@ var Flags = require("../flags"); var db = require("../database"); var CustomEmbedFilter = require("../customembed").filter; var XSS = require("../xss"); -import counters from '../counters'; import { Counter } from 'prom-client'; const LOGGER = require('@calzoneman/jsli')('playlist'); @@ -512,7 +511,6 @@ PlaylistModule.prototype.queueStandard = function (user, data) { const self = this; this.channel.refCounter.ref("PlaylistModule::queueStandard"); - counters.add("playlist:queue:count", 1); this.semaphore.queue(function (lock) { InfoGetter.getMedia(data.id, data.type, function (err, media) { if (err) { diff --git a/src/counters.js b/src/counters.js deleted file mode 100644 index 2a042f3c..00000000 --- a/src/counters.js +++ /dev/null @@ -1,55 +0,0 @@ -import io from 'socket.io'; -import Socket from 'socket.io/lib/socket'; -import * as Metrics from './metrics/metrics'; -import { JSONFileMetricsReporter } from './metrics/jsonfilemetricsreporter'; - -const LOGGER = require('@calzoneman/jsli')('counters'); - -var server = null; - -exports.add = Metrics.incCounter; - -Socket.prototype._packet = Socket.prototype.packet; -Socket.prototype.packet = function () { - this._packet.apply(this, arguments); - exports.add('socket.io:packet'); -}; - -function getConnectedSockets() { - var sockets = io.instance.sockets.sockets; - if (typeof sockets.length === 'number') { - return sockets.length; - } else { - return Object.keys(sockets).length; - } -} - -function setChannelCounts(metrics) { - if (server === null) { - server = require('./server').getServer(); - } - - try { - var publicCount = 0; - var allCount = 0; - server.channels.forEach(function (c) { - allCount++; - if (c.modules.options && c.modules.options.get("show_public")) { - publicCount++; - } - }); - - metrics.addProperty('channelCount:all', allCount); - metrics.addProperty('channelCount:public', publicCount); - } catch (error) { - LOGGER.error(error.stack); - } -} - -const reporter = new JSONFileMetricsReporter('counters.log'); -Metrics.setReporter(reporter); -Metrics.setReportInterval(60000); -Metrics.addReportHook((metrics) => { - metrics.addProperty('socket.io:count', getConnectedSockets()); - setChannelCounts(metrics); -}); diff --git a/src/database.js b/src/database.js index 77527c82..9103ad97 100644 --- a/src/database.js +++ b/src/database.js @@ -1,6 +1,5 @@ var Config = require("./config"); var tables = require("./database/tables"); -import * as Metrics from './metrics/metrics'; import knex from 'knex'; import { GlobalBanDB } from './db/globalban'; import { MetadataCacheDB } from './database/metadata_cache'; @@ -53,14 +52,12 @@ class Database { } runTransaction(fn) { - const timer = Metrics.startTimer('db:queryTime'); const end = queryLatency.startTimer(); return this.knex.transaction(fn).catch(error => { queryErrorCount.inc(1); throw error; }).finally(() => { end(); - Metrics.stopTimer(timer); queryCount.inc(1); }); } @@ -110,7 +107,6 @@ module.exports.getGlobalBanDB = function getGlobalBanDB() { * Execute a database query */ module.exports.query = function (query, sub, callback) { - const timer = Metrics.startTimer('db:queryTime'); // 2nd argument is optional if (typeof sub === "function") { callback = sub; @@ -157,7 +153,6 @@ module.exports.query = function (query, sub, callback) { process.nextTick(callback, 'Database failure', null); }).finally(() => { end(); - Metrics.stopTimer(timer); queryCount.inc(1); }); }; diff --git a/src/io/ioserver.js b/src/io/ioserver.js index 3c0a50f7..1da79f6a 100644 --- a/src/io/ioserver.js +++ b/src/io/ioserver.js @@ -7,7 +7,6 @@ const cookieParser = require("cookie-parser")(Config.get("http.cookie-secret")); import typecheck from 'json-typecheck'; import { isTorExit } from '../tor'; import session from '../session'; -import counters from '../counters'; import { verifyIPSessionCookie } from '../web/middleware/ipsessioncookie'; import Promise from 'bluebird'; const verifySession = Promise.promisify(session.verifySession); @@ -228,7 +227,6 @@ class IOServer { emitMetrics(socket); LOGGER.info('Accepted socket from %s', socket.context.ipAddress); - counters.add('socket.io:accept', 1); socket.once('disconnect', (reason, reasonDetail) => { LOGGER.info( '%s disconnected (%s%s)', @@ -236,7 +234,6 @@ class IOServer { reason, reasonDetail ? ` - ${reasonDetail}` : '' ); - counters.add('socket.io:disconnect', 1); }); const user = new User(socket, socket.context.ipAddress, socket.context.user); diff --git a/src/metrics/jsonfilemetricsreporter.js b/src/metrics/jsonfilemetricsreporter.js deleted file mode 100644 index c94a7f71..00000000 --- a/src/metrics/jsonfilemetricsreporter.js +++ /dev/null @@ -1,73 +0,0 @@ -import fs from 'fs'; - -/** MetricsReporter that records metrics as JSON objects in a file, one per line */ -class JSONFileMetricsReporter { - /** - * Create a new JSONFileMetricsReporter that writes to the given file path. - * - * @param {string} filename file path to write to - */ - constructor(filename) { - this.writeStream = fs.createWriteStream(filename, { flags: 'a' }); - this.metrics = {}; - this.timers = {}; - } - - /** - * @see {@link module:cytube-common/metrics/metrics.incCounter} - */ - incCounter(counter, value) { - if (!this.metrics.hasOwnProperty(counter)) { - this.metrics[counter] = 0; - } - - this.metrics[counter] += value; - } - - /** - * Add a time metric - * - * @param {string} timer name of the timer - * @param {number} ms milliseconds to record - */ - addTime(timer, ms) { - if (!this.timers.hasOwnProperty(timer)) { - this.timers[timer] = { - totalTime: 0, - count: 0, - p100: 0 - }; - } - - this.timers[timer].totalTime += ms; - this.timers[timer].count++; - if (ms > this.timers[timer].p100) { - this.timers[timer].p100 = ms; - } - } - - /** - * @see {@link module:cytube-common/metrics/metrics.addProperty} - */ - addProperty(property, value) { - this.metrics[property] = value; - } - - report() { - for (const timer in this.timers) { - this.metrics[timer+':avg'] = this.timers[timer].totalTime / this.timers[timer].count; - this.metrics[timer+':count'] = this.timers[timer].count; - this.metrics[timer+':p100'] = this.timers[timer].p100; - } - - const line = JSON.stringify(this.metrics) + '\n'; - try { - this.writeStream.write(line); - } finally { - this.metrics = {}; - this.timers = {}; - } - } -} - -export { JSONFileMetricsReporter }; diff --git a/src/metrics/metrics.js b/src/metrics/metrics.js deleted file mode 100644 index 6e1badb3..00000000 --- a/src/metrics/metrics.js +++ /dev/null @@ -1,136 +0,0 @@ -import os from 'os'; - -/** @module cytube-common/metrics/metrics */ - -const MEM_RSS = 'memory:rss'; -const LOAD_1MIN = 'load:1min'; -const TIMESTAMP = 'time'; -const logger = require('@calzoneman/jsli')('metrics'); - -var delegate = null; -var reportInterval = null; -var reportHooks = []; -let warnedNoReporter = false; - -function warnNoReporter() { - if (!warnedNoReporter) { - warnedNoReporter = true; - logger.warn('No metrics reporter configured. Metrics will not be recorded.'); - } -} - -/** - * Increment a metrics counter by the specified amount. - * - * @param {string} counter name of the counter to increment - * @param {number} value optional value to increment by (default 1) - */ -export function incCounter(counter, amount = 1) { - if (delegate === null) { - warnNoReporter(); - } else { - delegate.incCounter(counter, amount); - } -} - -/** - * Start a timer. Returns a handle to use to end the timer. - * - * @param {string} timer name - * @return {object} timer handle - */ -export function startTimer(timer) { - return { - timer: timer, - hrtime: process.hrtime() - }; -} - -/** - * Stop a timer and record the time (as an average) - * - * @param {object} handle timer handle to Stop - */ -export function stopTimer(handle) { - if (delegate === null) { - warnNoReporter(); - return; - } - const [seconds, ns] = process.hrtime(handle.hrtime); - delegate.addTime(handle.timer, seconds*1e3 + ns/1e6); -} - -/** - * Add a property to the current metrics period. - * - * @param {string} property property name to add - * @param {any} property value - */ -export function addProperty(property, value) { - if (delegate === null) { - warnNoReporter(); - } else { - delegate.addProperty(property, value); - } -} - -/** - * Set the metrics reporter to record to. - * - * @param {MetricsReporter} reporter reporter to record metrics to - */ -export function setReporter(reporter) { - delegate = reporter; -} - -/** - * Set the interval at which to report metrics. - * - * @param {number} interval time in milliseconds between successive reports - */ -export function setReportInterval(interval) { - clearInterval(reportInterval); - if (!isNaN(interval) && interval >= 0) { - reportInterval = setInterval(reportLoop, interval); - } -} - -/** - * Add a callback to add additional metrics before reporting. - * - * @param {function(metricsReporter)} hook callback to be invoked before reporting - */ -export function addReportHook(hook) { - reportHooks.push(hook); -} - -export function clearReportHooks() { - reportHooks = []; -} - -/** - * Force metrics to be reported right now. - */ -export function flush() { - reportLoop(); -} - -function addDefaults() { - addProperty(MEM_RSS, process.memoryUsage().rss / 1048576); - addProperty(LOAD_1MIN, os.loadavg()[0]); - addProperty(TIMESTAMP, new Date()); -} - -function reportLoop() { - if (delegate !== null) { - try { - addDefaults(); - reportHooks.forEach(hook => { - hook(delegate); - }); - delegate.report(); - } catch (error) { - logger.error(error.stack); - } - } -} diff --git a/src/web/webserver.js b/src/web/webserver.js index c94f7a4e..f81781cb 100644 --- a/src/web/webserver.js +++ b/src/web/webserver.js @@ -9,7 +9,6 @@ import morgan from 'morgan'; import csrf from './csrf'; import * as HTTPStatus from './httpstatus'; import { CSRFError, HTTPError } from '../errors'; -import counters from '../counters'; import { Summary, Counter } from 'prom-client'; import session from '../session'; const verifySessionAsync = require('bluebird').promisify(session.verifySession); @@ -150,10 +149,6 @@ module.exports = { const chanPath = Config.get('channel-path'); initPrometheus(app); - app.use((req, res, next) => { - counters.add("http:request", 1); - next(); - }); require('./middleware/x-forwarded-for').initialize(app, webConfig); app.use(bodyParser.urlencoded({ extended: false,