moon
/
sync
mirror of https://github.com/calzoneman/sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove legacy counters

This commit is contained in:
Calvin Montgomery 2021-08-12 22:11:10 -07:00
parent 11a0cd79bb
commit 1b7e7c74f5
9 changed files with 9 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
NEWS.md
View File

@ -1,6 +1,15 @@
2021-08-12 2021-08-12
========== ==========
The legacy metrics recorder (`counters.log` file) has been removed. For over 4
years now, CyTube has integrated with [Prometheus](https://prometheus.io/),
which provides a superior way to monitor the application. Copy
`conf/example/prometheus.toml` to `conf/prometheus.toml` and edit it to
configure CyTube's Prometheus support.
2021-08-12
==========
Due to changes in Soundcloud's authorization scheme, support has been dropped Due to changes in Soundcloud's authorization scheme, support has been dropped
from core due to requiring each server owner to register an API key (which is from core due to requiring each server owner to register an API key (which is
currently impossible as they have not accepted new API key registrations for currently impossible as they have not accepted new API key registrations for

3
src/channel/chat.js
View File

@ -3,7 +3,6 @@ var XSS = require("../xss");
var ChannelModule = require("./module"); var ChannelModule = require("./module");
var util = require("../utilities"); var util = require("../utilities");
var Flags = require("../flags"); var Flags = require("../flags");
var counters = require("../counters");
import { transformImgTags } from '../camo'; import { transformImgTags } from '../camo';
import { Counter } from 'prom-client'; import { Counter } from 'prom-client';
@ -157,7 +156,6 @@ const chatIncomingCount = new Counter({
}); });
ChatModule.prototype.handleChatMsg = function (user, data) { ChatModule.prototype.handleChatMsg = function (user, data) {
var self = this; var self = this;
counters.add("chat:incoming");
chatIncomingCount.inc(1, new Date()); chatIncomingCount.inc(1, new Date());
if (!this.channel || !this.channel.modules.permissions.canChat(user)) { if (!this.channel || !this.channel.modules.permissions.canChat(user)) {
@ -358,7 +356,6 @@ ChatModule.prototype.processChatMsg = function (user, data) {
return; return;
} }
this.sendMessage(msgobj); this.sendMessage(msgobj);
counters.add("chat:sent");
chatSentCount.inc(1, new Date()); chatSentCount.inc(1, new Date());
}; };

2
src/channel/playlist.js
View File

@ -8,7 +8,6 @@ var Flags = require("../flags");
var db = require("../database"); var db = require("../database");
var CustomEmbedFilter = require("../customembed").filter; var CustomEmbedFilter = require("../customembed").filter;
var XSS = require("../xss"); var XSS = require("../xss");
import counters from '../counters';
import { Counter } from 'prom-client'; import { Counter } from 'prom-client';
const LOGGER = require('@calzoneman/jsli')('playlist'); const LOGGER = require('@calzoneman/jsli')('playlist');
@ -512,7 +511,6 @@ PlaylistModule.prototype.queueStandard = function (user, data) {
const self = this; const self = this;
this.channel.refCounter.ref("PlaylistModule::queueStandard"); this.channel.refCounter.ref("PlaylistModule::queueStandard");
counters.add("playlist:queue:count", 1);
this.semaphore.queue(function (lock) { this.semaphore.queue(function (lock) {
InfoGetter.getMedia(data.id, data.type, function (err, media) { InfoGetter.getMedia(data.id, data.type, function (err, media) {
if (err) { if (err) {

55
src/counters.js
View File

@ -1,55 +0,0 @@
import io from 'socket.io';
import Socket from 'socket.io/lib/socket';
import * as Metrics from './metrics/metrics';
import { JSONFileMetricsReporter } from './metrics/jsonfilemetricsreporter';
const LOGGER = require('@calzoneman/jsli')('counters');
var server = null;
exports.add = Metrics.incCounter;
Socket.prototype._packet = Socket.prototype.packet;
Socket.prototype.packet = function () {
this._packet.apply(this, arguments);
exports.add('socket.io:packet');
};
function getConnectedSockets() {
var sockets = io.instance.sockets.sockets;
if (typeof sockets.length === 'number') {
return sockets.length;
} else {
return Object.keys(sockets).length;
}
}
function setChannelCounts(metrics) {
if (server === null) {
server = require('./server').getServer();
}
try {
var publicCount = 0;
var allCount = 0;
server.channels.forEach(function (c) {
allCount++;
if (c.modules.options && c.modules.options.get("show_public")) {
publicCount++;
}
});
metrics.addProperty('channelCount:all', allCount);
metrics.addProperty('channelCount:public', publicCount);
} catch (error) {
LOGGER.error(error.stack);
}
}
const reporter = new JSONFileMetricsReporter('counters.log');
Metrics.setReporter(reporter);
Metrics.setReportInterval(60000);
Metrics.addReportHook((metrics) => {
metrics.addProperty('socket.io:count', getConnectedSockets());
setChannelCounts(metrics);
});

5
src/database.js
View File

@ -1,6 +1,5 @@
var Config = require("./config"); var Config = require("./config");
var tables = require("./database/tables"); var tables = require("./database/tables");
import * as Metrics from './metrics/metrics';
import knex from 'knex'; import knex from 'knex';
import { GlobalBanDB } from './db/globalban'; import { GlobalBanDB } from './db/globalban';
import { MetadataCacheDB } from './database/metadata_cache'; import { MetadataCacheDB } from './database/metadata_cache';
@ -53,14 +52,12 @@ class Database {
} }
runTransaction(fn) { runTransaction(fn) {
const timer = Metrics.startTimer('db:queryTime');
const end = queryLatency.startTimer(); const end = queryLatency.startTimer();
return this.knex.transaction(fn).catch(error => { return this.knex.transaction(fn).catch(error => {
queryErrorCount.inc(1); queryErrorCount.inc(1);
throw error; throw error;
}).finally(() => { }).finally(() => {
end(); end();
Metrics.stopTimer(timer);
queryCount.inc(1); queryCount.inc(1);
}); });
} }
@ -110,7 +107,6 @@ module.exports.getGlobalBanDB = function getGlobalBanDB() {
* Execute a database query * Execute a database query
*/ */
module.exports.query = function (query, sub, callback) { module.exports.query = function (query, sub, callback) {
const timer = Metrics.startTimer('db:queryTime');
// 2nd argument is optional // 2nd argument is optional
if (typeof sub === "function") { if (typeof sub === "function") {
callback = sub; callback = sub;
@ -157,7 +153,6 @@ module.exports.query = function (query, sub, callback) {
process.nextTick(callback, 'Database failure', null); process.nextTick(callback, 'Database failure', null);
}).finally(() => { }).finally(() => {
end(); end();
Metrics.stopTimer(timer);
queryCount.inc(1); queryCount.inc(1);
}); });
}; };

3
src/io/ioserver.js
View File

@ -7,7 +7,6 @@ const cookieParser = require("cookie-parser")(Config.get("http.cookie-secret"));
import typecheck from 'json-typecheck'; import typecheck from 'json-typecheck';
import { isTorExit } from '../tor'; import { isTorExit } from '../tor';
import session from '../session'; import session from '../session';
import counters from '../counters';
import { verifyIPSessionCookie } from '../web/middleware/ipsessioncookie'; import { verifyIPSessionCookie } from '../web/middleware/ipsessioncookie';
import Promise from 'bluebird'; import Promise from 'bluebird';
const verifySession = Promise.promisify(session.verifySession); const verifySession = Promise.promisify(session.verifySession);
@ -228,7 +227,6 @@ class IOServer {
emitMetrics(socket); emitMetrics(socket);
LOGGER.info('Accepted socket from %s', socket.context.ipAddress); LOGGER.info('Accepted socket from %s', socket.context.ipAddress);
counters.add('socket.io:accept', 1);
socket.once('disconnect', (reason, reasonDetail) => { socket.once('disconnect', (reason, reasonDetail) => {
LOGGER.info( LOGGER.info(
'%s disconnected (%s%s)', '%s disconnected (%s%s)',
@ -236,7 +234,6 @@ class IOServer {
reason, reason,
reasonDetail ? ` - ${reasonDetail}` : '' reasonDetail ? ` - ${reasonDetail}` : ''
); );
counters.add('socket.io:disconnect', 1);
}); });
const user = new User(socket, socket.context.ipAddress, socket.context.user); const user = new User(socket, socket.context.ipAddress, socket.context.user);

73
src/metrics/jsonfilemetricsreporter.js
View File

@ -1,73 +0,0 @@
import fs from 'fs';
/** MetricsReporter that records metrics as JSON objects in a file, one per line */
class JSONFileMetricsReporter {
/**
* Create a new JSONFileMetricsReporter that writes to the given file path.
*
* @param {string} filename file path to write to
*/
constructor(filename) {
this.writeStream = fs.createWriteStream(filename, { flags: 'a' });
this.metrics = {};
this.timers = {};
}
/**
* @see {@link module:cytube-common/metrics/metrics.incCounter}
*/
incCounter(counter, value) {
if (!this.metrics.hasOwnProperty(counter)) {
this.metrics[counter] = 0;
}
this.metrics[counter] += value;
}
/**
* Add a time metric
*
* @param {string} timer name of the timer
* @param {number} ms milliseconds to record
*/
addTime(timer, ms) {
if (!this.timers.hasOwnProperty(timer)) {
this.timers[timer] = {
totalTime: 0,
count: 0,
p100: 0
};
}
this.timers[timer].totalTime += ms;
this.timers[timer].count++;
if (ms > this.timers[timer].p100) {
this.timers[timer].p100 = ms;
}
}
/**
* @see {@link module:cytube-common/metrics/metrics.addProperty}
*/
addProperty(property, value) {
this.metrics[property] = value;
}
report() {
for (const timer in this.timers) {
this.metrics[timer+':avg'] = this.timers[timer].totalTime / this.timers[timer].count;
this.metrics[timer+':count'] = this.timers[timer].count;
this.metrics[timer+':p100'] = this.timers[timer].p100;
}
const line = JSON.stringify(this.metrics) + '\n';
try {
this.writeStream.write(line);
} finally {
this.metrics = {};
this.timers = {};
}
}
}
export { JSONFileMetricsReporter };

136
src/metrics/metrics.js
View File

@ -1,136 +0,0 @@
import os from 'os';
/** @module cytube-common/metrics/metrics */
const MEM_RSS = 'memory:rss';
const LOAD_1MIN = 'load:1min';
const TIMESTAMP = 'time';
const logger = require('@calzoneman/jsli')('metrics');
var delegate = null;
var reportInterval = null;
var reportHooks = [];
let warnedNoReporter = false;
function warnNoReporter() {
if (!warnedNoReporter) {
warnedNoReporter = true;
logger.warn('No metrics reporter configured. Metrics will not be recorded.');
}
}
/**
* Increment a metrics counter by the specified amount.
*
* @param {string} counter name of the counter to increment
* @param {number} value optional value to increment by (default 1)
*/
export function incCounter(counter, amount = 1) {
if (delegate === null) {
warnNoReporter();
} else {
delegate.incCounter(counter, amount);
}
}
/**
* Start a timer. Returns a handle to use to end the timer.
*
* @param {string} timer name
* @return {object} timer handle
*/
export function startTimer(timer) {
return {
timer: timer,
hrtime: process.hrtime()
};
}
/**
* Stop a timer and record the time (as an average)
*
* @param {object} handle timer handle to Stop
*/
export function stopTimer(handle) {
if (delegate === null) {
warnNoReporter();
return;
}
const [seconds, ns] = process.hrtime(handle.hrtime);
delegate.addTime(handle.timer, seconds*1e3 + ns/1e6);
}
/**
* Add a property to the current metrics period.
*
* @param {string} property property name to add
* @param {any} property value
*/
export function addProperty(property, value) {
if (delegate === null) {
warnNoReporter();
} else {
delegate.addProperty(property, value);
}
}
/**
* Set the metrics reporter to record to.
*
* @param {MetricsReporter} reporter reporter to record metrics to
*/
export function setReporter(reporter) {
delegate = reporter;
}
/**
* Set the interval at which to report metrics.
*
* @param {number} interval time in milliseconds between successive reports
*/
export function setReportInterval(interval) {
clearInterval(reportInterval);
if (!isNaN(interval) && interval >= 0) {
reportInterval = setInterval(reportLoop, interval);
}
}
/**
* Add a callback to add additional metrics before reporting.
*
* @param {function(metricsReporter)} hook callback to be invoked before reporting
*/
export function addReportHook(hook) {
reportHooks.push(hook);
}
export function clearReportHooks() {
reportHooks = [];
}
/**
* Force metrics to be reported right now.
*/
export function flush() {
reportLoop();
}
function addDefaults() {
addProperty(MEM_RSS, process.memoryUsage().rss / 1048576);
addProperty(LOAD_1MIN, os.loadavg()[0]);
addProperty(TIMESTAMP, new Date());
}
function reportLoop() {
if (delegate !== null) {
try {
addDefaults();
reportHooks.forEach(hook => {
hook(delegate);
});
delegate.report();
} catch (error) {
logger.error(error.stack);
}
}
}

5
src/web/webserver.js
View File

@ -9,7 +9,6 @@ import morgan from 'morgan';
import csrf from './csrf'; import csrf from './csrf';
import * as HTTPStatus from './httpstatus'; import * as HTTPStatus from './httpstatus';
import { CSRFError, HTTPError } from '../errors'; import { CSRFError, HTTPError } from '../errors';
import counters from '../counters';
import { Summary, Counter } from 'prom-client'; import { Summary, Counter } from 'prom-client';
import session from '../session'; import session from '../session';
const verifySessionAsync = require('bluebird').promisify(session.verifySession); const verifySessionAsync = require('bluebird').promisify(session.verifySession);
@ -150,10 +149,6 @@ module.exports = {
const chanPath = Config.get('channel-path'); const chanPath = Config.get('channel-path');
initPrometheus(app); initPrometheus(app);
app.use((req, res, next) => {
counters.add("http:request", 1);
next();
});
require('./middleware/x-forwarded-for').initialize(app, webConfig); require('./middleware/x-forwarded-for').initialize(app, webConfig);
app.use(bodyParser.urlencoded({ app.use(bodyParser.urlencoded({
extended: false, extended: false,