Begin prometheus integration

Add a dependency on `prom-client` and emit a basic latency metric for
testing purposes.  Add a new configuration file for enabling/disabling
prometheus exporter and configuring the listen address.
This commit is contained in:
Calvin Montgomery 2017-07-16 22:35:33 -07:00
parent dd770137e5
commit c7bec6251e
10 changed files with 222 additions and 1 deletions

View File

@ -0,0 +1,14 @@
# Configuration for binding an HTTP server to export prometheus metrics.
# See https://prometheus.io/ and https://github.com/siimon/prom-client
# for more details.
[prometheus]
enabled = true
# Host, port to bind. This is separate from the main CyTube HTTP server
# because it may be desirable to bind a different IP/port for monitoring
# purposes. Default: localhost port 19820 (arbitrary port chosen not to
# conflict with existing prometheus exporters).
host = '127.0.0.1'
port = 19820
# Request path to serve metrics. All other paths are rejected with
# 400 Bad Request.
path = '/metrics'

View File

@ -32,6 +32,7 @@
"mysql": "^2.9.0",
"nodemailer": "^1.4.0",
"oauth": "^0.9.12",
"prom-client": "^10.0.2",
"proxy-addr": "^1.1.4",
"pug": "^2.0.0-beta3",
"q": "^1.4.1",

View File

@ -6,6 +6,7 @@ var YAML = require("yamljs");
import { loadFromToml } from 'cytube-common/lib/configuration/configloader';
import { CamoConfig } from './configuration/camoconfig';
import { PrometheusConfig } from './configuration/prometheusconfig';
const LOGGER = require('@calzoneman/jsli')('config');
@ -149,6 +150,7 @@ function merge(obj, def, path) {
var cfg = defaults;
let camoConfig = new CamoConfig();
let prometheusConfig = new PrometheusConfig();
/**
* Initializes the configuration from the given YAML file
@ -191,6 +193,7 @@ exports.load = function (file) {
LOGGER.info("Loaded configuration from " + file);
loadCamoConfig();
loadPrometheusConfig();
};
function loadCamoConfig() {
@ -214,6 +217,28 @@ function loadCamoConfig() {
}
}
function loadPrometheusConfig() {
try {
prometheusConfig = loadFromToml(PrometheusConfig,
path.resolve(__dirname, '..', 'conf', 'prometheus.toml'));
const enabled = prometheusConfig.isEnabled() ? 'ENABLED' : 'DISABLED';
LOGGER.info('Loaded prometheus configuration from conf/prometheus.toml. '
+ `Prometheus listener is ${enabled}`);
} catch (error) {
if (error.code === 'ENOENT') {
LOGGER.info('No prometheus configuration found, defaulting to disabled');
prometheusConfig = new PrometheusConfig();
return;
}
if (typeof error.line !== 'undefined') {
LOGGER.error(`Error in conf/prometheus.toml: ${error} (line ${error.line})`);
} else {
LOGGER.error(`Error loading conf/prometheus.toml: ${error.stack}`);
}
}
}
// I'm sorry
function preprocessConfig(cfg) {
/* Detect 3.0.0-style config and warng the user about it */
@ -483,3 +508,7 @@ exports.set = function (key, value) {
exports.getCamoConfig = function getCamoConfig() {
return camoConfig;
};
exports.getPrometheusConfig = function getPrometheusConfig() {
return prometheusConfig;
};

View File

@ -0,0 +1,23 @@
class PrometheusConfig {
constructor(config = { prometheus: { enabled: false } }) {
this.config = config.prometheus;
}
isEnabled() {
return this.config.enabled;
}
getPort() {
return this.config.port;
}
getHost() {
return this.config.host;
}
getPath() {
return this.config.path;
}
}
export { PrometheusConfig };

View File

@ -588,6 +588,7 @@ module.exports = {
Getters: Getters,
getMedia: function (id, type, callback) {
if(type in this.Getters) {
LOGGER.info("Looking up %s:%s", type, id);
this.Getters[type](id, callback);
} else {
callback("Unknown media type '" + type + "'", null);

47
src/prometheus-server.js Normal file
View File

@ -0,0 +1,47 @@
import http from 'http';
import { register } from 'prom-client';
import { parse as parseURL } from 'url';
const LOGGER = require('@calzoneman/jsli')('prometheus-server');
let server = null;
export function init(prometheusConfig) {
if (server !== null) {
LOGGER.error('init() called but server is already initialized! %s',
new Error().stack);
return;
}
server = http.createServer((req, res) => {
if (req.method !== 'GET'
|| parseURL(req.url).pathname !== prometheusConfig.getPath()) {
res.writeHead(400, { 'Content-Type': 'text/plain' });
res.end('Bad Request');
return;
}
res.writeHead(200, {
'Content-Type': register.contentType
});
res.end(register.metrics());
});
server.on('error', error => {
LOGGER.error('Server error: %s', error.stack);
});
server.once('listening', () => {
LOGGER.info('Prometheus metrics reporter listening on %s:%s',
prometheusConfig.getHost(),
prometheusConfig.getPort());
});
server.listen(prometheusConfig.getPort(), prometheusConfig.getHost());
return { once: server.once.bind(server) };
}
export function shutdown() {
server.close();
server = null;
}

View File

@ -147,6 +147,12 @@ var Server = function () {
// background tasks init ----------------------------------------------
require("./bgtask")(self);
// prometheus server
const prometheusConfig = Config.getPrometheusConfig();
if (prometheusConfig.isEnabled()) {
require("./prometheus-server").init(prometheusConfig);
}
// setuid
require("./setuid");

View File

@ -10,7 +10,8 @@ import morgan from 'morgan';
import csrf from './csrf';
import * as HTTPStatus from './httpstatus';
import { CSRFError, HTTPError } from '../errors';
import counters from "../counters";
import counters from '../counters';
import { Summary } from 'prom-client';
const LOGGER = require('@calzoneman/jsli')('webserver');
@ -27,6 +28,29 @@ function initializeLog(app) {
}));
}
function initPrometheus(app) {
const latency = new Summary({
name: 'cytube_http_req_latency',
help: 'HTTP Request latency from execution of the first middleware '
+ 'until the "finish" event on the response object.',
labelNames: ['method', 'statusCode']
});
app.use((req, res, next) => {
const startTime = process.hrtime();
res.on('finish', () => {
try {
const diff = process.hrtime(startTime);
const diffMs = diff[0]*1e3 + diff[1]*1e-6;
latency.labels(req.method, res.statusCode).observe(diffMs);
} catch (error) {
LOGGER.error('Failed to record HTTP Prometheus metrics: %s', error.stack);
}
});
next();
});
}
/**
* Redirects a request to HTTPS if the server supports it
*/
@ -133,6 +157,7 @@ module.exports = {
init: function (app, webConfig, ioConfig, clusterClient, channelIndex, session) {
const chanPath = Config.get('channel-path');
initPrometheus(app);
app.use((req, res, next) => {
counters.add("http:request", 1);
next();

View File

@ -0,0 +1,10 @@
const assert = require('assert');
const PrometheusConfig = require('../../lib/configuration/prometheusconfig').PrometheusConfig;
describe('PrometheusConfig', () => {
describe('#constructor', () => {
it('defaults to enabled=false', () => {
assert.strictEqual(new PrometheusConfig().isEnabled(), false);
});
});
});

65
test/prometheus-server.js Normal file
View File

@ -0,0 +1,65 @@
const assert = require('assert');
const http = require('http');
const server = require('../lib/prometheus-server');
const PrometheusConfig = require('../lib/configuration/prometheusconfig').PrometheusConfig;
describe('prometheus-server', () => {
before(done => {
let inst = server.init(new PrometheusConfig({
prometheus: {
enabled: true,
port: 19820,
host: '127.0.0.1',
path: '/metrics'
}
}));
inst.once('listening', () => done());
});
function checkReq(options, done) {
const req = http.request({
method: options.method,
host: '127.0.0.1',
port: 19820,
path: options.path
}, res => {
assert.strictEqual(res.statusCode, options.expectedStatusCode);
assert.strictEqual(res.headers['content-type'], options.expectedContentType);
res.on('data', () => {});
res.on('end', () => done());
});
req.end();
}
it('rejects a non-GET request', done => {
checkReq({
method: 'POST',
path: '/metrics',
expectedStatusCode: 400,
expectedContentType: 'text/plain'
}, done);
});
it('rejects a request for the wrong path', done => {
checkReq({
method: 'GET',
path: '/qwerty',
expectedStatusCode: 400,
expectedContentType: 'text/plain'
}, done);
});
it('accepts a request for the configured path', done => {
checkReq({
method: 'GET',
path: '/metrics',
expectedStatusCode: 200,
expectedContentType: 'text/plain; version=0.0.4; charset=utf-8'
}, done);
});
after(() => {
server.shutdown();
});
});