Begin prometheus integration

Add a dependency on `prom-client` and emit a basic latency metric for
testing purposes.  Add a new configuration file for enabling/disabling
prometheus exporter and configuring the listen address.

conf/example/prometheus.toml

@@ -0,0 +1,14 @@
+# Configuration for binding an HTTP server to export prometheus metrics.
+# See https://prometheus.io/ and https://github.com/siimon/prom-client
+# for more details.
+[prometheus]
+enabled = true
+# Host, port to bind.  This is separate from the main CyTube HTTP server
+# because it may be desirable to bind a different IP/port for monitoring
+# purposes.  Default: localhost port 19820 (arbitrary port chosen not to
+# conflict with existing prometheus exporters).
+host = '127.0.0.1'
+port = 19820
+# Request path to serve metrics.  All other paths are rejected with
+# 400 Bad Request.
+path = '/metrics'

package.json

@@ -32,6 +32,7 @@
     "mysql": "^2.9.0",
     "nodemailer": "^1.4.0",
     "oauth": "^0.9.12",
+    "prom-client": "^10.0.2",
     "proxy-addr": "^1.1.4",
     "pug": "^2.0.0-beta3",
     "q": "^1.4.1",

src/config.js

@@ -6,6 +6,7 @@ var YAML = require("yamljs");
 
 import { loadFromToml } from 'cytube-common/lib/configuration/configloader';
 import { CamoConfig } from './configuration/camoconfig';
+import { PrometheusConfig } from './configuration/prometheusconfig';
 
 const LOGGER = require('@calzoneman/jsli')('config');
 
@@ -149,6 +150,7 @@ function merge(obj, def, path) {
 
 var cfg = defaults;
 let camoConfig = new CamoConfig();
+let prometheusConfig = new PrometheusConfig();
 
 /**
  * Initializes the configuration from the given YAML file
@@ -191,6 +193,7 @@ exports.load = function (file) {
     LOGGER.info("Loaded configuration from " + file);
 
     loadCamoConfig();
+    loadPrometheusConfig();
 };
 
 function loadCamoConfig() {
@@ -214,6 +217,28 @@ function loadCamoConfig() {
     }
 }
 
+function loadPrometheusConfig() {
+    try {
+        prometheusConfig = loadFromToml(PrometheusConfig,
+                path.resolve(__dirname, '..', 'conf', 'prometheus.toml'));
+        const enabled = prometheusConfig.isEnabled() ? 'ENABLED' : 'DISABLED';
+        LOGGER.info('Loaded prometheus configuration from conf/prometheus.toml.  '
+                + `Prometheus listener is ${enabled}`);
+    } catch (error) {
+        if (error.code === 'ENOENT') {
+            LOGGER.info('No prometheus configuration found, defaulting to disabled');
+            prometheusConfig = new PrometheusConfig();
+            return;
+        }
+
+        if (typeof error.line !== 'undefined') {
+            LOGGER.error(`Error in conf/prometheus.toml: ${error} (line ${error.line})`);
+        } else {
+            LOGGER.error(`Error loading conf/prometheus.toml: ${error.stack}`);
+        }
+    }
+}
+
 // I'm sorry
 function preprocessConfig(cfg) {
     /* Detect 3.0.0-style config and warng the user about it */
@@ -483,3 +508,7 @@ exports.set = function (key, value) {
 exports.getCamoConfig = function getCamoConfig() {
     return camoConfig;
 };
+
+exports.getPrometheusConfig = function getPrometheusConfig() {
+    return prometheusConfig;
+};

src/configuration/prometheusconfig.js

@@ -0,0 +1,23 @@
+class PrometheusConfig {
+    constructor(config = { prometheus: { enabled: false } }) {
+        this.config = config.prometheus;
+    }
+
+    isEnabled() {
+        return this.config.enabled;
+    }
+
+    getPort() {
+        return this.config.port;
+    }
+
+    getHost() {
+        return this.config.host;
+    }
+
+    getPath() {
+        return this.config.path;
+    }
+}
+
+export { PrometheusConfig };

src/get-info.js

@@ -588,6 +588,7 @@ module.exports = {
     Getters: Getters,
     getMedia: function (id, type, callback) {
         if(type in this.Getters) {
+            LOGGER.info("Looking up %s:%s", type, id);
             this.Getters[type](id, callback);
         } else {
             callback("Unknown media type '" + type + "'", null);

src/prometheus-server.js

@@ -0,0 +1,47 @@
+import http from 'http';
+import { register } from 'prom-client';
+import { parse as parseURL } from 'url';
+
+const LOGGER = require('@calzoneman/jsli')('prometheus-server');
+
+let server = null;
+
+export function init(prometheusConfig) {
+    if (server !== null) {
+        LOGGER.error('init() called but server is already initialized! %s',
+                new Error().stack);
+        return;
+    }
+
+    server = http.createServer((req, res) => {
+        if (req.method !== 'GET'
+                || parseURL(req.url).pathname !== prometheusConfig.getPath()) {
+            res.writeHead(400, { 'Content-Type': 'text/plain' });
+            res.end('Bad Request');
+            return;
+        }
+
+        res.writeHead(200, {
+            'Content-Type': register.contentType
+        });
+        res.end(register.metrics());
+    });
+
+    server.on('error', error => {
+        LOGGER.error('Server error: %s', error.stack);
+    });
+
+    server.once('listening', () => {
+        LOGGER.info('Prometheus metrics reporter listening on %s:%s',
+                prometheusConfig.getHost(),
+                prometheusConfig.getPort());
+    });
+
+    server.listen(prometheusConfig.getPort(), prometheusConfig.getHost());
+    return { once: server.once.bind(server) };
+}
+
+export function shutdown() {
+    server.close();
+    server = null;
+}

src/server.js

@@ -147,6 +147,12 @@ var Server = function () {
     // background tasks init ----------------------------------------------
     require("./bgtask")(self);
 
+    // prometheus server
+    const prometheusConfig = Config.getPrometheusConfig();
+    if (prometheusConfig.isEnabled()) {
+        require("./prometheus-server").init(prometheusConfig);
+    }
+
     // setuid
     require("./setuid");
 

src/web/webserver.js

@@ -10,7 +10,8 @@ import morgan from 'morgan';
 import csrf from './csrf';
 import * as HTTPStatus from './httpstatus';
 import { CSRFError, HTTPError } from '../errors';
-import counters from "../counters";
+import counters from '../counters';
+import { Summary } from 'prom-client';
 
 const LOGGER = require('@calzoneman/jsli')('webserver');
 
@@ -27,6 +28,29 @@ function initializeLog(app) {
     }));
 }
 
+function initPrometheus(app) {
+    const latency = new Summary({
+        name: 'cytube_http_req_latency',
+        help: 'HTTP Request latency from execution of the first middleware '
+                + 'until the "finish" event on the response object.',
+        labelNames: ['method', 'statusCode']
+    });
+
+    app.use((req, res, next) => {
+        const startTime = process.hrtime();
+        res.on('finish', () => {
+            try {
+                const diff = process.hrtime(startTime);
+                const diffMs = diff[0]*1e3 + diff[1]*1e-6;
+                latency.labels(req.method, res.statusCode).observe(diffMs);
+            } catch (error) {
+                LOGGER.error('Failed to record HTTP Prometheus metrics: %s', error.stack);
+            }
+        });
+        next();
+    });
+}
+
 /**
  * Redirects a request to HTTPS if the server supports it
  */
@@ -133,6 +157,7 @@ module.exports = {
     init: function (app, webConfig, ioConfig, clusterClient, channelIndex, session) {
         const chanPath = Config.get('channel-path');
 
+        initPrometheus(app);
         app.use((req, res, next) => {
             counters.add("http:request", 1);
             next();

test/configuration/prometheusconfig.js

@@ -0,0 +1,10 @@
+const assert = require('assert');
+const PrometheusConfig = require('../../lib/configuration/prometheusconfig').PrometheusConfig;
+
+describe('PrometheusConfig', () => {
+    describe('#constructor', () => {
+        it('defaults to enabled=false', () => {
+            assert.strictEqual(new PrometheusConfig().isEnabled(), false);
+        });
+    });
+});

test/prometheus-server.js

@@ -0,0 +1,65 @@
+const assert = require('assert');
+const http = require('http');
+const server = require('../lib/prometheus-server');
+const PrometheusConfig = require('../lib/configuration/prometheusconfig').PrometheusConfig;
+
+describe('prometheus-server', () => {
+    before(done => {
+        let inst = server.init(new PrometheusConfig({
+            prometheus: {
+                enabled: true,
+                port: 19820,
+                host: '127.0.0.1',
+                path: '/metrics'
+            }
+        }));
+        inst.once('listening', () => done());
+    });
+
+    function checkReq(options, done) {
+        const req = http.request({
+            method: options.method,
+            host: '127.0.0.1',
+            port: 19820,
+            path: options.path
+        }, res => {
+            assert.strictEqual(res.statusCode, options.expectedStatusCode);
+            assert.strictEqual(res.headers['content-type'], options.expectedContentType);
+            res.on('data', () => {});
+            res.on('end', () => done());
+        });
+
+        req.end();
+    }
+
+    it('rejects a non-GET request', done => {
+        checkReq({
+            method: 'POST',
+            path: '/metrics',
+            expectedStatusCode: 400,
+            expectedContentType: 'text/plain'
+        }, done);
+    });
+
+    it('rejects a request for the wrong path', done => {
+        checkReq({
+            method: 'GET',
+            path: '/qwerty',
+            expectedStatusCode: 400,
+            expectedContentType: 'text/plain'
+        }, done);
+    });
+
+    it('accepts a request for the configured path', done => {
+        checkReq({
+            method: 'GET',
+            path: '/metrics',
+            expectedStatusCode: 200,
+            expectedContentType: 'text/plain; version=0.0.4; charset=utf-8'
+        }, done);
+    });
+
+    after(() => {
+        server.shutdown();
+    });
+});