From 106065184f4fd0c8b10842ef9118d56897a7ea5c Mon Sep 17 00:00:00 2001 From: Calvin Montgomery Date: Sat, 15 Feb 2020 16:17:49 -0800 Subject: [PATCH] Remove old flatfile chandump storage --- NEWS.md | 30 +++++ config.template.yaml | 3 - package.json | 2 +- src/channel-storage/channelstore.js | 14 +-- src/channel-storage/filestore.js | 78 ------------ src/channel-storage/migrator.js | 187 ---------------------------- src/channel/channel.js | 17 --- src/config.js | 7 +- src/database/channels.js | 10 -- src/server.js | 5 - src/setuid.js | 1 - 11 files changed, 42 insertions(+), 312 deletions(-) delete mode 100644 src/channel-storage/filestore.js delete mode 100644 src/channel-storage/migrator.js diff --git a/NEWS.md b/NEWS.md index 1ee728a4..e15e0e90 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,33 @@ +2020-02-15 +========== + +Old versions of CyTube defaulted to storing channel state in flatfiles located +in the `chandump` directory. The default was changed a while ago, and the +flatfile storage mechanism has now been removed. + +Admins who have not already migrated their installation to the "database" +channel storage type can do so by following these instructions: + + 1. Run `git checkout e3a9915b454b32e49d3871c94c839899f809520a` to temporarily + switch to temporarily revert to the previous version of the code that + supports the "file" channel storage type + 2. Run `npm run build-server` to build the old version + 3. Run `node lib/channel-storage/migrator.js |& tee migration.log` to migrate + channel state from files to the database + 4. Inspect the output of the migration tool for errors + 5. Set `channel-storage`/`type` to `"database"` in `config.yaml` and start the + server. Load a channel to verify the migration worked as expected + 6. Upgrade back to the latest version with `git checkout 3.0` and `npm run + build-server` + 7. Remove the `channel-storage` block from `config.yaml` and remove the + `chandump` directory since it is no longer needed (you may wish to archive + it somewhere in case you later discover the migration didn't work as + expected). + +If you encounter any errors during the process, please file an issue on GitHub +and attach the output of the migration tool (which if you use the above commands +will be written to `migration.log`). + 2019-12-01 ========== diff --git a/config.template.yaml b/config.template.yaml index 6eae8bbb..a315ec78 100644 --- a/config.template.yaml +++ b/config.template.yaml @@ -133,9 +133,6 @@ channel-path: 'r' channel-blacklist: [] # Minutes between saving channel state to disk channel-save-interval: 5 -# Determines channel data storage mechanism. -channel-storage: - type: 'database' # Configure periodic clearing of old alias data aliases: diff --git a/package.json b/package.json index 641592be..4c7afc2a 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "author": "Calvin Montgomery", "name": "CyTube", "description": "Online media synchronizer and chat", - "version": "3.68.0", + "version": "3.69.0", "repository": { "url": "http://github.com/calzoneman/sync" }, diff --git a/src/channel-storage/channelstore.js b/src/channel-storage/channelstore.js index 6ad9d6e0..d8013b01 100644 --- a/src/channel-storage/channelstore.js +++ b/src/channel-storage/channelstore.js @@ -1,4 +1,3 @@ -import { FileStore } from './filestore'; import { DatabaseStore } from './dbstore'; import Config from '../config'; import Promise from 'bluebird'; @@ -26,11 +25,12 @@ export function save(id, channelName, data) { } function loadChannelStore() { - switch (Config.get('channel-storage.type')) { - case 'database': - return new DatabaseStore(); - case 'file': - default: - return new FileStore(); + if (Config.get('channel-storage.type') === 'file') { + throw new Error( + 'channel-storage type "file" is no longer supported. Please see ' + + 'NEWS.md for instructions on upgrading.' + ); } + + return new DatabaseStore(); } diff --git a/src/channel-storage/filestore.js b/src/channel-storage/filestore.js deleted file mode 100644 index 7c03dfed..00000000 --- a/src/channel-storage/filestore.js +++ /dev/null @@ -1,78 +0,0 @@ -import Promise from 'bluebird'; -import { stat } from 'fs'; -import * as fs from 'graceful-fs'; -import path from 'path'; -import { ChannelStateSizeError } from '../errors'; - -const readFileAsync = Promise.promisify(fs.readFile); -const writeFileAsync = Promise.promisify(fs.writeFile); -const readdirAsync = Promise.promisify(fs.readdir); -const statAsync = Promise.promisify(stat); -const SIZE_LIMIT = 1048576; -const CHANDUMP_DIR = path.resolve(__dirname, '..', '..', 'chandump'); - -export class FileStore { - filenameForChannel(channelName) { - return path.join(CHANDUMP_DIR, channelName); - } - - load(id, channelName) { - const filename = this.filenameForChannel(channelName); - return statAsync(filename).then(stats => { - if (stats.size > SIZE_LIMIT) { - return Promise.reject( - new ChannelStateSizeError( - 'Channel state file is too large', - { - limit: SIZE_LIMIT, - actual: stats.size - } - ) - ); - } else { - return readFileAsync(filename); - } - }).then(fileContents => { - try { - return JSON.parse(fileContents); - } catch (e) { - return Promise.reject(new Error('Channel state file is not valid JSON: ' + e)); - } - }); - } - - async save(id, channelName, data) { - let original; - try { - original = await this.load(id, channelName); - } catch (error) { - if (error.code !== 'ENOENT') { - throw error; - } else { - original = {}; - } - } - - Object.keys(data).forEach(key => { - original[key] = data[key]; - }); - - const filename = this.filenameForChannel(channelName); - const fileContents = new Buffer(JSON.stringify(original), 'utf8'); - if (fileContents.length > SIZE_LIMIT) { - throw new ChannelStateSizeError( - 'Channel state size is too large', - { - limit: SIZE_LIMIT, - actual: fileContents.length - } - ); - } - - return await writeFileAsync(filename, fileContents); - } - - listChannels() { - return readdirAsync(CHANDUMP_DIR); - } -} diff --git a/src/channel-storage/migrator.js b/src/channel-storage/migrator.js deleted file mode 100644 index d8c7c005..00000000 --- a/src/channel-storage/migrator.js +++ /dev/null @@ -1,187 +0,0 @@ -import Config from '../config'; -import Promise from 'bluebird'; -import db from '../database'; -import { FileStore } from './filestore'; -import { DatabaseStore } from './dbstore'; -import { sanitizeHTML } from '../xss'; -import { ChannelNotFoundError } from '../errors'; - -const lookupAsync = Promise.promisify(require('../database/channels').lookup); - -/* eslint no-console: off */ - -const EXPECTED_KEYS = [ - 'chatbuffer', - 'chatmuted', - 'css', - 'emotes', - 'filters', - 'js', - 'motd', - 'openPlaylist', - 'opts', - 'permissions', - 'playlist', - 'poll' -]; - -function fixOldChandump(data) { - const converted = {}; - EXPECTED_KEYS.forEach(key => { - converted[key] = data[key]; - }); - - if (data.queue) { - converted.playlist = { - pl: data.queue.map(item => { - return { - media: { - id: item.id, - title: item.title, - seconds: item.seconds, - duration: item.duration, - type: item.type, - meta: {} - }, - queueby: item.queueby, - temp: item.temp - }; - }), - pos: data.position, - time: data.currentTime - }; - } - - if (data.hasOwnProperty('openqueue')) { - converted.openPlaylist = data.openqueue; - } - - if (data.hasOwnProperty('playlistLock')) { - converted.openPlaylist = !data.playlistLock; - } - - if (data.chatbuffer) { - converted.chatbuffer = data.chatbuffer.map(entry => { - return { - username: entry.username, - msg: entry.msg, - meta: entry.meta || { - addClass: entry.msgclass ? entry.msgclass : undefined - }, - time: entry.time - }; - }); - } - - if (data.motd && data.motd.motd) { - converted.motd = sanitizeHTML(data.motd.motd).replace(/\n/g, '
\n'); - } - - if (data.opts && data.opts.customcss) { - converted.opts.externalcss = data.opts.customcss; - } - - if (data.opts && data.opts.customjs) { - converted.opts.externaljs = data.opts.customjs; - } - - if (data.filters && data.filters.length > 0 && Array.isArray(data.filters[0])) { - converted.filters = data.filters.map(filter => { - let [source, replace, active] = filter; - return { - source: source, - replace: replace, - flags: 'g', - active: active, - filterlinks: false - }; - }); - } - - return converted; -} - -function migrate(src, dest, opts) { - const chanPath = Config.get('channel-path'); - - return src.listChannels().then(names => { - return Promise.reduce(names, (_, name) => { - // A long time ago there was a bug where CyTube would save a different - // chandump depending on the capitalization of the channel name in the URL. - // This was fixed, but there are still some really old chandumps with - // uppercase letters in the name. - // - // If another chandump exists which is all lowercase, then that one is - // canonical. Otherwise, it's safe to load the existing capitalization, - // convert it, and save. - if (name !== name.toLowerCase()) { - if (names.indexOf(name.toLowerCase()) >= 0) { - return Promise.resolve(); - } - } - - let id; - return lookupAsync(name).then(chan => { - id = chan.id; - return src.load(id, name); - }).then(data => { - data = fixOldChandump(data); - Object.keys(data).forEach(key => { - if (opts.keyWhitelist.length > 0 && - opts.keyWhitelist.indexOf(key) < 0) { - delete data[key]; - } else if (opts.keyBlacklist.length > 0 && - opts.keyBlacklist.indexOf(key) >= 0) { - delete data[key]; - } - }); - return dest.save(id, name, data); - }).then(() => { - console.log(`Migrated /${chanPath}/${name}`); - }).catch(ChannelNotFoundError, _err => { - console.log(`Skipping /${chanPath}/${name} (not present in the database)`); - }).catch(err => { - console.error(`Failed to migrate /${chanPath}/${name}: ${err.stack}`); - }); - }, 0); - }); -} - -function loadOpts(argv) { - const opts = { - keyWhitelist: [], - keyBlacklist: [] - }; - - for (let i = 0; i < argv.length; i++) { - if (argv[i] === '-w') { - opts.keyWhitelist = (argv[i+1] || '').split(','); - i++; - } else if (argv[i] === '-b') { - opts.keyBlacklist = (argv[i+1] || '').split(','); - i++; - } - } - - return opts; -} - -function main() { - Config.load('config.yaml'); - db.init(); - const src = new FileStore(); - const dest = new DatabaseStore(); - const opts = loadOpts(process.argv.slice(2)); - - Promise.delay(1000).then(() => { - return migrate(src, dest, opts); - }).then(() => { - console.log('Migration complete'); - process.exit(0); - }).catch(err => { - console.error(`Migration failed: ${err.stack}`); - process.exit(1); - }); -} - -main(); diff --git a/src/channel/channel.js b/src/channel/channel.js index c1d9c68d..3abcd259 100644 --- a/src/channel/channel.js +++ b/src/channel/channel.js @@ -174,23 +174,6 @@ Channel.prototype.initModules = function () { self.logger.log("[init] Loaded modules: " + inited.join(", ")); }; -Channel.prototype.getDiskSize = function (cb) { - if (this._getDiskSizeTimeout > Date.now()) { - return cb(null, this._cachedDiskSize); - } - - var self = this; - var file = path.join(__dirname, "..", "..", "chandump", self.uniqueName); - fs.stat(file, function (err, stats) { - if (err) { - return cb(err); - } - - self._cachedDiskSize = stats.size; - cb(null, self._cachedDiskSize); - }); -}; - Channel.prototype.loadState = function () { /* Don't load from disk if not registered */ if (!this.is(Flags.C_REGISTERED)) { diff --git a/src/config.js b/src/config.js index b13111ee..1a6bb498 100644 --- a/src/config.js +++ b/src/config.js @@ -65,9 +65,6 @@ var defaults = { "channel-blacklist": [], "channel-path": "r", "channel-save-interval": 5, - "channel-storage": { - type: "file" - }, "max-channels-per-user": 5, "max-accounts-per-ip": 5, "guest-login-delay": 60, @@ -436,6 +433,10 @@ function preprocessConfig(cfg) { 'bucket-capacity': cfg.io.throttle['in-rate-limit'] }, cfg.io.throttle); + if (!cfg['channel-storage']) { + cfg['channel-storage'] = { type: undefined }; + } + return cfg; } diff --git a/src/database/channels.js b/src/database/channels.js index bbbd020b..90180356 100644 --- a/src/database/channels.js +++ b/src/database/channels.js @@ -1,7 +1,5 @@ var db = require("../database"); var valid = require("../utilities").isValidChannelName; -var fs = require("fs"); -var path = require("path"); var Flags = require("../flags"); var util = require("../utilities"); import { createMySQLDuplicateKeyUpdate } from '../util/on-duplicate-key-update'; @@ -199,14 +197,6 @@ module.exports = { } }); - fs.unlink(path.join(__dirname, "..", "..", "chandump", name), - function (err) { - if (err && err.code !== "ENOENT") { - LOGGER.error("Deleting chandump failed:"); - LOGGER.error(err); - } - }); - callback(err, !err); }); }, diff --git a/src/server.js b/src/server.js index 4f999153..a45053d4 100644 --- a/src/server.js +++ b/src/server.js @@ -15,11 +15,6 @@ module.exports = { exists || fs.mkdirSync(chanlogpath); }); - var chandumppath = path.join(__dirname, "../chandump"); - fs.exists(chandumppath, function (exists) { - exists || fs.mkdirSync(chandumppath); - }); - var gdvttpath = path.join(__dirname, "../google-drive-subtitles"); fs.exists(gdvttpath, function (exists) { exists || fs.mkdirSync(gdvttpath); diff --git a/src/setuid.js b/src/setuid.js index 516c88bf..bb52e1c8 100644 --- a/src/setuid.js +++ b/src/setuid.js @@ -7,7 +7,6 @@ const LOGGER = require('@calzoneman/jsli')('setuid'); var needPermissionsFixed = [ path.join(__dirname, "..", "chanlogs"), - path.join(__dirname, "..", "chandump"), path.join(__dirname, "..", "google-drive-subtitles") ];