moon
/
sync
mirror of https://github.com/calzoneman/sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove old flatfile chandump storage

This commit is contained in:
Calvin Montgomery 2020-02-15 16:17:49 -08:00
parent e3a9915b45
commit 106065184f
11 changed files with 42 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
NEWS.md
View File

@ -1,3 +1,33 @@
2020-02-15
==========
Old versions of CyTube defaulted to storing channel state in flatfiles located
in the `chandump` directory. The default was changed a while ago, and the
flatfile storage mechanism has now been removed.
Admins who have not already migrated their installation to the "database"
channel storage type can do so by following these instructions:
1. Run `git checkout e3a9915b454b32e49d3871c94c839899f809520a` to temporarily
switch to temporarily revert to the previous version of the code that
supports the "file" channel storage type
2. Run `npm run build-server` to build the old version
3. Run `node lib/channel-storage/migrator.js |& tee migration.log` to migrate
channel state from files to the database
4. Inspect the output of the migration tool for errors
5. Set `channel-storage`/`type` to `"database"` in `config.yaml` and start the
server. Load a channel to verify the migration worked as expected
6. Upgrade back to the latest version with `git checkout 3.0` and `npm run
build-server`
7. Remove the `channel-storage` block from `config.yaml` and remove the
`chandump` directory since it is no longer needed (you may wish to archive
it somewhere in case you later discover the migration didn't work as
expected).
If you encounter any errors during the process, please file an issue on GitHub
and attach the output of the migration tool (which if you use the above commands
will be written to `migration.log`).
2019-12-01
==========

3
config.template.yaml
View File

@ -133,9 +133,6 @@ channel-path: 'r'
channel-blacklist: []
# Minutes between saving channel state to disk
channel-save-interval: 5
# Determines channel data storage mechanism.
channel-storage:
type: 'database'
# Configure periodic clearing of old alias data
aliases:

2
package.json
View File

@ -2,7 +2,7 @@
"author": "Calvin Montgomery",
"name": "CyTube",
"description": "Online media synchronizer and chat",
"version": "3.68.0",
"version": "3.69.0",
"repository": {
"url": "http://github.com/calzoneman/sync"
},

14
src/channel-storage/channelstore.js
View File

@ -1,4 +1,3 @@
import { FileStore } from './filestore';
import { DatabaseStore } from './dbstore';
import Config from '../config';
import Promise from 'bluebird';
@ -26,11 +25,12 @@ export function save(id, channelName, data) {
}
function loadChannelStore() {
switch (Config.get('channel-storage.type')) {
case 'database':
return new DatabaseStore();
case 'file':
default:
return new FileStore();
if (Config.get('channel-storage.type') === 'file') {
throw new Error(
'channel-storage type "file" is no longer supported. Please see ' +
'NEWS.md for instructions on upgrading.'
);
}
return new DatabaseStore();
}

78
src/channel-storage/filestore.js
View File

@ -1,78 +0,0 @@
import Promise from 'bluebird';
import { stat } from 'fs';
import * as fs from 'graceful-fs';
import path from 'path';
import { ChannelStateSizeError } from '../errors';
const readFileAsync = Promise.promisify(fs.readFile);
const writeFileAsync = Promise.promisify(fs.writeFile);
const readdirAsync = Promise.promisify(fs.readdir);
const statAsync = Promise.promisify(stat);
const SIZE_LIMIT = 1048576;
const CHANDUMP_DIR = path.resolve(__dirname, '..', '..', 'chandump');
export class FileStore {
filenameForChannel(channelName) {
return path.join(CHANDUMP_DIR, channelName);
}
load(id, channelName) {
const filename = this.filenameForChannel(channelName);
return statAsync(filename).then(stats => {
if (stats.size > SIZE_LIMIT) {
return Promise.reject(
new ChannelStateSizeError(
'Channel state file is too large',
{
limit: SIZE_LIMIT,
actual: stats.size
}
)
);
} else {
return readFileAsync(filename);
}
}).then(fileContents => {
try {
return JSON.parse(fileContents);
} catch (e) {
return Promise.reject(new Error('Channel state file is not valid JSON: ' + e));
}
});
}
async save(id, channelName, data) {
let original;
try {
original = await this.load(id, channelName);
} catch (error) {
if (error.code !== 'ENOENT') {
throw error;
} else {
original = {};
}
}
Object.keys(data).forEach(key => {
original[key] = data[key];
});
const filename = this.filenameForChannel(channelName);
const fileContents = new Buffer(JSON.stringify(original), 'utf8');
if (fileContents.length > SIZE_LIMIT) {
throw new ChannelStateSizeError(
'Channel state size is too large',
{
limit: SIZE_LIMIT,
actual: fileContents.length
}
);
}
return await writeFileAsync(filename, fileContents);
}
listChannels() {
return readdirAsync(CHANDUMP_DIR);
}
}

187
src/channel-storage/migrator.js
View File

@ -1,187 +0,0 @@
import Config from '../config';
import Promise from 'bluebird';
import db from '../database';
import { FileStore } from './filestore';
import { DatabaseStore } from './dbstore';
import { sanitizeHTML } from '../xss';
import { ChannelNotFoundError } from '../errors';
const lookupAsync = Promise.promisify(require('../database/channels').lookup);
/* eslint no-console: off */
const EXPECTED_KEYS = [
'chatbuffer',
'chatmuted',
'css',
'emotes',
'filters',
'js',
'motd',
'openPlaylist',
'opts',
'permissions',
'playlist',
'poll'
];
function fixOldChandump(data) {
const converted = {};
EXPECTED_KEYS.forEach(key => {
converted[key] = data[key];
});
if (data.queue) {
converted.playlist = {
pl: data.queue.map(item => {
return {
media: {
id: item.id,
title: item.title,
seconds: item.seconds,
duration: item.duration,
type: item.type,
meta: {}
},
queueby: item.queueby,
temp: item.temp
};
}),
pos: data.position,
time: data.currentTime
};
}
if (data.hasOwnProperty('openqueue')) {
converted.openPlaylist = data.openqueue;
}
if (data.hasOwnProperty('playlistLock')) {
converted.openPlaylist = !data.playlistLock;
}
if (data.chatbuffer) {
converted.chatbuffer = data.chatbuffer.map(entry => {
return {
username: entry.username,
msg: entry.msg,
meta: entry.meta || {
addClass: entry.msgclass ? entry.msgclass : undefined
},
time: entry.time
};
});
}
if (data.motd && data.motd.motd) {
converted.motd = sanitizeHTML(data.motd.motd).replace(/\n/g, '<br>\n');
}
if (data.opts && data.opts.customcss) {
converted.opts.externalcss = data.opts.customcss;
}
if (data.opts && data.opts.customjs) {
converted.opts.externaljs = data.opts.customjs;
}
if (data.filters && data.filters.length > 0 && Array.isArray(data.filters[0])) {
converted.filters = data.filters.map(filter => {
let [source, replace, active] = filter;
return {
source: source,
replace: replace,
flags: 'g',
active: active,
filterlinks: false
};
});
}
return converted;
}
function migrate(src, dest, opts) {
const chanPath = Config.get('channel-path');
return src.listChannels().then(names => {
return Promise.reduce(names, (_, name) => {
// A long time ago there was a bug where CyTube would save a different
// chandump depending on the capitalization of the channel name in the URL.
// This was fixed, but there are still some really old chandumps with
// uppercase letters in the name.
//
// If another chandump exists which is all lowercase, then that one is
// canonical. Otherwise, it's safe to load the existing capitalization,
// convert it, and save.
if (name !== name.toLowerCase()) {
if (names.indexOf(name.toLowerCase()) >= 0) {
return Promise.resolve();
}
}
let id;
return lookupAsync(name).then(chan => {
id = chan.id;
return src.load(id, name);
}).then(data => {
data = fixOldChandump(data);
Object.keys(data).forEach(key => {
if (opts.keyWhitelist.length > 0 &&
opts.keyWhitelist.indexOf(key) < 0) {
delete data[key];
} else if (opts.keyBlacklist.length > 0 &&
opts.keyBlacklist.indexOf(key) >= 0) {
delete data[key];
}
});
return dest.save(id, name, data);
}).then(() => {
console.log(`Migrated /${chanPath}/${name}`);
}).catch(ChannelNotFoundError, _err => {
console.log(`Skipping /${chanPath}/${name} (not present in the database)`);
}).catch(err => {
console.error(`Failed to migrate /${chanPath}/${name}: ${err.stack}`);
});
}, 0);
});
}
function loadOpts(argv) {
const opts = {
keyWhitelist: [],
keyBlacklist: []
};
for (let i = 0; i < argv.length; i++) {
if (argv[i] === '-w') {
opts.keyWhitelist = (argv[i+1] || '').split(',');
i++;
} else if (argv[i] === '-b') {
opts.keyBlacklist = (argv[i+1] || '').split(',');
i++;
}
}
return opts;
}
function main() {
Config.load('config.yaml');
db.init();
const src = new FileStore();
const dest = new DatabaseStore();
const opts = loadOpts(process.argv.slice(2));
Promise.delay(1000).then(() => {
return migrate(src, dest, opts);
}).then(() => {
console.log('Migration complete');
process.exit(0);
}).catch(err => {
console.error(`Migration failed: ${err.stack}`);
process.exit(1);
});
}
main();

17
src/channel/channel.js
View File

@ -174,23 +174,6 @@ Channel.prototype.initModules = function () {
self.logger.log("[init] Loaded modules: " + inited.join(", "));
};
Channel.prototype.getDiskSize = function (cb) {
if (this._getDiskSizeTimeout > Date.now()) {
return cb(null, this._cachedDiskSize);
}
var self = this;
var file = path.join(__dirname, "..", "..", "chandump", self.uniqueName);
fs.stat(file, function (err, stats) {
if (err) {
return cb(err);
}
self._cachedDiskSize = stats.size;
cb(null, self._cachedDiskSize);
});
};
Channel.prototype.loadState = function () {
/* Don't load from disk if not registered */
if (!this.is(Flags.C_REGISTERED)) {

7
src/config.js
View File

@ -65,9 +65,6 @@ var defaults = {
"channel-blacklist": [],
"channel-path": "r",
"channel-save-interval": 5,
"channel-storage": {
type: "file"
},
"max-channels-per-user": 5,
"max-accounts-per-ip": 5,
"guest-login-delay": 60,
@ -436,6 +433,10 @@ function preprocessConfig(cfg) {
'bucket-capacity': cfg.io.throttle['in-rate-limit']
}, cfg.io.throttle);
if (!cfg['channel-storage']) {
cfg['channel-storage'] = { type: undefined };
}
return cfg;
}

10
src/database/channels.js
View File

@ -1,7 +1,5 @@
var db = require("../database");
var valid = require("../utilities").isValidChannelName;
var fs = require("fs");
var path = require("path");
var Flags = require("../flags");
var util = require("../utilities");
import { createMySQLDuplicateKeyUpdate } from '../util/on-duplicate-key-update';
@ -199,14 +197,6 @@ module.exports = {
}
});
fs.unlink(path.join(__dirname, "..", "..", "chandump", name),
function (err) {
if (err && err.code !== "ENOENT") {
LOGGER.error("Deleting chandump failed:");
LOGGER.error(err);
}
});
callback(err, !err);
});
},

5
src/server.js
View File

@ -15,11 +15,6 @@ module.exports = {
exists || fs.mkdirSync(chanlogpath);
});
var chandumppath = path.join(__dirname, "../chandump");
fs.exists(chandumppath, function (exists) {
exists || fs.mkdirSync(chandumppath);
});
var gdvttpath = path.join(__dirname, "../google-drive-subtitles");
fs.exists(gdvttpath, function (exists) {
exists || fs.mkdirSync(gdvttpath);

1
src/setuid.js
View File

@ -7,7 +7,6 @@ const LOGGER = require('@calzoneman/jsli')('setuid');
var needPermissionsFixed = [
path.join(__dirname, "..", "chanlogs"),
path.join(__dirname, "..", "chandump"),
path.join(__dirname, "..", "google-drive-subtitles")
];