Fixes; initial migrator work

src/channel-storage/dbstore.js

@@ -1,6 +1,6 @@
 import Promise from 'bluebird';
 import { ChannelStateSizeError,
-         ChannelDataNotFoundError } from '../errors';
+         ChannelNotFoundError } from '../errors';
 import db from '../database';
 import Logger from '../logger';
 
@@ -25,6 +25,16 @@ function queryAsync(query, substitutions) {
     });
 }
 
+function buildUpdateQuery(numEntries) {
+    const values = [];
+    for (let i = 0; i < numEntries; i++) {
+        values.push('(?, ?, ?)');
+    }
+
+    return `INSERT INTO channel_data VALUES ${values.join(', ')} ` +
+           'ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)';
+}
+
 export class DatabaseStore {
     load(channelName) {
         return queryAsync(QUERY_CHANNEL_ID_FOR_NAME, [channelName]).then((rows) => {
@@ -55,17 +65,16 @@ export class DatabaseStore {
             }
 
             let totalSize = 0;
+            let rowCount = 0;
             const id = rows[0].id;
             const substitutions = [];
             for (const key of Object.keys(data)) {
+                rowCount++;
                 const value = JSON.stringify(data[key]);
                 totalSize += value.length;
-                substitutions.push([
-                    id,
-                    key,
-                    value,
-                    value // Extra substitution var necessary for ON DUPLICATE KEY UPDATE
-                ]);
+                substitutions.push(id);
+                substitutions.push(key);
+                substitutions.push(value);
             }
 
             if (totalSize > SIZE_LIMIT) {
@@ -75,9 +84,7 @@ export class DatabaseStore {
                 });
             }
 
-            return Promise.map(substitutions, entry => {
-                return queryAsync(QUERY_UPDATE_CHANNEL_DATA, entry);
-            });
+            return queryAsync(buildUpdateQuery(rowCount), substitutions);
         });
     }
 }

src/channel-storage/filestore.js

@@ -6,6 +6,7 @@ import { ChannelStateSizeError } from '../errors';
 
 const readFileAsync = Promise.promisify(fs.readFile);
 const writeFileAsync = Promise.promisify(fs.writeFile);
+const readdirAsync = Promise.promisify(fs.readdir);
 const statAsync = Promise.promisify(stat);
 const SIZE_LIMIT = 1048576;
 const CHANDUMP_DIR = path.resolve(__dirname, '..', '..', 'chandump');
@@ -48,4 +49,8 @@ export class FileStore {
 
         return writeFileAsync(filename, fileContents);
     }
+
+    listChannels() {
+        return readdirAsync(CHANDUMP_DIR);
+    }
 }

src/channel-storage/migrator.js

@@ -0,0 +1,74 @@
+import Config from '../config';
+import Promise from 'bluebird';
+import db from '../database';
+import { FileStore } from './filestore';
+import { DatabaseStore } from './dbstore';
+
+const QUERY_CHANNEL_NAMES = 'SELECT name FROM channels WHERE 1';
+
+function queryAsync(query, substitutions) {
+    return new Promise((resolve, reject) => {
+        db.query(query, substitutions, (err, res) => {
+            if (err) {
+                if (!(err instanceof Error)) {
+                    err = new Error(err);
+                }
+                reject(err);
+            } else {
+                resolve(res);
+            }
+        });
+    });
+}
+
+function fixOldChandump(data) {
+    return data;
+}
+
+function migrate(src, dest) {
+    return src.listChannels().then(names => {
+        return Promise.reduce(names, (_, name) => {
+            // A long time ago there was a bug where CyTube would save a different
+            // chandump depending on the capitalization of the channel name in the URL.
+            // This was fixed, but there are still some really old chandumps with
+            // uppercase letters in the name.
+            //
+            // If another chandump exists which is all lowercase, then that one is
+            // canonical.  Otherwise, it's safe to just lowercase the name and convert
+            // it.
+            if (name !== name.toLowerCase()) {
+                if (names.indexOf(name.toLowerCase()) >= 0) {
+                    return Promise.resolve();
+                } else {
+                    name = name.toLowerCase();
+                }
+            }
+
+            return src.load(name).then(data => {
+                data = fixOldChandump(data);
+                return dest.save(name, data);
+            }).then(() => {
+                console.log(`Migrated /r/${name}`);
+            }).catch(err => {
+                console.error(`Failed to migrate /r/${name}: ${err.stack}`);
+            });
+        });
+    });
+}
+
+function main() {
+    Config.load('config.yaml');
+    db.init();
+    const src = new FileStore();
+    const dest = new DatabaseStore();
+
+    migrate(src, dest).then(() => {
+        console.log('Migration complete');
+        process.exit(0);
+    }).catch(err => {
+        console.error(`Migration failed: ${err.stack}`);
+        process.exit(1);
+    });
+}
+
+main();