From 67e2ff987f40c7360966d40c4fec4fcccf773b16 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 6 Oct 2016 21:03:04 -0600 Subject: [PATCH] * getISOTimestampString() * More file entry load/persist --- core/database.js | 8 +++- core/file_area.js | 96 +++++++++++++++++++++++++++++----------------- core/file_entry.js | 64 ++++++++++++++++++++++++++++--- core/message.js | 29 +++++++------- 4 files changed, 139 insertions(+), 58 deletions(-) diff --git a/core/database.js b/core/database.js index f4bfc752..608c263a 100644 --- a/core/database.js +++ b/core/database.js @@ -10,11 +10,13 @@ const paths = require('path'); const async = require('async'); const _ = require('lodash'); const assert = require('assert'); +const moment = require('moment'); // database handles let dbs = {}; exports.getModDatabasePath = getModDatabasePath; +exports.getISOTimestampString = getISOTimestampString; exports.initializeDatabases = initializeDatabases; exports.dbs = dbs; @@ -46,6 +48,11 @@ function getModDatabasePath(moduleInfo, suffix) { return paths.join(conf.config.paths.modsDb, `${full}.sqlite3`); } +function getISOTimestampString(ts) { + ts = ts || moment(); + return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ'); +} + function initializeDatabases(cb) { async.each( [ 'system', 'user', 'message', 'file' ], (dbName, next) => { dbs[dbName] = new sqlite3.Database(getDatabasePath(dbName), err => { @@ -257,7 +264,6 @@ const DB_INIT_TABLE = { file_name, /* FTS @ file_fts */ desc, /* FTS @ file_fts */ desc_long, /* FTS @ file_fts */ - upload_by_username VARCHAR NOT NULL, upload_timestamp DATETIME NOT NULL );` ); diff --git a/core/file_area.js b/core/file_area.js index af5c3075..36d93078 100644 --- a/core/file_area.js +++ b/core/file_area.js @@ -156,28 +156,35 @@ function sliceAtSauceMarker(data) { return data.slice(0, eof); } -function getEstYear(input) { +function attemptSetEstimatedReleaseDate(fileEntry) { // :TODO: yearEstPatterns RegExp's should be cached - we can do this @ Config (re)load time const patterns = Config.fileBase.yearEstPatterns.map( p => new RegExp(p, 'gmi')); - let match; - for(let i = 0; i < patterns.length; ++i) { - match = patterns[i].exec(input); - if(match) { - break; + function getMatch(input) { + if(input) { + let m; + for(let i = 0; i < patterns.length; ++i) { + m = patterns[i].exec(input); + if(m) { + return m; + } + } } } - if(match) { - if(2 == match[1].length) { - return parseInt('19' + match[1]); - } else { - return parseInt(match[1]); + // + // We attempt deteciton in short -> long order + // + const match = getMatch(fileEntry.desc) || getMatch(fileEntry.descLong); + if(match && match[1]) { + const year = (2 === match[1].length) ? parseInt('19' + match[1]) : parseInt(match[1]); + if(year) { + fileEntry.meta.est_release_year = year; } } } -function addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb) { +function populateFileEntryWithArchive(fileEntry, filePath, archiveType, cb) { const archiveUtil = ArchiveUtil.getInstance(); async.waterfall( @@ -258,10 +265,8 @@ function addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb) { }); }, function attemptReleaseYearEstimation(callback) { - let estYear; - if(fileEntry.descLong) { - estYear = getEstYear(fileEntry.descLong); - } + attemptSetEstimatedReleaseDate(fileEntry); + return callback(null); } ], err => { @@ -270,17 +275,47 @@ function addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb) { ); } +function populateFileEntry(fileEntry, filePath, archiveType, cb) { + // :TODO: implement me! + return cb(null); +} + function addNewFileEntry(fileEntry, filePath, cb) { const archiveUtil = ArchiveUtil.getInstance(); // :TODO: Use detectTypeWithBuf() once avail - we *just* read some file data - archiveUtil.detectType(filePath, (err, archiveType) => { - if(archiveType) { - return addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb); - } else { - // :TODO:addNewNonArchiveFileEntry - } - }); + + async.series( + [ + function populateInfo(callback) { + archiveUtil.detectType(filePath, (err, archiveType) => { + if(archiveType) { + populateFileEntryWithArchive(fileEntry, filePath, archiveType, err => { + if(err) { + populateFileEntry(fileEntry, filePath, err => { + // :TODO: log err + return callback(null); // ignore err + }); + } + return callback(null); + }); + } else { + populateFileEntry(fileEntry, filePath, err => { + // :TODO: log err + return callback(null); // ignore err + }); + } + }); + }, + function addNewDbRecord(callback) { + return fileEntry.persist(callback); + } + ] + ); +} + +function updateFileEntry(fileEntry, filePath, cb) { + } function addOrUpdateFileEntry(areaInfo, fileName, options, cb) { @@ -289,6 +324,7 @@ function addOrUpdateFileEntry(areaInfo, fileName, options, cb) { areaTag : areaInfo.areaTag, meta : options.meta, hashTags : options.hashTags, // Set() or Array + fileName : fileName, }); const filePath = paths.join(getAreaStorageDirectory(areaInfo), fileName); @@ -302,8 +338,7 @@ function addOrUpdateFileEntry(areaInfo, fileName, options, cb) { const sha1 = crypto.createHash('sha1'); const sha256 = crypto.createHash('sha256'); const md5 = crypto.createHash('md5'); - - + // :TODO: crc32 stream.on('data', data => { @@ -340,17 +375,6 @@ function addOrUpdateFileEntry(areaInfo, fileName, options, cb) { if(existingEntries.length > 0) { } else { - // - // Some basics for new entries - // - fileEntry.meta.user_rating = 0; - if(options.uploadByUserName) { - fileEntry.meta.upload_by_username = options.uploadByUserName; - } - if(options.uploadByUserId) { - fileEntry.meta.upload_by_user_id = options.uploadByUserId; - } - return addNewFileEntry(fileEntry, filePath, callback); } }, diff --git a/core/file_entry.js b/core/file_entry.js index e4c44d11..25e6e81c 100644 --- a/core/file_entry.js +++ b/core/file_entry.js @@ -1,16 +1,17 @@ /* jslint node: true */ 'use strict'; -const fileDb = require('./database.js').dbs.file; -const Errors = require('./enig_error.js').Errors; +const fileDb = require('./database.js').dbs.file; +const Errors = require('./enig_error.js').Errors; +const getISOTimestampString = require('./database.js').getISOTimestampString; // deps -const async = require('async'); -const _ = require('lodash'); +const async = require('async'); +const _ = require('lodash'); const FILE_TABLE_MEMBERS = [ 'file_id', 'area_tag', 'file_sha1', 'file_name', - 'desc', 'desc_long', 'upload_by_username', 'upload_timestamp' // :TODO: remove upload_by_username -- and from database.js, etc. + 'desc', 'desc_long', 'upload_timestamp' ]; const FILE_WELL_KNOWN_META = { @@ -34,6 +35,7 @@ module.exports = class FileEntry { this.areaTag = options.areaTag || ''; this.meta = {}; this.hashTags = new Set(); + this.fileName = options.fileName; } load(fileId, cb) { @@ -79,6 +81,58 @@ module.exports = class FileEntry { ); } + persist(cb) { + const self = this; + + async.series( + [ + function startTrans(callback) { + return fileDb.run('BEGIN;', callback); + }, + function storeEntry(callback) { + fileDb.run( + `REPLACE INTO file (area_tag, file_sha1, file_name, desc, desc_long, upload_timestamp) + VALUES(?, ?, ?, ?, ?, ?);`, + [ self.areaTag, self.fileSha1, self.fileName, self.desc, self.descLong, getISOTimestampString() ], + function inserted(err) { // use non-arrow func for 'this' scope / lastID + if(!err) { + self.fileId = this.lastID; + } + return callback(err); + } + ); + }, + function storeMeta(callback) { + async.each(Object.keys(self.meta), (n, next) => { + const v = self.meta[n]; + return FileEntry.persistMetaValue(self.fileId, n, v, next); + }, + err => { + return callback(err); + }); + }, + function storeHashTags(callback) { + return callback(null); + } + ], + err => { + // :TODO: Log orig err + fileDb.run(err ? 'ROLLBACK;' : 'COMMIT;', err => { + return cb(err); + }); + } + ); + } + + static persistMetaValue(fileId, name, value, cb) { + fileDb.run( + `REPLACE INTO file_meta (file_id, meta_name, meta_value) + VALUES(?, ?, ?);`, + [ fileId, name, value ], + cb + ); + } + loadMeta(cb) { fileDb.each( `SELECT meta_name, meta_value diff --git a/core/message.js b/core/message.js index e1d3e4fd..ce6a7c8f 100644 --- a/core/message.js +++ b/core/message.js @@ -1,17 +1,19 @@ /* jslint node: true */ 'use strict'; -let msgDb = require('./database.js').dbs.message; -let wordWrapText = require('./word_wrap.js').wordWrapText; -let ftnUtil = require('./ftn_util.js'); -let createNamedUUID = require('./uuid_util.js').createNamedUUID; +const msgDb = require('./database.js').dbs.message; +const wordWrapText = require('./word_wrap.js').wordWrapText; +const ftnUtil = require('./ftn_util.js'); +const createNamedUUID = require('./uuid_util.js').createNamedUUID; +const getISOTimestampString = require('./database.js').getISOTimestampString; -let uuid = require('node-uuid'); -let async = require('async'); -let _ = require('lodash'); -let assert = require('assert'); -let moment = require('moment'); -const iconvEncode = require('iconv-lite').encode; +// deps +const uuid = require('node-uuid'); +const async = require('async'); +const _ = require('lodash'); +const assert = require('assert'); +const moment = require('moment'); +const iconvEncode = require('iconv-lite').encode; module.exports = Message; @@ -64,11 +66,6 @@ function Message(options) { this.isPrivate = function() { return Message.isPrivateAreaTag(this.areaTag); }; - - this.getMessageTimestampString = function(ts) { - ts = ts || moment(); - return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ'); - }; } Message.WellKnownAreaTags = { @@ -374,7 +371,7 @@ Message.prototype.persist = function(cb) { msgDb.run( `INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?);`, - [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(msgTimestamp) ], + [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, getISOTimestampString(msgTimestamp) ], function inserted(err) { // use non-arrow function for 'this' scope if(!err) { self.messageId = this.lastID;