/* jslint node: true */ 'use strict'; // ENiGMA½ const Log = require('../../logger.js').log; const { ServerModule } = require('../../server_module.js'); const Config = require('../../config.js').get; const { getTransactionDatabase, getModDatabasePath } = require('../../database.js'); const { getMessageAreaByTag, getMessageConferenceByTag, } = require('../../message_area.js'); const User = require('../../user.js'); const Errors = require('../../enig_error.js').Errors; const Message = require('../../message.js'); const FTNAddress = require('../../ftn_address.js'); const { isAnsi, stripAnsiControlCodes, splitTextAtTerms, } = require('../../string_util.js'); const AnsiPrep = require('../../ansi_prep.js'); const { stripMciColorCodes } = require('../../color_codes.js'); // deps const NNTPServerBase = require('nntp-server'); const _ = require('lodash'); const fs = require('fs-extra'); const forEachSeries = require('async/forEachSeries'); const asyncReduce = require('async/reduce'); const asyncMap = require('async/map'); const asyncSeries = require('async/series'); const asyncWaterfall = require('async/waterfall'); const LRU = require('lru-cache'); const sqlite3 = require('sqlite3'); const paths = require('path'); // // Network News Transfer Protocol (NNTP) // // RFCS // - https://www.w3.org/Protocols/rfc977/rfc977 // - https://tools.ietf.org/html/rfc3977 // - https://tools.ietf.org/html/rfc2980 // - https://tools.ietf.org/html/rfc5536 // exports.moduleInfo = { name : 'NNTP', desc : 'Network News Transfer Protocol (NNTP) Server', author : 'NuSkooler', packageName : 'codes.l33t.enigma.nntp.server', }; exports.performMaintenanceTask = performMaintenanceTask; /* General TODO - ACS checks need worked out. Currently ACS relies on |client|. We need a client spec that can be created even without a login server. Some checks and simply return false/fail. */ // simple DB maps NNTP Message-ID's which are // sequential per group -> ENiG messages // A single instance is shared across NNTP and/or NNTPS class NNTPDatabase { constructor() { } init(cb) { asyncSeries( [ (callback) => { this.db = getTransactionDatabase(new sqlite3.Database( getModDatabasePath(exports.moduleInfo), err => { return callback(err); } )); }, (callback) => { this.db.serialize( () => { this.db.run( `CREATE TABLE IF NOT EXISTS nntp_area_message ( nntp_message_id INTEGER NOT NULL, message_id INTEGER NOT NULL, message_area_tag VARCHAR NOT NULL, message_uuid VARCHAR NOT NULL, UNIQUE(nntp_message_id, message_area_tag) );` ); this.db.run( `CREATE INDEX IF NOT EXISTS nntp_area_message_by_uuid_index ON nntp_area_message (message_uuid);` ); return callback(null); }); } ], err => { return cb(err); } ); } } let nntpDatabase; class NNTPServer extends NNTPServerBase { constructor(options, serverName) { super(options); this.log = Log.child( { server : serverName } ); const config = Config(); this.groupCache = new LRU({ max : _.get(config, 'contentServers.nntp.cache.maxItems', 200), maxAge : _.get(config, 'contentServers.nntp.cache.maxAge', 1000 * 30), // default=30s }); } _needAuth(session, command) { return super._needAuth(session, command); } _authenticate(session) { const username = session.authinfo_user; const password = session.authinfo_pass; this.log.trace( { username }, 'Authentication request'); return new Promise( resolve => { const user = new User(); user.authenticateFactor1({ type : User.AuthFactor1Types.Password, username, password }, err => { if(err) { // :TODO: Log IP address this.log.debug( { username, reason : err.message }, 'Authentication failure'); return resolve(false); } session.authUser = user; this.log.debug( { username }, 'User authenticated successfully'); return resolve(true); }); }); } isGroupSelected(session) { return Array.isArray(_.get(session, 'groupInfo.messageList')); } getJAMStyleFrom(message, fromName) { // // Try to to create a (JamNTTPd) JAM style "From" field: // // - If we're dealing with a FTN address, create an email-like format // but do not include ':' or '/' characters as it may cause clients // to puke. FTN addresses are formatted how JamNTTPd does it for // some sort of compliance. We also extend up to 5D addressing. // - If we have an email address, then it's ready to go. // const remoteFrom = _.get(message.meta, [ 'System', Message.SystemMetaNames.RemoteFromUser ]); let jamStyleFrom; if(remoteFrom) { const flavor = _.get(message.meta, [ 'System', Message.SystemMetaNames.ExternalFlavor ]); switch(flavor) { case [ Message.AddressFlavor.FTN ] : { let ftnAddr = FTNAddress.fromString(remoteFrom); if(ftnAddr && ftnAddr.isValid()) { // In general, addresses are in point, node, net, zone, domain order if(ftnAddr.domain) { // 5D // point.node.net.zone@domain or node.net.zone@domain jamStyleFrom = `${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}@${ftnAddr.domain}`; if(ftnAddr.point) { jamStyleFrom = `${ftnAddr.point}.` + jamStyleFrom; } } else { if(ftnAddr.point) { jamStyleFrom = `${ftnAddr.point}@${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}`; } else { jamStyleFrom = `0@${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}`; } } } } break; case [ Message.AddressFlavor.Email ] : jamStyleFrom = `${fromName} <${remoteFrom}>`; break; } } if(!jamStyleFrom) { jamStyleFrom = fromName; } return jamStyleFrom; } populateNNTPHeaders(session, message, cb) { // // Build compliant headers // // Resources: // - https://tools.ietf.org/html/rfc5536#section-3.1 // - https://github.com/ftnapps/jamnntpd/blob/master/src/nntpserv.c#L962 // const toName = this.getMessageTo(message); const fromName = this.getMessageFrom(message); message.nntpHeaders = { From : this.getJAMStyleFrom(message, fromName), 'X-Comment-To' : toName, Newsgroups : session.group.name, Subject : message.subject, Date : this.getMessageDate(message), 'Message-ID' : this.getMessageIdentifier(message), Path : 'ENiGMA1/2!not-for-mail', 'Content-Type' : 'text/plain; charset=utf-8', }; const externalFlavor = _.get(message.meta.System, [ Message.SystemMetaNames.ExternalFlavor ]); if(externalFlavor) { message.nntpHeaders['X-ENiG-MessageFlavor'] = externalFlavor; } // Any FTN properties -> X-FTN-* _.each(message.meta.FtnProperty, (v, k) => { const suffix = { [ Message.FtnPropertyNames.FtnTearLine ] : 'Tearline', [ Message.FtnPropertyNames.FtnOrigin ] : 'Origin', [ Message.FtnPropertyNames.FtnArea ] : 'AREA', [ Message.FtnPropertyNames.FtnSeenBy ] : 'SEEN-BY', }[k]; if(suffix) { // some special treatment. if('Tearline' === suffix) { v = v.replace(/^--- /, ''); } else if('Origin' === suffix) { v = v.replace(/^[ ]{1,2}\* Origin: /, ''); } if(Array.isArray(v)) { // ie: SEEN-BY[] -> one big list v = v.join(' '); } message.nntpHeaders[`X-FTN-${suffix}`] = v.trim(); } }); // Other FTN kludges _.each(message.meta.FtnKludge, (v, k) => { if(Array.isArray(v)) { v = v.join(' '); // same as above } message.nntpHeaders[`X-FTN-${k.toUpperCase()}`] = v.toString().trim(); }); // // Set X-FTN-To and X-FTN-From: // - If remote to/from : joeuser // - Without remote : joeuser // const remoteFrom = _.get(message.meta, [ 'System', Message.SystemMetaNames.RemoteFromUser ]); message.nntpHeaders['X-FTN-From'] = remoteFrom ? `${fromName} <${remoteFrom}>` : fromName; const remoteTo = _.get(message.meta [ 'System', Message.SystemMetaNames.RemoteToUser ]); message.nntpHeaders['X-FTN-To'] = remoteTo ? `${toName} <${remoteTo}>` : toName; if(!message.replyToMsgId) { return cb(null); } // replyToMessageId -> Message-ID formatted ID const filter = { resultType : 'uuid', ids : [ parseInt(message.replyToMsgId) ], limit : 1, }; Message.findMessages(filter, (err, uuids) => { if(!err && Array.isArray(uuids)) { message.nntpHeaders.References = this.makeMessageIdentifier(message.replyToMsgId, uuids[0]); } return cb(null); }); } getMessageUUIDFromMessageID(session, messageId) { let messageUuid; // Direct ID request if((_.isString(messageId) && '<' !== messageId.charAt(0)) || _.isNumber(messageId)) { // group must be in session if(!this.isGroupSelected(session)) { return null; } messageId = parseInt(messageId); if(isNaN(messageId)) { return null; } const msg = session.groupInfo.messageList.find(m => { return m.index === messageId; }); messageUuid = msg && msg.messageUuid; } else { // request [ , messageUuid ] = this.getMessageIdentifierParts(messageId); } if(!_.isString(messageUuid)) { return null; } return messageUuid; } _getArticle(session, messageId) { return new Promise( resolve => { this.log.trace( { messageId }, 'Get article'); const messageUuid = this.getMessageUUIDFromMessageID(session, messageId); if(!messageUuid) { this.log.debug( { messageId }, 'Unable to retrieve message UUID for article request'); return resolve(null); } const message = new Message(); asyncSeries( [ (callback) => { return message.load( { uuid : messageUuid }, callback); }, (callback) => { if(!_.has(session, 'groupInfo.areaTag')) { // :TODO: if this is needed, how to validate properly? this.log.warn( { messageUuid, messageId }, 'Get article request without group selection'); return resolve(null); } if(session.groupInfo.areaTag !== message.areaTag) { return resolve(null); } if(!this.hasConfAndAreaReadAccess(session, session.groupInfo.confTag, session.groupInfo.areaTag)) { this.log.info( { messageUuid, messageId}, 'Access denied for message'); return resolve(null); } return callback(null); }, (callback) => { return this.populateNNTPHeaders(session, message, callback); }, (callback) => { return this.prepareMessageBody(message, callback); } ], err => { if(err) { this.log.error( { error : err.message, messageUuid }, 'Failed to load article'); return resolve(null); } this.log.info( { messageUuid, messageId, areaTag : message.areaTag }, 'Serving article'); return resolve(message); } ); }); } _getRange(session, first, last /*options*/) { return new Promise(resolve => { // // Build an array of message objects that can later // be used with the various _build* methods. // // :TODO: Handle |options| if(!this.isGroupSelected(session)) { return resolve(null); } const uuids = session.groupInfo.messageList.filter(m => { if(m.areaTag !== session.groupInfo.areaTag) { return false; } if(m.index < first || m.index > last) { return false; } return true; }).map(m => { return { uuid : m.messageUuid, index : m.index }; }); asyncMap(uuids, (msgInfo, nextMessageUuid) => { const message = new Message(); message.load( { uuid : msgInfo.uuid }, err => { if(err) { return nextMessageUuid(err); } message.index = msgInfo.index; this.populateNNTPHeaders(session, message, () => { this.prepareMessageBody(message, () => { return nextMessageUuid(null, message); }); }); }); }, (err, messages) => { return resolve(err ? null : messages); }); }); } _selectGroup (session, groupName) { this.log.trace( { groupName }, 'Select group request'); return new Promise( resolve => { this.getGroup(session, groupName, (err, group) => { if(err) { return resolve(false); } session.group = Object.assign( {}, // start clean { description : group.friendlyDesc || group.friendlyName, current_article : group.nntp.total ? group.nntp.min_index : 0, }, group.nntp ); session.groupInfo = group; // full set of info return resolve(true); }); }); } _getGroups(session, time, wildmat) { this.log.trace( { time, wildmat }, 'Get groups request'); // :TODO: handle time - probably use as caching mechanism - must consider user/auth/rights // :TODO: handle |time| if possible. return new Promise( (resolve, reject) => { const config = Config(); // :TODO: merge confs avail to authenticated user const publicConfs = _.get(config, 'contentServers.nntp.publicMessageConferences', {}); asyncReduce(Object.keys(publicConfs), [], (groups, confTag, nextConfTag) => { const areaTags = publicConfs[confTag]; // :TODO: merge area tags available to authenticated user asyncMap(areaTags, (areaTag, nextAreaTag) => { const groupName = this.getGroupName(confTag, areaTag); // filter on |wildmat| if supplied. We will remove // empty areas below in the final results. if(wildmat && !wildmat.test(groupName)) { return nextAreaTag(null, null); } this.getGroup(session, groupName, (err, group) => { if(err) { return nextAreaTag(null, null); // try others } return nextAreaTag(null, group.nntp); }); }, (err, areas) => { if(err) { return nextConfTag(err); } areas = areas.filter(a => a && Object.keys(a).length > 0); // remove empty groups.push(...areas); return nextConfTag(null, groups); }); }, (err, groups) => { if(err) { return reject(err); } return resolve(groups); }); }); } isConfAndAreaPubliclyExposed(confTag, areaTag) { const publicAreaTags = _.get(Config(), [ 'contentServers', 'nntp', 'publicMessageConferences', confTag ] ); return Array.isArray(publicAreaTags) && publicAreaTags.includes(areaTag); } hasConfAndAreaReadAccess(session, confTag, areaTag) { if(Message.isPrivateAreaTag(areaTag)) { return false; } if(this.isConfAndAreaPubliclyExposed(confTag, areaTag)) { return true; } // further checks require an authenticated user & ACS if(!session || !session.authUser) { return false; } const conf = getMessageConferenceByTag(confTag); if(!conf) { return false; } // :TODO: validate ACS const area = getMessageAreaByTag(areaTag, confTag); if(!area) { return false; } // :TODO: validate ACS return false; } getGroup(session, groupName, cb) { let group = this.groupCache.get(groupName); if(group) { return cb(null, group); } const [ confTag, areaTag ] = groupName.split('.'); if(!confTag || !areaTag) { return cb(Errors.UnexpectedState(`Invalid NNTP group name: ${groupName}`)); } if(!this.hasConfAndAreaReadAccess(session, confTag, areaTag)) { return cb(Errors.AccessDenied(`No access to conference ${confTag} and/or area ${areaTag}`)); } const area = getMessageAreaByTag(areaTag, confTag); if(!area) { return cb(Errors.DoesNotExist(`No area for areaTag "${areaTag}" / confTag "${confTag}"`)); } this.getMappedMessageListForArea(areaTag, (err, messageList) => { if(err) { return cb(err); } if(0 === messageList.length) { // // Handle empty group // See https://tools.ietf.org/html/rfc3977#section-6.1.1.2 // return cb(null, { messageList : [], confTag, areaTag, friendlyName : area.name, friendlyDesc : area.desc, nntp : { name : groupName, description : area.desc, min_index : 0, max_index : 0, total : 0, } }); } group = { messageList, confTag, areaTag, friendlyName : area.name, friendlyDesc : area.desc, nntp : { name : groupName, min_index : messageList[0].index, max_index : messageList[messageList.length - 1].index, total : messageList.length, }, }; this.groupCache.set(groupName, group); return cb(null, group); }); } getMappedMessageListForArea(areaTag, cb) { // // Get all messages in mapped database. Then, find any messages that are not // yet mapped with ID's > the highest ID we have. Any new messages will have // new mappings created. // // :TODO: introduce caching asyncWaterfall( [ (callback) => { nntpDatabase.db.all( `SELECT nntp_message_id, message_id, message_uuid FROM nntp_area_message WHERE message_area_tag = ? ORDER BY nntp_message_id;`, [ areaTag ], (err, rows) => { if(err) { return callback(err); } let messageList; const lastMessageId = rows.length > 0 ? rows[rows.length - 1].message_id : 0; if(!lastMessageId) { messageList = []; } else { messageList = rows.map(r => { return { areaTag, index : r.nntp_message_id, // node-nntp wants this name messageUuid : r.message_uuid, }; }); } return callback(null, messageList, lastMessageId); } ); }, (messageList, lastMessageId, callback) => { // Find any new entries const filter = { areaTag, newerThanMessageId : lastMessageId, sort : 'messageId', order : 'ascending', resultType : 'messageList', }; Message.findMessages(filter, (err, newMessageList) => { if(err) { return callback(err); } let index = messageList.length > 0 ? messageList[messageList.length - 1].index + 1 : 1; newMessageList = newMessageList.map(m => { return Object.assign(m, { index : index++ } ); }); if(0 === newMessageList.length) { return callback(null, messageList); } // populate mapping DB with any new entries nntpDatabase.db.beginTransaction( (err, trans) => { if(err) { return callback(err); } forEachSeries(newMessageList, (newMessage, nextNewMessage) => { trans.run( `INSERT INTO nntp_area_message (nntp_message_id, message_id, message_area_tag, message_uuid) VALUES (?, ?, ?, ?);`, [ newMessage.index, newMessage.messageId, areaTag, newMessage.messageUuid ], err => { return nextNewMessage(err); } ); }, err => { if(err) { return trans.rollback( () => { return callback(err); }); } trans.commit( () => { messageList.push(...newMessageList.map(m => { return { areaTag, index : m.nntpMessageId, messageUuid : m.messageUuid, }; })); return callback(null, messageList); }); }); }); }); } ], (err, messageList) => { return cb(err, messageList); } ); } _buildHead(session, message) { return _.map(message.nntpHeaders, (v, k) => `${k}: ${v}`).join('\r\n'); } _buildBody(session, message) { return message.preparedBody; } _buildHeaderField(session, message, field) { const body = message.preparedBody || message.message; const value = { ':bytes' : Buffer.byteLength(body).toString(), ':lines' : splitTextAtTerms(body).length.toString(), }[field] || _.find(message.nntpHeaders, (v, k) => { return k.toLowerCase() === field; }); if(!value) { // // Clients will check some headers just to see if they exist. // Don't spam logs with these. For others, it's good to know. // if(!['references', 'xref'].includes(field)) { this.log.trace(`No value for requested header field "${field}"`); } } return value; } _getOverviewFmt(session) { return super._getOverviewFmt(session); } _getNewNews(session, time, wildmat) { // Currently seems pointless to implement. No semi-modern clients seem to use it anyway. this.log.debug( { time, wildmat }, 'Request made using unsupported NEWNEWS command'); throw new Errors.Invalid('NEWNEWS is not enabled on this server'); } getMessageDate(message) { // https://tools.ietf.org/html/rfc5536#section-3.1.1 -> https://tools.ietf.org/html/rfc5322#section-3.3 return message.modTimestamp.format('ddd, D MMM YYYY HH:mm:ss ZZ'); } makeMessageIdentifier(messageId, messageUuid) { // // Spec : RFC-5536 Section 3.1.3 @ https://tools.ietf.org/html/rfc5536#section-3.1.3 // Example : <2456.0f6587f7-5512-4d03-8740-4d592190145a@enigma-bbs> // return `<${messageId}.${messageUuid}@enigma-bbs>`; } getMessageIdentifier(message) { // note that we use the *real* message ID here, not the NNTP-specific index. return this.makeMessageIdentifier(message.messageId, message.messageUuid); } getMessageIdentifierParts(messageId) { const m = messageId.match(/<([0-9]+)\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})@enigma-bbs>/); if(m) { return [ m[1], m[2] ]; } return []; } getMessageTo(message) { // :TODO: same as From -- check config return message.toUserName; } getMessageFrom(message) { // :TODO: NNTP config > conf > area config for real names return message.fromUserName; } prepareMessageBody(message, cb) { if(isAnsi(message.message)) { AnsiPrep( message.message, { rows : 'auto', cols : 79, forceLineTerm : true, asciiMode : true, fillLines : false, }, (err, prepped) => { message.preparedBody = prepped || message.message; return cb(null); } ); } else { message.preparedBody = stripMciColorCodes(stripAnsiControlCodes(message.message, { all : true })); return cb(null); } } getGroupName(confTag, areaTag) { // // Example: // input : fsxNet (confTag) fsx_bbs (areaTag) // output: fsx_net.fsx_bbs // // Note also that periods are replaced in conf and area // tags such that we *only* have a period separator // between the two for a group name! // return `${_.snakeCase(confTag).replace(/\./g, '_')}.${_.snakeCase(areaTag).replace(/\./g, '_')}`; } } exports.getModule = class NNTPServerModule extends ServerModule { constructor() { super(); } isEnabled() { return this.enableNntp || this.enableNttps; } get enableNntp() { return _.get(Config(), 'contentServers.nntp.nntp.enabled', false); } get enableNttps() { return _.get(Config(), 'contentServers.nntp.nntps.enabled', false); } isConfigured() { const config = Config(); // // Any conf/areas exposed? // const publicConfs = _.get(config, 'contentServers.nntp.publicMessageConferences', {}); const areasExposed = _.some(publicConfs, areas => { return Array.isArray(areas) && areas.length > 0; }); if(!areasExposed) { return false; } const nntp = _.get(config, 'contentServers.nntp.nntp'); if(nntp && this.enableNntp) { if(isNaN(nntp.port)) { return false; } } const nntps = _.get(config, 'contentServers.nntp.nntps'); if(nntps && this.enableNttps) { if(isNaN(nntps.port)) { return false; } if(!_.isString(nntps.certPem) || !_.isString(nntps.keyPem)) { return false; } } return true; } createServer(cb) { if(!this.isEnabled() || !this.isConfigured()) { return cb(null); } const config = Config(); const commonOptions = { //requireAuth : true, // :TODO: re-enable! // :TODO: override |session| - use our own debug to Bunyan, etc. }; if(this.enableNntp) { this.nntpServer = new NNTPServer( // :TODO: according to docs: if connection is non-tls, but behind proxy (assuming TLS termination?!!) then set this to true Object.assign( { secure : false }, commonOptions), 'NNTP' ); } if(this.enableNttps) { this.nntpsServer = new NNTPServer( Object.assign( { secure : true, tls : { cert : fs.readFileSync(config.contentServers.nntp.nntps.certPem), key : fs.readFileSync(config.contentServers.nntp.nntps.keyPem), } }, commonOptions ), 'NTTPS' ); } nntpDatabase = new NNTPDatabase(); nntpDatabase.init(err => { return cb(err); }); } listen(cb) { const config = Config(); forEachSeries([ 'nntp', 'nntps' ], (service, nextService) => { const server = this[`${service}Server`]; if(server) { const port = config.contentServers.nntp[service].port; server.listen(this.listenURI(port, service)) .catch(e => { Log.warn( { error : e.message, port }, `${service.toUpperCase()} failed to listen`); return nextService(null); // try next anyway }).then( () => { return nextService(null); }); } else { return nextService(null); } }, err => { return cb(err); }); } listenURI(port, service = 'nntp') { return `${service}://0.0.0.0:${port}`; } }; function performMaintenanceTask(args, cb) { // // Delete any message mapping that no longer have // an actual message associated with them. // if(!nntpDatabase) { Log.trace('Cannot perform NNTP maintenance without NNTP database initialized'); return cb(null); } let attached = false; asyncSeries( [ (callback) => { const messageDbPath = paths.join(Config().paths.db, 'message.sqlite3'); nntpDatabase.db.run( `ATTACH DATABASE "${messageDbPath}" AS msgdb;`, err => { attached = !err; return callback(err); } ); }, (callback) => { nntpDatabase.db.run( `DELETE FROM nntp_area_message WHERE message_uuid NOT IN ( SELECT message_uuid FROM msgdb.message );`, function result(err) { // no arrow func; need |this.changes| if(err) { Log.warn( { error : err.message }, 'Failed to delete from NNTP database'); } else { Log.debug( { count : this.changes }, 'Deleted mapped message IDs from NNTP database'); } return callback(err); } ); } ], err => { if(attached) { nntpDatabase.db.run('DETACH DATABASE msgdb;'); } return cb(err); } ); }