/* jslint node: true */ 'use strict'; // ENiGMA½ const Log = require('../../logger.js').log; const { ServerModule } = require('../../server_module.js'); const Config = require('../../config.js').get; const { getTransactionDatabase, getModDatabasePath } = require('../../database.js'); const { getMessageAreaByTag, getMessageConferenceByTag, persistMessage, } = require('../../message_area.js'); const User = require('../../user.js'); const Errors = require('../../enig_error.js').Errors; const Message = require('../../message.js'); const FTNAddress = require('../../ftn_address.js'); const { isAnsi, stripAnsiControlCodes, splitTextAtTerms, } = require('../../string_util.js'); const AnsiPrep = require('../../ansi_prep.js'); const { stripMciColorCodes } = require('../../color_codes.js'); const ACS = require('../../acs'); // deps const NNTPServerBase = require('nntp-server'); const _ = require('lodash'); const fs = require('fs-extra'); const forEachSeries = require('async/forEachSeries'); const asyncReduce = require('async/reduce'); const asyncMap = require('async/map'); const asyncSeries = require('async/series'); const asyncWaterfall = require('async/waterfall'); const LRU = require('lru-cache'); const sqlite3 = require('sqlite3'); const paths = require('path'); // // Network News Transfer Protocol (NNTP) // // RFCS // - https://www.w3.org/Protocols/rfc977/rfc977 // - https://tools.ietf.org/html/rfc3977 // - https://tools.ietf.org/html/rfc2980 // - https://tools.ietf.org/html/rfc5536 // exports.moduleInfo = { name: 'NNTP', desc: 'Network News Transfer Protocol (NNTP) Server', author: 'NuSkooler', packageName: 'codes.l33t.enigma.nntp.server', }; exports.performMaintenanceTask = performMaintenanceTask; /* General TODO - ACS checks need worked out. Currently ACS relies on |client|. We need a client spec that can be created even without a login server. Some checks and simply return false/fail. */ // simple DB maps NNTP Message-ID's which are // sequential per group -> ENiG messages // A single instance is shared across NNTP and/or NNTPS class NNTPDatabase { constructor() {} init(cb) { asyncSeries( [ callback => { this.db = getTransactionDatabase( new sqlite3.Database( getModDatabasePath(exports.moduleInfo), err => { return callback(err); } ) ); }, callback => { this.db.serialize(() => { this.db.run( `CREATE TABLE IF NOT EXISTS nntp_area_message ( nntp_message_id INTEGER NOT NULL, message_id INTEGER NOT NULL, message_area_tag VARCHAR NOT NULL, message_uuid VARCHAR NOT NULL, UNIQUE(nntp_message_id, message_area_tag) );` ); this.db.run( `CREATE INDEX IF NOT EXISTS nntp_area_message_by_uuid_index ON nntp_area_message (message_uuid);` ); return callback(null); }); }, ], err => { return cb(err); } ); } } let nntpDatabase; const AuthCommands = 'POST'; // these aren't exported by the NNTP module, unfortunantely const Responses = { ArticlePostedOk: '240 article posted ok', SendArticle: '340 send article to be posted', PostingNotAllowed: '440 posting not allowed', ArticlePostFailed: '441 posting failed', AuthRequired: '480 authentication required', }; const PostCommand = { head: 'POST', validate: /^POST$/i, run(session, cmd) { if (!session.authenticated) { session.receivingPostArticle = false; // ensure reset return Responses.AuthRequired; } session.receivingPostArticle = true; return Responses.SendArticle; }, capability(session, report) { report.push('POST'); }, }; class NNTPServer extends NNTPServerBase { constructor(options, serverName) { super(options); this.log = Log.child({ server: serverName }); const config = Config(); this.groupCache = new LRU({ max: _.get(config, 'contentServers.nntp.cache.maxItems', 200), ttl: _.get(config, 'contentServers.nntp.cache.maxAge', 1000 * 30), // default=30s }); } _needAuth(session, command) { if (AuthCommands.includes(command)) { return !session.authenticated && !session.authUser; } return super._needAuth(session, command); } _address(session) { const addr = session.in_stream.remoteAddress; return addr ? addr.replace(/^::ffff:/, '').replace(/^::1$/, 'localhost') : 'N/A'; } _authenticate(session) { const username = session.authinfo_user; const password = session.authinfo_pass; this.log.debug( { username, ip: this._address(session) }, `NNTP authentication request for "${username}"` ); return new Promise(resolve => { const user = new User(); user.authenticateFactor1( { type: User.AuthFactor1Types.Password, username, password }, err => { if (err) { this.log.warn( { username, reason: err.message, ip: this._address(session) }, `NNTP authentication failure for "${username}"` ); return resolve(false); } session.authUser = user; this.log.info( { username, ip: this._address(session) }, `NTTP authentication success for "${username}"` ); return resolve(true); } ); }); } isGroupSelected(session) { return Array.isArray(_.get(session, 'groupInfo.messageList')); } getJAMStyleFrom(message, fromName) { // // Try to to create a (JamNTTPd) JAM style "From" field: // // - If we're dealing with a FTN address, create an email-like format // but do not include ':' or '/' characters as it may cause clients // to puke. FTN addresses are formatted how JamNTTPd does it for // some sort of compliance. We also extend up to 5D addressing. // - If we have an email address, then it's ready to go. // const remoteFrom = _.get(message.meta, [ 'System', Message.SystemMetaNames.RemoteFromUser, ]); let jamStyleFrom; if (remoteFrom) { const flavor = _.get(message.meta, [ 'System', Message.SystemMetaNames.ExternalFlavor, ]); switch (flavor) { case [Message.AddressFlavor.FTN]: { let ftnAddr = FTNAddress.fromString(remoteFrom); if (ftnAddr && ftnAddr.isValid()) { // In general, addresses are in point, node, net, zone, domain order if (ftnAddr.domain) { // 5D // point.node.net.zone@domain or node.net.zone@domain jamStyleFrom = `${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}@${ftnAddr.domain}`; if (ftnAddr.point) { jamStyleFrom = `${ftnAddr.point}.` + jamStyleFrom; } } else { if (ftnAddr.point) { jamStyleFrom = `${ftnAddr.point}@${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}`; } else { jamStyleFrom = `0@${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}`; } } } } break; case [Message.AddressFlavor.Email]: jamStyleFrom = `${fromName} <${remoteFrom}>`; break; } } if (!jamStyleFrom) { jamStyleFrom = fromName; } return jamStyleFrom; } populateNNTPHeaders(session, message, cb) { // // Build compliant headers // // Resources: // - https://tools.ietf.org/html/rfc5536#section-3.1 // - https://github.com/ftnapps/jamnntpd/blob/master/src/nntpserv.c#L962 // const toName = this.getMessageTo(message); const fromName = this.getMessageFrom(message); message.nntpHeaders = { From: this.getJAMStyleFrom(message, fromName), 'X-Comment-To': toName, To: toName, // JAM-ish Newsgroups: session.group.name, Subject: message.subject, Date: this.getMessageDate(message), 'Message-ID': this.getMessageIdentifier(message), Path: 'ENiGMA1/2!not-for-mail', 'Content-Type': 'text/plain; charset=utf-8', }; const externalFlavor = _.get(message.meta.System, [ Message.SystemMetaNames.ExternalFlavor, ]); if (externalFlavor) { message.nntpHeaders['X-ENiG-MessageFlavor'] = externalFlavor; } // Any FTN properties -> X-FTN-* _.each(message.meta.FtnProperty, (v, k) => { const suffix = { [Message.FtnPropertyNames.FtnTearLine]: 'Tearline', [Message.FtnPropertyNames.FtnOrigin]: 'Origin', [Message.FtnPropertyNames.FtnArea]: 'AREA', [Message.FtnPropertyNames.FtnSeenBy]: 'SEEN-BY', }[k]; if (suffix) { // some special treatment. if ('Tearline' === suffix) { v = v.replace(/^--- /, ''); } else if ('Origin' === suffix) { v = v.replace(/^[ ]{1,2}\* Origin: /, ''); } if (Array.isArray(v)) { // ie: SEEN-BY[] -> one big list v = v.join(' '); } message.nntpHeaders[`X-FTN-${suffix}`] = v.trim(); } }); // Other FTN kludges _.each(message.meta.FtnKludge, (v, k) => { if (Array.isArray(v)) { v = v.join(' '); // same as above } message.nntpHeaders[`X-FTN-${k.toUpperCase()}`] = v.toString().trim(); }); // // Set X-FTN-To and X-FTN-From: // - If remote to/from : joeuser // - Without remote : joeuser // const remoteFrom = _.get(message.meta, [ 'System', Message.SystemMetaNames.RemoteFromUser, ]); message.nntpHeaders['X-FTN-From'] = remoteFrom ? `${fromName} <${remoteFrom}>` : fromName; const remoteTo = _.get(message.meta, [ 'System', Message.SystemMetaNames.RemoteToUser, ]); message.nntpHeaders['X-FTN-To'] = remoteTo ? `${toName} <${remoteTo}>` : toName; if (!message.replyToMsgId) { return cb(null); } // replyToMessageId -> Message-ID formatted ID const filter = { resultType: 'uuid', ids: [parseInt(message.replyToMsgId)], limit: 1, }; Message.findMessages(filter, (err, uuids) => { if (!err && Array.isArray(uuids)) { message.nntpHeaders.References = this.makeMessageIdentifier( message.replyToMsgId, uuids[0] ); } return cb(null); }); } getMessageUUIDFromMessageID(session, messageId) { let messageUuid; // Direct ID request if ( (_.isString(messageId) && '<' !== messageId.charAt(0)) || _.isNumber(messageId) ) { // group must be in session if (!this.isGroupSelected(session)) { return null; } messageId = parseInt(messageId); if (isNaN(messageId)) { return null; } const msg = session.groupInfo.messageList.find(m => { return m.index === messageId; }); messageUuid = msg && msg.messageUuid; } else { // request [, messageUuid] = NNTPServer.getMessageIdentifierParts(messageId); } if (!_.isString(messageUuid)) { return null; } return messageUuid; } _getArticle(session, messageId) { return new Promise(resolve => { this.log.trace({ messageId }, 'Get article'); const messageUuid = this.getMessageUUIDFromMessageID(session, messageId); if (!messageUuid) { this.log.debug( { messageId }, 'Unable to retrieve message UUID for article request' ); return resolve(null); } const message = new Message(); asyncSeries( [ callback => { return message.load({ uuid: messageUuid }, callback); }, callback => { if (!_.has(session, 'groupInfo.areaTag')) { // :TODO: if this is needed, how to validate properly? this.log.warn( { messageUuid, messageId }, 'Get article request without group selection' ); return resolve(null); } if (session.groupInfo.areaTag !== message.areaTag) { return resolve(null); } if ( !this.hasConfAndAreaReadAccess( session, session.groupInfo.confTag, session.groupInfo.areaTag ) ) { this.log.info( { messageUuid, messageId, ip: this._address(session) }, 'Access denied for message' ); return resolve(null); } return callback(null); }, callback => { return this.populateNNTPHeaders(session, message, callback); }, callback => { return this.prepareMessageBody(message, callback); }, ], err => { if (err) { this.log.error( { error: err.message, messageUuid }, 'Failed to load article' ); return resolve(null); } this.log.info( { messageUuid, messageId, areaTag: message.areaTag }, 'Serving article' ); return resolve(message); } ); }); } _getRange(session, first, last /*options*/) { return new Promise(resolve => { // // Build an array of message objects that can later // be used with the various _build* methods. // // :TODO: Handle |options| if (!this.isGroupSelected(session)) { return resolve(null); } const uuids = session.groupInfo.messageList .filter(m => { if (m.areaTag !== session.groupInfo.areaTag) { return false; } if (m.index < first || m.index > last) { return false; } return true; }) .map(m => { return { uuid: m.messageUuid, index: m.index }; }); asyncMap( uuids, (msgInfo, nextMessageUuid) => { const message = new Message(); message.load({ uuid: msgInfo.uuid }, err => { if (err) { return nextMessageUuid(err); } message.index = msgInfo.index; this.populateNNTPHeaders(session, message, () => { this.prepareMessageBody(message, () => { return nextMessageUuid(null, message); }); }); }); }, (err, messages) => { return resolve(err ? null : messages); } ); }); } _selectGroup(session, groupName) { this.log.trace({ groupName }, 'Select group request'); return new Promise(resolve => { this.getGroup(session, groupName, (err, group) => { if (err) { return resolve(false); } session.group = Object.assign( {}, // start clean { description: group.friendlyDesc || group.friendlyName, current_article: group.nntp.total ? group.nntp.min_index : 0, }, group.nntp ); session.groupInfo = group; // full set of info return resolve(true); }); }); } _getGroups(session, time, wildmat) { this.log.trace({ time, wildmat }, 'Get groups request'); // :TODO: handle time - probably use as caching mechanism - must consider user/auth/rights // :TODO: handle |time| if possible. return new Promise((resolve, reject) => { const config = Config(); // :TODO: merge confs avail to authenticated user const publicConfs = _.get( config, 'contentServers.nntp.publicMessageConferences', {} ); asyncReduce( Object.keys(publicConfs), [], (groups, confTag, nextConfTag) => { const areaTags = publicConfs[confTag]; // :TODO: merge area tags available to authenticated user asyncMap( areaTags, (areaTag, nextAreaTag) => { const groupName = this.getGroupName(confTag, areaTag); // filter on |wildmat| if supplied. We will remove // empty areas below in the final results. if (wildmat && !wildmat.test(groupName)) { return nextAreaTag(null, null); } this.getGroup(session, groupName, (err, group) => { if (err) { return nextAreaTag(null, null); // try others } return nextAreaTag(null, group.nntp); }); }, (err, areas) => { if (err) { return nextConfTag(err); } areas = areas.filter(a => a && Object.keys(a).length > 0); // remove empty groups.push(...areas); return nextConfTag(null, groups); } ); }, (err, groups) => { if (err) { return reject(err); } return resolve(groups); } ); }); } isConfAndAreaPubliclyExposed(confTag, areaTag) { const publicAreaTags = _.get(Config(), [ 'contentServers', 'nntp', 'publicMessageConferences', confTag, ]); return Array.isArray(publicAreaTags) && publicAreaTags.includes(areaTag); } hasConfAndAreaReadAccess(session, confTag, areaTag) { if (Message.isPrivateAreaTag(areaTag)) { return false; } if (this.isConfAndAreaPubliclyExposed(confTag, areaTag)) { return true; } // further checks require an authenticated user & ACS if (!session || !session.authUser) { return false; } const conf = getMessageConferenceByTag(confTag); if (!conf) { return false; } const area = getMessageAreaByTag(areaTag, confTag); if (!area) { return false; } const acs = new ACS({ client: null, user: session.authUser }); return acs.hasMessageConfRead(conf) && acs.hasMessageAreaRead(area); } static hasConfAndAreaWriteAccess(session, confTag, areaTag) { if (Message.isPrivateAreaTag(areaTag)) { return false; } const conf = getMessageConferenceByTag(confTag); if (!conf) { return false; } const area = getMessageAreaByTag(areaTag, confTag); if (!area) { return false; } const acs = new ACS({ client: null, user: session.authUser }); return acs.hasMessageConfWrite(conf) && acs.hasMessageAreaWrite(area); } getGroup(session, groupName, cb) { let group = this.groupCache.get(groupName); if (group) { return cb(null, group); } const [confTag, areaTag] = groupName.split('.'); if (!confTag || !areaTag) { return cb(Errors.UnexpectedState(`Invalid NNTP group name: ${groupName}`)); } if (!this.hasConfAndAreaReadAccess(session, confTag, areaTag)) { return cb( Errors.AccessDenied( `No access to conference ${confTag} and/or area ${areaTag}` ) ); } const area = getMessageAreaByTag(areaTag, confTag); if (!area) { return cb( Errors.DoesNotExist( `No area for areaTag "${areaTag}" / confTag "${confTag}"` ) ); } this.getMappedMessageListForArea(areaTag, (err, messageList) => { if (err) { return cb(err); } if (0 === messageList.length) { // // Handle empty group // See https://tools.ietf.org/html/rfc3977#section-6.1.1.2 // return cb(null, { messageList: [], confTag, areaTag, friendlyName: area.name, friendlyDesc: area.desc, nntp: { name: groupName, description: area.desc, min_index: 0, max_index: 0, total: 0, }, }); } group = { messageList, confTag, areaTag, friendlyName: area.name, friendlyDesc: area.desc, nntp: { name: groupName, min_index: messageList[0].index, max_index: messageList[messageList.length - 1].index, total: messageList.length, }, }; this.groupCache.set(groupName, group); return cb(null, group); }); } getMappedMessageListForArea(areaTag, cb) { // // Get all messages in mapped database. Then, find any messages that are not // yet mapped with ID's > the highest ID we have. Any new messages will have // new mappings created. // // :TODO: introduce caching asyncWaterfall( [ callback => { nntpDatabase.db.all( `SELECT nntp_message_id, message_id, message_uuid FROM nntp_area_message WHERE message_area_tag = ? ORDER BY nntp_message_id;`, [areaTag], (err, rows) => { if (err) { return callback(err); } let messageList; const lastMessageId = rows.length > 0 ? rows[rows.length - 1].message_id : 0; if (!lastMessageId) { messageList = []; } else { messageList = rows.map(r => { return { areaTag, index: r.nntp_message_id, // node-nntp wants this name messageUuid: r.message_uuid, }; }); } return callback(null, messageList, lastMessageId); } ); }, (messageList, lastMessageId, callback) => { // Find any new entries const filter = { areaTag, newerThanMessageId: lastMessageId, sort: 'messageId', order: 'ascending', resultType: 'messageList', }; Message.findMessages(filter, (err, newMessageList) => { if (err) { return callback(err); } let index = messageList.length > 0 ? messageList[messageList.length - 1].index + 1 : 1; newMessageList = newMessageList.map(m => { return Object.assign(m, { index: index++ }); }); if (0 === newMessageList.length) { return callback(null, messageList); } // populate mapping DB with any new entries nntpDatabase.db.beginTransaction((err, trans) => { if (err) { return callback(err); } forEachSeries( newMessageList, (newMessage, nextNewMessage) => { trans.run( `INSERT INTO nntp_area_message (nntp_message_id, message_id, message_area_tag, message_uuid) VALUES (?, ?, ?, ?);`, [ newMessage.index, newMessage.messageId, areaTag, newMessage.messageUuid, ], err => { return nextNewMessage(err); } ); }, err => { if (err) { return trans.rollback(() => { return callback(err); }); } trans.commit(() => { messageList.push( ...newMessageList.map(m => { return { areaTag, index: m.nntpMessageId, messageUuid: m.messageUuid, }; }) ); return callback(null, messageList); }); } ); }); }); }, ], (err, messageList) => { return cb(err, messageList); } ); } _buildHead(session, message) { return _.map(message.nntpHeaders, (v, k) => `${k}: ${v}`).join('\r\n'); } _buildBody(session, message) { return message.preparedBody; } _buildHeaderField(session, message, field) { const body = message.preparedBody || message.message; const value = { ':bytes': Buffer.byteLength(body).toString(), ':lines': splitTextAtTerms(body).length.toString(), }[field] || _.find(message.nntpHeaders, (v, k) => { return k.toLowerCase() === field; }); if (!value) { // // Clients will check some headers just to see if they exist. // Don't spam logs with these. For others, it's good to know. // if (!['references', 'xref'].includes(field)) { this.log.trace(`No value for requested header field "${field}"`); } } return value; } _getOverviewFmt(session) { return super._getOverviewFmt(session); } _getNewNews(session, time, wildmat) { // Currently seems pointless to implement. No semi-modern clients seem to use it anyway. this.log.debug( { time, wildmat }, 'Request made using unsupported NEWNEWS command' ); throw new Errors.Invalid('NEWNEWS is not enabled on this server'); } getMessageDate(message) { // https://tools.ietf.org/html/rfc5536#section-3.1.1 -> https://tools.ietf.org/html/rfc5322#section-3.3 return message.modTimestamp.format('ddd, D MMM YYYY HH:mm:ss ZZ'); } makeMessageIdentifier(messageId, messageUuid) { // // Spec : RFC-5536 Section 3.1.3 @ https://tools.ietf.org/html/rfc5536#section-3.1.3 // Example : <2456.0f6587f7-5512-4d03-8740-4d592190145a@enigma-bbs> // return `<${messageId}.${messageUuid}@enigma-bbs>`; } getMessageIdentifier(message) { // note that we use the *real* message ID here, not the NNTP-specific index. return this.makeMessageIdentifier(message.messageId, message.messageUuid); } static getMessageIdentifierParts(messageId) { const m = messageId.match( /<([0-9]+)\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})@enigma-bbs>/ ); if (m) { return [m[1], m[2]]; } return []; } getMessageTo(message) { // :TODO: same as From -- check config return message.toUserName; } getMessageFrom(message) { // :TODO: NNTP config > conf > area config for real names return message.fromUserName; } prepareMessageBody(message, cb) { if (isAnsi(message.message)) { AnsiPrep( message.message, { rows: 'auto', cols: 79, forceLineTerm: true, asciiMode: true, fillLines: false, }, (err, prepped) => { message.preparedBody = prepped || message.message; return cb(null); } ); } else { message.preparedBody = stripMciColorCodes( stripAnsiControlCodes(message.message, { all: true }) ); return cb(null); } } getGroupName(confTag, areaTag) { // // Example: // input : fsxNet (confTag) fsx_bbs (areaTag) // output: fsx_net.fsx_bbs // // Note also that periods are replaced in conf and area // tags such that we *only* have a period separator // between the two for a group name! // return `${_.snakeCase(confTag).replace(/\./g, '_')}.${_.snakeCase( areaTag ).replace(/\./g, '_')}`; } static _importMessage(session, articleLines, cb) { const tidyFrom = f => { if (f) { // remove quotes around name, if present let m = /^"([^"]+)" <([^>]+)>$/.exec(f); if (m && m[1] && m[2]) { f = `${m[1]} <${m[2]}>`; } } return f; }; asyncWaterfall( [ callback => { return NNTPServer._parseArticleLines(articleLines, callback); }, (parsed, callback) => { // gather some initially important bits const subject = parsed.header.get('subject'); const to = parsed.header.get('to') || parsed.header.get('x-jam-to'); // non-standard, may be missing const from = tidyFrom( parsed.header.get('from') || parsed.header.get('sender') || parsed.header.get('x-jam-from') ); const date = parsed.header.get('date'); // if not present we'll use 'now' const newsgroups = parsed.header .get('newsgroups') .split(',') .map(ng => { const [confTag, areaTag] = ng.split('.'); return { confTag, areaTag }; }); // validate areaTag exists -- currently only a single area/post; no x-posts // :TODO: look into x-posting const area = getMessageAreaByTag(newsgroups[0].areaTag); if (!area) { return callback( Errors.DoesNotExist( `No area by tag "${newsgroups[0].areaTag}" exists!` ) ); } // NOTE: Not all ACS checks work with NNTP since we don't have a standard client; // If a particular ACS requires a |client|, it will return false! if ( !NNTPServer.hasConfAndAreaWriteAccess( session, area.confTag, area.areaTag ) ) { return callback( Errors.AccessDenied( `No ACS to ${area.confTag}/${area.areaTag}` ) ); } if ( !_.isString(subject) || !_.isString(from) || !Array.isArray(newsgroups) ) { return callback( Errors.Invalid('Missing one or more required article fields') ); } return callback(null, { subject, from, date, newsgroups, to, parsed, }); }, (msgData, callback) => { if (msgData.to) { return callback(null, msgData); } // // We don't have a 'to' field, try to derive if this is a // response to a message. If not, just fall back 'All' // // 'References' // - https://www.rfc-editor.org/rfc/rfc5536#section-3.2.10 // - https://www.rfc-editor.org/rfc/rfc5322.html // // 'In-Reply-To' // - https://www.rfc-editor.org/rfc/rfc5322.html // // Both may contain 1:N, "optionally" separated by CFWS; by this // point in the code, they should be space separated at most. // // Each entry is in msg-id format. That is: // "<" id-left "@" id-right ">" // msgData.to = 'All'; // fallback let parentMessageId = ( msgData.parsed.header.get('in-reply-to') || msgData.parsed.header.get('references') || '' ).split(' ')[0]; if (parentMessageId) { let [_, messageUuid] = NNTPServer.getMessageIdentifierParts(parentMessageId); if (messageUuid) { const filter = { resultType: 'messageList', uuids: messageUuid, limit: 1, }; return Message.findMessages(filter, (err, messageList) => { if (err) { return callback(err); } // current message/article is a reply to this message: msgData.to = messageList[0].fromUserName; msgData.replyToMsgId = messageList[0].replyToMsgId; // may not be present return callback(null, msgData); }); } } return callback(null, msgData); }, (msgData, callback) => { const message = new Message({ toUserName: msgData.to, fromUserName: msgData.from, subject: msgData.subject, replyToMsgId: msgData.replyToMsgId || 0, modTimestamp: msgData.date, // moment can generally parse these // :TODO: inspect Content-Type 'charset' if present & attempt to properly decode if not UTF-8 message: msgData.parsed.body.join('\n'), areaTag: msgData.newsgroups[0].areaTag, }); message.meta.System[Message.SystemMetaNames.ExternalFlavor] = Message.AddressFlavor.NNTP; // :TODO: investigate JAMNTTP clients/etc. // :TODO: slurp in various X-XXXX kludges/etc. and bring them in persistMessage(message, err => { if (!err) { Log.info( `NNTP post to "${message.areaTag}" by "${session.authUser.username}": "${message.subject}"` ); } return callback(err); }); }, ], err => { return cb(err); } ); } static _parseArticleLines(articleLines, cb) { // // Split articleLines into: // - Header split into N:V pairs // - Message Body lines // - const header = new Map(); const body = []; let inHeader = true; let currentHeaderName; forEachSeries( articleLines, (line, nextLine) => { if (inHeader) { if (line === '.' || line === '') { inHeader = false; return nextLine(null); } const sep = line.indexOf(':'); if (sep < 1) { // at least a single char name // entries can split across lines -- they will be prefixed with a single space. if ( currentHeaderName && (line.startsWith(' ') || line.startsWith('\t')) ) { let v = header.get(currentHeaderName); v += line .replace(/^\t/, ' ') // if we're dealign with a legacy tab .trimRight(); header.set(currentHeaderName, v); return nextLine(null); } return nextLine( Errors.Invalid( `"${line}" is not a valid NNTP message header!` ) ); } currentHeaderName = line.slice(0, sep).trim().toLowerCase(); const value = line.slice(sep + 1).trim(); header.set(currentHeaderName, value); return nextLine(null); } // body if (line !== '.') { // lines consisting of a single '.' are escaped to '..' if (line.startsWith('..')) { body.push(line.slice(1)); } else { body.push(line); } } return nextLine(null); }, err => { return cb(err, { header, body }); } ); } } exports.getModule = class NNTPServerModule extends ServerModule { constructor() { super(); } isEnabled() { return this.enableNntp || this.enableNttps; } get enableNntp() { return _.get(Config(), 'contentServers.nntp.nntp.enabled', false); } get enableNttps() { return _.get(Config(), 'contentServers.nntp.nntps.enabled', false); } isConfigured() { const config = Config(); // // Any conf/areas exposed? // const publicConfs = _.get( config, 'contentServers.nntp.publicMessageConferences', {} ); const areasExposed = _.some(publicConfs, areas => { return Array.isArray(areas) && areas.length > 0; }); if (!areasExposed) { return false; } const nntp = _.get(config, 'contentServers.nntp.nntp'); if (nntp && this.enableNntp) { if (isNaN(nntp.port)) { return false; } } const nntps = _.get(config, 'contentServers.nntp.nntps'); if (nntps && this.enableNttps) { if (isNaN(nntps.port)) { return false; } if (!_.isString(nntps.certPem) || !_.isString(nntps.keyPem)) { return false; } } return true; } createServer(cb) { if (!this.isEnabled() || !this.isConfigured()) { return cb(null); } const config = Config(); // :TODO: nntp-server doesn't currently allow posting in a nice way, so this is kludged in. Fork+MR something cleaner at some point class ProxySession extends NNTPServerBase.Session { constructor(server, stream) { super(server, stream); this.articleLinesBuffer = []; } parse(data) { if (this.receivingPostArticle) { return this.receivePostArticleData(data); } super.parse(data); } receivePostArticleData(data) { this.articleLinesBuffer.push(...data.split(/r?\n/)); const endOfPost = data.length === 1 && data[0] === '.'; if (endOfPost) { this.receivingPostArticle = false; // Command is not exported currently; maybe submit a MR to allow posting in a nicer way... function Command(runner, articleLines, session) { this.state = 0; // CMD_WAIT this.cmd_line = 'POST'; this.resolved_value = null; this.rejected_value = null; this.run = runner; this.articleLines = articleLines; this.session = session; } this.pipeline.push( new Command( this._processarticleLinesBuffer, this.articleLinesBuffer, this ) ); this.articleLinesBuffer = []; this.tick(); } } _processarticleLinesBuffer() { return new Promise(resolve => { NNTPServer._importMessage(this.session, this.articleLines, err => { if (err) { this.rejected_value = err; // will be serialized and 403 sent back currently; not really ideal as we want ArticlePostFailed // :TODO: tick() needs updated in session.js such that we can write back a proper code this.state = 3; // CMD_REJECTED Log.error( { error: err.message }, `NNTP post failed: ${err.message}` ); } else { this.resolved_value = Responses.ArticlePostedOk; this.state = 2; // CMD_RESOLVED } return resolve(); }); }); } static create(server, stream) { return new ProxySession(server, stream); } } const commonOptions = { // :TODO: How to hook into debugging?! }; if (true === _.get(config, 'contentServers.nntp.allowPosts')) { // add in some additional supported commands const commands = Object.assign({}, NNTPServerBase.commands, { POST: PostCommand, }); commonOptions.commands = commands; commonOptions.session = ProxySession; } if (this.enableNntp) { this.nntpServer = new NNTPServer( // :TODO: according to docs: if connection is non-tls, but behind proxy (assuming TLS termination?!!) then set this to true Object.assign({ secure: false }, commonOptions), 'NNTP' ); } if (this.enableNttps) { this.nntpsServer = new NNTPServer( Object.assign( { secure: true, tls: { cert: fs.readFileSync( config.contentServers.nntp.nntps.certPem ), key: fs.readFileSync(config.contentServers.nntp.nntps.keyPem), }, }, commonOptions ), 'NTTPS' ); } nntpDatabase = new NNTPDatabase(); nntpDatabase.init(err => { return cb(err); }); } listen(cb) { const config = Config(); forEachSeries( ['nntp', 'nntps'], (service, nextService) => { const server = this[`${service}Server`]; if (server) { const port = config.contentServers.nntp[service].port; server .listen(this.listenURI(port, service)) .catch(e => { Log.warn( { error: e.message, port }, `${service.toUpperCase()} failed to listen` ); return nextService(null); // try next anyway }) .then(() => { return nextService(null); }); } else { return nextService(null); } }, err => { return cb(err); } ); } listenURI(port, service = 'nntp') { return `${service}://0.0.0.0:${port}`; } }; function performMaintenanceTask(args, cb) { // // Delete any message mapping that no longer have // an actual message associated with them. // if (!nntpDatabase) { Log.trace('Cannot perform NNTP maintenance without NNTP database initialized'); return cb(null); } let attached = false; asyncSeries( [ callback => { const messageDbPath = paths.join(Config().paths.db, 'message.sqlite3'); nntpDatabase.db.run( `ATTACH DATABASE "${messageDbPath}" AS msgdb;`, err => { attached = !err; return callback(err); } ); }, callback => { nntpDatabase.db.run( `DELETE FROM nntp_area_message WHERE message_uuid NOT IN ( SELECT message_uuid FROM msgdb.message );`, function result(err) { // no arrow func; need |this.changes| if (err) { Log.warn( { error: err.message }, 'Failed to delete from NNTP database' ); } else { Log.debug( { count: this.changes }, 'Deleted mapped message IDs from NNTP database' ); } return callback(err); } ); }, ], err => { if (attached) { nntpDatabase.db.run('DETACH DATABASE msgdb;'); } return cb(err); } ); }