enigma-bbs/core/servers/content/nntp.js

1498 lines
52 KiB
JavaScript
Raw Normal View History

/* jslint node: true */
'use strict';
// ENiGMA½
const Log = require('../../logger.js').log;
const { ServerModule } = require('../../server_module.js');
const Config = require('../../config.js').get;
const { getTransactionDatabase, getModDatabasePath } = require('../../database.js');
const {
getMessageAreaByTag,
getMessageConferenceByTag,
2022-09-23 03:03:53 +00:00
persistMessage,
} = require('../../message_area.js');
const User = require('../../user.js');
const Errors = require('../../enig_error.js').Errors;
const Message = require('../../message.js');
const FTNAddress = require('../../ftn_address.js');
const {
isAnsi,
stripAnsiControlCodes,
splitTextAtTerms,
} = require('../../string_util.js');
const AnsiPrep = require('../../ansi_prep.js');
const { stripMciColorCodes } = require('../../color_codes.js');
2022-09-23 03:03:53 +00:00
const ACS = require('../../acs');
// deps
const NNTPServerBase = require('nntp-server');
const _ = require('lodash');
const fs = require('fs-extra');
const forEachSeries = require('async/forEachSeries');
const asyncReduce = require('async/reduce');
const asyncMap = require('async/map');
const asyncSeries = require('async/series');
const asyncWaterfall = require('async/waterfall');
const LRU = require('lru-cache');
const sqlite3 = require('sqlite3');
const paths = require('path');
//
// Network News Transfer Protocol (NNTP)
//
// RFCS
// - https://www.w3.org/Protocols/rfc977/rfc977
// - https://tools.ietf.org/html/rfc3977
// - https://tools.ietf.org/html/rfc2980
// - https://tools.ietf.org/html/rfc5536
//
exports.moduleInfo = {
name: 'NNTP',
desc: 'Network News Transfer Protocol (NNTP) Server',
author: 'NuSkooler',
packageName: 'codes.l33t.enigma.nntp.server',
};
exports.performMaintenanceTask = performMaintenanceTask;
/*
General TODO
- ACS checks need worked out. Currently ACS relies on |client|. We need a client
spec that can be created even without a login server. Some checks and simply
return false/fail.
*/
// simple DB maps NNTP Message-ID's which are
// sequential per group -> ENiG messages
// A single instance is shared across NNTP and/or NNTPS
class NNTPDatabase {
constructor() {}
init(cb) {
asyncSeries(
[
callback => {
this.db = getTransactionDatabase(
new sqlite3.Database(
getModDatabasePath(exports.moduleInfo),
err => {
return callback(err);
}
)
);
},
callback => {
this.db.serialize(() => {
this.db.run(
`CREATE TABLE IF NOT EXISTS nntp_area_message (
nntp_message_id INTEGER NOT NULL,
message_id INTEGER NOT NULL,
message_area_tag VARCHAR NOT NULL,
message_uuid VARCHAR NOT NULL,
UNIQUE(nntp_message_id, message_area_tag)
);`
);
this.db.run(
`CREATE INDEX IF NOT EXISTS nntp_area_message_by_uuid_index
ON nntp_area_message (message_uuid);`
);
return callback(null);
});
},
],
err => {
return cb(err);
}
);
}
}
let nntpDatabase;
2022-09-23 03:03:53 +00:00
const AuthCommands = 'POST';
// these aren't exported by the NNTP module, unfortunantely
const Responses = {
ArticlePostedOk: '240 article posted ok',
SendArticle: '340 send article to be posted',
2022-09-23 03:24:24 +00:00
PostingNotAllowed: '440 posting not allowed',
2022-09-23 03:03:53 +00:00
ArticlePostFailed: '441 posting failed',
AuthRequired: '480 authentication required',
};
const PostCommand = {
head: 'POST',
validate: /^POST$/i,
run(session, cmd) {
if (!session.authenticated) {
session.receivingPostArticle = false; // ensure reset
return Responses.AuthRequired;
}
session.receivingPostArticle = true;
return Responses.SendArticle;
},
capability(session, report) {
report.push('POST');
},
};
class NNTPServer extends NNTPServerBase {
constructor(options, serverName) {
super(options);
this.log = Log.child({ server: serverName });
const config = Config();
this.groupCache = new LRU({
max: _.get(config, 'contentServers.nntp.cache.maxItems', 200),
ttl: _.get(config, 'contentServers.nntp.cache.maxAge', 1000 * 30), // default=30s
});
}
_needAuth(session, command) {
2022-09-23 03:03:53 +00:00
if (AuthCommands.includes(command)) {
return !session.authenticated && !session.authUser;
}
return super._needAuth(session, command);
}
2022-09-26 00:29:00 +00:00
_address(session) {
const addr = session.in_stream.remoteAddress;
return addr ? addr.replace(/^::ffff:/, '').replace(/^::1$/, 'localhost') : 'N/A';
}
_authenticate(session) {
const username = session.authinfo_user;
const password = session.authinfo_pass;
2022-09-26 00:29:00 +00:00
this.log.debug(
{ username, ip: this._address(session) },
`NNTP authentication request for "${username}"`
);
return new Promise(resolve => {
const user = new User();
user.authenticateFactor1(
{ type: User.AuthFactor1Types.Password, username, password },
err => {
if (err) {
2022-09-23 03:03:53 +00:00
this.log.warn(
2022-09-26 00:29:00 +00:00
{ username, reason: err.message, ip: this._address(session) },
2022-09-23 03:03:53 +00:00
`NNTP authentication failure for "${username}"`
);
return resolve(false);
}
session.authUser = user;
2022-09-23 03:03:53 +00:00
this.log.info(
2022-09-26 00:29:00 +00:00
{ username, ip: this._address(session) },
2022-09-23 03:03:53 +00:00
`NTTP authentication success for "${username}"`
);
return resolve(true);
}
);
});
}
isGroupSelected(session) {
return Array.isArray(_.get(session, 'groupInfo.messageList'));
}
getJAMStyleFrom(message, fromName) {
//
// Try to to create a (JamNTTPd) JAM style "From" field:
//
// - If we're dealing with a FTN address, create an email-like format
// but do not include ':' or '/' characters as it may cause clients
// to puke. FTN addresses are formatted how JamNTTPd does it for
// some sort of compliance. We also extend up to 5D addressing.
// - If we have an email address, then it's ready to go.
//
const remoteFrom = _.get(message.meta, [
'System',
Message.SystemMetaNames.RemoteFromUser,
]);
let jamStyleFrom;
if (remoteFrom) {
const flavor = _.get(message.meta, [
'System',
Message.SystemMetaNames.ExternalFlavor,
]);
switch (flavor) {
case [Message.AddressFlavor.FTN]:
{
let ftnAddr = FTNAddress.fromString(remoteFrom);
if (ftnAddr && ftnAddr.isValid()) {
// In general, addresses are in point, node, net, zone, domain order
if (ftnAddr.domain) {
// 5D
// point.node.net.zone@domain or node.net.zone@domain
jamStyleFrom = `${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}@${ftnAddr.domain}`;
if (ftnAddr.point) {
jamStyleFrom = `${ftnAddr.point}.` + jamStyleFrom;
}
} else {
if (ftnAddr.point) {
jamStyleFrom = `${ftnAddr.point}@${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}`;
} else {
jamStyleFrom = `0@${ftnAddr.node}.${ftnAddr.net}.${ftnAddr.zone}`;
}
}
}
}
break;
case [Message.AddressFlavor.Email]:
jamStyleFrom = `${fromName} <${remoteFrom}>`;
break;
}
}
if (!jamStyleFrom) {
jamStyleFrom = fromName;
}
return jamStyleFrom;
}
populateNNTPHeaders(session, message, cb) {
//
// Build compliant headers
//
// Resources:
// - https://tools.ietf.org/html/rfc5536#section-3.1
// - https://github.com/ftnapps/jamnntpd/blob/master/src/nntpserv.c#L962
//
const toName = this.getMessageTo(message);
const fromName = this.getMessageFrom(message);
message.nntpHeaders = {
From: this.getJAMStyleFrom(message, fromName),
'X-Comment-To': toName,
2022-09-25 20:00:52 +00:00
To: toName, // JAM-ish
Newsgroups: session.group.name,
Subject: message.subject,
Date: this.getMessageDate(message),
'Message-ID': this.getMessageIdentifier(message),
Path: 'ENiGMA1/2!not-for-mail',
'Content-Type': 'text/plain; charset=utf-8',
};
const externalFlavor = _.get(message.meta.System, [
Message.SystemMetaNames.ExternalFlavor,
]);
if (externalFlavor) {
message.nntpHeaders['X-ENiG-MessageFlavor'] = externalFlavor;
}
// Any FTN properties -> X-FTN-*
_.each(message.meta.FtnProperty, (v, k) => {
const suffix = {
[Message.FtnPropertyNames.FtnTearLine]: 'Tearline',
[Message.FtnPropertyNames.FtnOrigin]: 'Origin',
[Message.FtnPropertyNames.FtnArea]: 'AREA',
[Message.FtnPropertyNames.FtnSeenBy]: 'SEEN-BY',
}[k];
if (suffix) {
// some special treatment.
if ('Tearline' === suffix) {
v = v.replace(/^--- /, '');
} else if ('Origin' === suffix) {
v = v.replace(/^[ ]{1,2}\* Origin: /, '');
}
if (Array.isArray(v)) {
// ie: SEEN-BY[] -> one big list
v = v.join(' ');
}
message.nntpHeaders[`X-FTN-${suffix}`] = v.trim();
}
});
// Other FTN kludges
_.each(message.meta.FtnKludge, (v, k) => {
if (Array.isArray(v)) {
v = v.join(' '); // same as above
}
message.nntpHeaders[`X-FTN-${k.toUpperCase()}`] = v.toString().trim();
});
//
// Set X-FTN-To and X-FTN-From:
// - If remote to/from : joeuser <remoteAddr>
// - Without remote : joeuser
//
const remoteFrom = _.get(message.meta, [
'System',
Message.SystemMetaNames.RemoteFromUser,
]);
message.nntpHeaders['X-FTN-From'] = remoteFrom
? `${fromName} <${remoteFrom}>`
: fromName;
const remoteTo = _.get(message.meta, [
'System',
Message.SystemMetaNames.RemoteToUser,
]);
message.nntpHeaders['X-FTN-To'] = remoteTo ? `${toName} <${remoteTo}>` : toName;
if (!message.replyToMsgId) {
return cb(null);
}
// replyToMessageId -> Message-ID formatted ID
const filter = {
resultType: 'uuid',
ids: [parseInt(message.replyToMsgId)],
limit: 1,
};
Message.findMessages(filter, (err, uuids) => {
if (!err && Array.isArray(uuids)) {
message.nntpHeaders.References = this.makeMessageIdentifier(
message.replyToMsgId,
uuids[0]
);
}
return cb(null);
});
}
getMessageUUIDFromMessageID(session, messageId) {
let messageUuid;
// Direct ID request
if (
(_.isString(messageId) && '<' !== messageId.charAt(0)) ||
_.isNumber(messageId)
) {
// group must be in session
if (!this.isGroupSelected(session)) {
return null;
}
messageId = parseInt(messageId);
if (isNaN(messageId)) {
return null;
}
const msg = session.groupInfo.messageList.find(m => {
return m.index === messageId;
});
messageUuid = msg && msg.messageUuid;
} else {
// <Message-ID> request
2022-09-25 20:00:52 +00:00
[, messageUuid] = NNTPServer.getMessageIdentifierParts(messageId);
}
if (!_.isString(messageUuid)) {
return null;
}
return messageUuid;
}
_getArticle(session, messageId) {
return new Promise(resolve => {
this.log.trace({ messageId }, 'Get article');
const messageUuid = this.getMessageUUIDFromMessageID(session, messageId);
if (!messageUuid) {
this.log.debug(
{ messageId },
'Unable to retrieve message UUID for article request'
);
return resolve(null);
}
const message = new Message();
asyncSeries(
[
callback => {
return message.load({ uuid: messageUuid }, callback);
},
callback => {
if (!_.has(session, 'groupInfo.areaTag')) {
2018-12-15 09:40:36 +00:00
// :TODO: if this is needed, how to validate properly?
this.log.warn(
{ messageUuid, messageId },
'Get article request without group selection'
);
2018-12-15 09:40:36 +00:00
return resolve(null);
}
if (session.groupInfo.areaTag !== message.areaTag) {
return resolve(null);
}
if (
!this.hasConfAndAreaReadAccess(
session,
session.groupInfo.confTag,
session.groupInfo.areaTag
)
) {
this.log.info(
2022-09-26 00:29:00 +00:00
{ messageUuid, messageId, ip: this._address(session) },
'Access denied for message'
);
return resolve(null);
}
return callback(null);
},
callback => {
return this.populateNNTPHeaders(session, message, callback);
},
callback => {
return this.prepareMessageBody(message, callback);
},
],
err => {
if (err) {
this.log.error(
{ error: err.message, messageUuid },
'Failed to load article'
);
return resolve(null);
}
2018-12-15 06:08:53 +00:00
this.log.info(
{ messageUuid, messageId, areaTag: message.areaTag },
'Serving article'
);
return resolve(message);
}
);
});
}
_getRange(session, first, last /*options*/) {
return new Promise(resolve => {
//
// Build an array of message objects that can later
// be used with the various _build* methods.
//
// :TODO: Handle |options|
if (!this.isGroupSelected(session)) {
return resolve(null);
}
const uuids = session.groupInfo.messageList
.filter(m => {
if (m.areaTag !== session.groupInfo.areaTag) {
return false;
}
if (m.index < first || m.index > last) {
return false;
}
return true;
})
.map(m => {
return { uuid: m.messageUuid, index: m.index };
});
asyncMap(
uuids,
(msgInfo, nextMessageUuid) => {
const message = new Message();
message.load({ uuid: msgInfo.uuid }, err => {
if (err) {
return nextMessageUuid(err);
}
message.index = msgInfo.index;
this.populateNNTPHeaders(session, message, () => {
this.prepareMessageBody(message, () => {
return nextMessageUuid(null, message);
});
});
});
},
(err, messages) => {
return resolve(err ? null : messages);
}
);
});
}
_selectGroup(session, groupName) {
this.log.trace({ groupName }, 'Select group request');
return new Promise(resolve => {
this.getGroup(session, groupName, (err, group) => {
if (err) {
return resolve(false);
}
session.group = Object.assign(
{}, // start clean
{
description: group.friendlyDesc || group.friendlyName,
current_article: group.nntp.total ? group.nntp.min_index : 0,
},
group.nntp
);
session.groupInfo = group; // full set of info
return resolve(true);
});
});
}
_getGroups(session, time, wildmat) {
this.log.trace({ time, wildmat }, 'Get groups request');
// :TODO: handle time - probably use as caching mechanism - must consider user/auth/rights
// :TODO: handle |time| if possible.
return new Promise((resolve, reject) => {
const config = Config();
// :TODO: merge confs avail to authenticated user
const publicConfs = _.get(
config,
'contentServers.nntp.publicMessageConferences',
{}
);
asyncReduce(
Object.keys(publicConfs),
[],
(groups, confTag, nextConfTag) => {
const areaTags = publicConfs[confTag];
// :TODO: merge area tags available to authenticated user
asyncMap(
areaTags,
(areaTag, nextAreaTag) => {
const groupName = this.getGroupName(confTag, areaTag);
// filter on |wildmat| if supplied. We will remove
// empty areas below in the final results.
if (wildmat && !wildmat.test(groupName)) {
return nextAreaTag(null, null);
}
this.getGroup(session, groupName, (err, group) => {
if (err) {
return nextAreaTag(null, null); // try others
}
return nextAreaTag(null, group.nntp);
});
},
(err, areas) => {
if (err) {
return nextConfTag(err);
}
areas = areas.filter(a => a && Object.keys(a).length > 0); // remove empty
groups.push(...areas);
return nextConfTag(null, groups);
}
);
},
(err, groups) => {
if (err) {
return reject(err);
}
return resolve(groups);
}
);
});
}
isConfAndAreaPubliclyExposed(confTag, areaTag) {
const publicAreaTags = _.get(Config(), [
'contentServers',
'nntp',
'publicMessageConferences',
confTag,
]);
return Array.isArray(publicAreaTags) && publicAreaTags.includes(areaTag);
}
hasConfAndAreaReadAccess(session, confTag, areaTag) {
if (Message.isPrivateAreaTag(areaTag)) {
return false;
}
if (this.isConfAndAreaPubliclyExposed(confTag, areaTag)) {
return true;
}
// further checks require an authenticated user & ACS
if (!session || !session.authUser) {
return false;
}
const conf = getMessageConferenceByTag(confTag);
if (!conf) {
return false;
}
const area = getMessageAreaByTag(areaTag, confTag);
if (!area) {
return false;
}
2022-09-23 03:03:53 +00:00
const acs = new ACS({ client: null, user: session.authUser });
return acs.hasMessageConfRead(conf) && acs.hasMessageAreaRead(area);
}
static hasConfAndAreaWriteAccess(session, confTag, areaTag) {
if (Message.isPrivateAreaTag(areaTag)) {
return false;
}
const conf = getMessageConferenceByTag(confTag);
if (!conf) {
return false;
}
const area = getMessageAreaByTag(areaTag, confTag);
if (!area) {
return false;
}
const acs = new ACS({ client: null, user: session.authUser });
return acs.hasMessageConfWrite(conf) && acs.hasMessageAreaWrite(area);
}
getGroup(session, groupName, cb) {
let group = this.groupCache.get(groupName);
if (group) {
return cb(null, group);
}
const [confTag, areaTag] = groupName.split('.');
if (!confTag || !areaTag) {
return cb(Errors.UnexpectedState(`Invalid NNTP group name: ${groupName}`));
}
if (!this.hasConfAndAreaReadAccess(session, confTag, areaTag)) {
return cb(
Errors.AccessDenied(
`No access to conference ${confTag} and/or area ${areaTag}`
)
);
}
const area = getMessageAreaByTag(areaTag, confTag);
if (!area) {
return cb(
Errors.DoesNotExist(
`No area for areaTag "${areaTag}" / confTag "${confTag}"`
)
);
}
this.getMappedMessageListForArea(areaTag, (err, messageList) => {
if (err) {
return cb(err);
}
if (0 === messageList.length) {
//
// Handle empty group
// See https://tools.ietf.org/html/rfc3977#section-6.1.1.2
//
return cb(null, {
messageList: [],
confTag,
areaTag,
friendlyName: area.name,
friendlyDesc: area.desc,
nntp: {
name: groupName,
description: area.desc,
min_index: 0,
max_index: 0,
total: 0,
},
});
}
group = {
messageList,
confTag,
areaTag,
friendlyName: area.name,
friendlyDesc: area.desc,
nntp: {
name: groupName,
min_index: messageList[0].index,
max_index: messageList[messageList.length - 1].index,
total: messageList.length,
},
};
this.groupCache.set(groupName, group);
return cb(null, group);
});
}
getMappedMessageListForArea(areaTag, cb) {
//
// Get all messages in mapped database. Then, find any messages that are not
// yet mapped with ID's > the highest ID we have. Any new messages will have
// new mappings created.
//
// :TODO: introduce caching
asyncWaterfall(
[
callback => {
nntpDatabase.db.all(
`SELECT nntp_message_id, message_id, message_uuid
FROM nntp_area_message
WHERE message_area_tag = ?
ORDER BY nntp_message_id;`,
[areaTag],
(err, rows) => {
if (err) {
return callback(err);
}
let messageList;
const lastMessageId =
rows.length > 0 ? rows[rows.length - 1].message_id : 0;
if (!lastMessageId) {
messageList = [];
} else {
messageList = rows.map(r => {
return {
areaTag,
index: r.nntp_message_id, // node-nntp wants this name
messageUuid: r.message_uuid,
};
});
}
return callback(null, messageList, lastMessageId);
}
);
},
(messageList, lastMessageId, callback) => {
// Find any new entries
const filter = {
areaTag,
newerThanMessageId: lastMessageId,
sort: 'messageId',
order: 'ascending',
resultType: 'messageList',
};
Message.findMessages(filter, (err, newMessageList) => {
if (err) {
return callback(err);
}
let index =
messageList.length > 0
? messageList[messageList.length - 1].index + 1
: 1;
newMessageList = newMessageList.map(m => {
return Object.assign(m, { index: index++ });
});
if (0 === newMessageList.length) {
return callback(null, messageList);
}
// populate mapping DB with any new entries
nntpDatabase.db.beginTransaction((err, trans) => {
if (err) {
return callback(err);
}
forEachSeries(
newMessageList,
(newMessage, nextNewMessage) => {
trans.run(
`INSERT INTO nntp_area_message (nntp_message_id, message_id, message_area_tag, message_uuid)
VALUES (?, ?, ?, ?);`,
[
newMessage.index,
newMessage.messageId,
areaTag,
newMessage.messageUuid,
],
err => {
return nextNewMessage(err);
}
);
},
err => {
if (err) {
return trans.rollback(() => {
return callback(err);
});
}
trans.commit(() => {
messageList.push(
...newMessageList.map(m => {
return {
areaTag,
index: m.nntpMessageId,
messageUuid: m.messageUuid,
};
})
);
return callback(null, messageList);
});
}
);
});
});
},
],
(err, messageList) => {
return cb(err, messageList);
}
);
}
_buildHead(session, message) {
return _.map(message.nntpHeaders, (v, k) => `${k}: ${v}`).join('\r\n');
}
_buildBody(session, message) {
return message.preparedBody;
}
_buildHeaderField(session, message, field) {
const body = message.preparedBody || message.message;
const value =
{
':bytes': Buffer.byteLength(body).toString(),
':lines': splitTextAtTerms(body).length.toString(),
}[field] ||
_.find(message.nntpHeaders, (v, k) => {
2018-12-15 09:40:36 +00:00
return k.toLowerCase() === field;
});
if (!value) {
2018-12-17 18:20:14 +00:00
//
// Clients will check some headers just to see if they exist.
// Don't spam logs with these. For others, it's good to know.
//
if (!['references', 'xref'].includes(field)) {
2018-12-17 18:20:14 +00:00
this.log.trace(`No value for requested header field "${field}"`);
}
}
return value;
}
_getOverviewFmt(session) {
return super._getOverviewFmt(session);
}
2018-12-17 18:20:14 +00:00
_getNewNews(session, time, wildmat) {
2018-12-16 06:52:59 +00:00
// Currently seems pointless to implement. No semi-modern clients seem to use it anyway.
this.log.debug(
{ time, wildmat },
'Request made using unsupported NEWNEWS command'
);
2018-12-16 06:52:59 +00:00
throw new Errors.Invalid('NEWNEWS is not enabled on this server');
}
getMessageDate(message) {
// https://tools.ietf.org/html/rfc5536#section-3.1.1 -> https://tools.ietf.org/html/rfc5322#section-3.3
return message.modTimestamp.format('ddd, D MMM YYYY HH:mm:ss ZZ');
}
makeMessageIdentifier(messageId, messageUuid) {
//
// Spec : RFC-5536 Section 3.1.3 @ https://tools.ietf.org/html/rfc5536#section-3.1.3
// Example : <2456.0f6587f7-5512-4d03-8740-4d592190145a@enigma-bbs>
//
return `<${messageId}.${messageUuid}@enigma-bbs>`;
}
getMessageIdentifier(message) {
// note that we use the *real* message ID here, not the NNTP-specific index.
return this.makeMessageIdentifier(message.messageId, message.messageUuid);
}
2022-09-25 20:00:52 +00:00
static getMessageIdentifierParts(messageId) {
const m = messageId.match(
/<([0-9]+)\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})@enigma-bbs>/
);
if (m) {
return [m[1], m[2]];
}
return [];
}
getMessageTo(message) {
// :TODO: same as From -- check config
return message.toUserName;
}
getMessageFrom(message) {
// :TODO: NNTP config > conf > area config for real names
return message.fromUserName;
}
prepareMessageBody(message, cb) {
if (isAnsi(message.message)) {
AnsiPrep(
message.message,
{
rows: 'auto',
cols: 79,
forceLineTerm: true,
asciiMode: true,
fillLines: false,
},
(err, prepped) => {
message.preparedBody = prepped || message.message;
return cb(null);
}
);
} else {
message.preparedBody = stripMciColorCodes(
stripAnsiControlCodes(message.message, { all: true })
);
return cb(null);
}
}
getGroupName(confTag, areaTag) {
//
// Example:
// input : fsxNet (confTag) fsx_bbs (areaTag)
// output: fsx_net.fsx_bbs
//
// Note also that periods are replaced in conf and area
// tags such that we *only* have a period separator
// between the two for a group name!
//
return `${_.snakeCase(confTag).replace(/\./g, '_')}.${_.snakeCase(
areaTag
).replace(/\./g, '_')}`;
}
2022-09-23 03:03:53 +00:00
static _importMessage(session, articleLines, cb) {
const tidyFrom = f => {
if (f) {
// remove quotes around name, if present
let m = /^"([^"]+)" <([^>]+)>$/.exec(f);
if (m && m[1] && m[2]) {
f = `${m[1]} <${m[2]}>`;
}
}
return f;
};
2022-09-23 03:03:53 +00:00
asyncWaterfall(
[
callback => {
return NNTPServer._parseArticleLines(articleLines, callback);
},
(parsed, callback) => {
// gather some initially important bits
const subject = parsed.header.get('subject');
2022-09-25 20:00:52 +00:00
const to = parsed.header.get('to') || parsed.header.get('x-jam-to'); // non-standard, may be missing
const from = tidyFrom(
2022-09-25 20:00:52 +00:00
parsed.header.get('from') ||
parsed.header.get('sender') ||
parsed.header.get('x-jam-from')
);
const date = parsed.header.get('date'); // if not present we'll use 'now'
2022-09-23 03:03:53 +00:00
const newsgroups = parsed.header
.get('newsgroups')
.split(',')
.map(ng => {
const [confTag, areaTag] = ng.split('.');
return { confTag, areaTag };
});
// validate areaTag exists -- currently only a single area/post; no x-posts
// :TODO: look into x-posting
const area = getMessageAreaByTag(newsgroups[0].areaTag);
if (!area) {
return callback(
Errors.DoesNotExist(
`No area by tag "${newsgroups[0].areaTag}" exists!`
)
);
}
// NOTE: Not all ACS checks work with NNTP since we don't have a standard client;
// If a particular ACS requires a |client|, it will return false!
if (
!NNTPServer.hasConfAndAreaWriteAccess(
session,
area.confTag,
area.areaTag
)
) {
return callback(
Errors.AccessDenied(
`No ACS to ${area.confTag}/${area.areaTag}`
)
);
}
if (
!_.isString(subject) ||
!_.isString(from) ||
!Array.isArray(newsgroups)
) {
return callback(
Errors.Invalid('Missing one or more required article fields')
);
}
return callback(null, {
subject,
from,
date,
newsgroups,
to,
parsed,
});
},
(msgData, callback) => {
if (msgData.to) {
return callback(null, msgData);
}
//
// We don't have a 'to' field, try to derive if this is a
// response to a message. If not, just fall back 'All'
//
// 'References'
// - https://www.rfc-editor.org/rfc/rfc5536#section-3.2.10
// - https://www.rfc-editor.org/rfc/rfc5322.html
//
// 'In-Reply-To'
// - https://www.rfc-editor.org/rfc/rfc5322.html
//
// Both may contain 1:N, "optionally" separated by CFWS; by this
// point in the code, they should be space separated at most.
//
// Each entry is in msg-id format. That is:
// "<" id-left "@" id-right ">"
//
msgData.to = 'All'; // fallback
let parentMessageId = (
msgData.parsed.header.get('in-reply-to') ||
msgData.parsed.header.get('references') ||
''
2022-09-25 20:00:52 +00:00
).split(' ')[0];
2022-09-23 03:03:53 +00:00
if (parentMessageId) {
2022-09-25 20:00:52 +00:00
let [_, messageUuid] =
NNTPServer.getMessageIdentifierParts(parentMessageId);
if (messageUuid) {
2022-09-23 03:03:53 +00:00
const filter = {
resultType: 'messageList',
2022-09-25 20:00:52 +00:00
uuids: messageUuid,
2022-09-23 03:03:53 +00:00
limit: 1,
};
return Message.findMessages(filter, (err, messageList) => {
if (err) {
return callback(err);
}
// current message/article is a reply to this message:
msgData.to = messageList[0].fromUserName;
msgData.replyToMsgId = messageList[0].replyToMsgId; // may not be present
return callback(null, msgData);
});
}
}
return callback(null, msgData);
},
(msgData, callback) => {
const message = new Message({
toUserName: msgData.to,
fromUserName: msgData.from,
subject: msgData.subject,
replyToMsgId: msgData.replyToMsgId || 0,
modTimestamp: msgData.date, // moment can generally parse these
// :TODO: inspect Content-Type 'charset' if present & attempt to properly decode if not UTF-8
message: msgData.parsed.body.join('\n'),
2022-09-23 03:03:53 +00:00
areaTag: msgData.newsgroups[0].areaTag,
});
message.meta.System[Message.SystemMetaNames.ExternalFlavor] =
Message.AddressFlavor.NNTP;
// :TODO: investigate JAMNTTP clients/etc.
2022-09-25 20:00:52 +00:00
// :TODO: slurp in various X-XXXX kludges/etc. and bring them in
2022-09-23 03:03:53 +00:00
persistMessage(message, err => {
if (!err) {
Log.info(
`NNTP post to "${message.areaTag}" by "${session.authUser.username}": "${message.subject}"`
);
}
2022-09-23 03:03:53 +00:00
return callback(err);
});
},
],
err => {
return cb(err);
}
);
}
static _parseArticleLines(articleLines, cb) {
//
// Split articleLines into:
// - Header split into N:V pairs
// - Message Body lines
// -
const header = new Map();
const body = [];
let inHeader = true;
let currentHeaderName;
forEachSeries(
articleLines,
(line, nextLine) => {
if (inHeader) {
if (line === '.' || line === '') {
inHeader = false;
return nextLine(null);
}
const sep = line.indexOf(':');
if (sep < 1) {
// at least a single char name
// entries can split across lines -- they will be prefixed with a single space.
if (
currentHeaderName &&
(line.startsWith(' ') || line.startsWith('\t'))
) {
let v = header.get(currentHeaderName);
v += line
.replace(/^\t/, ' ') // if we're dealign with a legacy tab
.trimRight();
header.set(currentHeaderName, v);
return nextLine(null);
}
return nextLine(
Errors.Invalid(
`"${line}" is not a valid NNTP message header!`
)
);
}
currentHeaderName = line.slice(0, sep).trim().toLowerCase();
const value = line.slice(sep + 1).trim();
header.set(currentHeaderName, value);
return nextLine(null);
}
// body
if (line !== '.') {
// lines consisting of a single '.' are escaped to '..'
if (line.startsWith('..')) {
body.push(line.slice(1));
} else {
body.push(line);
}
2022-09-23 03:03:53 +00:00
}
return nextLine(null);
},
err => {
return cb(err, { header, body });
}
);
}
}
exports.getModule = class NNTPServerModule extends ServerModule {
constructor() {
super();
}
isEnabled() {
return this.enableNntp || this.enableNttps;
}
get enableNntp() {
return _.get(Config(), 'contentServers.nntp.nntp.enabled', false);
}
get enableNttps() {
return _.get(Config(), 'contentServers.nntp.nntps.enabled', false);
}
isConfigured() {
const config = Config();
//
// Any conf/areas exposed?
//
const publicConfs = _.get(
config,
'contentServers.nntp.publicMessageConferences',
{}
);
const areasExposed = _.some(publicConfs, areas => {
return Array.isArray(areas) && areas.length > 0;
});
if (!areasExposed) {
return false;
}
const nntp = _.get(config, 'contentServers.nntp.nntp');
if (nntp && this.enableNntp) {
if (isNaN(nntp.port)) {
return false;
}
}
const nntps = _.get(config, 'contentServers.nntp.nntps');
if (nntps && this.enableNttps) {
if (isNaN(nntps.port)) {
return false;
}
if (!_.isString(nntps.certPem) || !_.isString(nntps.keyPem)) {
return false;
}
}
return true;
}
createServer(cb) {
if (!this.isEnabled() || !this.isConfigured()) {
return cb(null);
}
const config = Config();
2022-09-23 03:03:53 +00:00
// :TODO: nntp-server doesn't currently allow posting in a nice way, so this is kludged in. Fork+MR something cleaner at some point
class ProxySession extends NNTPServerBase.Session {
constructor(server, stream) {
super(server, stream);
this.articleLinesBuffer = [];
}
parse(data) {
if (this.receivingPostArticle) {
return this.receivePostArticleData(data);
}
super.parse(data);
}
receivePostArticleData(data) {
this.articleLinesBuffer.push(...data.split(/r?\n/));
const endOfPost = data.length === 1 && data[0] === '.';
2022-09-23 03:03:53 +00:00
if (endOfPost) {
this.receivingPostArticle = false;
// Command is not exported currently; maybe submit a MR to allow posting in a nicer way...
function Command(runner, articleLines, session) {
this.state = 0; // CMD_WAIT
this.cmd_line = 'POST';
this.resolved_value = null;
this.rejected_value = null;
this.run = runner;
this.articleLines = articleLines;
this.session = session;
}
this.pipeline.push(
new Command(
this._processarticleLinesBuffer,
this.articleLinesBuffer,
this
)
);
this.articleLinesBuffer = [];
this.tick();
}
}
_processarticleLinesBuffer() {
return new Promise(resolve => {
NNTPServer._importMessage(this.session, this.articleLines, err => {
if (err) {
this.rejected_value = err; // will be serialized and 403 sent back currently; not really ideal as we want ArticlePostFailed
// :TODO: tick() needs updated in session.js such that we can write back a proper code
2022-09-23 03:03:53 +00:00
this.state = 3; // CMD_REJECTED
Log.error(
{ error: err.message },
`NNTP post failed: ${err.message}`
);
2022-09-23 03:03:53 +00:00
} else {
this.resolved_value = Responses.ArticlePostedOk;
this.state = 2; // CMD_RESOLVED
}
return resolve();
});
});
}
static create(server, stream) {
return new ProxySession(server, stream);
}
}
const commonOptions = {
2022-09-23 03:03:53 +00:00
// :TODO: How to hook into debugging?!
};
2022-09-23 03:24:24 +00:00
if (true === _.get(config, 'contentServers.nntp.allowPosts')) {
// add in some additional supported commands
const commands = Object.assign({}, NNTPServerBase.commands, {
POST: PostCommand,
});
commonOptions.commands = commands;
commonOptions.session = ProxySession;
}
if (this.enableNntp) {
this.nntpServer = new NNTPServer(
// :TODO: according to docs: if connection is non-tls, but behind proxy (assuming TLS termination?!!) then set this to true
Object.assign({ secure: false }, commonOptions),
'NNTP'
);
}
if (this.enableNttps) {
this.nntpsServer = new NNTPServer(
Object.assign(
{
secure: true,
tls: {
cert: fs.readFileSync(
config.contentServers.nntp.nntps.certPem
),
key: fs.readFileSync(config.contentServers.nntp.nntps.keyPem),
},
},
commonOptions
),
'NTTPS'
);
}
nntpDatabase = new NNTPDatabase();
nntpDatabase.init(err => {
return cb(err);
});
}
2018-12-27 09:46:16 +00:00
listen(cb) {
const config = Config();
forEachSeries(
['nntp', 'nntps'],
(service, nextService) => {
const server = this[`${service}Server`];
if (server) {
const port = config.contentServers.nntp[service].port;
server
.listen(this.listenURI(port, service))
.catch(e => {
Log.warn(
{ error: e.message, port },
`${service.toUpperCase()} failed to listen`
);
return nextService(null); // try next anyway
})
.then(() => {
return nextService(null);
});
} else {
return nextService(null);
}
},
err => {
return cb(err);
}
);
}
listenURI(port, service = 'nntp') {
2018-12-15 06:03:10 +00:00
return `${service}://0.0.0.0:${port}`;
}
};
function performMaintenanceTask(args, cb) {
//
// Delete any message mapping that no longer have
// an actual message associated with them.
//
if (!nntpDatabase) {
Log.trace('Cannot perform NNTP maintenance without NNTP database initialized');
return cb(null);
}
let attached = false;
asyncSeries(
[
callback => {
const messageDbPath = paths.join(Config().paths.db, 'message.sqlite3');
nntpDatabase.db.run(
`ATTACH DATABASE "${messageDbPath}" AS msgdb;`,
err => {
attached = !err;
return callback(err);
}
);
},
callback => {
nntpDatabase.db.run(
`DELETE FROM nntp_area_message
WHERE message_uuid NOT IN (
SELECT message_uuid
FROM msgdb.message
);`,
function result(err) {
// no arrow func; need |this.changes|
if (err) {
Log.warn(
{ error: err.message },
'Failed to delete from NNTP database'
);
} else {
Log.debug(
{ count: this.changes },
'Deleted mapped message IDs from NNTP database'
);
}
return callback(err);
}
);
},
],
err => {
if (attached) {
nntpDatabase.db.run('DETACH DATABASE msgdb;');
}
return cb(err);
}
);
}