Better handling of to/from HTML and BBS message formats, Note handling esp with inReplyTo, etc.

core/activitypub/note.js

@@ -4,6 +4,7 @@ const { Errors } = require('../enig_error');
 const { getISOTimestampString } = require('../database');
 const User = require('../user');
 const { messageBodyToHtml, htmlToMessageBody } = require('./util');
+const { isAnsi } = require('../string_util');
 
 // deps
 const { v5: UUIDv5 } = require('uuid');
@@ -11,7 +12,7 @@ const Actor = require('./actor');
 const moment = require('moment');
 const Collection = require('./collection');
 const async = require('async');
-const { isString, isObject } = require('lodash');
+const { isString, isObject, truncate } = require('lodash');
 
 const APMessageIdNamespace = '307bc7b3-3735-4573-9a20-e3f9eaac29c5';
 const APDefaultSummary = '[ActivityPub]';
@@ -77,26 +78,57 @@ module.exports = class Note extends ActivityPubObject {
                     });
                 },
                 (fromUser, fromActor, remoteActor, callback) => {
-                    const to = message.isPrivate()
+                    if (!message.replyToMsgId) {
-                        ? remoteActor.id
+                        return callback(null, null, fromUser, fromActor, remoteActor);
-                        : Collection.PublicCollectionId;
+                    }
 
-                    // Refs
+                    Message.getMetaValuesByMessageId(
-                    // - https://docs.joinmastodon.org/spec/activitypub/#properties-used
+                        message.replyToMsgId,
+                        Message.WellKnownMetaCategories.ActivityPub,
+                        Message.ActivityPubPropertyNames.NoteId,
+                        (err, replyToNoteId) => {
+                            // (ignore error)
+                            return callback(
+                                null,
+                                replyToNoteId,
+                                fromUser,
+                                fromActor,
+                                remoteActor
+                            );
+                        }
+                    );
+                },
+                (replyToNoteId, fromUser, fromActor, remoteActor, callback) => {
+                    const to = [
+                        message.isPrivate()
+                            ? remoteActor.id
+                            : Collection.PublicCollectionId,
+                    ];
+
+                    const sourceMediaType = isAnsi(message.message)
+                        ? 'text/x-ansi' // ye ol' https://lists.freedesktop.org/archives/xdg/2006-March/006214.html
+                        : 'text/plain';
+
+                    // https://docs.joinmastodon.org/spec/activitypub/#properties-used
                     const obj = {
                         id: ActivityPubObject.makeObjectId(webServer, 'note'),
                         type: 'Note',
                         published: getISOTimestampString(message.modTimestamp),
                         to,
                         attributedTo: fromActor.id,
-                        audience: [message.isPrivate() ? 'as:Private' : 'as:Public'],
-
-                        // :TODO: inReplyto if this is a reply; we need this store in message meta.
                         content: messageBodyToHtml(message.message.trim()),
+                        source: {
+                            content: message.message,
+                            mediaType: sourceMediaType,
+                        },
                     };
 
-                    //  Filter out replace replacement
+                    if (replyToNoteId) {
-                    if (message.subject !== APDefaultSummary) {
+                        obj.inReplyTo = replyToNoteId;
+                    }
+
+                    //  ignore the subject if it's our default summary value for replies
+                    if (message.subject !== `RE: ${APDefaultSummary}`) {
                         obj.summary = message.subject;
                     }
 
@@ -137,10 +169,12 @@ module.exports = class Note extends ActivityPubObject {
                 options.toUser.userId;
             message.areaTag = options.areaTag || Message.WellKnownAreaTags.Private;
 
-            message.subject = this.summary || APDefaultSummary;
-
             //  :TODO: it would be better to do some basic HTML to ANSI or pipe codes perhaps
             message.message = htmlToMessageBody(this.content);
+            message.subject =
+                this.summary ||
+                truncate(message.message, { length: 32, omission: '...' }) ||
+                APDefaultSummary;
 
             try {
                 message.modTimestamp = moment(this.published);
@@ -155,10 +189,12 @@ module.exports = class Note extends ActivityPubObject {
 
             message.meta.ActivityPub = message.meta.ActivityPub || {};
             message.meta.ActivityPub[Message.ActivityPubPropertyNames.ActivityId] =
-                this.id;
+                options.activityId || 0;
-            if (this.InReplyTo) {
+            message.meta.ActivityPub[Message.ActivityPubPropertyNames.NoteId] = this.id;
+
+            if (this.inReplyTo) {
                 message.meta.ActivityPub[Message.ActivityPubPropertyNames.InReplyTo] =
-                    this.InReplyTo;
+                    this.inReplyTo;
             }
 
             return cb(null, message);

core/activitypub/util.js

@@ -3,6 +3,7 @@ const User = require('../user');
 const { Errors, ErrorReasons } = require('../enig_error');
 const UserProps = require('../user_property');
 const ActivityPubSettings = require('./settings');
+const { stripAnsiControlCodes } = require('../string_util');
 
 // deps
 const _ = require('lodash');
@@ -12,6 +13,7 @@ const fs = require('graceful-fs');
 const paths = require('path');
 const moment = require('moment');
 const { striptags } = require('striptags');
+const { encode, decode } = require('html-entities');
 
 exports.ActivityStreamsContext = 'https://www.w3.org/ns/activitystreams';
 exports.isValidLink = isValidLink;
@@ -173,14 +175,23 @@ function getUserProfileTemplatedBody(
 //
 //  Apply very basic HTML to a message following
 //  Mastodon's supported tags of 'p', 'br', 'a', and 'span':
-//  https://blog.joinmastodon.org/2018/06/how-to-implement-a-basic-activitypub-server/
+//  - https://docs.joinmastodon.org/spec/activitypub/#sanitization
+//  - https://blog.joinmastodon.org/2018/06/how-to-implement-a-basic-activitypub-server/
 //
+//  :TODO: https://docs.joinmastodon.org/spec/microformats/
 function messageBodyToHtml(body) {
-    return `<p>${body.replace(/\r?\n/g, '<br>')}</p>`;
+    body = encode(stripAnsiControlCodes(body), { mode: 'nonAsciiPrintable' }).replace(
+        /\r?\n/g,
+        '<br>'
+    );
+
+    return `<p>${body}</p>`;
 }
 
 function htmlToMessageBody(html) {
-    return striptags(html);
+    // <br>, </br>, and <br/> -> \r\n
+    html = html.replace(/<\/?br?\/?>/g, '\r\n');
+    return striptags(decode(html));
 }
 
 function userNameFromSubject(subject) {

core/message.js

@@ -117,6 +117,7 @@ const QWKPropertyNames = {
 const ActivityPubPropertyNames = {
     ActivityId: 'activitypub_activity_id', //  Activity ID; FK to AP table entries
     InReplyTo: 'activitypub_in_reply_to', //  Activity ID from 'inReplyTo' field
+    NoteId: 'activitypub_note_id', // Note ID specific to Note Activities
 };
 
 //  :TODO: this is a ugly hack due to bad variable names - clean it up & just _.camelCase(k)!

core/scanner_tossers/activitypub.js

@@ -132,6 +132,16 @@ exports.getModule = class ActivityPubScannerTosser extends MessageScanTossModule
                         }
                     );
                 },
+                (activity, callback) => {
+                    return message.persistMetaValue(
+                        Message.WellKnownMetaCategories.ActivityPub,
+                        Message.ActivityPubPropertyNames.NoteId,
+                        activity.object.id,
+                        err => {
+                            return callback(err, activity);
+                        }
+                    );
+                },
             ],
             (err, activity) => {
                 if (err) {

core/servers/content/web.js

@@ -313,6 +313,11 @@ exports.getModule = class WebServerModule extends ServerModule {
         });
     }
 
+    accepted(resp, body = '', headers = { 'Content-Type:': 'text/html' }) {
+        resp.writeHead(202, 'Accepted', body ? headers : null);
+        return resp.end(body);
+    }
+
     badRequest(resp) {
         return this.respondWithError(resp, 400, 'Bad request.', 'Bad Request');
     }

core/servers/content/web_handlers/activitypub.js

@@ -299,11 +299,8 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule {
         async.forEach(
             toActors,
             (actorId, nextActor) => {
-                //  :TODO: verify this - if *any* audience/actor is public, then this message is public I believe.
                 if (Collection.PublicCollectionId === actorId) {
-                    // Deliver to inbox for "everyone":
+                    //  :TODO: we should probably land this in a public areaTag as well for AP; allowing Message objects to be used/etc.
-                    // - Add to 'sharedInbox' collection
-                    //
                     Collection.addPublicInboxItem(note, err => {
                         return nextActor(err);
                     });
@@ -312,29 +309,23 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule {
                         req,
                         resp,
                         actorId,
+                        activity,
                         note,
                         nextActor
                     );
                 }
             },
             err => {
-                if (err) {
+                if (err && 'SQLITE_CONSTRAINT' !== err.code) {
-                    // if we get a dupe, just tell the remote everything is A-OK
-                    if ('SQLITE_CONSTRAINT' === err.code) {
-                        resp.writeHead(202);
-                        return resp.end('');
-                    }
-
                     return this.webServer.internalServerError(resp, err);
                 }
 
-                resp.writeHead(202);
+                return this.webServer.accepted(resp);
-                return resp.end('');
             }
         );
     }
 
-    _deliverInboxNoteToLocalActor(req, resp, actorId, note, cb) {
+    _deliverInboxNoteToLocalActor(req, resp, actorId, activity, note, cb) {
         const localUserName = accountFromSelfUrl(actorId);
         if (!localUserName) {
             this.log.debug({ url: req.url }, 'Could not get username from URL');
@@ -360,6 +351,7 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule {
                 //
                 const messageOpts = {
                     //  Notes can have 1:N 'to' relationships while a Message is 1:1;
+                    activityId: activity.id,
                     toUser: localUser,
                     areaTag: Message.WellKnownAreaTags.Private,
                 };
@@ -564,8 +556,7 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule {
                         'Undo "Follow" (un-follow) success'
                     );
 
-                    resp.writeHead(202);
+                    return this.webServer.accepted(resp);
-                    return resp.end('');
                 }
             );
         });

package.json

@@ -43,6 +43,7 @@
         "graceful-fs": "^4.2.10",
         "hashids": "^2.2.10",
         "hjson": "3.2.2",
+        "html-entities": "^2.3.3",
         "http-signature": "^1.3.6",
         "iconv-lite": "0.6.3",
         "ini-config-parser": "^1.0.4",

yarn.lock

@@ -1230,6 +1230,11 @@ hjson@3.2.2:
   resolved "https://registry.npmjs.org/hjson/-/hjson-3.2.2.tgz"
   integrity sha512-MkUeB0cTIlppeSsndgESkfFD21T2nXPRaBStLtf3cAYA2bVEFdXlodZB0TukwZiobPD1Ksax5DK4RTZeaXCI3Q==
 
+html-entities@^2.3.3:
+  version "2.3.3"
+  resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-2.3.3.tgz#117d7626bece327fc8baace8868fa6f5ef856e46"
+  integrity sha512-DV5Ln36z34NNTDgnz0EWGBLZENelNAtkiFA4kyNOG2tDI6Mz1uSWiq1wAKdyjnJwyDiDO7Fa2SO1CTxPXL8VxA==
+
 http-cache-semantics@^4.1.0:
   version "4.1.0"
   resolved "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz"