Many WebFinger improvements, can now round trip private messages

core/activitypub/actor.js

@@ -11,6 +11,7 @@ const {
     selfUrl,
     isValidLink,
     makeSharedInboxUrl,
+    userNameFromSubject,
 } = require('./util');
 const Log = require('../logger').log;
 const { queryWebFinger } = require('../webfinger');
@@ -24,6 +25,9 @@ const _ = require('lodash');
 const mimeTypes = require('mime-types');
 const { getJson } = require('../http_util.js');
 const { getISOTimestampString } = require('../database.js');
+const moment = require('moment');
+
+const ActorCacheTTL = moment.duration(1, 'day');
 
 // https://www.w3.org/TR/activitypub/#actor-objects
 module.exports = class Actor extends ActivityPubObject {
@@ -138,28 +142,30 @@ module.exports = class Actor extends ActivityPubObject {
         return cb(null, new Actor(obj));
     }
 
-    static fromId(id, forceRefresh, cb) {
+    static fromId(id, cb) {
-        if (_.isFunction(forceRefresh) && !cb) {
+        Actor._fromCache(id, (err, actor, subject) => {
-            cb = forceRefresh;
+            if (!err) {
-            forceRefresh = false;
+                // cache hit
+                return cb(null, actor, subject);
             }
 
-        Actor.fromCache(id, (err, actor) => {
+            // cache miss: attempt to fetch & populate
-            if (err) {
+            Actor._fromWebFinger(id, (err, actor, subject) => {
-                if (forceRefresh) {
+                if (subject) {
-                    return cb(err);
+                    subject = `@${userNameFromSubject(subject)}`; // e.g. @Username@host.com
+                } else {
+                    subject = actor.id; //   best we can do for now
                 }
 
-                Actor.fromRemoteQuery(id, (err, actor) => {
                 // deliver result to caller
-                    cb(err, actor);
+                cb(err, actor, subject);
 
                 // cache our entry
                 if (actor) {
                     apDb.run(
-                            `INSERT INTO actor_cache (actor_id, actor_json, timestamp)
+                        `REPLACE INTO actor_cache (actor_id, actor_json, subject, timestamp)
-                            VALUES (?, ?, ?);`,
+                        VALUES (?, ?, ?, ?);`,
-                            [id, JSON.stringify(actor), getISOTimestampString()],
+                        [id, JSON.stringify(actor), subject, getISOTimestampString()],
                         err => {
                             if (err) {
                                 //  :TODO: log me
@@ -168,13 +174,10 @@ module.exports = class Actor extends ActivityPubObject {
                     );
                 }
             });
-            } else {
-                return cb(null, actor);
-            }
         });
     }
 
-    static fromRemoteQuery(id, cb) {
+    static _fromRemoteQuery(id, cb) {
         const headers = {
             Accept: 'application/activity+json',
         };
@@ -194,13 +197,13 @@ module.exports = class Actor extends ActivityPubObject {
         });
     }
 
-    static fromCache(id, cb) {
+    static _fromCache(id, cb) {
         apDb.get(
-            `SELECT actor_json
+            `SELECT actor_json, subject, timestamp
             FROM actor_cache
-            WHERE actor_id = ?
+            WHERE actor_id = ? OR subject = ?
             LIMIT 1;`,
-            [id],
+            [id, id],
             (err, row) => {
                 if (err) {
                     return cb(err);
@@ -210,6 +213,11 @@ module.exports = class Actor extends ActivityPubObject {
                     return cb(Errors.DoesNotExist());
                 }
 
+                const timestamp = moment(row.timestamp);
+                if (moment().isAfter(timestamp.add(ActorCacheTTL))) {
+                    return cb(Errors.Expired('The cache entry is expired'));
+                }
+
                 const obj = ActivityPubObject.fromJsonString(row.actor_json);
                 if (!obj || !obj.isValid()) {
                     return cb(Errors.Invalid('Failed to create ActivityPub object'));
@@ -220,20 +228,14 @@ module.exports = class Actor extends ActivityPubObject {
                     return cb(Errors.Invalid('Failed to create Actor object'));
                 }
 
-                return cb(null, actor);
+                const subject = row.subject || actor.id;
+                return cb(null, actor, subject);
             }
         );
     }
 
-    static fromAccountName(actorName, cb) {
+    static _fromWebFinger(actorQuery, cb) {
-        //  :TODO: cache first -- do we have an Actor for this account already with a OK TTL?
+        queryWebFinger(actorQuery, (err, res) => {
-
-        //  account names can come in multiple forms, so need a cache mapping of that as well
-        //  actor_alias_cache
-        //  actor_alias | actor_id
-        //
-
-        queryWebFinger(actorName, (err, res) => {
             if (err) {
                 return cb(err);
             }
@@ -255,7 +257,9 @@ module.exports = class Actor extends ActivityPubObject {
             }
 
             // we can now query the href value for an Actor
-            return Actor.fromId(activityLink.href, cb);
+            return Actor._fromRemoteQuery(activityLink.href, (err, actor) => {
+                return cb(err, actor, res.subject);
+            });
         });
     }
 };

core/activitypub/note.js

@@ -14,6 +14,7 @@ const async = require('async');
 const { isString, isObject } = require('lodash');
 
 const APMessageIdNamespace = '307bc7b3-3735-4573-9a20-e3f9eaac29c5';
+const APDefaultSummary = '[ActivityPub]';
 
 module.exports = class Note extends ActivityPubObject {
     constructor(obj) {
@@ -71,7 +72,7 @@ module.exports = class Note extends ActivityPubObject {
                     });
                 },
                 (fromUser, fromActor, callback) => {
-                    Actor.fromAccountName(remoteActorAccount, (err, remoteActor) => {
+                    Actor.fromId(remoteActorAccount, (err, remoteActor) => {
                         return callback(err, fromUser, fromActor, remoteActor);
                     });
                 },
@@ -91,10 +92,14 @@ module.exports = class Note extends ActivityPubObject {
                         audience: [message.isPrivate() ? 'as:Private' : 'as:Public'],
 
                         // :TODO: inReplyto if this is a reply; we need this store in message meta.
-                        summary: message.subject,
                         content: messageBodyToHtml(message.message.trim()),
                     };
 
+                    //  Filter out replace replacement
+                    if (message.subject !== APDefaultSummary) {
+                        obj.summary = message.subject;
+                    }
+
                     const note = new Note(obj);
                     return callback(null, { note, fromUser, remoteActor });
                 },
@@ -116,25 +121,23 @@ module.exports = class Note extends ActivityPubObject {
         });
 
         // Fetch the remote actor info to get their user info
-        Actor.fromId(this.attributedTo, false, (err, attributedToActor) => {
+        Actor.fromId(this.attributedTo, (err, attributedToActor, fromActorSubject) => {
             if (err) {
-                //  :TODO: Log me
+                return cb(err);
-                message.fromUserName = this.attributedTo; // have some sort of value =/
-            } else {
-                message.fromUserName =
-                    attributedToActor.preferredUsername || this.attributedTo;
             }
 
+            message.fromUserName = fromActorSubject || this.attributedTo;
+
             //
             //  Note's can be addressed to 1:N users, but a Message is a 1:1
             //  relationship. This method requires the mapping up front via options
             //
-            (message.toUserName = options.toUser.username),
+            message.toUserName = options.toUser.username;
-                (message.meta.System[Message.SystemMetaNames.LocalToUserID] =
+            message.meta.System[Message.SystemMetaNames.LocalToUserID] =
-                    options.toUser.userId);
+                options.toUser.userId;
             message.areaTag = options.areaTag || Message.WellKnownAreaTags.Private;
 
-            message.subject = this.summary || '-ActivityPub-';
+            message.subject = this.summary || APDefaultSummary;
 
             //  :TODO: it would be better to do some basic HTML to ANSI or pipe codes perhaps
             message.message = htmlToMessageBody(this.content);

core/activitypub/util.js

@@ -24,6 +24,7 @@ exports.accountFromSelfUrl = accountFromSelfUrl;
 exports.getUserProfileTemplatedBody = getUserProfileTemplatedBody;
 exports.messageBodyToHtml = messageBodyToHtml;
 exports.htmlToMessageBody = htmlToMessageBody;
+exports.userNameFromSubject = userNameFromSubject;
 
 //  :TODO: more info in default
 // this profile template is the *default* for both WebFinger
@@ -181,3 +182,7 @@ function messageBodyToHtml(body) {
 function htmlToMessageBody(html) {
     return striptags(html);
 }
+
+function userNameFromSubject(subject) {
+    return subject.replace(/^acct:(.+)$/, '$1');
+}

core/database.js

@@ -505,9 +505,10 @@ dbs.message.run(
         // Actors we know about and have cached
         dbs.activitypub.run(
             `CREATE TABLE IF NOT EXISTS actor_cache (
-                id                  INTEGER PRIMARY KEY,    -- Local DB ID
+                actor_cache_id      INTEGER PRIMARY KEY,    -- Local DB ID
                 actor_id            VARCHAR NOT NULL,       -- Fully qualified Actor ID/URL
                 actor_json          VARCHAR NOT NULL,       -- Actor document
+                subject             VARCHAR,                -- Subject obtained from WebFinger, e.g. @Username@some.domain
                 timestamp           DATETIME NOT NULL,      -- Timestamp in which this Actor was cached
 
                 UNIQUE(actor_id)

core/enig_error.js

@@ -56,6 +56,7 @@ exports.Errors = {
     Timeout: (reason, reasonCode) => new EnigError('Timeout', -32014, reason, reasonCode),
     MissingProperty: (reason, reasonCode) =>
         new EnigError('Missing property', -32014, reason, reasonCode),
+    Expired: (reason, reasonCode) => new EnigError('Expired', -32015, reason, reasonCode),
 };
 
 exports.ErrorReasons = {

core/scanner_tossers/activitypub.js

@@ -61,10 +61,10 @@ exports.getModule = class ActivityPubScannerTosser extends MessageScanTossModule
                     );
 
                     //  :TODO: Implement retry logic (connection issues, retryable HTTP status) ??
-                    //const inbox = remoteActor.inbox;
+                    const inbox = remoteActor.inbox;
 
-                    const inbox = remoteActor.endpoints.sharedInbox;
+                    // const inbox = remoteActor.endpoints.sharedInbox;
-                    activity.object.to = 'https://www.w3.org/ns/activitystreams#Public';
+                    // activity.object.to = 'https://www.w3.org/ns/activitystreams#Public';
 
                     activity.sendTo(
                         inbox,

core/servers/content/web_handlers/activitypub.js

@@ -299,6 +299,7 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule {
         async.forEach(
             toActors,
             (actorId, nextActor) => {
+                //  :TODO: verify this - if *any* audience/actor is public, then this message is public I believe.
                 if (Collection.PublicCollectionId === actorId) {
                     // Deliver to inbox for "everyone":
                     // - Add to 'sharedInbox' collection
@@ -318,7 +319,12 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule {
             },
             err => {
                 if (err) {
-                    //  :TODO: If sqlite constraint, just return OK -- it's a dupe
+                    // if we get a dupe, just tell the remote everything is A-OK
+                    if ('SQLITE_CONSTRAINT' === err.code) {
+                        resp.writeHead(202);
+                        return resp.end('');
+                    }
+
                     return this.webServer.internalServerError(resp, err);
                 }
 

core/webfinger.js

@@ -7,23 +7,45 @@ const { getJson } = require('./http_util');
 
 exports.queryWebFinger = queryWebFinger;
 
-function queryWebFinger(account, cb) {
+function queryWebFinger(query, cb) {
+    //
+    //  Accept a variety of formats to query via WebFinger
+    //  1) @Username@foo.bar -> query with acct:Username resource
+    //  2) http/https URL -> query with resource = URL
+    //  3) If not one of the above and a '/' is present in the query,
+    //     assume https:// and try #2
+    //
+
     // ex: @NuSkooler@toot.community -> https://toot.community/.well-known/webfinger with acct:NuSkooler resource
-    const addrInfo = getAddressedToInfo(account);
+    const addrInfo = getAddressedToInfo(query);
+    let resource;
+    let host;
     if (
-        addrInfo.flavor !== Message.AddressFlavor.ActivityPub &&
+        addrInfo.flavor === Message.AddressFlavor.ActivityPub ||
-        addrInfo.flavor !== Message.AddressFlavor.Email
+        addrInfo.flavor === Message.AddressFlavor.Email
     ) {
-        return cb(Errors.Invalid(`Cannot WebFinger "${account.remote}"; Missing domain`));
+        host = addrInfo.remote.slice(addrInfo.remote.lastIndexOf('@') + 1);
+        if (!host) {
+            return cb(Errors.Invalid(`Unsure how to WebFinger "${query}"`));
+        }
+        resource = `acct:${addrInfo.name}@${host}`;
+    } else {
+        if (!/^https?:\/\/.+$/.test(query)) {
+            resource = `https://${query}`;
+        } else {
+            resource = query;
         }
 
-    const domain = addrInfo.remote.slice(addrInfo.remote.lastIndexOf('@') + 1);
+        try {
-    if (!domain) {
+            const url = new URL(resource);
-        return cb(Errors.Invalid(`Cannot WebFinger "${account.remote}"; Missing domain`));
+            host = url.host;
+        } catch (e) {
+            return cb(Errors.Invalid(`Cannot WebFinger "${query}": ${e.message}`));
+        }
     }
 
-    const resource = encodeURIComponent(`acct:${account.slice(1)}`); // we need drop the initial '@' prefix
+    resource = encodeURIComponent(resource);
-    const webFingerUrl = `https://${domain}/.well-known/webfinger?resource=${resource}`;
+    const webFingerUrl = `https://${host}/.well-known/webfinger?resource=${resource}`;
     getJson(webFingerUrl, {}, (err, json, res) => {
         if (err) {
             return cb(err);