diff --git a/core/activitypub/activity.js b/core/activitypub/activity.js index f6c9db5d..83dc3e28 100644 --- a/core/activitypub/activity.js +++ b/core/activitypub/activity.js @@ -1,17 +1,11 @@ -const { messageBodyToHtml, selfUrl } = require('./util'); -const { ActivityStreamsContext, WellKnownActivityTypes } = require('./const'); +const { selfUrl } = require('./util'); +const { WellKnownActivityTypes } = require('./const'); const ActivityPubObject = require('./object'); -const User = require('../user'); -const Actor = require('./actor'); const { Errors } = require('../enig_error'); -const { getISOTimestampString } = require('../database'); const UserProps = require('../user_property'); const { postJson } = require('../http_util'); -const { WellKnownLocations } = require('../servers/content/web'); // deps -const { v4: UUIDv4 } = require('uuid'); -const async = require('async'); const _ = require('lodash'); module.exports = class Activity extends ActivityPubObject { @@ -23,10 +17,9 @@ module.exports = class Activity extends ActivityPubObject { return WellKnownActivityTypes; } - static makeFollow(webServer, localActor, remoteActor, id = null) { - id = id || Activity._makeFullId(webServer, 'follow'); + static makeFollow(webServer, localActor, remoteActor) { return new Activity({ - id, + id: Activity.activityObjectId(webServer), type: 'Follow', actor: localActor, object: remoteActor.id, @@ -34,92 +27,22 @@ module.exports = class Activity extends ActivityPubObject { } // https://www.w3.org/TR/activitypub/#accept-activity-inbox - static makeAccept(webServer, localActor, followRequest, id = null) { - id = id || Activity._makeFullId(webServer, 'accept'); - + static makeAccept(webServer, localActor, followRequest) { return new Activity({ - id, + id: Activity.activityObjectId(webServer), type: 'Accept', actor: localActor, object: followRequest, // previous request Activity }); } - static noteFromLocalMessage(webServer, message, cb) { - const localUserId = message.getLocalFromUserId(); - if (!localUserId) { - return cb(Errors.UnexpectedState('Invalid user ID for local user!')); - } - - async.waterfall( - [ - callback => { - return User.getUser(localUserId, callback); - }, - (localUser, callback) => { - const remoteActorAccount = message.getRemoteToUser(); - if (!remoteActorAccount) { - return callback( - Errors.UnexpectedState( - 'Message does not contain a remote address' - ) - ); - } - - const opts = {}; - Actor.fromAccountName( - remoteActorAccount, - opts, - (err, remoteActor) => { - return callback(err, localUser, remoteActor); - } - ); - }, - (localUser, remoteActor, callback) => { - Actor.fromLocalUser(localUser, webServer, (err, localActor) => { - return callback(err, localUser, localActor, remoteActor); - }); - }, - (localUser, localActor, remoteActor, callback) => { - // we'll need the entire |activityId| as a linked reference later - const activityId = Activity._makeFullId(webServer, 'create'); - - const obj = { - '@context': ActivityStreamsContext, - id: activityId, - type: 'Create', - actor: localActor.id, - object: { - id: Activity._makeFullId(webServer, 'note'), - type: 'Note', - published: getISOTimestampString(message.modTimestamp), - attributedTo: localActor.id, - audience: [message.isPrivate() ? 'as:Private' : 'as:Public'], - // :TODO: inReplyto if this is a reply; we need this store in message meta. - - content: messageBodyToHtml(message.message.trim()), - }, - }; - - // :TODO: this probably needs to change quite a bit based on "groups" - // :TODO: verify we need both 'to' fields: https://socialhub.activitypub.rocks/t/problems-posting-to-mastodon-inbox/801/4 - if (message.isPrivate()) { - //obj.to = remoteActor.id; - obj.object.to = remoteActor.id; - } else { - const publicInbox = `${ActivityStreamsContext}#Public`; - //obj.to = publicInbox; - obj.object.to = publicInbox; - } - - const activity = new Activity(obj); - return callback(null, activity, localUser, remoteActor); - }, - ], - (err, activity, fromUser, remoteActor) => { - return cb(err, { activity, fromUser, remoteActor }); - } - ); + static makeCreate(webServer, actor, obj) { + return new Activity({ + id: Activity.activityObjectId(webServer), + type: 'Create', + actor, + object: obj, + }); } sendTo(actorUrl, fromUser, webServer, cb) { @@ -149,10 +72,7 @@ module.exports = class Activity extends ActivityPubObject { return postJson(actorUrl, activityJson, reqOpts, cb); } - static _makeFullId(webServer, prefix) { - // e.g. http://some.host/_enig/ap/note/bf81a22e-cb3e-41c8-b114-21f375b61124 - return webServer.buildUrl( - WellKnownLocations.Internal + `/ap/${prefix}/${UUIDv4()}` - ); + static activityObjectId(webServer) { + return ActivityPubObject.makeObjectId(webServer, 'activity'); } }; diff --git a/core/activitypub/actor.js b/core/activitypub/actor.js index b88ce060..5466107f 100644 --- a/core/activitypub/actor.js +++ b/core/activitypub/actor.js @@ -17,11 +17,13 @@ const { queryWebFinger } = require('../webfinger'); const EnigAssert = require('../enigma_assert'); const ActivityPubSettings = require('./settings'); const ActivityPubObject = require('./object'); +const apDb = require('../database').dbs.activitypub; // deps const _ = require('lodash'); const mimeTypes = require('mime-types'); const { getJson } = require('../http_util.js'); +const { getISOTimestampString } = require('../database.js'); // https://www.w3.org/TR/activitypub/#actor-objects module.exports = class Actor extends ActivityPubObject { @@ -136,13 +138,48 @@ module.exports = class Actor extends ActivityPubObject { return cb(null, new Actor(obj)); } - static fromRemoteUrl(url, cb) { - // :TODO: cache first + static fromId(id, forceRefresh, cb) { + if (_.isFunction(forceRefresh) && !cb) { + cb = forceRefresh; + forceRefresh = false; + } + + Actor.fromCache(id, (err, actor) => { + if (err) { + if (forceRefresh) { + return cb(err); + } + + Actor.fromRemoteQuery(id, (err, actor) => { + // deliver result to caller + cb(err, actor); + + // cache our entry + if (actor) { + apDb.run( + `INSERT INTO actor_cache (actor_id, actor_json, timestamp) + VALUES (?, ?, ?);`, + [id, JSON.stringify(actor), getISOTimestampString()], + err => { + if (err) { + // :TODO: log me + } + } + ); + } + }); + } else { + return cb(null, actor); + } + }); + } + + static fromRemoteQuery(id, cb) { const headers = { Accept: 'application/activity+json', }; - getJson(url, { headers }, (err, actor) => { + getJson(id, { headers }, (err, actor) => { if (err) { return cb(err); } @@ -157,9 +194,45 @@ module.exports = class Actor extends ActivityPubObject { }); } - static fromAccountName(actorName, options, cb) { + static fromCache(id, cb) { + apDb.get( + `SELECT actor_json + FROM actor_cache + WHERE actor_id = ? + LIMIT 1;`, + [id], + (err, row) => { + if (err) { + return cb(err); + } + + if (!row) { + return cb(Errors.DoesNotExist()); + } + + const obj = ActivityPubObject.fromJsonString(row.actor_json); + if (!obj || !obj.isValid()) { + return cb(Errors.Invalid('Failed to create ActivityPub object')); + } + + const actor = new Actor(obj); + if (!actor.isValid()) { + return cb(Errors.Invalid('Failed to create Actor object')); + } + + return cb(null, actor); + } + ); + } + + static fromAccountName(actorName, cb) { // :TODO: cache first -- do we have an Actor for this account already with a OK TTL? + // account names can come in multiple forms, so need a cache mapping of that as well + // actor_alias_cache + // actor_alias | actor_id + // + queryWebFinger(actorName, (err, res) => { if (err) { return cb(err); @@ -182,12 +255,7 @@ module.exports = class Actor extends ActivityPubObject { } // we can now query the href value for an Actor - return Actor.fromRemoteUrl(activityLink.href, cb); + return Actor.fromId(activityLink.href, cb); }); } - - static fromJsonString(json) { - const parsed = JSON.parse(json); - return new Actor(parsed); - } }; diff --git a/core/activitypub/collection.js b/core/activitypub/collection.js index b40b8573..8bef5727 100644 --- a/core/activitypub/collection.js +++ b/core/activitypub/collection.js @@ -2,7 +2,9 @@ const { makeUserUrl } = require('./util'); const ActivityPubObject = require('./object'); const apDb = require('../database').dbs.activitypub; const { getISOTimestampString } = require('../database'); +const { Errors } = require('../enig_error.js'); +// deps const { isString, get, isObject } = require('lodash'); const APPublicCollectionId = 'https://www.w3.org/ns/activitystreams#Public'; @@ -86,18 +88,62 @@ module.exports = class Collection extends ActivityPubObject { ); } - static getOrdered(name, owningUser, includePrivate, page, mapper, webServer, cb) { + static embeddedObjById(collectionName, includePrivate, objectId, cb) { + const privateQuery = includePrivate ? '' : ' AND is_private = FALSE'; + + apDb.get( + `SELECT obj_json + FROM collection + WHERE name = ? + ${privateQuery} + AND json_extract(obj_json, '$.object.id') = ?;`, + [collectionName, objectId], + (err, row) => { + if (err) { + return cb(err); + } + + if (!row) { + return cb( + Errors.DoesNotExist( + `No embedded Object with object.id of "${objectId}" found` + ) + ); + } + + const obj = ActivityPubObject.fromJsonString(row.obj_json); + if (!obj) { + return cb(Errors.Invalid('Failed to parse Object JSON')); + } + + return cb(null, obj); + } + ); + } + + static getOrdered( + collectionName, + owningUser, + includePrivate, + page, + mapper, + webServer, + cb + ) { const privateQuery = includePrivate ? '' : ' AND is_private = FALSE'; - const followersUrl = - makeUserUrl(webServer, owningUser, '/ap/users/') + `/${name}`; const owningUserId = isObject(owningUser) ? owningUser.userId : owningUser; + // e.g. http://some.host/_enig/ap/collections/1234/followers + const collectionIdBase = + makeUserUrl(webServer, owningUser, `/ap/collections/${owningUserId}`) + + `/${collectionName}`; + if (!page) { return apDb.get( `SELECT COUNT(id) AS count FROM collection WHERE user_id = ? AND name = ?${privateQuery};`, - [owningUserId, name], + [owningUserId, collectionName], (err, row) => { if (err) { return cb(err); @@ -112,14 +158,14 @@ module.exports = class Collection extends ActivityPubObject { let obj; if (row.count > 0) { obj = { - id: followersUrl, + id: collectionIdBase, type: 'OrderedCollection', - first: `${followersUrl}?page=1`, + first: `${collectionIdBase}?page=1`, totalItems: row.count, }; } else { obj = { - id: followersUrl, + id: collectionIdBase, type: 'OrderedCollection', totalItems: 0, orderedItems: [], @@ -137,7 +183,7 @@ module.exports = class Collection extends ActivityPubObject { FROM collection WHERE user_id = ? AND name = ?${privateQuery} ORDER BY timestamp;`, - [owningUserId, name], + [owningUserId, collectionName], (err, entries) => { if (err) { return cb(err); @@ -149,11 +195,11 @@ module.exports = class Collection extends ActivityPubObject { } const obj = { - id: `${followersUrl}/page=${page}`, + id: `${collectionIdBase}/page=${page}`, type: 'OrderedCollectionPage', totalItems: entries.length, orderedItems: entries, - partOf: followersUrl, + partOf: collectionIdBase, }; return cb(null, new Collection(obj)); @@ -161,7 +207,7 @@ module.exports = class Collection extends ActivityPubObject { ); } - static addToCollection(name, owningUser, objectId, obj, isPrivate, cb) { + static addToCollection(collectionName, owningUser, objectId, obj, isPrivate, cb) { if (!isString(obj)) { obj = JSON.stringify(obj); } @@ -171,7 +217,14 @@ module.exports = class Collection extends ActivityPubObject { apDb.run( `INSERT OR IGNORE INTO collection (name, timestamp, user_id, obj_id, obj_json, is_private) VALUES (?, ?, ?, ?, ?, ?);`, - [name, getISOTimestampString(), owningUserId, objectId, obj, isPrivate], + [ + collectionName, + getISOTimestampString(), + owningUserId, + objectId, + obj, + isPrivate, + ], function res(err) { // non-arrow for 'this' scope if (err) { @@ -182,12 +235,12 @@ module.exports = class Collection extends ActivityPubObject { ); } - static removeFromCollectionById(name, owningUser, objectId, cb) { + static removeFromCollectionById(collectionName, owningUser, objectId, cb) { const owningUserId = isObject(owningUser) ? owningUser.userId : owningUser; apDb.run( `DELETE FROM collection WHERE user_id = ? AND name = ? AND obj_id = ?;`, - [owningUserId, name, objectId], + [owningUserId, collectionName, objectId], err => { return cb(err); } diff --git a/core/activitypub/note.js b/core/activitypub/note.js new file mode 100644 index 00000000..29e0b96f --- /dev/null +++ b/core/activitypub/note.js @@ -0,0 +1,152 @@ +const Message = require('../message'); +const ActivityPubObject = require('./object'); +const { Errors } = require('../enig_error'); +const { getISOTimestampString } = require('../database'); +const User = require('../user'); +const { messageBodyToHtml } = require('./util'); + +// deps +const { v5: UUIDv5 } = require('uuid'); +const Actor = require('./actor'); +const moment = require('moment'); +const Collection = require('./collection'); +const async = require('async'); + +const APMessageIdNamespace = '307bc7b3-3735-4573-9a20-e3f9eaac29c5'; + +module.exports = class Note extends ActivityPubObject { + constructor(obj) { + super(obj); + } + + isValid() { + if (!super.isValid()) { + return false; + } + + // :TODO: validate required properties + + return true; + } + + static fromPublicNoteId(noteId, cb) { + Collection.embeddedObjById('outbox', false, noteId, (err, obj) => { + if (err) { + return cb(err); + } + + return cb(null, new Note(obj.object)); + }); + } + + // A local Message bound for ActivityPub + static fromLocalOutgoingMessage(message, webServer, cb) { + const localUserId = message.getLocalFromUserId(); + if (!localUserId) { + return cb(Errors.UnexpectedState('Invalid user ID for local user!')); + } + + if (Message.AddressFlavor.ActivityPub !== message.getAddressFlavor()) { + return cb( + Errors.Invalid('Cannot build note for non-ActivityPub addressed message') + ); + } + + const remoteActorAccount = message.getRemoteToUser(); + if (!remoteActorAccount) { + return cb( + Errors.UnexpectedState('Message does not contain a remote address') + ); + } + + async.waterfall( + [ + callback => { + return User.getUser(localUserId, callback); + }, + (fromUser, callback) => { + Actor.fromLocalUser(fromUser, webServer, (err, fromActor) => { + return callback(err, fromUser, fromActor); + }); + }, + (fromUser, fromActor, callback) => { + Actor.fromAccountName(remoteActorAccount, (err, remoteActor) => { + return callback(err, fromUser, fromActor, remoteActor); + }); + }, + (fromUser, fromActor, remoteActor, callback) => { + const to = message.isPrivate() + ? remoteActor.id + : Collection.PublicCollectionId; + + // Refs + // - https://docs.joinmastodon.org/spec/activitypub/#properties-used + const obj = { + id: ActivityPubObject.makeObjectId(webServer, 'note'), + type: 'Note', + published: getISOTimestampString(message.modTimestamp), + to, + attributedTo: fromActor.id, + audience: [message.isPrivate() ? 'as:Private' : 'as:Public'], + + // :TODO: inReplyto if this is a reply; we need this store in message meta. + + content: messageBodyToHtml(message.message.trim()), + }; + + const note = new Note(obj); + return callback(null, { note, fromUser, remoteActor }); + }, + ], + (err, noteInfo) => { + return cb(err, noteInfo); + } + ); + } + + toMessage(cb) { + // stable ID based on Note ID + const message = new Message({ + uuid: UUIDv5(this.id, APMessageIdNamespace), + }); + + // Fetch the remote actor + Actor.fromId(this.attributedTo, false, (err, attributedToActor) => { + if (err) { + // :TODO: Log me + message.toUserName = this.attributedTo; // have some sort of value =/ + } else { + message.toUserName = + attributedToActor.preferredUsername || this.attributedTo; + } + + message.subject = this.summary || '-ActivityPub-'; + message.message = this.content; // :TODO: HTML to suitable format, or even strip + + try { + message.modTimestamp = moment(this.published); + } catch (e) { + // :TODO: Log warning + message.modTimestamp = moment(); + } + + // :TODO: areaTag + // :TODO: replyToMsgId from 'inReplyTo' + // :TODO: RemoteFromUser + + message.meta[Message.WellKnownMetaCategories.ActivityPub] = + message.meta[Message.WellKnownMetaCategories.ActivityPub] || {}; + const apMeta = message.meta[Message.WellKnownAreaTags.ActivityPub]; + + apMeta[Message.ActivityPubPropertyNames.ActivityId] = this.id; + if (this.InReplyTo) { + apMeta[Message.ActivityPubPropertyNames.InReplyTo] = this.InReplyTo; + } + + message.setRemoteFromUser(this.attributedTo); + message.setExternalFlavor(Message.ExternalFlavor.ActivityPub); + + return cb(null, message); + }); + } +}; diff --git a/core/activitypub/object.js b/core/activitypub/object.js new file mode 100644 index 00000000..e8149757 --- /dev/null +++ b/core/activitypub/object.js @@ -0,0 +1,45 @@ +const { ActivityStreamsContext } = require('./const'); +const { WellKnownLocations } = require('../servers/content/web'); + +// deps +const { isString } = require('lodash'); +const { v4: UUIDv4 } = require('uuid'); + +module.exports = class ActivityPubObject { + constructor(obj) { + this['@context'] = ActivityStreamsContext; + Object.assign(this, obj); + } + + static fromJsonString(s) { + let obj; + try { + obj = JSON.parse(s); + obj = new ActivityPubObject(obj); + } catch (e) { + return null; + } + return obj; + } + + isValid() { + const nes = s => isString(s) && s.length > 1; + // :TODO: Additional validation + if ( + (this['@context'] === ActivityStreamsContext || + this['@context'][0] === ActivityStreamsContext) && + nes(this.id) && + nes(this.type) + ) { + return true; + } + return false; + } + + static makeObjectId(webServer, suffix) { + // e.g. http://some.host/_enig/ap/bf81a22e-cb3e-41c8-b114-21f375b61124/activity + return webServer.buildUrl( + WellKnownLocations.Internal + `/ap/${UUIDv4()}/${suffix}` + ); + } +}; diff --git a/core/database.js b/core/database.js index 8427b592..dc82b705 100644 --- a/core/database.js +++ b/core/database.js @@ -519,11 +519,19 @@ dbs.message.run( ON actor_cache (actor_id);` ); + // Mapping of known aliases for a fully qualified Actor ID + // generally obtained via WebFinger dbs.activitypub.run( - `CREATE INDEX IF NOT EXISTS outbox_activity_json_type_index0 - ON outbox (json_extract(activity_json, '$.type'));` + `CREATE TABLE IF NOT EXISTS actor_alias_cache ( + id INTEGER PRIMARY KEY, + alias VARCHAR NOT NULL, + actor_id VARCHAR NOT NULL, -- Fully qualified Actor ID/URL + + UNIQUE(alias) + );` ); + // ActivityPub Collections of various types such as followers, following, likes, ... dbs.activitypub.run( `CREATE TABLE IF NOT EXISTS collection ( id INTEGER PRIMARY KEY, -- Auto-generated key diff --git a/core/message.js b/core/message.js index 2e11e0f0..43bc7517 100644 --- a/core/message.js +++ b/core/message.js @@ -178,7 +178,25 @@ module.exports = class Message { } isFromRemoteUser() { - return null !== _.get(this, 'meta.System.remote_from_user', null); + return null !== this.getRemoteFromUser(); + } + + setRemoteFromUser(remoteFrom) { + this.meta[Message.WellKnownMetaCategories.System][ + Message.SystemMetaNames.RemoteFromUser + ] = remoteFrom; + } + + getRemoteFromUser() { + return _.get( + this, + [ + 'meta', + Message.WellKnownMetaCategories.System, + Message.SystemMetaNames.RemoteFromUser, + ], + null + ); } isCP437Encodable() { diff --git a/core/scanner_tossers/activitypub.js b/core/scanner_tossers/activitypub.js index 029cda36..3ffc2a49 100644 --- a/core/scanner_tossers/activitypub.js +++ b/core/scanner_tossers/activitypub.js @@ -8,6 +8,7 @@ const Log = require('../logger').log; const async = require('async'); const _ = require('lodash'); const Collection = require('../activitypub/collection'); +const Note = require('../activitypub/note'); exports.moduleInfo = { name: 'ActivityPub', @@ -42,18 +43,31 @@ exports.getModule = class ActivityPubScannerTosser extends MessageScanTossModule async.waterfall( [ callback => { - return Activity.noteFromLocalMessage( - this._webServer(), + Note.fromLocalOutgoingMessage( message, - callback + this._webServer(), + (err, noteInfo) => { + return callback(err, noteInfo); + } ); }, (noteInfo, callback) => { - const { activity, fromUser, remoteActor } = noteInfo; + const { note, fromUser, remoteActor } = noteInfo; + + const activity = Activity.makeCreate( + this._webServer(), + note.attributedTo, + note + ); // :TODO: Implement retry logic (connection issues, retryable HTTP status) ?? + //const inbox = remoteActor.inbox; + + const inbox = remoteActor.endpoints.sharedInbox; + activity.object.to = 'https://www.w3.org/ns/activitystreams#Public'; + activity.sendTo( - remoteActor.inbox, + inbox, fromUser, this._webServer(), (err, respBody, res) => { diff --git a/core/servers/content/web_handlers/activitypub.js b/core/servers/content/web_handlers/activitypub.js index 81be485a..65bab44f 100644 --- a/core/servers/content/web_handlers/activitypub.js +++ b/core/servers/content/web_handlers/activitypub.js @@ -17,6 +17,7 @@ const _ = require('lodash'); const enigma_assert = require('../../../enigma_assert'); const httpSignature = require('http-signature'); const async = require('async'); +const Note = require('../../../activitypub/note'); exports.moduleInfo = { name: 'ActivityPub', @@ -98,6 +99,13 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule { }, }); + this.webServer.addRoute({ + method: 'GET', + // e.g. http://some.host/_enig/ap/bf81a22e-cb3e-41c8-b114-21f375b61124/note + path: /^\/_enig\/ap\/[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}\/note$/, + handler: this._singlePublicNoteGetHandler.bind(this), + }); + // :TODO: NYI // this.webServer.addRoute({ // method: 'GET', @@ -359,6 +367,22 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule { return this._getCollectionHandler('outbox', req, resp, signature); } + _singlePublicNoteGetHandler(req, resp) { + this.log.debug({ url: req.url }, 'Request for "Note"'); + + const url = new URL(req.url, `https://${req.headers.host}`); + const noteId = url.toString(); + + Note.fromPublicNoteId(noteId, (err, note) => { + if (err) { + return this.webServer.internalServerError(resp, err); + } + + resp.writeHead(200, { 'Content-Type': 'text/html' }); + return resp.end(note.content); + }); + } + _accountNameFromUserPath(url, suffix) { const re = new RegExp(`^/_enig/ap/users/(.+)/${suffix}(\\?page=[0-9]+)?$`); const m = url.pathname.match(re); @@ -478,7 +502,7 @@ exports.getModule = class ActivityPubWebHandler extends WebHandlerModule { return this.webServer.resourceNotFound(resp); } - Actor.fromRemoteUrl(activity.actor, (err, actor) => { + Actor.fromId(activity.actor, (err, actor) => { if (err) { return this.webServer.internalServerError(resp, err); }