Remove actor_cache; use a special collection

core/activitypub/actor.js

@@ -12,16 +12,15 @@ const EnigAssert = require('../enigma_assert');
 const ActivityPubSettings = require('./settings');
 const ActivityPubObject = require('./object');
 const { ActivityStreamMediaType, Collections } = require('./const');
-const apDb = require('../database').dbs.activitypub;
 const Config = require('../config').get;
 
 //  deps
 const _ = require('lodash');
 const mimeTypes = require('mime-types');
 const { getJson } = require('../http_util.js');
-const { getISOTimestampString } = require('../database.js');
 const moment = require('moment');
 const paths = require('path');
+const Collection = require('./collection.js');
 
 const ActorCacheExpiration = moment.duration(15, 'days');
 const ActorCacheMaxAgeDays = 125; // hasn't been used in >= 125 days, nuke it.
@@ -204,16 +203,11 @@ module.exports = class Actor extends ActivityPubObject {
 
                 // cache our entry
                 if (actor) {
-                    apDb.run(
+                    Collection.addActor(actor, subject, err => {
-                        `REPLACE INTO actor_cache (actor_id, actor_json, subject, timestamp)
+                        if (err) {
-                        VALUES (?, ?, ?, ?);`,
+                            //  :TODO: Log me
-                        [id, JSON.stringify(actor), subject, getISOTimestampString()],
-                        err => {
-                            if (err) {
-                                //  :TODO: log me
-                            }
                         }
-                    );
+                    });
                 }
             });
         });
@@ -228,17 +222,13 @@ module.exports = class Actor extends ActivityPubObject {
             return;
         }
 
-        apDb.run(
+        Collection.removeOldActorEntries(ActorCacheMaxAgeDays, err => {
-            `DELETE FROM actor_cache
+            if (err) {
-            WHERE DATETIME(timestamp, "+${ActorCacheMaxAgeDays} days") > DATETIME("now");`,
+                //  :TODO: log me
-            err => {
-                if (err) {
-                    //  :TODO: log me
-                }
-
-                return cb(null); // always non-fatal
             }
-        );
+
+            return cb(null); // always non-fatal
+        });
     }
 
     static _fromRemoteQuery(id, cb) {
@@ -262,40 +252,22 @@ module.exports = class Actor extends ActivityPubObject {
     }
 
     static _fromCache(actorIdOrSubject, cb) {
-        apDb.get(
+        Collection.actor(actorIdOrSubject, (err, actor, info) => {
-            `SELECT actor_json, subject, timestamp
+            if (err) {
-            FROM actor_cache
+                return cb(err);
-            WHERE actor_id = ? OR subject = ?
-            LIMIT 1;`,
-            [actorIdOrSubject, actorIdOrSubject],
-            (err, row) => {
-                if (err) {
-                    return cb(err);
-                }
-
-                if (!row) {
-                    return cb(Errors.DoesNotExist());
-                }
-
-                const timestamp = moment(row.timestamp);
-                const needsRefresh = moment().isAfter(
-                    timestamp.add(ActorCacheExpiration)
-                );
-
-                const obj = ActivityPubObject.fromJsonString(row.actor_json);
-                if (!obj || !obj.isValid()) {
-                    return cb(Errors.Invalid('Failed to create ActivityPub object'));
-                }
-
-                const actor = new Actor(obj);
-                if (!actor.isValid()) {
-                    return cb(Errors.Invalid('Failed to create Actor object'));
-                }
-
-                const subject = row.subject || actor.id;
-                return cb(null, actor, subject, needsRefresh);
             }
-        );
+
+            const needsRefresh = moment().isAfter(
+                info.timestamp.add(ActorCacheExpiration)
+            );
+
+            actor = new Actor(actor);
+            if (!actor.isValid()) {
+                return cb(Errors.Invalid('Failed to create Actor object'));
+            }
+
+            return cb(null, actor, info.subject, needsRefresh);
+        });
     }
 
     static _fromWebFinger(actorQuery, cb) {

core/activitypub/collection.js

@@ -8,6 +8,7 @@ const {
     PublicCollectionId: APPublicCollectionId,
     ActivityStreamMediaType,
     Collections,
+    ActorCollectionId,
 } = require('./const');
 const UserProps = require('../user_property');
 const { getJson } = require('../http_util');
@@ -15,6 +16,7 @@ const { getJson } = require('../http_util');
 // deps
 const { isString } = require('lodash');
 const Log = require('../logger');
+const async = require('async');
 
 module.exports = class Collection extends ActivityPubObject {
     constructor(obj) {
@@ -168,6 +170,119 @@ module.exports = class Collection extends ActivityPubObject {
         );
     }
 
+    //  Actors is a special collection
+    static actor(actorIdOrSubject, cb) {
+        // We always store subjects prefixed with '@'
+        if (!/^https?:\/\//.test(actorIdOrSubject) && '@' !== actorIdOrSubject[0]) {
+            actorIdOrSubject = `@${actorIdOrSubject}`;
+        }
+
+        apDb.get(
+            `SELECT c.name, c.timestamp, c.owner_actor_id, c.is_private, c.object_json, m.meta_value
+            FROM collection c, collection_object_meta m
+            WHERE c.collection_id = ? AND c.name = ? AND (c.object_id LIKE ? OR (m.object_id = c.object_id AND m.meta_name = ? AND m.meta_value LIKE ?))
+            LIMIT 1;`,
+            [
+                ActorCollectionId,
+                Collections.Actors,
+                actorIdOrSubject,
+                'actor_subject',
+                actorIdOrSubject,
+            ],
+            (err, row) => {
+                if (err) {
+                    return cb(err);
+                }
+
+                if (!row) {
+                    return cb(
+                        Errors.DoesNotExist(`No Actor found for "${actorIdOrSubject}"`)
+                    );
+                }
+
+                const obj = ActivityPubObject.fromJsonString(row.object_json);
+                if (!obj) {
+                    return cb(Errors.Invalid('Failed to parse Object JSON'));
+                }
+
+                const info = Collection._rowToObjectInfo(row);
+                if (row.meta_value) {
+                    info.subject = row.meta_value;
+                } else {
+                    info.subject = obj.id;
+                }
+
+                return cb(null, obj, info);
+            }
+        );
+    }
+
+    static addActor(actor, subject, cb) {
+        async.waterfall(
+            [
+                callback => {
+                    return apDb.beginTransaction(callback);
+                },
+                (trans, callback) => {
+                    trans.run(
+                        `REPLACE INTO collection (collection_id, name, timestamp, owner_actor_id, object_id, object_json, is_private)
+                        VALUES(?, ?, ?, ?, ?, ?, ?);`,
+                        [
+                            ActorCollectionId,
+                            Collections.Actors,
+                            getISOTimestampString(),
+                            APPublicCollectionId,
+                            actor.id,
+                            JSON.stringify(actor),
+                            false,
+                        ],
+                        err => {
+                            return callback(err, trans);
+                        }
+                    );
+                },
+                (trans, callback) => {
+                    trans.run(
+                        `REPLACE INTO collection_object_meta (collection_id, name, object_id, meta_name, meta_value)
+                        VALUES(?, ?, ?, ?, ?);`,
+                        [
+                            ActorCollectionId,
+                            Collections.Actors,
+                            actor.id,
+                            'actor_subject',
+                            subject,
+                        ],
+                        err => {
+                            return callback(err, trans);
+                        }
+                    );
+                },
+            ],
+            (err, trans) => {
+                if (err) {
+                    trans.rollback(err => {
+                        return cb(err);
+                    });
+                } else {
+                    trans.commit(err => {
+                        return cb(err);
+                    });
+                }
+            }
+        );
+    }
+
+    static removeOldActorEntries(maxAgeDays, cb) {
+        apDb.run(
+            `DELETE FROM collection
+            WHERE collection_id = ? AND name = ? AND DATETIME(timestamp, "+${maxAgeDays} days") > DATETIME("now");`,
+            [ActorCollectionId, Collections.Actors],
+            err => {
+                return cb(err);
+            }
+        );
+    }
+
     //  Get Object(s) by ID; There may be multiples as they may be
     //  e.g. Actors belonging to multiple followers collections.
     //  This method also returns information about the objects

core/activitypub/const.js

@@ -2,6 +2,8 @@ exports.ActivityStreamsContext = 'https://www.w3.org/ns/activitystreams';
 exports.PublicCollectionId = 'https://www.w3.org/ns/activitystreams#Public';
 exports.ActivityStreamMediaType = 'application/activity+json';
 
+exports.ActorCollectionId = exports.PublicCollectionId + 'Actors';
+
 const WellKnownActivity = {
     Create: 'Create',
     Update: 'Update',
@@ -39,5 +41,6 @@ const Collections = {
     Outbox: 'outbox',
     Inbox: 'inbox',
     SharedInbox: 'sharedInbox',
+    Actors: 'actors',
 };
 exports.Collections = Collections;

core/database.js

@@ -473,23 +473,6 @@ dbs.message.run(
     activitypub: cb => {
         enableForeignKeys(dbs.activitypub);
 
-        // Actors we know about and have cached
-        dbs.activitypub.run(
-            `CREATE TABLE IF NOT EXISTS actor_cache (
-                actor_id            VARCHAR NOT NULL,       -- Fully qualified Actor ID/URL
-                actor_json          VARCHAR NOT NULL,       -- Actor document
-                subject             VARCHAR,                -- Subject obtained from WebFinger, e.g. @Username@some.domain
-                timestamp           DATETIME NOT NULL,      -- Timestamp in which this Actor was cached
-
-                UNIQUE(actor_id)
-            );`
-        );
-
-        dbs.activitypub.run(
-            `CREATE INDEX IF NOT EXISTS actor_cache_actor_id_index0
-            ON actor_cache (actor_id);`
-        );
-
         //  ActivityPub Collections of various types such as followers, following, likes, ...
         dbs.activitypub.run(
             `CREATE TABLE IF NOT EXISTS collection (
@@ -515,6 +498,20 @@ dbs.message.run(
             ON collection (name, collection_id);`
         );
 
+        //  Collection meta contains 0:N additional metadata records for a object_id in a collection
+        dbs.activitypub.run(
+            `CREATE TABLE IF NOT EXISTS collection_object_meta (
+                collection_id   VARCHAR NOT NULL,
+                name            VARCHAR NOT NULL,
+                object_id       VARCHAR NOT NULL,
+                meta_name       VARCHAR NOT NULL,
+                meta_value      VARCHAR NOT NULL,
+
+                UNIQUE(collection_id, object_id, meta_name),
+                FOREIGN KEY(name, collection_id, object_id) REFERENCES collection(name, collection_id, object_id) ON DELETE CASCADE
+            );`
+        );
+
         return cb(null);
     },
 };