Remove actor_cache; use a special collection
This commit is contained in:
parent
51c58b5d8a
commit
036a3dcd58
|
@ -12,16 +12,15 @@ const EnigAssert = require('../enigma_assert');
|
|||
const ActivityPubSettings = require('./settings');
|
||||
const ActivityPubObject = require('./object');
|
||||
const { ActivityStreamMediaType, Collections } = require('./const');
|
||||
const apDb = require('../database').dbs.activitypub;
|
||||
const Config = require('../config').get;
|
||||
|
||||
// deps
|
||||
const _ = require('lodash');
|
||||
const mimeTypes = require('mime-types');
|
||||
const { getJson } = require('../http_util.js');
|
||||
const { getISOTimestampString } = require('../database.js');
|
||||
const moment = require('moment');
|
||||
const paths = require('path');
|
||||
const Collection = require('./collection.js');
|
||||
|
||||
const ActorCacheExpiration = moment.duration(15, 'days');
|
||||
const ActorCacheMaxAgeDays = 125; // hasn't been used in >= 125 days, nuke it.
|
||||
|
@ -204,16 +203,11 @@ module.exports = class Actor extends ActivityPubObject {
|
|||
|
||||
// cache our entry
|
||||
if (actor) {
|
||||
apDb.run(
|
||||
`REPLACE INTO actor_cache (actor_id, actor_json, subject, timestamp)
|
||||
VALUES (?, ?, ?, ?);`,
|
||||
[id, JSON.stringify(actor), subject, getISOTimestampString()],
|
||||
err => {
|
||||
if (err) {
|
||||
// :TODO: log me
|
||||
}
|
||||
Collection.addActor(actor, subject, err => {
|
||||
if (err) {
|
||||
// :TODO: Log me
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
@ -228,17 +222,13 @@ module.exports = class Actor extends ActivityPubObject {
|
|||
return;
|
||||
}
|
||||
|
||||
apDb.run(
|
||||
`DELETE FROM actor_cache
|
||||
WHERE DATETIME(timestamp, "+${ActorCacheMaxAgeDays} days") > DATETIME("now");`,
|
||||
err => {
|
||||
if (err) {
|
||||
// :TODO: log me
|
||||
}
|
||||
|
||||
return cb(null); // always non-fatal
|
||||
Collection.removeOldActorEntries(ActorCacheMaxAgeDays, err => {
|
||||
if (err) {
|
||||
// :TODO: log me
|
||||
}
|
||||
);
|
||||
|
||||
return cb(null); // always non-fatal
|
||||
});
|
||||
}
|
||||
|
||||
static _fromRemoteQuery(id, cb) {
|
||||
|
@ -262,40 +252,22 @@ module.exports = class Actor extends ActivityPubObject {
|
|||
}
|
||||
|
||||
static _fromCache(actorIdOrSubject, cb) {
|
||||
apDb.get(
|
||||
`SELECT actor_json, subject, timestamp
|
||||
FROM actor_cache
|
||||
WHERE actor_id = ? OR subject = ?
|
||||
LIMIT 1;`,
|
||||
[actorIdOrSubject, actorIdOrSubject],
|
||||
(err, row) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
if (!row) {
|
||||
return cb(Errors.DoesNotExist());
|
||||
}
|
||||
|
||||
const timestamp = moment(row.timestamp);
|
||||
const needsRefresh = moment().isAfter(
|
||||
timestamp.add(ActorCacheExpiration)
|
||||
);
|
||||
|
||||
const obj = ActivityPubObject.fromJsonString(row.actor_json);
|
||||
if (!obj || !obj.isValid()) {
|
||||
return cb(Errors.Invalid('Failed to create ActivityPub object'));
|
||||
}
|
||||
|
||||
const actor = new Actor(obj);
|
||||
if (!actor.isValid()) {
|
||||
return cb(Errors.Invalid('Failed to create Actor object'));
|
||||
}
|
||||
|
||||
const subject = row.subject || actor.id;
|
||||
return cb(null, actor, subject, needsRefresh);
|
||||
Collection.actor(actorIdOrSubject, (err, actor, info) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
);
|
||||
|
||||
const needsRefresh = moment().isAfter(
|
||||
info.timestamp.add(ActorCacheExpiration)
|
||||
);
|
||||
|
||||
actor = new Actor(actor);
|
||||
if (!actor.isValid()) {
|
||||
return cb(Errors.Invalid('Failed to create Actor object'));
|
||||
}
|
||||
|
||||
return cb(null, actor, info.subject, needsRefresh);
|
||||
});
|
||||
}
|
||||
|
||||
static _fromWebFinger(actorQuery, cb) {
|
||||
|
|
|
@ -8,6 +8,7 @@ const {
|
|||
PublicCollectionId: APPublicCollectionId,
|
||||
ActivityStreamMediaType,
|
||||
Collections,
|
||||
ActorCollectionId,
|
||||
} = require('./const');
|
||||
const UserProps = require('../user_property');
|
||||
const { getJson } = require('../http_util');
|
||||
|
@ -15,6 +16,7 @@ const { getJson } = require('../http_util');
|
|||
// deps
|
||||
const { isString } = require('lodash');
|
||||
const Log = require('../logger');
|
||||
const async = require('async');
|
||||
|
||||
module.exports = class Collection extends ActivityPubObject {
|
||||
constructor(obj) {
|
||||
|
@ -168,6 +170,119 @@ module.exports = class Collection extends ActivityPubObject {
|
|||
);
|
||||
}
|
||||
|
||||
// Actors is a special collection
|
||||
static actor(actorIdOrSubject, cb) {
|
||||
// We always store subjects prefixed with '@'
|
||||
if (!/^https?:\/\//.test(actorIdOrSubject) && '@' !== actorIdOrSubject[0]) {
|
||||
actorIdOrSubject = `@${actorIdOrSubject}`;
|
||||
}
|
||||
|
||||
apDb.get(
|
||||
`SELECT c.name, c.timestamp, c.owner_actor_id, c.is_private, c.object_json, m.meta_value
|
||||
FROM collection c, collection_object_meta m
|
||||
WHERE c.collection_id = ? AND c.name = ? AND (c.object_id LIKE ? OR (m.object_id = c.object_id AND m.meta_name = ? AND m.meta_value LIKE ?))
|
||||
LIMIT 1;`,
|
||||
[
|
||||
ActorCollectionId,
|
||||
Collections.Actors,
|
||||
actorIdOrSubject,
|
||||
'actor_subject',
|
||||
actorIdOrSubject,
|
||||
],
|
||||
(err, row) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
if (!row) {
|
||||
return cb(
|
||||
Errors.DoesNotExist(`No Actor found for "${actorIdOrSubject}"`)
|
||||
);
|
||||
}
|
||||
|
||||
const obj = ActivityPubObject.fromJsonString(row.object_json);
|
||||
if (!obj) {
|
||||
return cb(Errors.Invalid('Failed to parse Object JSON'));
|
||||
}
|
||||
|
||||
const info = Collection._rowToObjectInfo(row);
|
||||
if (row.meta_value) {
|
||||
info.subject = row.meta_value;
|
||||
} else {
|
||||
info.subject = obj.id;
|
||||
}
|
||||
|
||||
return cb(null, obj, info);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
static addActor(actor, subject, cb) {
|
||||
async.waterfall(
|
||||
[
|
||||
callback => {
|
||||
return apDb.beginTransaction(callback);
|
||||
},
|
||||
(trans, callback) => {
|
||||
trans.run(
|
||||
`REPLACE INTO collection (collection_id, name, timestamp, owner_actor_id, object_id, object_json, is_private)
|
||||
VALUES(?, ?, ?, ?, ?, ?, ?);`,
|
||||
[
|
||||
ActorCollectionId,
|
||||
Collections.Actors,
|
||||
getISOTimestampString(),
|
||||
APPublicCollectionId,
|
||||
actor.id,
|
||||
JSON.stringify(actor),
|
||||
false,
|
||||
],
|
||||
err => {
|
||||
return callback(err, trans);
|
||||
}
|
||||
);
|
||||
},
|
||||
(trans, callback) => {
|
||||
trans.run(
|
||||
`REPLACE INTO collection_object_meta (collection_id, name, object_id, meta_name, meta_value)
|
||||
VALUES(?, ?, ?, ?, ?);`,
|
||||
[
|
||||
ActorCollectionId,
|
||||
Collections.Actors,
|
||||
actor.id,
|
||||
'actor_subject',
|
||||
subject,
|
||||
],
|
||||
err => {
|
||||
return callback(err, trans);
|
||||
}
|
||||
);
|
||||
},
|
||||
],
|
||||
(err, trans) => {
|
||||
if (err) {
|
||||
trans.rollback(err => {
|
||||
return cb(err);
|
||||
});
|
||||
} else {
|
||||
trans.commit(err => {
|
||||
return cb(err);
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
static removeOldActorEntries(maxAgeDays, cb) {
|
||||
apDb.run(
|
||||
`DELETE FROM collection
|
||||
WHERE collection_id = ? AND name = ? AND DATETIME(timestamp, "+${maxAgeDays} days") > DATETIME("now");`,
|
||||
[ActorCollectionId, Collections.Actors],
|
||||
err => {
|
||||
return cb(err);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Get Object(s) by ID; There may be multiples as they may be
|
||||
// e.g. Actors belonging to multiple followers collections.
|
||||
// This method also returns information about the objects
|
||||
|
|
|
@ -2,6 +2,8 @@ exports.ActivityStreamsContext = 'https://www.w3.org/ns/activitystreams';
|
|||
exports.PublicCollectionId = 'https://www.w3.org/ns/activitystreams#Public';
|
||||
exports.ActivityStreamMediaType = 'application/activity+json';
|
||||
|
||||
exports.ActorCollectionId = exports.PublicCollectionId + 'Actors';
|
||||
|
||||
const WellKnownActivity = {
|
||||
Create: 'Create',
|
||||
Update: 'Update',
|
||||
|
@ -39,5 +41,6 @@ const Collections = {
|
|||
Outbox: 'outbox',
|
||||
Inbox: 'inbox',
|
||||
SharedInbox: 'sharedInbox',
|
||||
Actors: 'actors',
|
||||
};
|
||||
exports.Collections = Collections;
|
||||
|
|
|
@ -473,23 +473,6 @@ dbs.message.run(
|
|||
activitypub: cb => {
|
||||
enableForeignKeys(dbs.activitypub);
|
||||
|
||||
// Actors we know about and have cached
|
||||
dbs.activitypub.run(
|
||||
`CREATE TABLE IF NOT EXISTS actor_cache (
|
||||
actor_id VARCHAR NOT NULL, -- Fully qualified Actor ID/URL
|
||||
actor_json VARCHAR NOT NULL, -- Actor document
|
||||
subject VARCHAR, -- Subject obtained from WebFinger, e.g. @Username@some.domain
|
||||
timestamp DATETIME NOT NULL, -- Timestamp in which this Actor was cached
|
||||
|
||||
UNIQUE(actor_id)
|
||||
);`
|
||||
);
|
||||
|
||||
dbs.activitypub.run(
|
||||
`CREATE INDEX IF NOT EXISTS actor_cache_actor_id_index0
|
||||
ON actor_cache (actor_id);`
|
||||
);
|
||||
|
||||
// ActivityPub Collections of various types such as followers, following, likes, ...
|
||||
dbs.activitypub.run(
|
||||
`CREATE TABLE IF NOT EXISTS collection (
|
||||
|
@ -515,6 +498,20 @@ dbs.message.run(
|
|||
ON collection (name, collection_id);`
|
||||
);
|
||||
|
||||
// Collection meta contains 0:N additional metadata records for a object_id in a collection
|
||||
dbs.activitypub.run(
|
||||
`CREATE TABLE IF NOT EXISTS collection_object_meta (
|
||||
collection_id VARCHAR NOT NULL,
|
||||
name VARCHAR NOT NULL,
|
||||
object_id VARCHAR NOT NULL,
|
||||
meta_name VARCHAR NOT NULL,
|
||||
meta_value VARCHAR NOT NULL,
|
||||
|
||||
UNIQUE(collection_id, object_id, meta_name),
|
||||
FOREIGN KEY(name, collection_id, object_id) REFERENCES collection(name, collection_id, object_id) ON DELETE CASCADE
|
||||
);`
|
||||
);
|
||||
|
||||
return cb(null);
|
||||
},
|
||||
};
|
||||
|
|
Loading…
Reference in New Issue